From 9cad451a88f27afe80f41499029a3268dd1dddff Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 25 Jul 2023 16:43:36 +0100
Subject: [PATCH 001/110] Bump dev version 8.3.0.dev

---
 cylc/flow/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py
index ac1743a704f..bc9f24aa573 100644
--- a/cylc/flow/__init__.py
+++ b/cylc/flow/__init__.py
@@ -53,7 +53,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.2.1.dev'
+__version__ = '8.3.0.dev'
 
 
 def iter_entry_points(entry_point_name):

From 69e08792d86a4dd40f4d85f37e5fd88908bc9680 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Wed, 7 Jun 2023 11:33:58 +1200
Subject: [PATCH 002/110] Add CYLC_ variables to template engine globals.

---
 cylc/flow/parsec/empysupport.py   |  7 +++
 cylc/flow/parsec/fileparse.py     | 24 +++++++++-
 cylc/flow/parsec/jinja2support.py | 23 ++++------
 cylc/flow/scripts/view.py         | 37 ++++++++++------
 tests/unit/test_config.py         | 73 +++++++++++++++++++++++++++++++
 5 files changed, 133 insertions(+), 31 deletions(-)

diff --git a/cylc/flow/parsec/empysupport.py b/cylc/flow/parsec/empysupport.py
index 7bc8a69e3c4..b4164894e0f 100644
--- a/cylc/flow/parsec/empysupport.py
+++ b/cylc/flow/parsec/empysupport.py
@@ -24,6 +24,7 @@
 import typing as t
 
 from cylc.flow.parsec.exceptions import EmPyError
+from cylc.flow.parsec.fileparse import get_cylc_env_vars
 
 
 def empyprocess(
@@ -52,6 +53,12 @@ def empyprocess(
     ftempl = StringIO('\n'.join(flines))
     xtempl = StringIO()
     interpreter = em.Interpreter(output=em.UncloseableFile(xtempl))
+
+    # Add `CYLC_` environment variables to the global namespace.
+    interpreter.updateGlobals(
+        get_cylc_env_vars()
+    )
+
     try:
         interpreter.file(ftempl, '<template>', template_vars)
     except Exception as exc:
diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py
index 9dd757c7cdb..3c954821ea4 100644
--- a/cylc/flow/parsec/fileparse.py
+++ b/cylc/flow/parsec/fileparse.py
@@ -105,6 +105,27 @@
 )
 
 
+def get_cylc_env_vars() -> t.Dict[str, str]:
+    """Return a restricted dict of CYLC_ environment variables for templating.
+
+    The following variables are ignored because the do not necessarily reflect
+    the running code version (I might not use the "cylc" wrapper, or it might
+    select a different version):
+
+    CYLC_VERSION
+        Set as a template variable elsewhere, from the hardwired code version.
+
+    CYLC_ENV_NAME
+        Providing it as a template variable would just be misleading.
+    """
+    return {
+        key: val
+        for key, val in os.environ.items()
+        if key.startswith('CYLC_')
+        if key not in ["CYLC_VERSION", "CYLC_ENV_NAME"]
+    }
+
+
 def _concatenate(lines):
     """concatenate continuation lines"""
     index = 0
@@ -446,10 +467,9 @@ def read_and_proc(
         flines = inline(
             flines, fdir, fpath, viewcfg=viewcfg)
 
+    # Add the hardwired code version to template vars as CYLC_VERSION
     template_vars['CYLC_VERSION'] = __version__
-
     template_vars = merge_template_vars(template_vars, extra_vars)
-
     template_vars['CYLC_TEMPLATE_VARS'] = template_vars
 
     # Fail if templating_detected ≠ hashbang
diff --git a/cylc/flow/parsec/jinja2support.py b/cylc/flow/parsec/jinja2support.py
index 5a39dbd01fe..34f92c11519 100644
--- a/cylc/flow/parsec/jinja2support.py
+++ b/cylc/flow/parsec/jinja2support.py
@@ -39,6 +39,7 @@
 
 from cylc.flow import LOG
 from cylc.flow.parsec.exceptions import Jinja2Error
+from cylc.flow.parsec.fileparse import get_cylc_env_vars
 
 TRACEBACK_LINENO = re.compile(
     r'\s+?File "(?P<file>.*)", line (?P<line>\d+), in .*template'
@@ -186,11 +187,15 @@ def jinja2environment(dir_=None):
                     envnsp[fname] = getattr(module, fname)
 
     # Import WORKFLOW HOST USER ENVIRONMENT into template:
-    # (usage e.g.: {{environ['HOME']}}).
+    # (Usage e.g.: {{environ['HOME']}}).
     env.globals['environ'] = os.environ
     env.globals['raise'] = raise_helper
     env.globals['assert'] = assert_helper
 
+    # Add `CYLC_` environment variables to the global namespace.
+    env.globals.update(
+        get_cylc_env_vars()
+    )
     return env
 
 
@@ -269,7 +274,6 @@ def jinja2process(
     # CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
     # AND TYPEERROR (e.g. for not using "|int" filter on number inputs.
     # Convert unicode to plain str, ToDo - still needed for parsec?)
-
     try:
         env = jinja2environment(dir_)
         template = env.from_string('\n'.join(flines[1:]))
@@ -300,16 +304,5 @@ def jinja2process(
             lines=get_error_lines(fpath, flines),
         )
 
-    flow_config = []
-    for line in lines:
-        # Jinja2 leaves blank lines where source lines contain
-        # only Jinja2 code; this matters if line continuation
-        # markers are involved, so we remove blank lines here.
-        if not line.strip():
-            continue
-            # restoring newlines here is only necessary for display by
-        # the cylc view command:
-        # ##flow_config.append(line + '\n')
-        flow_config.append(line)
-
-    return flow_config
+    # Ignore blank lines (lone Jinja2 statements leave blank lines behind)
+    return [line for line in lines if line.strip()]
diff --git a/cylc/flow/scripts/view.py b/cylc/flow/scripts/view.py
index 495af07a412..3fe5c59e49d 100644
--- a/cylc/flow/scripts/view.py
+++ b/cylc/flow/scripts/view.py
@@ -20,9 +20,17 @@
 
 Print a processed workflow configuration.
 
-Note:
-  This is different to `cylc config` which displays the parsed
-  configuration (as Cylc would see it).
+Print workflow configurations as processed before full parsing by Cylc. This
+includes Jinja2 or Empy template processing, and inlining of include-files.
+Some explanatory markup may also be requested.
+
+Warning:
+  This command will fail if `CYLC_` template variables are referenced
+  without default values, because they are only defined for full parsing.
+  E.g. (Jinja2): `{{CYLC_WORKFLOW_ID | default("not defined")}}`.
+
+See also `cylc config`, which displays the fully parsed configuration.
+
 """
 
 import asyncio
@@ -115,20 +123,21 @@ async def _main(options: 'Values', workflow_id: str) -> None:
         constraint='workflows',
     )
     # read in the flow.cylc file
-    viewcfg = {
-        'mark': options.mark,
-        'single': options.single,
-        'label': options.label,
-        'empy': options.empy or options.process,
-        'jinja2': options.jinja2 or options.process,
-        'contin': options.cat or options.process,
-        'inline': (options.inline or options.jinja2 or options.empy
-                   or options.process),
-    }
     for line in read_and_proc(
         flow_file,
         get_template_vars(options),
-        viewcfg=viewcfg,
+        viewcfg={
+            'mark': options.mark,
+            'single': options.single,
+            'label': options.label,
+            'empy': options.empy or options.process,
+            'jinja2': options.jinja2 or options.process,
+            'contin': options.cat or options.process,
+            'inline': (
+                options.jinja2 or options.empy or
+                options.inline or options.process
+            ),
+        },
         opts=options,
     ):
         print(line)
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 736bd34ed9d..34c596a770c 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import sys
 from optparse import Values
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type
 from pathlib import Path
@@ -34,6 +35,7 @@
     WorkflowConfigError,
     XtriggerConfigError,
 )
+from cylc.flow.parsec.exceptions import Jinja2Error, EmPyError
 from cylc.flow.scheduler_cli import RunOptions
 from cylc.flow.scripts.validate import ValidateOptions
 from cylc.flow.workflow_files import WorkflowFiles
@@ -1019,6 +1021,77 @@ def test_rsync_includes_will_not_accept_sub_directories(tmp_flow_config):
     assert "Directories can only be from the top level" in str(exc.value)
 
 
+@pytest.mark.parametrize(
+    'cylc_var, expected_err',
+    [
+        ["CYLC_WORKFLOW_NAME", None],
+        ["CYLC_BEEF_WELLINGTON", (Jinja2Error, "is undefined")],
+    ]
+)
+def test_jinja2_cylc_vars(tmp_flow_config, cylc_var, expected_err):
+    """Defined CYLC_ variables should be available to Jinja2 during parsing.
+
+    This test is not located in the jinja2_support unit test module because
+    CYLC_ variables are only defined during workflow config parsing.
+    """
+    reg = 'nodule'
+    flow_file = tmp_flow_config(reg, """#!Jinja2
+    # {{""" + cylc_var + """}}
+    [scheduler]
+        allow implicit tasks = True
+    [scheduling]
+        [[graph]]
+            R1 = foo
+    """)
+    if expected_err is None:
+        WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+    else:
+        with pytest.raises(expected_err[0]) as exc:
+            WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+        assert expected_err[1] in str(exc)
+
+
+@pytest.mark.parametrize(
+    'cylc_var, expected_err',
+    [
+        ["CYLC_WORKFLOW_NAME", None],
+        ["CYLC_BEEF_WELLINGTON", (EmPyError, "is not defined")],
+    ]
+)
+def test_empy_cylc_vars(tmp_flow_config, cylc_var, expected_err):
+    """Defined CYLC_ variables should be available to empy during parsing.
+
+    This test is not located in the empy_support unit test module because
+    CYLC_ variables are only defined during workflow config parsing.
+    """
+    reg = 'nodule'
+    flow_file = tmp_flow_config(reg, """#!empy
+    # @(""" + cylc_var + """)
+    [scheduler]
+        allow implicit tasks = True
+    [scheduling]
+        [[graph]]
+            R1 = foo
+    """)
+
+    # empy replaces sys.stdout with a "proxy". And pytest needs it for capture?
+    # (clue: "pytest --capture=no" avoids the error)
+    stdout = sys.stdout
+    sys.stdout._testProxy = lambda: ''
+    sys.stdout.pop = lambda _: ''
+    sys.stdout.push = lambda _: ''
+    sys.stdout.clear = lambda _: ''
+
+    if expected_err is None:
+        WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+    else:
+        with pytest.raises(expected_err[0]) as exc:
+            WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+        assert expected_err[1] in str(exc)
+
+    sys.stdout = stdout
+
+
 def test_valid_rsync_includes_returns_correct_list(tmp_flow_config):
     """Test that the rsync includes in the correct """
     id_ = 'rsynctest'

From 7f0d80badd8e45f1966039fe15627aa4450c5b7f Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Mon, 19 Jun 2023 19:48:04 +1200
Subject: [PATCH 003/110] Update change log.

---
 changes.d/5571.feat.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes.d/5571.feat.md

diff --git a/changes.d/5571.feat.md b/changes.d/5571.feat.md
new file mode 100644
index 00000000000..4bda6c6af4b
--- /dev/null
+++ b/changes.d/5571.feat.md
@@ -0,0 +1 @@
+Make workflow `CYLC_` variables available to the template processor during parsing.

From e1fdcf017f7c5057f99469a1e8816a9dce227686 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Wed, 26 Jul 2023 18:31:31 +1200
Subject: [PATCH 004/110] Add new unit test.

---
 tests/unit/parsec/test_fileparse.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/unit/parsec/test_fileparse.py b/tests/unit/parsec/test_fileparse.py
index 04b7245f492..a4bdbcb8ea8 100644
--- a/tests/unit/parsec/test_fileparse.py
+++ b/tests/unit/parsec/test_fileparse.py
@@ -34,6 +34,7 @@
 from cylc.flow.parsec.fileparse import (
     _prepend_old_templatevars,
     _get_fpath_for_source,
+    get_cylc_env_vars,
     addict,
     addsect,
     multiline,
@@ -736,3 +737,23 @@ def test_get_fpath_for_source(tmp_path):
     opts.against_source = True
     assert _get_fpath_for_source(
         rundir / 'flow.cylc', opts) == str(srcdir / 'flow.cylc')
+
+
+def test_get_cylc_env_vars(monkeypatch):
+    """It should return CYLC env vars but not CYLC_VERSION or CYLC_ENV_NAME."""
+    monkeypatch.setattr(
+        'os.environ',
+        {
+            "CYLC_VERSION": "betwixt",
+            "CYLC_ENV_NAME": "between",
+            "CYLC_QUESTION": "que?", 
+            "CYLC_ANSWER": "42", 
+            "FOO": "foo"
+        }
+    )
+    assert (
+        get_cylc_env_vars() == {
+            "CYLC_QUESTION": "que?", 
+            "CYLC_ANSWER": "42", 
+        }
+    )

From bffda20383c792dea57875acd791b1dddd5c4b6a Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Tue, 25 Jul 2023 13:22:31 +1200
Subject: [PATCH 005/110] Fix functional test.

---
 tests/functional/cylc-cat-log/01-remote.t | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/functional/cylc-cat-log/01-remote.t b/tests/functional/cylc-cat-log/01-remote.t
index cec9de4d8ab..1e61c6ca2c9 100755
--- a/tests/functional/cylc-cat-log/01-remote.t
+++ b/tests/functional/cylc-cat-log/01-remote.t
@@ -30,7 +30,9 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
 run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
 #-------------------------------------------------------------------------------
 TEST_NAME="${TEST_NAME_BASE}-run"
-workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}"
+workflow_run_ok "${TEST_NAME}" \
+    cylc play --debug --no-detach \
+        -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}"
 #-------------------------------------------------------------------------------
 TEST_NAME=${TEST_NAME_BASE}-task-out
 cylc cat-log -f o "${WORKFLOW_NAME}//1/a-task" >"${TEST_NAME}.out"
@@ -56,7 +58,7 @@ grep_ok "jumped over the lazy dog" "${TEST_NAME}.out"
 # remote
 TEST_NAME=${TEST_NAME_BASE}-task-status
 cylc cat-log -f s "${WORKFLOW_NAME}//1/a-task" >"${TEST_NAME}.out"
-grep_ok "CYLC_JOB_RUNNER_NAME=background" "${TEST_NAME}.out"
+grep_ok "CYLC_JOB_RUNNER_NAME=at" "${TEST_NAME}.out"
 #-------------------------------------------------------------------------------
 # local
 TEST_NAME=${TEST_NAME_BASE}-task-activity

From d658eb6e972abc3ad08a2e0ebccabbef22f500ea Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Aug 2023 09:41:34 +0100
Subject: [PATCH 006/110] Bump pypa/gh-action-pypi-publish from 1.8.8 to 1.8.10
 (#5684)

---
 .github/workflows/2_auto_publish_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 99aac4991d6..ba8e03df093 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.8
+      uses: pypa/gh-action-pypi-publish@v1.8.10
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

From 47203266119874167ab494dcb380e5408c8400d8 Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Mon, 28 Aug 2023 16:47:11 +1000
Subject: [PATCH 007/110] Forward arbitrary environment variables over SSH

Allow users to specify environment variables that get forwarded from the
submit host to the scheduler and task submission.

Define variables to forward in `global.cylc` like:
```
[platforms]
    [[localhost]]
        ssh forward environment variables = PROJECT, LUSTRE_DISK
```

This will add `PROJECT` and `LUSTRE_DISK` to the list of variables
exported in SSH commands to launch the scheduler on remote hosts (if
they have been set in the current environment).

Once they are available on the scheduler they can further be forwarded
to other platforms, where they may interact with the scheduler to
set a default project code or be used to set storage paths:
```
[platforms]
    [[mtu]]
        ssh forward environment variables = PROJECT, LUSTRE_DISK
        install target = mtu

[install]
    [[symlink dirs]]
        [[[mtu]]]
            share = $LUSTRE_DISK/$PROJECT/$USER
```
---
 CONTRIBUTING.md                |  2 +-
 cylc/flow/cfgspec/globalcfg.py |  7 +++++++
 cylc/flow/remote.py            |  3 ++-
 tests/unit/test_remote.py      | 29 ++++++++++++++++++++++++++++-
 4 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6fbc4283968..911dff536ba 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -64,7 +64,7 @@ requests_).
  - Prasanna Challuri
  - David Matthews
  - Tim Whitcomb
- - (Scott Wales)
+ - Scott Wales
  - Tomek Trzeciak
  - Thomas Coleman
  - Bruno Kinoshita
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 3c01ff52729..b3a25252c09 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -1416,6 +1416,13 @@ def default_for(
                    {REPLACES}``global.rc[hosts][<host>]copyable
                    environment variables``.
             ''')
+            Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
+                 desc=f'''
+                A list containing the names of the environment variables to
+                forward with SSH connections to the server and run hosts
+
+                .. versionchanged:: 8.3.0
+            ''')
             Conf('retrieve job logs', VDR.V_BOOLEAN,
                  desc=f'''
                 {LOG_RETR_SETTINGS['retrieve job logs']}
diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py
index 62cd77fd9af..8a72d56b6d6 100644
--- a/cylc/flow/remote.py
+++ b/cylc/flow/remote.py
@@ -298,7 +298,8 @@ def construct_ssh_cmd(
         'CYLC_CONF_PATH',
         'CYLC_COVERAGE',
         'CLIENT_COMMS_METH',
-        'CYLC_ENV_NAME'
+        'CYLC_ENV_NAME',
+        *platform['ssh forward environment variables'],
     ]:
         if envvar in os.environ:
             command.append(
diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py
index 7be01de2e20..6e86ac1a410 100644
--- a/tests/unit/test_remote.py
+++ b/tests/unit/test_remote.py
@@ -15,7 +15,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Test the cylc.flow.remote module."""
 
-from cylc.flow.remote import run_cmd, construct_rsync_over_ssh_cmd
+from cylc.flow.remote import run_cmd, construct_rsync_over_ssh_cmd, construct_ssh_cmd
+from unittest import mock
+import cylc.flow
 
 
 def test_run_cmd_stdin_str():
@@ -86,3 +88,28 @@ def test_construct_rsync_over_ssh_cmd():
         '/foo/',
         'miklegard:/bar/',
     ]
+
+
+def test_construct_ssh_cmd_forward_env():
+    """ Test for 'ssh forward environment variables'
+    """
+    import os
+
+    host = 'example.com'
+    config = {
+        'ssh command': 'ssh',
+        'use login shell': None,
+        'cylc path': None,
+        'ssh forward environment variables': ['FOO'],
+        }
+
+    # Variable isn't set, no change to command
+    expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'cylc', 'play']
+    cmd = construct_ssh_cmd(['play'], config, host)
+    assert cmd == expect
+
+    # Variable is set, appears in `env` list
+    with mock.patch.dict(os.environ, {'FOO': 'BAR'}):
+        expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'FOO=BAR', 'cylc', 'play']
+        cmd = construct_ssh_cmd(['play'], config, host)
+        assert cmd == expect

From 833f89c5411a60734044b2d948603c0dcdbce2a0 Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Tue, 29 Aug 2023 10:29:43 +1000
Subject: [PATCH 008/110] Flake8 fix

---
 cylc/flow/cfgspec/globalcfg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index b3a25252c09..2182ee32edb 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -1417,7 +1417,7 @@ def default_for(
                    environment variables``.
             ''')
             Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
-                 desc=f'''
+                 desc='''
                 A list containing the names of the environment variables to
                 forward with SSH connections to the server and run hosts
 

From 253674559a458dc3764b826432f4f55f25282cfb Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Tue, 29 Aug 2023 11:03:40 +1000
Subject: [PATCH 009/110] Add changelog

---
 changes.d/5709.feat.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes.d/5709.feat.md

diff --git a/changes.d/5709.feat.md b/changes.d/5709.feat.md
new file mode 100644
index 00000000000..11aeabcf81d
--- /dev/null
+++ b/changes.d/5709.feat.md
@@ -0,0 +1 @@
+Forward arbitrary environment variables over SSH connections

From 5b76dbaa3d3acc0e3bb11810e9501fb5c2d2fdc0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 6 Sep 2023 12:02:31 +0100
Subject: [PATCH 010/110] Bump actions/checkout from 3 to 4 (#5717)

---
 .github/workflows/1_create_release_pr.yml    | 2 +-
 .github/workflows/2_auto_publish_release.yml | 2 +-
 .github/workflows/bash.yml                   | 2 +-
 .github/workflows/branch_sync.yml            | 2 +-
 .github/workflows/build.yml                  | 2 +-
 .github/workflows/shortlog.yml               | 2 +-
 .github/workflows/test_conda-build.yml       | 2 +-
 .github/workflows/test_fast.yml              | 4 ++--
 .github/workflows/test_functional.yml        | 4 ++--
 .github/workflows/test_manylinux.yml         | 2 +-
 .github/workflows/test_tutorial_workflow.yml | 2 +-
 .github/workflows/update_copyright.yml       | 2 +-
 12 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml
index f94b8d60873..6d792304196 100644
--- a/.github/workflows/1_create_release_pr.yml
+++ b/.github/workflows/1_create_release_pr.yml
@@ -21,7 +21,7 @@ jobs:
         uses: cylc/release-actions/stage-1/sanitize-inputs@v1
 
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE_REF }}
           fetch-depth: 0  # need to fetch all commits to check contributors
diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index ba8e03df093..02ea52d678c 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ env.MERGE_SHA }}
 
diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml
index cf0e49c362b..4d740df23ac 100644
--- a/.github/workflows/bash.yml
+++ b/.github/workflows/bash.yml
@@ -44,7 +44,7 @@ jobs:
           - '5.0'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Docker container
         run: |
diff --git a/.github/workflows/branch_sync.yml b/.github/workflows/branch_sync.yml
index 0056722926f..d0e5be512e5 100644
--- a/.github/workflows/branch_sync.yml
+++ b/.github/workflows/branch_sync.yml
@@ -48,7 +48,7 @@ jobs:
             print(f'SYNC_BRANCH={branch}-sync', file=F)
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: master
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 041b6e50a70..51bf49500f9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -24,7 +24,7 @@ jobs:
             python: '3.7'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4
diff --git a/.github/workflows/shortlog.yml b/.github/workflows/shortlog.yml
index 7998d92ad3f..25e2b0427e3 100644
--- a/.github/workflows/shortlog.yml
+++ b/.github/workflows/shortlog.yml
@@ -14,7 +14,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # need to fetch all commits to check contributors
 
diff --git a/.github/workflows/test_conda-build.yml b/.github/workflows/test_conda-build.yml
index a077e798366..91d1ac2ad6b 100644
--- a/.github/workflows/test_conda-build.yml
+++ b/.github/workflows/test_conda-build.yml
@@ -19,7 +19,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: checkout cylc-flow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: build conda env
         run: |
diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 6cc1ebb6c14..8d6a5041c66 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -28,7 +28,7 @@ jobs:
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
         uses: actions/setup-python@v4
@@ -112,7 +112,7 @@ jobs:
     timeout-minutes: 2
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download coverage artifacts
         uses: actions/download-artifact@v3
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index 2c3915b6caf..75dac06a964 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -96,7 +96,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
         uses: actions/setup-python@v4
@@ -306,7 +306,7 @@ jobs:
     timeout-minutes: 2
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download coverage artifacts
         uses: actions/download-artifact@v3
diff --git a/.github/workflows/test_manylinux.yml b/.github/workflows/test_manylinux.yml
index e3d21cee291..5b57b4973e7 100644
--- a/.github/workflows/test_manylinux.yml
+++ b/.github/workflows/test_manylinux.yml
@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
diff --git a/.github/workflows/test_tutorial_workflow.yml b/.github/workflows/test_tutorial_workflow.yml
index 1e1fec1746e..c95439923d6 100644
--- a/.github/workflows/test_tutorial_workflow.yml
+++ b/.github/workflows/test_tutorial_workflow.yml
@@ -31,7 +31,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install docs/tutorial dependencies
         uses: cylc/cylc-doc/.github/actions/install-dependencies@master
diff --git a/.github/workflows/update_copyright.yml b/.github/workflows/update_copyright.yml
index 603973f4d83..878412a1a27 100644
--- a/.github/workflows/update_copyright.yml
+++ b/.github/workflows/update_copyright.yml
@@ -12,7 +12,7 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Configure git
       uses: cylc/release-actions/configure-git@v1

From 9707f0a77e79da6798026241af4932b1d86ddfe8 Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Wed, 13 Sep 2023 16:37:31 +1000
Subject: [PATCH 011/110] Move config option to under 'run hosts'

---
 cylc/flow/cfgspec/globalcfg.py | 15 ++++++++-------
 cylc/flow/remote.py            |  5 ++++-
 tests/unit/test_remote.py      | 12 ++++++++++--
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 2182ee32edb..1b97e8258c0 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -784,6 +784,14 @@ def default_for(
 
                    {REPLACES}``[suite servers][run host select]rank``.
             ''')
+            Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
+                 desc='''
+                A list containing the names of the environment variables to
+                forward with SSH connections to the run and job hosts from
+                the host running 'cylc play'
+
+                .. versionchanged:: 8.3.0
+            ''')
 
         with Conf('host self-identification', desc=f'''
             How Cylc determines and shares the identity of the workflow host.
@@ -1416,13 +1424,6 @@ def default_for(
                    {REPLACES}``global.rc[hosts][<host>]copyable
                    environment variables``.
             ''')
-            Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
-                 desc='''
-                A list containing the names of the environment variables to
-                forward with SSH connections to the server and run hosts
-
-                .. versionchanged:: 8.3.0
-            ''')
             Conf('retrieve job logs', VDR.V_BOOLEAN,
                  desc=f'''
                 {LOG_RETR_SETTINGS['retrieve job logs']}
diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py
index 8a72d56b6d6..5d9ed7c3937 100644
--- a/cylc/flow/remote.py
+++ b/cylc/flow/remote.py
@@ -35,6 +35,7 @@
 from cylc.flow.option_parsers import verbosity_to_opts
 from cylc.flow.platforms import get_platform, get_host_from_platform
 from cylc.flow.util import format_cmd
+from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 
 
 def get_proc_ancestors():
@@ -299,7 +300,9 @@ def construct_ssh_cmd(
         'CYLC_COVERAGE',
         'CLIENT_COMMS_METH',
         'CYLC_ENV_NAME',
-        *platform['ssh forward environment variables'],
+        *(glbl_cfg().get(['scheduler'])
+            ['run hosts']
+            ['ssh forward environment variables']),
     ]:
         if envvar in os.environ:
             command.append(
diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py
index 6e86ac1a410..c3c0c1ca9ff 100644
--- a/tests/unit/test_remote.py
+++ b/tests/unit/test_remote.py
@@ -90,17 +90,25 @@ def test_construct_rsync_over_ssh_cmd():
     ]
 
 
-def test_construct_ssh_cmd_forward_env():
+def test_construct_ssh_cmd_forward_env(mock_glbl_cfg):
     """ Test for 'ssh forward environment variables'
     """
     import os
 
+    mock_glbl_cfg(
+        'cylc.flow.remote.glbl_cfg',
+        '''
+        [scheduler]
+            [[run hosts]]
+                ssh forward environment variables = FOO, BAR
+        '''
+    )
+
     host = 'example.com'
     config = {
         'ssh command': 'ssh',
         'use login shell': None,
         'cylc path': None,
-        'ssh forward environment variables': ['FOO'],
         }
 
     # Variable isn't set, no change to command

From a06688ad01edf9d3567db06b837c78a73b0c2d77 Mon Sep 17 00:00:00 2001
From: Mark Dawson <mark.dawson@metoffice.gov.uk>
Date: Thu, 21 Sep 2023 12:10:18 +0100
Subject: [PATCH 012/110] added hub url config to globalcfg (#5733)

* added hub url config to globalcfg
---------

Co-authored-by: Mark Dawson <mark.dawson@metoffice.gov.ukpython>
Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Co-authored-by: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
---
 CONTRIBUTING.md                |  1 +
 cylc/flow/cfgspec/globalcfg.py | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6fbc4283968..c6d27175ff9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -92,6 +92,7 @@ requests_).
  - John Haiducek
  - (Andrew Huang)
  - Cheng Da
+ - Mark Dawson
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 3c01ff52729..86c639af113 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -610,6 +610,26 @@ def default_for(
        Prior to Cylc 8, ``global.cylc`` was named ``global.rc``, but that name
        is no longer supported.
 ''') as SPEC:
+    with Conf('hub', desc='''
+        Configure the public URL of Jupyter Hub.
+
+        If configured, the ``cylc gui`` command will open a web browser at this
+        location rather than starting a standalone server when called.
+
+
+        .. seealso::
+
+           * The cylc hub :ref:`architecture-reference` for fuller details.
+           * :ref:`UI_Server_config` for practical details.
+
+    '''):
+        Conf('url', VDR.V_STRING, '', desc='''
+            .. versionadded:: 8.3.0
+
+            Where Jupyter Hub is used a url can be provided for routing on
+            execution of ``cylc gui`` command.
+        ''')
+
     with Conf('scheduler', desc=(
         default_for(SCHEDULER_DESCR, "[scheduler]", section=True)
     )):

From 90c31b6b89a8935d594afefd7dc91fe1b4b07e73 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 22 Sep 2023 15:13:46 +0100
Subject: [PATCH 013/110] client: don't attempt to clean up old contact files
 in task messages

* The `detect_old_contact_file` check serves two purposes:
  1. It removes the contact files of crashed workflows.
  2. It helps the client to differentiate between a communication
     timeout (e.g. network error or blocked port) and a stopped
     workflow.
* Remote platforms don't necessarily have SSH access back to the
  scheduler host which can cause confusing errors.
* Due to the potentially large number of jobs, this check can
  cause a flurry of SSH'es which isn't great.
* The consequence of this change is that WorkflowStopped errors
  will be replaced by ClientTimeout errors in job stderr.
---
 cylc/flow/network/client.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py
index f8b499332e9..baf65168db7 100644
--- a/cylc/flow/network/client.py
+++ b/cylc/flow/network/client.py
@@ -143,6 +143,10 @@ def timeout_handler(self) -> None:
                 f'It has moved to {contact_host}:{contact_port}'
             )
 
+        if os.getenv('CYLC_TASK_COMMS_METHOD'):
+            # don't attempt to clean up old contact files in task messages
+            return
+
         # Cannot connect, perhaps workflow is no longer running and is leaving
         # behind a contact file?
         try:

From 8ff3082d7641afe822ce7ae1fc9e962bb26cdf91 Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Wed, 27 Sep 2023 14:28:31 +1000
Subject: [PATCH 014/110] Remove reference to job host

---
 cylc/flow/cfgspec/globalcfg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 1b97e8258c0..8038d374988 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -787,7 +787,7 @@ def default_for(
             Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
                  desc='''
                 A list containing the names of the environment variables to
-                forward with SSH connections to the run and job hosts from
+                forward with SSH connections to the workflow host from
                 the host running 'cylc play'
 
                 .. versionchanged:: 8.3.0

From 8053ead4dfb3e06db49338083176c8d0444ef4fa Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Thu, 28 Sep 2023 13:49:05 +1000
Subject: [PATCH 015/110] Fix metadata

---
 cylc/flow/cfgspec/globalcfg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 8038d374988..d42415d2a9c 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -790,7 +790,7 @@ def default_for(
                 forward with SSH connections to the workflow host from
                 the host running 'cylc play'
 
-                .. versionchanged:: 8.3.0
+                .. versionadded:: 8.3.0
             ''')
 
         with Conf('host self-identification', desc=f'''

From e5b3302ecd7da013eadb060cd0b348856f25a017 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Thu, 5 Oct 2023 18:54:34 +1300
Subject: [PATCH 016/110] Improve CLI 'Done' stdout.

---
 cylc/flow/network/multi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/network/multi.py b/cylc/flow/network/multi.py
index 86c89c5f71d..a11842d6391 100644
--- a/cylc/flow/network/multi.py
+++ b/cylc/flow/network/multi.py
@@ -107,4 +107,4 @@ def _report_single(report, workflow, result):
 
 
 def _report(_):
-    print('Done')
+    print('Command submitted; the scheduler will log any problems.')

From fc5ee96ca75b7fe4af86c87ab163995b59b50e49 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 5 Oct 2023 11:48:52 +0100
Subject: [PATCH 017/110] dump: increase default comms timeout

---
 cylc/flow/scripts/dump.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py
index 7c3264f3f19..636f0c81947 100755
--- a/cylc/flow/scripts/dump.py
+++ b/cylc/flow/scripts/dump.py
@@ -184,7 +184,12 @@ def main(_, options: 'Values', workflow_id: str) -> None:
         workflow_id,
         constraint='workflows',
     )
-    pclient = get_client(workflow_id, timeout=options.comms_timeout)
+    pclient = get_client(
+        workflow_id,
+        # set the default timeout a bit higher for "cylc dump" as it can
+        # take a little while for Cylc to formulate the response
+        timeout=options.comms_timeout or 15,
+    )
 
     if options.sort_by_cycle:
         sort_args = {'keys': ['cyclePoint', 'name']}

From 09975aabd70c1027dcc078c72271ff4df7bcf23d Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 5 Oct 2023 11:57:14 +0100
Subject: [PATCH 018/110] ClientTimeout: improve error message

---
 cylc/flow/network/client.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py
index baf65168db7..b3955d3b058 100644
--- a/cylc/flow/network/client.py
+++ b/cylc/flow/network/client.py
@@ -303,7 +303,9 @@ async def async_request(
             raise ClientTimeout(
                 'Timeout waiting for server response.'
                 ' This could be due to network or server issues.'
-                ' Check the workflow log.'
+                '\n* You might want to increase the timeout using the'
+                ' --comms-timeout option;'
+                '\n* or check the workflow log.'
             )
 
         if msg['command'] in PB_METHOD_MAP:

From 571b27f40f3c7350b4cc00e7f35f0ab7f6674c12 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 1 Sep 2023 12:03:50 +0100
Subject: [PATCH 019/110] Tui 1.0

* Move the updater into another process.
  This removes the limitation that caused Tui to crash with active workflows.
  Closes #3527
* Add multi-workflow capability.
  Closes #3464
* Add visual regression testing framework.
  Closes #3530
---
 changes.d/5731.feat.md                  |   1 +
 cylc/flow/option_parsers.py             |   1 +
 cylc/flow/scripts/tui.py                |  87 ++--
 cylc/flow/tui/__init__.py               |   3 -
 cylc/flow/tui/app.py                    | 534 ++++++++++++++----------
 cylc/flow/tui/data.py                   |  71 +++-
 cylc/flow/tui/overlay.py                | 132 +++++-
 cylc/flow/tui/tree.py                   | 109 ++++-
 cylc/flow/tui/updater.py                | 320 ++++++++++++++
 cylc/flow/tui/util.py                   | 352 ++++++++++------
 tests/integration/tui/conftest.py       | 266 ++++++++++++
 tests/integration/tui/test_app.py       | 378 +++++++++++++++++
 tests/integration/tui/test_mutations.py | 216 ++++++++++
 tests/integration/tui/test_updater.py   | 286 +++++++++++++
 tests/unit/tui/test_data.py             |   2 +-
 tests/unit/tui/test_overlay.py          |  14 +-
 tests/unit/tui/test_util.py             | 134 +++---
 17 files changed, 2380 insertions(+), 526 deletions(-)
 create mode 100644 changes.d/5731.feat.md
 create mode 100644 cylc/flow/tui/updater.py
 create mode 100644 tests/integration/tui/conftest.py
 create mode 100644 tests/integration/tui/test_app.py
 create mode 100644 tests/integration/tui/test_mutations.py
 create mode 100644 tests/integration/tui/test_updater.py

diff --git a/changes.d/5731.feat.md b/changes.d/5731.feat.md
new file mode 100644
index 00000000000..b0c28a01ac1
--- /dev/null
+++ b/changes.d/5731.feat.md
@@ -0,0 +1 @@
+Major upgrade to `cylc tui` which now supports larger workflows and can browse installed workflows.
diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py
index cb21f5e8bcf..96073ca4aef 100644
--- a/cylc/flow/option_parsers.py
+++ b/cylc/flow/option_parsers.py
@@ -44,6 +44,7 @@
 )
 
 WORKFLOW_ID_ARG_DOC = ('WORKFLOW', 'Workflow ID')
+OPT_WORKFLOW_ID_ARG_DOC = ('[WORKFLOW]', 'Workflow ID')
 WORKFLOW_ID_MULTI_ARG_DOC = ('WORKFLOW ...', 'Workflow ID(s)')
 WORKFLOW_ID_OR_PATH_ARG_DOC = ('WORKFLOW | PATH', 'Workflow ID or path')
 ID_MULTI_ARG_DOC = ('ID ...', 'Workflow/Cycle/Family/Task ID(s)')
diff --git a/cylc/flow/scripts/tui.py b/cylc/flow/scripts/tui.py
index 86970052b46..2185efbf7e4 100644
--- a/cylc/flow/scripts/tui.py
+++ b/cylc/flow/scripts/tui.py
@@ -15,34 +15,36 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""cylc tui WORKFLOW
+"""cylc tui [WORKFLOW]
 
 View and control running workflows in the terminal.
 
 (Tui = Terminal User Interface)
 
-WARNING: Tui is experimental and may break with large flows.
-An upcoming change to the way Tui receives data from the scheduler will make it
-much more efficient in the future.
+Tui allows you to monitor and interact with workflows in a manner similar
+to the GUI.
+
+Press "h" whilst running Tui to bring up the help screen, use the arrow
+keys to navigage.
+
 """
-# TODO: remove this warning once Tui is delta-driven
-# https://github.com/cylc/cylc-flow/issues/3527
 
+from getpass import getuser
 from textwrap import indent
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 from urwid import html_fragment
 
+from cylc.flow.id import Tokens
 from cylc.flow.id_cli import parse_id
 from cylc.flow.option_parsers import (
-    WORKFLOW_ID_ARG_DOC,
+    OPT_WORKFLOW_ID_ARG_DOC,
     CylcOptionParser as COP,
 )
 from cylc.flow.terminal import cli_function
 from cylc.flow.tui import TUI
+from cylc.flow.tui.util import suppress_logging
 from cylc.flow.tui.app import (
     TuiApp,
-    TREE_EXPAND_DEPTH
-    # ^ a nasty solution
 )
 
 if TYPE_CHECKING:
@@ -55,57 +57,36 @@
 def get_option_parser() -> COP:
     parser = COP(
         __doc__,
-        argdoc=[WORKFLOW_ID_ARG_DOC],
+        argdoc=[OPT_WORKFLOW_ID_ARG_DOC],
         # auto_add=False,  NOTE: at present auto_add can not be turned off
         color=False
     )
 
-    parser.add_option(
-        '--display',
-        help=(
-            'Specify the display technology to use.'
-            ' "raw" for interactive in-terminal display.'
-            ' "html" for non-interactive html output.'
-        ),
-        action='store',
-        choices=['raw', 'html'],
-        default='raw',
-    )
-    parser.add_option(
-        '--v-term-size',
-        help=(
-            'The virtual terminal size for non-interactive'
-            '--display options.'
-        ),
-        action='store',
-        default='80,24'
-    )
-
     return parser
 
 
-@cli_function(get_option_parser)
-def main(_, options: 'Values', workflow_id: str) -> None:
-    workflow_id, *_ = parse_id(
-        workflow_id,
-        constraint='workflows',
+def configure_screenshot(v_term_size):
+    screen = html_fragment.HtmlGenerator()
+    screen.set_terminal_properties(256)
+    screen.register_palette(TuiApp.palette)
+    html_fragment.screenshot_init(
+        [tuple(map(int, v_term_size.split(',')))],
+        []
     )
-    screen = None
-    if options.display == 'html':
-        TREE_EXPAND_DEPTH[0] = -1  # expand tree fully
-        screen = html_fragment.HtmlGenerator()
-        screen.set_terminal_properties(256)
-        screen.register_palette(TuiApp.palette)
-        html_fragment.screenshot_init(
-            [tuple(map(int, options.v_term_size.split(',')))],
-            []
-        )
+    return screen, html_fragment
+
 
-    try:
-        TuiApp(workflow_id, screen=screen).main()
+@cli_function(get_option_parser)
+def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None:
+    # get workflow ID if specified
+    if workflow_id:
+        workflow_id, *_ = parse_id(
+            workflow_id,
+            constraint='workflows',
+        )
+        tokens = Tokens(workflow_id)
+        workflow_id = tokens.duplicate(user=getuser()).id
 
-        if options.display == 'html':
-            for fragment in html_fragment.screenshot_collect():
-                print(fragment)
-    except KeyboardInterrupt:
+    # start Tui
+    with suppress_logging(), TuiApp().main(workflow_id):
         pass
diff --git a/cylc/flow/tui/__init__.py b/cylc/flow/tui/__init__.py
index 92e3bce0268..78ce0f36d62 100644
--- a/cylc/flow/tui/__init__.py
+++ b/cylc/flow/tui/__init__.py
@@ -143,6 +143,3 @@ def list_groups(self):
                     if binding['group'] == name
                 ]
             )
-
-
-BINDINGS = Bindings()
diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py
index cf09f75db0e..86b5f0dcf98 100644
--- a/cylc/flow/tui/app.py
+++ b/cylc/flow/tui/app.py
@@ -16,32 +16,23 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """The application control logic for Tui."""
 
-import sys
+from copy import deepcopy
+from contextlib import contextmanager
+from multiprocessing import Process
+import re
 
 import urwid
-from urwid import html_fragment
 from urwid.wimp import SelectableIcon
-from pathlib import Path
 
-from cylc.flow.network.client_factory import get_client
-from cylc.flow.exceptions import (
-    ClientError,
-    ClientTimeout,
-    WorkflowStopped
-)
-from cylc.flow.pathutil import get_workflow_run_dir
+from cylc.flow.id import Tokens
 from cylc.flow.task_state import (
-    TASK_STATUSES_ORDERED,
     TASK_STATUS_SUBMITTED,
     TASK_STATUS_RUNNING,
     TASK_STATUS_FAILED,
 )
-from cylc.flow.tui.data import (
-    QUERY
-)
 import cylc.flow.tui.overlay as overlay
 from cylc.flow.tui import (
-    BINDINGS,
+    Bindings,
     FORE,
     BACK,
     JOB_COLOURS,
@@ -49,32 +40,36 @@
 )
 from cylc.flow.tui.tree import (
     find_closest_focus,
-    translate_collapsing
+    translate_collapsing,
+    expand_tree,
+)
+from cylc.flow.tui.updater import (
+    Updater,
+    get_default_filters,
 )
 from cylc.flow.tui.util import (
-    compute_tree,
     dummy_flow,
-    get_task_status_summary,
-    get_workflow_status_str,
     render_node
 )
-from cylc.flow.workflow_files import WorkflowFiles
+from cylc.flow.workflow_status import (
+    WorkflowStatus,
+)
 
 
-urwid.set_encoding('utf8')  # required for unicode task icons
+DEFAULT_FILTERS = get_default_filters()
 
-TREE_EXPAND_DEPTH = [2]
+
+urwid.set_encoding('utf8')  # required for unicode task icons
 
 
 class TuiWidget(urwid.TreeWidget):
     """Display widget for tree nodes.
 
     Arguments:
+        app (TuiApp):
+            Reference to the application.
         node (TuiNode):
             The root tree node.
-        max_depth (int):
-            Determines which nodes are unfolded by default.
-            The maximum tree depth to unfold.
 
     """
 
@@ -82,16 +77,12 @@ class TuiWidget(urwid.TreeWidget):
     # will skip rows when the user navigates
     unexpandable_icon = SelectableIcon(' ', 0)
 
-    def __init__(self, node, max_depth=None):
-        if not max_depth:
-            max_depth = TREE_EXPAND_DEPTH[0]
+    def __init__(self, app, node):
+        self.app = app
         self._node = node
         self._innerwidget = None
         self.is_leaf = not node.get_child_keys()
-        if max_depth > 0:
-            self.expanded = node.get_depth() < max_depth
-        else:
-            self.expanded = True
+        self.expanded = False
         widget = self.get_indented_widget()
         urwid.WidgetWrap.__init__(self, widget)
 
@@ -103,18 +94,6 @@ def selectable(self):
         """
         return self.get_node().get_value()['type_'] != 'job_info'
 
-    def _is_leaf(self):
-        """Return True if this node has no children
-
-        Note: the `is_leaf` attribute doesn't seem to give the right
-              answer.
-
-        """
-        return (
-            not hasattr(self, 'git_first_child')
-            or not self.get_first_child()
-        )
-
     def get_display_text(self):
         """Compute the text to display for a given node.
 
@@ -158,40 +137,74 @@ def get_indented_widget(self):
             )
         return self.__super.get_indented_widget()
 
+    def update_expanded_icon(self, subscribe=True):
+        """Update the +/- icon.
 
-class TuiNode(urwid.TreeNode):
-    """Data storage object for leaf nodes."""
-
-    def load_widget(self):
-        return TuiWidget(self)
+        This method overrides the built-in urwid update_expanded_icon method
+        in order to add logic for subscribing and unsubscribing to workflows.
 
+        Args:
+            subscribe:
+                If True, then we will [un]subscribe to workflows when workflow
+                nodes are expanded/collapsed. If False, then these events will
+                be ignored. Note we set subscribe=False when we are translating
+                the expand/collapse status when rebuilding the tree after an
+                update.
 
-class TuiParentNode(urwid.ParentNode):
-    """Data storage object for interior/parent nodes."""
+        """
+        if subscribe:
+            node = self.get_node()
+            value = node.get_value()
+            data = value['data']
+            type_ = value['type_']
+            if type_ == 'workflow':
+                if self.expanded:
+                    self.app.updater.subscribe(data['id'])
+                    self.app.expand_on_load.add(data['id'])
+                else:
+                    self.app.updater.unsubscribe(data['id'])
+        return urwid.TreeWidget.update_expanded_icon(self)
+
+
+class TuiNode(urwid.ParentNode):
+    """Data storage object for Tui tree nodes."""
+
+    def __init__(self, app, *args, **kwargs):
+        self.app = app
+        urwid.ParentNode.__init__(self, *args, **kwargs)
 
     def load_widget(self):
-        return TuiWidget(self)
+        return TuiWidget(self.app, self)
 
     def load_child_keys(self):
         # Note: keys are really indices.
-        data = self.get_value()
-        return range(len(data['children']))
+        return range(len(self.get_value()['children']))
 
     def load_child_node(self, key):
-        """Return either an TuiNode or TuiParentNode"""
-        childdata = self.get_value()['children'][key]
-        if 'children' in childdata:
-            childclass = TuiParentNode
-        else:
-            childclass = TuiNode
-        return childclass(
-            childdata,
+        """Return a TuiNode instance for child "key"."""
+        return TuiNode(
+            self.app,
+            self.get_value()['children'][key],
             parent=self,
             key=key,
             depth=self.get_depth() + 1
         )
 
 
+@contextmanager
+def updater_subproc(filters):
+    """Runs the Updater in its own process."""
+    # start the updater
+    updater = Updater()
+    p = Process(target=updater.start, args=(filters,))
+    try:
+        p.start()
+        yield updater
+    finally:
+        updater.terminate()
+        p.join()
+
+
 class TuiApp:
     """An application to display a single Cylc workflow.
 
@@ -206,16 +219,25 @@ class TuiApp:
 
     """
 
-    UPDATE_INTERVAL = 1
-    CLIENT_TIMEOUT = 1
+    # the UI update interval
+    # NOTE: this is different from the data update interval
+    UPDATE_INTERVAL = 0.1
 
+    # colours to be used throughout the application
     palette = [
         ('head', FORE, BACK),
         ('body', FORE, BACK),
         ('foot', 'white', 'dark blue'),
         ('key', 'light cyan', 'dark blue'),
-        ('title', FORE, BACK, 'bold'),
+        ('title', 'default, bold', BACK),
+        ('header', 'dark gray', BACK),
+        ('header_key', 'dark gray, bold', BACK),
         ('overlay', 'black', 'light gray'),
+        # cylc logo colours
+        ('R', 'light red, bold', BACK),
+        ('Y', 'yellow, bold', BACK),
+        ('G', 'light green, bold', BACK),
+        ('B', 'light blue, bold', BACK),
     ] + [  # job colours
         (f'job_{status}', colour, BACK)
         for status, colour in JOB_COLOURS.items()
@@ -227,49 +249,117 @@ class TuiApp:
         for status, spec in WORKFLOW_COLOURS.items()
     ]
 
-    def __init__(self, id_, screen=None):
-        self.id_ = id_
-        self.client = None
+    def __init__(self, screen=None):
         self.loop = None
-        self.screen = None
+        self.screen = screen
         self.stack = 0
         self.tree_walker = None
 
+        # store a reference to the bindings on the app to avoid cicular import
+        self.bindings = BINDINGS
+
         # create the template
-        topnode = TuiParentNode(dummy_flow({'id': 'Loading...'}))
+        topnode = TuiNode(self, dummy_flow({'id': 'Loading...'}))
         self.listbox = urwid.TreeListBox(urwid.TreeWalker(topnode))
         header = urwid.Text('\n')
-        footer = urwid.AttrWrap(
-            # urwid.Text(self.FOOTER_TEXT),
-            urwid.Text(list_bindings()),
-            'foot'
-        )
+        footer = urwid.AttrWrap(urwid.Text(list_bindings()), 'foot')
         self.view = urwid.Frame(
             urwid.AttrWrap(self.listbox, 'body'),
             header=urwid.AttrWrap(header, 'head'),
             footer=footer
         )
-        self.filter_states = {
-            state: True
-            for state in TASK_STATUSES_ORDERED
-        }
-        if isinstance(screen, html_fragment.HtmlGenerator):
-            # the HtmlGenerator only captures one frame
-            # so we need to pre-populate the GUI before
-            # starting the event loop
-            self.update()
+        self.filters = get_default_filters()
 
-    def main(self):
-        """Start the event loop."""
-        self.loop = urwid.MainLoop(
-            self.view,
-            self.palette,
-            unhandled_input=self.unhandled_input,
-            screen=self.screen
-        )
-        # schedule the first update
-        self.loop.set_alarm_in(0, self._update)
-        self.loop.run()
+    @contextmanager
+    def main(self, w_id=None, id_filter=None, interactive=True):
+        """Start the Tui app.
+
+        With interactive=False, this does not start the urwid event loop to
+        make testing more deterministic. If you want Tui to update (i.e.
+        display the latest data), you must call the update method manually.
+
+        Note, we still run the updater asynchronously in a subprocess so that
+        we can test the interactions between the Tui application and the
+        updater processes.
+
+        """
+        self.set_initial_filters(w_id, id_filter)
+
+        with updater_subproc(self.filters) as updater:
+            self.updater = updater
+
+            # pre-subscribe to the provided workflow if requested
+            self.expand_on_load = {w_id or 'root'}
+            if w_id:
+                self.updater.subscribe(w_id)
+
+            # configure the urwid main loop
+            self.loop = urwid.MainLoop(
+                self.view,
+                self.palette,
+                unhandled_input=self.unhandled_input,
+                screen=self.screen
+            )
+
+            if interactive:
+                # Tui is being run normally as an interactive application
+                # schedule the first update
+                self.loop.set_alarm_in(0, self.update)
+
+                # start the urwid main loop
+                try:
+                    self.loop.run()
+                except KeyboardInterrupt:
+                    yield
+                    return
+            else:
+                # wait for the first full update
+                self.wait_until_loaded(w_id or 'root')
+
+            yield self
+
+    def set_initial_filters(self, w_id, id_filter):
+        """Set the default workflow/task filters on startup."""
+        if w_id:
+            # Tui has been configured to look at a single workflow
+            # => filter everything else out
+            workflow = str(Tokens(w_id)['workflow'])
+            self.filters['workflows']['id'] = rf'^{re.escape(workflow)}$'
+        elif id_filter:
+            # a custom workflow ID filter has been provided
+            self.filters['workflows']['id'] = id_filter
+
+    def wait_until_loaded(self, *ids, retries=None):
+        """Wait for any requested nodes to be created.
+
+        Warning:
+            This method is blocking! It's for HTML / testing purposes only!
+
+        Args:
+            ids:
+                Iterable containing the node IDs you want to wait for.
+                Note, these should be full IDs i.e. they should include the
+                user.
+                To wait for the root node to load, use "root".
+            retries:
+                The maximum number of updates to perform whilst waiting
+                for the specified IDs to appear in the tree.
+
+        Returns:
+            A list of the IDs which NOT not appear in the store.
+
+        """
+        from time import sleep
+        ids = set(ids)
+        self.expand_on_load.update(ids)
+        try_ = 0
+        while ids & self.expand_on_load:
+            self.update()
+            sleep(0.1)  # blocking
+            try_ += 1
+            if retries is not None and try_ > retries:
+                return list(self.expand_on_load)
+        return None
 
     def unhandled_input(self, key):
         """Catch key presses, uncaught events are passed down the chain."""
@@ -285,74 +375,6 @@ def unhandled_input(self, key):
                 meth(self, *args)
                 return
 
-    def get_snapshot(self):
-        """Contact the workflow, return a tree structure
-
-        In the event of error contacting the workflow the
-        message is written to this Widget's header.
-
-        Returns:
-            dict if successful, else False
-
-        """
-        try:
-            if not self.client:
-                self.client = get_client(self.id_, timeout=self.CLIENT_TIMEOUT)
-            data = self.client(
-                'graphql',
-                {
-                    'request_string': QUERY,
-                    'variables': {
-                        # list of task states we want to see
-                        'taskStates': [
-                            state
-                            for state, is_on in self.filter_states.items()
-                            if is_on
-                        ]
-                    }
-                }
-            )
-        except WorkflowStopped:
-            # Distinguish stopped flow from non-existent flow.
-            self.client = None
-            full_path = Path(get_workflow_run_dir(self.id_))
-            if (
-                (full_path / WorkflowFiles.SUITE_RC).is_file()
-                or (full_path / WorkflowFiles.FLOW_FILE).is_file()
-            ):
-                message = "stopped"
-            else:
-                message = (
-                    f"No {WorkflowFiles.SUITE_RC} or {WorkflowFiles.FLOW_FILE}"
-                    f"found in {self.id_}."
-                )
-
-            return dummy_flow({
-                'name': self.id_,
-                'id': self.id_,
-                'status': message,
-                'stateTotals': {}
-            })
-        except (ClientError, ClientTimeout) as exc:
-            # catch network / client errors
-            self.set_header([('workflow_error', str(exc))])
-            return False
-
-        if isinstance(data, list):
-            # catch GraphQL errors
-            try:
-                message = data[0]['error']['message']
-            except (IndexError, KeyError):
-                message = str(data)
-            self.set_header([('workflow_error', message)])
-            return False
-
-        if len(data['workflows']) != 1:
-            # multiple workflows in returned data - shouldn't happen
-            raise ValueError()
-
-        return compute_tree(data['workflows'][0])
-
     @staticmethod
     def get_node_id(node):
         """Return a unique identifier for a node.
@@ -366,62 +388,82 @@ def get_node_id(node):
         """
         return node.get_value()['id_']
 
-    def set_header(self, message: list):
-        """Set the header message for this widget.
+    def update_header(self):
+        """Update the application header."""
+        header = [
+            # the Cylc Tui logo
+            ('R', 'C'),
+            ('Y', 'y'),
+            ('G', 'l'),
+            ('B', 'c'),
+            ('title', ' Tui')
+        ]
+        if self.filters['tasks'] != DEFAULT_FILTERS['tasks']:
+            # if task filters are active, display short help
+            header.extend([
+                ('header', '   tasks filtered ('),
+                ('header_key', 'F'),
+                ('header', ' - edit, '),
+                ('header_key', 'R'),
+                ('header', ' - reset)'),
+            ])
+        if self.filters['workflows'] != DEFAULT_FILTERS['workflows']:
+            # if workflow filters are active, display short help
+            header.extend([
+                ('header', '   workflows filtered ('),
+                ('header_key', 'W'),
+                ('header', ' - edit, '),
+                ('header_key', 'E'),
+                ('header', ' - reset)'),
+            ])
+        elif self.filters == DEFAULT_FILTERS:
+            # if not filters are available show application help
+            header.extend([
+                ('header', '   '),
+                ('header_key', 'h'),
+                ('header', ' to show help, '),
+                ('header_key', 'q'),
+                ('header', ' to quit'),
+            ])
 
-        Arguments:
-            message (object):
-                Text content for the urwid.Text widget,
-                may be a string, tuple or list, see urwid docs.
-
-        """
         # put in a one line gap
-        message.append('\n')
-
-        # TODO: remove once Tui is delta-driven
-        # https://github.com/cylc/cylc-flow/issues/3527
-        message.extend([
-            (
-                'workflow_error',
-                'TUI is experimental and may break with large flows'
-            ),
-            '\n'
-        ])
+        header.append('\n')
 
-        self.view.header = urwid.Text(message)
+        # replace the previous header
+        self.view.header = urwid.Text(header)
 
-    def _update(self, *_):
-        try:
-            self.update()
-        except Exception as exc:
-            sys.exit(exc)
+    def get_update(self):
+        """Fetch the most recent update.
 
-    def update(self):
+        Returns the update, or False if there is no update queued.
+        """
+        update = False
+        while not self.updater.update_queue.empty():
+            # fetch the most recent update
+            update = self.updater.update_queue.get()
+        return update
+
+    def update(self, *_):
         """Refresh the data and redraw this widget.
 
         Preserves the current focus and collapse/expand state.
 
         """
-        # update the data store
-        # TODO: this can be done incrementally using deltas
-        #       once this interface is available
-        snapshot = self.get_snapshot()
-        if snapshot is False:
+        # attempt to fetch an update
+        update = self.get_update()
+        if update is False:
+            # there was no update, try again later
+            if self.loop:
+                self.loop.set_alarm_in(self.UPDATE_INTERVAL, self.update)
             return False
 
-        # update the workflow status message
-        header = [get_workflow_status_str(snapshot['data'])]
-        status_summary = get_task_status_summary(snapshot['data'])
-        if status_summary:
-            header.extend([' ('] + status_summary + [' )'])
-        if not all(self.filter_states.values()):
-            header.extend([' ', '*filtered* "R" to reset', ' '])
-        self.set_header(header)
+        # update the application header
+        self.update_header()
 
         # global update - the nuclear option - slow but simple
         # TODO: this can be done incrementally by adding and
         #       removing nodes from the existing tree
-        topnode = TuiParentNode(snapshot)
+        topnode = TuiNode(self, update)
 
         # NOTE: because we are nuking the tree we need to manually
         # preserve the focus and collapse status of tree nodes
@@ -443,9 +485,15 @@ def update(self):
         #  preserve the collapse/expand status of all nodes
         translate_collapsing(self, old_node, new_node)
 
+        # expand any nodes which have been requested
+        for id_ in list(self.expand_on_load):
+            depth = 1 if id_ == 'root' else None
+            if expand_tree(self, new_node, id_, depth):
+                self.expand_on_load.remove(id_)
+
         # schedule the next run of this update method
         if self.loop:
-            self.loop.set_alarm_in(self.UPDATE_INTERVAL, self._update)
+            self.loop.set_alarm_in(self.UPDATE_INTERVAL, self.update)
 
         return True
 
@@ -457,13 +505,39 @@ def filter_by_task_state(self, filtered_state=None):
                 A task state to filter by or None.
 
         """
-        self.filter_states = {
+        self.filters['tasks'] = {
             state: (state == filtered_state) or not filtered_state
-            for state in self.filter_states
+            for state in self.filters['tasks']
         }
-        return
+        self.updater.update_filters(self.filters)
+
+    def reset_workflow_filters(self):
+        """Reset workflow state/id filters."""
+        self.filters['workflows'] = deepcopy(DEFAULT_FILTERS['workflows'])
+        self.updater.update_filters(self.filters)
+
+    def filter_by_workflow_state(self, *filtered_states):
+        """Filter workflows.
+
+        Args:
+            filtered_state (str):
+                A task state to filter by or None.
+
+        """
+        for state in self.filters['workflows']:
+            if state != 'id':
+                self.filters['workflows'][state] = (
+                    (state in filtered_states) or not filtered_states
+                )
+        self.updater.update_filters(self.filters)
 
     def open_overlay(self, fcn):
+        """Open an overlay over the application.
+
+        Args:
+            fcn: A function which returns an urwid widget to overlay.
+
+        """
         self.create_overlay(*fcn(self))
 
     def create_overlay(self, widget, kwargs):
@@ -521,6 +595,10 @@ def close_topmost(self):
         self.stack -= 1
 
 
+# register key bindings
+# * all bindings must belong to a group
+# * all keys are auto-documented in the help screen and application footer
+BINDINGS = Bindings()
 BINDINGS.add_group(
     '',
     'Application Controls'
@@ -603,40 +681,68 @@ def close_topmost(self):
 )
 
 BINDINGS.add_group(
-    'filter',
+    'filter tasks',
     'Filter by task state'
 )
 BINDINGS.bind(
-    ('F',),
-    'filter',
+    ('T',),
+    'filter tasks',
     'Select task states to filter by',
     (TuiApp.open_overlay, overlay.filter_task_state)
 )
 BINDINGS.bind(
     ('f',),
-    'filter',
+    'filter tasks',
     'Show only failed tasks',
     (TuiApp.filter_by_task_state, TASK_STATUS_FAILED)
 )
 BINDINGS.bind(
     ('s',),
-    'filter',
+    'filter tasks',
     'Show only submitted tasks',
     (TuiApp.filter_by_task_state, TASK_STATUS_SUBMITTED)
 )
 BINDINGS.bind(
     ('r',),
-    'filter',
+    'filter tasks',
     'Show only running tasks',
     (TuiApp.filter_by_task_state, TASK_STATUS_RUNNING)
 )
 BINDINGS.bind(
     ('R',),
-    'filter',
+    'filter tasks',
     'Reset task state filtering',
     (TuiApp.filter_by_task_state,)
 )
 
+BINDINGS.add_group(
+    'filter workflows',
+    'Filter by workflow state'
+)
+BINDINGS.bind(
+    ('W',),
+    'filter workflows',
+    'Select workflow states to filter by',
+    (TuiApp.open_overlay, overlay.filter_workflow_state)
+)
+BINDINGS.bind(
+    ('E',),
+    'filter workflows',
+    'Reset workflow filtering',
+    (TuiApp.reset_workflow_filters,)
+)
+BINDINGS.bind(
+    ('p',),
+    'filter workflows',
+    'Show only running workflows',
+    (
+        TuiApp.filter_by_workflow_state,
+        WorkflowStatus.RUNNING.value,
+        WorkflowStatus.PAUSED.value,
+        WorkflowStatus.STOPPING.value
+    )
+)
+
 
 def list_bindings():
     """Write out an in-line list of the key bindings."""
diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index 4fd538d8783..3a3ed4ee378 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -16,9 +16,9 @@
 
 from functools import partial
 from subprocess import Popen, PIPE
-import sys
 
 from cylc.flow.exceptions import ClientError
+from cylc.flow.id import Tokens
 from cylc.flow.tui.util import (
     extract_context
 )
@@ -29,6 +29,7 @@
     workflows {
       id
       name
+      port
       status
       stateTotals
       taskProxies(states: $taskStates) {
@@ -149,28 +150,25 @@ def cli_cmd(*cmd):
         stdout=PIPE,
         text=True,
     )
-    out, err = proc.communicate()
+    _out, err = proc.communicate()
     if proc.returncode != 0:
-        raise ClientError(f'Error in command {" ".join(cmd)}\n{err}')
-
-
-def _clean(workflow):
-    # for now we will exit tui when the workflow is cleaned
-    # this will change when tui supports multiple workflows
-    cli_cmd('clean', workflow)
-    sys.exit(0)
+        raise ClientError(f'Error in command cylc {" ".join(cmd)}\n{err}')
 
 
 OFFLINE_MUTATIONS = {
+    'user': {
+        'stop-all': partial(cli_cmd, 'stop', '*'),
+    },
     'workflow': {
         'play': partial(cli_cmd, 'play'),
-        'clean': _clean,
+        'clean': partial(cli_cmd, 'clean', '--yes'),
         'reinstall-reload': partial(cli_cmd, 'vr', '--yes'),
     }
 }
 
 
 def generate_mutation(mutation, arguments):
+    arguments.pop('user')
     graphql_args = ', '.join([
         f'${argument}: {ARGUMENT_TYPES[argument]}'
         for argument in arguments
@@ -206,28 +204,40 @@ def context_to_variables(context):
 
     Examples:
         >>> context_to_variables(extract_context(['~a/b//c/d']))
-        {'workflow': ['b'], 'task': ['c/d']}
+        {'user': ['a'], 'workflow': ['b'], 'task': ['c/d']}
 
         >>> context_to_variables(extract_context(['~a/b//c']))
-        {'workflow': ['b'], 'task': ['c/*']}
+        {'user': ['a'], 'workflow': ['b'], 'task': ['c/*']}
 
         >>> context_to_variables(extract_context(['~a/b']))
-        {'workflow': ['b']}
+        {'user': ['a'], 'workflow': ['b']}
 
     """
     # context_to_variables because it can only handle single-selection ATM
-    variables = {'workflow': context['workflow']}
+    variables = {'user': context['user']}
+
+    if 'workflow' in context:
+        variables['workflow'] = context['workflow']
     if 'task' in context:
         variables['task'] = [
-            f'{context["cycle"][0]}/{context["task"][0]}'
+            Tokens(
+                cycle=context['cycle'][0],
+                task=context['task'][0]
+            ).relative_id
         ]
     elif 'cycle' in context:
-        variables['task'] = [f'{context["cycle"][0]}/*']
+        variables['task'] = [
+            Tokens(cycle=context['cycle'][0], task='*').relative_id
+        ]
     return variables
 
 
 def mutate(client, mutation, selection):
-    if mutation in OFFLINE_MUTATIONS['workflow']:
+    if mutation in {
+        _mutation
+        for section in OFFLINE_MUTATIONS.values()
+        for _mutation in section
+    }:
         offline_mutate(mutation, selection)
     elif client:
         online_mutate(client, mutation, selection)
@@ -238,10 +248,24 @@ def mutate(client, mutation, selection):
         )
 
 
-def online_mutate(client, mutation, selection):
+def online_mutate(mutation, selection):
     """Issue a mutation over a network interface."""
     context = extract_context(selection)
     variables = context_to_variables(context)
+
+    # note this only supports single workflow mutations at present
+    workflow = variables['workflow'][0]
+    try:
+        client = get_client(workflow)
+    except WorkflowStopped:
+        raise Exception(
+            f'Cannot peform command {mutation} on a stopped workflow'
+        )
+    except (ClientError, ClientTimeout) as exc:
+        raise Exception(
+            f'Error connecting to workflow: {exc}'
+        )
+
     request_string = generate_mutation(mutation, variables)
     client(
         'graphql',
@@ -256,6 +280,9 @@ def offline_mutate(mutation, selection):
     """Issue a mutation over the CLI or other offline interface."""
     context = extract_context(selection)
     variables = context_to_variables(context)
-    for workflow in variables['workflow']:
-        # NOTE: this currently only supports workflow mutations
-        OFFLINE_MUTATIONS['workflow'][mutation](workflow)
+    if 'workflow' in variables:
+        for workflow in variables['workflow']:
+            # NOTE: this currently only supports workflow mutations
+            OFFLINE_MUTATIONS['workflow'][mutation](workflow)
+    else:
+        OFFLINE_MUTATIONS['user'][mutation]()
diff --git a/cylc/flow/tui/overlay.py b/cylc/flow/tui/overlay.py
index cddcc8d5093..f4d5ca02f94 100644
--- a/cylc/flow/tui/overlay.py
+++ b/cylc/flow/tui/overlay.py
@@ -37,55 +37,137 @@
 
 """
 
+from contextlib import suppress
 from functools import partial
+import re
 import sys
 
 import urwid
 
 from cylc.flow.exceptions import (
     ClientError,
+    ClientTimeout,
+    WorkflowStopped,
 )
+from cylc.flow.id import Tokens
+from cylc.flow.network.client_factory import get_client
 from cylc.flow.task_state import (
     TASK_STATUSES_ORDERED,
     TASK_STATUS_WAITING
 )
 from cylc.flow.tui import (
-    BINDINGS,
     JOB_COLOURS,
     JOB_ICON,
     TUI
 )
 from cylc.flow.tui.data import (
+    extract_context,
     list_mutations,
     mutate,
 )
 from cylc.flow.tui.util import (
-    get_task_icon
+    get_task_icon,
 )
 
 
+def _get_display_id(id_):
+    """Return an ID for display in context menus.
+
+    * Display the full ID for users/workflows
+    * Display the relative ID for everything else
+
+    """
+    tokens = Tokens(id_)
+    if tokens.is_task_like:
+        # if it's a cycle/task/job, then use the relative id
+        return tokens.relative_id
+    else:
+        # otherwise use the full id
+        return tokens.id
+
+
+def _toggle_filter(app, filter_group, status, *_):
+    """Toggle a filter state."""
+    app.filters[filter_group][status] = not app.filters[filter_group][status]
+    app.updater.update_filters(app.filters)
+
+
+def _invert_filter(checkboxes, *_):
+    """Invert the state of all filters."""
+    for checkbox in checkboxes:
+        checkbox.set_state(not checkbox.state)
+
+
+def filter_workflow_state(app):
+    """Return a widget for adjusting the workflow filter options."""
+    checkboxes = [
+        urwid.CheckBox(
+            [status],
+            state=is_on,
+            on_state_change=partial(_toggle_filter, app, 'workflows', status)
+        )
+        for status, is_on in app.filters['workflows'].items()
+        if status != 'id'
+    ]
+
+    workflow_id_prompt = 'id (regex)'
+
+    def update_id_filter(widget, value):
+        nonlocal app
+        try:
+            # ensure the filter is value before updating the filter
+            re.compile(value)
+        except re.error:
+            # error in the regex -> inform the user
+            widget.set_caption(f'{workflow_id_prompt} - error: \n')
+        else:
+            # valid regex -> update the filter
+            widget.set_caption(f'{workflow_id_prompt}: \n')
+            app.filters['workflows']['id'] = value
+            app.updater.update_filters(app.filters)
+
+    id_filter_widget = urwid.Edit(
+        caption=f'{workflow_id_prompt}: \n',
+        edit_text=app.filters['workflows']['id'],
+    )
+    urwid.connect_signal(id_filter_widget, 'change', update_id_filter)
+
+    widget = urwid.ListBox(
+        urwid.SimpleFocusListWalker([
+            urwid.Text('Filter Workflow States'),
+            urwid.Divider(),
+            urwid.Padding(
+                urwid.Button(
+                    'Invert',
+                    on_press=partial(_invert_filter, checkboxes)
+                ),
+                right=19
+            )
+        ] + checkboxes + [
+            urwid.Divider(),
+            id_filter_widget,
+        ])
+    )
+
+    return (
+        widget,
+        {'width': 35, 'height': 23}
+    )
+
+
 def filter_task_state(app):
     """Return a widget for adjusting the task state filter."""
 
-    def toggle(state, *_):
-        """Toggle a filter state."""
-        app.filter_states[state] = not app.filter_states[state]
-
     checkboxes = [
         urwid.CheckBox(
             get_task_icon(state)
             + [' ' + state],
             state=is_on,
-            on_state_change=partial(toggle, state)
+            on_state_change=partial(_toggle_filter, app, 'tasks', state)
         )
-        for state, is_on in app.filter_states.items()
+        for state, is_on in app.filters['tasks'].items()
     ]
 
-    def invert(*_):
-        """Invert the state of all filters."""
-        for checkbox in checkboxes:
-            checkbox.set_state(not checkbox.state)
-
     widget = urwid.ListBox(
         urwid.SimpleFocusListWalker([
             urwid.Text('Filter Task States'),
@@ -93,7 +175,7 @@ def invert(*_):
             urwid.Padding(
                 urwid.Button(
                     'Invert',
-                    on_press=invert
+                    on_press=partial(_invert_filter, checkboxes)
                 ),
                 right=19
             )
@@ -127,7 +209,7 @@ def help_info(app):
     ]
 
     # list key bindings
-    for group, bindings in BINDINGS.list_groups():
+    for group, bindings in app.bindings.list_groups():
         items.append(
             urwid.Text([
                 f'{group["desc"]}:'
@@ -214,22 +296,32 @@ def context(app):
     """An overlay for context menus."""
     value = app.tree_walker.get_focus()[0].get_node().get_value()
     selection = [value['id_']]  # single selection ATM
+    context = extract_context(selection)
+
+    client = None
+    if 'workflow' in context:
+        w_id = context['workflow'][0]
+        with suppress(WorkflowStopped, ClientError, ClientTimeout):
+            client = get_client(w_id)
 
     def _mutate(mutation, _):
-        nonlocal app
+        nonlocal app, client
         app.open_overlay(partial(progress, text='Running Command'))
         try:
-            mutate(app.client, mutation, selection)
+            mutate(client, mutation, selection)
         except ClientError as exc:
             app.open_overlay(partial(error, text=str(exc)))
         else:
             app.close_topmost()
             app.close_topmost()
 
+    # determine the ID to display for the context menu
+    display_id = _get_display_id(value['id_'])
+
     widget = urwid.ListBox(
         urwid.SimpleFocusListWalker(
             [
-                urwid.Text(f'id: {value["id_"]}'),
+                urwid.Text(f'id: {display_id}'),
                 urwid.Divider(),
                 urwid.Text('Action'),
                 urwid.Button(
@@ -242,14 +334,14 @@ def _mutate(mutation, _):
                     mutation,
                     on_press=partial(_mutate, mutation)
                 )
-                for mutation in list_mutations(app.client, selection)
+                for mutation in list_mutations(client, selection)
             ]
         )
     )
 
     return (
         widget,
-        {'width': 30, 'height': 20}
+        {'width': 50, 'height': 20}
     )
 
 
diff --git a/cylc/flow/tui/tree.py b/cylc/flow/tui/tree.py
index 84ed55ab53a..02cecee61ed 100644
--- a/cylc/flow/tui/tree.py
+++ b/cylc/flow/tui/tree.py
@@ -56,6 +56,78 @@ def find_closest_focus(app, old_node, new_node):
     )
 
 
+def expand_tree(app, tree_node, id_, depth=5, node_types=None):
+    """Expand the Tui tree to the desired level.
+
+    Arguments:
+        app:
+            The Tui application instance.
+        tree_node:
+            The Tui widget representing the tree view.
+        id_:
+            If specified, we will look within the tree for a node matching
+            this ID and the tree below this node will be expanded.
+        depth:
+            The max depth to expand nodes too.
+        node_types:
+            Whitelist of node types to expand, note "task", "job" and "spring"
+            nodes are excluded by default.
+
+    Returns:
+        True, if the node was found in the tree, is loaded and has been
+        expanded.
+
+    Examples:
+        # expand the top three levels of the tree
+        compute_tree(app, node, None, 3)
+
+        # expand the "root" node AND the top five levels of the tree under
+        # ~user/workflow
+        compute_tree(app, node, '~user/workflow')
+
+    """
+    if not node_types:
+        # don't auto-expand job nodes by default
+        node_types = {'root', 'workflow', 'cycle', 'family'}
+
+    root_node = tree_node.get_root()
+    requested_node = root_node
+
+    # locate the "id_" within the tree if specified
+    if id_:
+        for node in walk_tree(root_node):
+            key = app.get_node_id(node)
+            if key == id_:
+                requested_node = node
+                child_keys = node.get_child_keys()
+                if (
+                    # if the node only has one child
+                    len(child_keys) == 1
+                    # and that child is a "#spring" node (i.e. a loading node)
+                    and (
+                        node.get_child_node(0).get_value()['type_']
+                    ) == '#spring'
+                ):
+                    # then the content hasn't loaded yet so the node cannot be
+                    # expanded
+                    return False
+                break
+        else:
+            # the requested node does not exist yet
+            # it might still be loading
+            return False
+
+    # expand the specified nodes
+    for node in (*walk_tree(requested_node, depth), root_node):
+        if node.get_value()['type_'] not in node_types:
+            continue
+        widget = node.get_widget()
+        widget.expanded = True
+        widget.update_expanded_icon(False)
+
+    return True
+
+
 def translate_collapsing(app, old_node, new_node):
     """Transfer the collapse state from one tree to another.
 
@@ -81,29 +153,46 @@ def translate_collapsing(app, old_node, new_node):
     for node in walk_tree(new_root):
         key = app.get_node_id(node)
         if key in old_tree:
+            # this node was present before
+            # => translate its expansion to the new tree
             expanded = old_tree.get(key)
             widget = node.get_widget()
             if widget.expanded != expanded:
                 widget.expanded = expanded
-                widget.update_expanded_icon()
-
-
-def walk_tree(node):
+                widget.update_expanded_icon(False)
+        else:
+            # this node was not present before
+            # => apply the standard expansion logic
+            expand_tree(
+                app,
+                node,
+                key,
+                3,
+                # don't auto-expand workflows, only cycles/families
+                # and the root node to help expand the tree on startup
+                node_types={'root', 'cycle', 'family'}
+            )
+
+
+def walk_tree(node, depth=None):
     """Yield nodes in order.
 
     Arguments:
         node (urwid.TreeNode):
             Yield this node and all nodes beneath it.
+        depth:
+            The maximum depth to walk to or None to walk all children.
 
     Yields:
         urwid.TreeNode
 
     """
-    stack = [node]
+    stack = [(node, 1)]
     while stack:
-        node = stack.pop()
+        node, _depth = stack.pop()
         yield node
-        stack.extend([
-            node.get_child_node(index)
-            for index in node.get_child_keys()
-        ])
+        if depth is None or _depth < depth:
+            stack.extend([
+                (node.get_child_node(index), _depth + 1)
+                for index in node.get_child_keys()
+            ])
diff --git a/cylc/flow/tui/updater.py b/cylc/flow/tui/updater.py
new file mode 100644
index 00000000000..5321a4d6c1f
--- /dev/null
+++ b/cylc/flow/tui/updater.py
@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Contains the logic for updating the Tui app."""
+
+from asyncio import (
+    run,
+    sleep,
+    gather,
+)
+from contextlib import suppress
+from copy import deepcopy
+from getpass import getuser
+from multiprocessing import Queue
+from time import time
+
+from zmq.error import ZMQError
+
+from cylc.flow.exceptions import (
+    ClientError,
+    ClientTimeout,
+    CylcError,
+    WorkflowStopped,
+)
+from cylc.flow.id import Tokens
+from cylc.flow.network.client_factory import get_client
+from cylc.flow.network.scan import (
+    filter_name,
+    graphql_query,
+    is_active,
+    scan,
+)
+from cylc.flow.task_state import (
+    TASK_STATUSES_ORDERED,
+)
+from cylc.flow.tui.data import (
+    QUERY
+)
+from cylc.flow.tui.util import (
+    compute_tree,
+    suppress_logging,
+)
+from cylc.flow.workflow_status import (
+    WorkflowStatus,
+)
+
+
+ME = getuser()
+
+
+def get_default_filters():
+    """Return default task/workflow filters.
+
+    These filters show everything.
+    """
+    return {
+        'tasks': {
+            # filtered task statuses
+            state: True
+            for state in TASK_STATUSES_ORDERED
+        },
+        'workflows': {
+            # filtered workflow statuses
+            **{
+                state.value: True
+                for state in WorkflowStatus
+            },
+            # filtered workflow ids
+            'id': '.*',
+        }
+    }
+
+
+class Updater():
+    """The bit of Tui which provides the data.
+
+    It lists workflows using the "scan" interface, and provides detail using
+    the "GraphQL" interface.
+
+    """
+
+    # the maximum time to wait for a workflow update
+    CLIENT_TIMEOUT = 2
+
+    # the interval between workflow listing scans
+    BASE_SCAN_INTERVAL = 20
+
+    # the interval between workflow data updates
+    BASE_UPDATE_INTERVAL = 1
+
+    # the command signal used to tell the updater to shut down
+    SIGNAL_TERMINATE = 'terminate'
+
+    def __init__(self):
+        # Cylc comms clients for each workflow we're connected to
+        self._clients = {}
+
+        # iterate over this to get a list of workflows
+        self._scan_pipe = None
+        # the new pipe if the workflow filter options are changed
+        self.__scan_pipe = None
+
+        # task/workflow filters
+        self.filters = None  # note set on self.run()
+        # queue for pushing out updates
+        self.update_queue = Queue(
+            # block the updater if it gets too far ahead of the application
+            maxsize=10
+        )
+        # queue for commands to the updater
+        self._command_queue = Queue()
+
+    def subscribe(self, w_id):
+        """Subscribe to updates from a workflow."""
+        self._command_queue.put((self._subscribe.__name__, w_id))
+
+    def unsubscribe(self, w_id):
+        """Unsubscribe to updates from a workflow."""
+        self._command_queue.put((self._unsubscribe.__name__, w_id))
+
+    def update_filters(self, filters):
+        """Update the task state filter."""
+        self._command_queue.put((self._update_filters.__name__, filters))
+
+    def terminate(self):
+        """Stop the updater."""
+        self._command_queue.put((self.SIGNAL_TERMINATE, None))
+
+    def start(self, filters):
+        """Start the updater in a new asyncio.loop.
+
+        The Tui app will call this within a dedicated process.
+        """
+        with suppress(KeyboardInterrupt):
+            run(self.run(filters))
+
+    async def run(self, filters):
+        """Start the updater in an existing asyncio.loop.
+
+        The tests call this within the same process.
+        """
+        with suppress_logging():
+            self._update_filters(filters)
+            await self._update()
+
+    def _subscribe(self, w_id):
+        if w_id not in self._clients:
+            self._clients[w_id] = None
+
+    def _unsubscribe(self, w_id):
+        if w_id in self._clients:
+            self._clients.pop(w_id)
+
+    def _update_filters(self, filters):
+        if (
+            not self.filters
+            or filters['workflows']['id'] != self.filters['workflows']['id']
+        ):
+            # update the scan pipe
+            self.__scan_pipe = (
+                # scan all workflows
+                scan
+                | filter_name(filters['workflows']['id'])
+                # if the workflow is active, retrieve its status
+                | is_active(True, filter_stop=False)
+                | graphql_query({'status': None})
+            )
+
+        self.filters = filters
+
+    async def _update(self):
+        last_scan_time = 0
+        while True:
+            # process any pending commands
+            if not self._command_queue.empty():
+                (command, payload) = self._command_queue.get()
+                if command == self.SIGNAL_TERMINATE:
+                    break
+                getattr(self, command)(payload)
+                continue
+
+            # do a workflow scan if it's due
+            update_start_time = time()
+            if update_start_time - last_scan_time > self.BASE_SCAN_INTERVAL:
+                data = await self._scan()
+
+            # get the next snapshot from workflows we are subscribed to
+            self.update_queue.put(await self._run_update(data))
+
+            # schedule the next update
+            update_time = time() - update_start_time
+            await sleep(self.BASE_UPDATE_INTERVAL - update_time)
+
+    async def _run_update(self, data):
+        # copy the scanned data so it can be reused for future updates
+        data = deepcopy(data)
+
+        # connect to schedulers if needed
+        self._connect(data)
+
+        # update data with the response from each workflow
+        # NOTE: Currently we're bunching these updates together so Tui will
+        #       only update as fast as the slowest responding workflow.
+        #       We could run these updates separately if this is an issue.
+        await gather(
+            *(
+                self._update_workflow(w_id, client, data)
+                for w_id, client in self._clients.items()
+            )
+        )
+
+        return compute_tree(data)
+
+    async def _update_workflow(self, w_id, client, data):
+        if not client:
+            # we could not connect to this workflow
+            # e.g. workflow is shut down
+            return
+
+        try:
+            # fetch the data from the workflow
+            workflow_update = await client.async_request(
+                'graphql',
+                {
+                    'request_string': QUERY,
+                    'variables': {
+                        # list of task states we want to see
+                        'taskStates': [
+                            state
+                            for state, is_on in self.filters['tasks'].items()
+                            if is_on
+                        ]
+                    }
+                }
+            )
+        except WorkflowStopped:
+            # remove the client on any error, we'll reconnect next time
+            self._clients[w_id] = None
+            for workflow in data['workflows']:
+                if workflow['id'] == w_id:
+                    break
+            else:
+                # there's no entry here, create a stub
+                # NOTE: this handles the situation where we've connected to a
+                # workflow before it has appeared in a scan which matters to
+                # the tests as they use fine timings
+                data['workflows'].append({
+                    'id': w_id,
+                    'status': 'stopped',
+                })
+        except (CylcError, ZMQError):
+            # something went wrong :(
+            # remove the client on any error, we'll reconnect next time
+            self._clients[w_id] = None
+        else:
+            # the data arrived, add it to the update
+            workflow_data = workflow_update['workflows'][0]
+            for workflow in data['workflows']:
+                if workflow['id'] == workflow_data['id']:
+                    workflow.update(workflow_data)
+                    break
+
+    def _connect(self, data):
+        """Connect to all subscribed workflows."""
+        for w_id, client in self._clients.items():
+            if not client:
+                try:
+                    self._clients[w_id] = get_client(
+                        Tokens(w_id)['workflow'],
+                        timeout=self.CLIENT_TIMEOUT
+                    )
+                except WorkflowStopped:
+                    for workflow in data['workflows']:
+                        if workflow['id'] == w_id:
+                            workflow['_tui_data'] = 'Workflow is not running'
+                except (ZMQError, ClientError, ClientTimeout) as exc:
+                    for workflow in data['workflows']:
+                        if workflow['id'] == w_id:
+                            workflow['_tui_data'] = f'Error: {exc}'
+                            break
+
+    async def _scan(self):
+        """Scan for workflows on the filesystem."""
+        data = {'workflows': []}
+        workflow_filter_statuses = {
+            status
+            for status, filtered in self.filters['workflows'].items()
+            if filtered
+        }
+        if self.__scan_pipe:
+            # switch to the new pipe if it has been changed
+            self._scan_pipe = self.__scan_pipe
+        async for workflow in self._scan_pipe:
+            status = workflow.get('status', WorkflowStatus.STOPPED.value)
+            if status not in workflow_filter_statuses:
+                # this workflow is filtered out
+                continue
+            data['workflows'].append({
+                'id': f'~{ME}/{workflow["name"]}',
+                'name': workflow['name'],
+                'status': status,
+                'stateTotals': {},
+            })
+
+        data['workflows'].sort(key=lambda x: x['id'])
+        return data
diff --git a/cylc/flow/tui/util.py b/cylc/flow/tui/util.py
index 575fc693437..6587cd4be1d 100644
--- a/cylc/flow/tui/util.py
+++ b/cylc/flow/tui/util.py
@@ -16,10 +16,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Common utilities for Tui."""
 
+from contextlib import contextmanager
+from getpass import getuser
 from itertools import zip_longest
 import re
 from time import time
 
+from cylc.flow import LOG
 from cylc.flow.id import Tokens
 from cylc.flow.task_state import (
     TASK_STATUS_RUNNING
@@ -33,6 +36,26 @@
 from cylc.flow.wallclock import get_unix_time_from_time_string
 
 
+# the Tui user, note this is always the same as the workflow owner
+# (Tui doesn't do multi-user stuff)
+ME = getuser()
+
+
+@contextmanager
+def suppress_logging():
+    """Suppress Cylc logging.
+
+    Log goes to stdout/err which can pollute Urwid apps.
+    Patching sys.stdout/err is insufficient so we set the level to something
+    silly for the duration of this context manager then set it back again
+    afterwards.
+    """
+    level = LOG.getEffectiveLevel()
+    LOG.setLevel(99999)
+    yield
+    LOG.setLevel(level)
+
+
 def get_task_icon(
     status,
     *,
@@ -113,80 +136,90 @@ def idpop(id_):
     return tokens.id
 
 
-def compute_tree(flow):
-    """Digest GraphQL data to produce a tree.
+def compute_tree(data):
+    """Digest GraphQL data to produce a tree."""
+    root_node = add_node('root', 'root', {}, data={})
 
-    Arguments:
-        flow (dict):
-            A dictionary representing a single workflow.
+    for flow in data['workflows']:
+        nodes = {}
+        flow_node = add_node(
+            'workflow', flow['id'], nodes, data=flow)
+        root_node['children'].append(flow_node)
 
-    Returns:
-        dict - A top-level workflow node.
+        # populate cycle nodes
+        for cycle in flow.get('cyclePoints', []):
+            cycle['id'] = idpop(cycle['id'])  # strip the family off of the id
+            cycle_node = add_node('cycle', cycle['id'], nodes, data=cycle)
+            flow_node['children'].append(cycle_node)
 
-    """
-    nodes = {}
-    flow_node = add_node(
-        'workflow', flow['id'], nodes, data=flow)
-
-    # populate cycle nodes
-    for cycle in flow['cyclePoints']:
-        cycle['id'] = idpop(cycle['id'])  # strip the family off of the id
-        cycle_node = add_node('cycle', cycle['id'], nodes, data=cycle)
-        flow_node['children'].append(cycle_node)
-
-    # populate family nodes
-    for family in flow['familyProxies']:
-        add_node('family', family['id'], nodes, data=family)
-
-    # create cycle/family tree
-    for family in flow['familyProxies']:
-        family_node = add_node(
-            'family', family['id'], nodes)
-        first_parent = family['firstParent']
-        if (
-            first_parent
-            and first_parent['name'] != 'root'
-        ):
-            parent_node = add_node(
-                'family', first_parent['id'], nodes)
-            parent_node['children'].append(family_node)
-        else:
-            add_node(
-                'cycle', idpop(family['id']), nodes
-            )['children'].append(family_node)
-
-    # add leaves
-    for task in flow['taskProxies']:
-        # If there's no first parent, the child will have been deleted
-        # during/after API query resolution. So ignore.
-        if not task['firstParent']:
-            continue
-        task_node = add_node(
-            'task', task['id'], nodes, data=task)
-        if task['firstParent']['name'] == 'root':
-            family_node = add_node(
-                'cycle', idpop(task['id']), nodes)
-        else:
+        # populate family nodes
+        for family in flow.get('familyProxies', []):
+            add_node('family', family['id'], nodes, data=family)
+
+        # create cycle/family tree
+        for family in flow.get('familyProxies', []):
             family_node = add_node(
-                'family', task['firstParent']['id'], nodes)
-        family_node['children'].append(task_node)
-        for job in task['jobs']:
-            job_node = add_node(
-                'job', job['id'], nodes, data=job)
-            job_info_node = add_node(
-                'job_info', job['id'] + '_info', nodes, data=job)
-            job_node['children'] = [job_info_node]
-            task_node['children'].append(job_node)
-
-    # sort
-    for (type_, _), node in nodes.items():
-        if type_ != 'task':
-            # NOTE: jobs are sorted by submit-num in the GraphQL query
-            node['children'].sort(
-                key=lambda x: NaturalSort(x['id_'])
+                'family', family['id'], nodes)
+            first_parent = family['firstParent']
+            if (
+                first_parent
+                and first_parent['name'] != 'root'
+            ):
+                parent_node = add_node(
+                    'family', first_parent['id'], nodes)
+                parent_node['children'].append(family_node)
+            else:
+                add_node(
+                    'cycle', idpop(family['id']), nodes
+                )['children'].append(family_node)
+
+        # add leaves
+        for task in flow.get('taskProxies', []):
+            # If there's no first parent, the child will have been deleted
+            # during/after API query resolution. So ignore.
+            if not task['firstParent']:
+                continue
+            task_node = add_node(
+                'task', task['id'], nodes, data=task)
+            if task['firstParent']['name'] == 'root':
+                family_node = add_node(
+                    'cycle', idpop(task['id']), nodes)
+            else:
+                family_node = add_node(
+                    'family', task['firstParent']['id'], nodes)
+            family_node['children'].append(task_node)
+            for job in task['jobs']:
+                job_node = add_node(
+                    'job', job['id'], nodes, data=job)
+                job_info_node = add_node(
+                    'job_info', job['id'] + '_info', nodes, data=job)
+                job_node['children'] = [job_info_node]
+                task_node['children'].append(job_node)
+
+        # sort
+        for (type_, _), node in nodes.items():
+            if type_ != 'task':
+                # NOTE: jobs are sorted by submit-num in the GraphQL query
+                node['children'].sort(
+                    key=lambda x: NaturalSort(x['id_'])
+                )
+
+        # spring nodes
+        if 'port' not in flow:
+            # the "port" field is only available via GraphQL
+            # so we are not connected to this workflow yet
+            flow_node['children'].append(
+                add_node(
+                    '#spring',
+                    '#spring',
+                    nodes,
+                    data={
+                        'id': flow.get('_tui_data', 'Loading ...'),
+                    }
+                )
             )
 
-    return flow_node
+    return root_node
 
 
 class NaturalSort:
@@ -361,20 +394,113 @@ def get_workflow_status_str(flow):
         list - Text list for the urwid.Text widget.
 
     """
-    status = flow['status']
+
+
+def _render_user(node, data):
+    return f'~{ME}'
+
+
+def _render_job_info(node, data):
+    key_len = max(len(key) for key in data)
+    ret = [
+        f'{key} {" " * (key_len - len(key))} {value}\n'
+        for key, value in data.items()
+    ]
+    ret[-1] = ret[-1][:-1]  # strip trailing newline
+    return ret
+
+
+def _render_job(node, data):
     return [
-        (
-            'title',
-            flow['name'],
+        f'#{data["submitNum"]:02d} ',
+        get_job_icon(data['state'])
+    ]
+
+
+def _render_task(node, data):
+    start_time = None
+    mean_time = None
+    try:
+        # due to sorting this is the most recent job
+        first_child = node.get_child_node(0)
+    except IndexError:
+        first_child = None
+
+    # progress information
+    if data['state'] == TASK_STATUS_RUNNING and first_child:
+        start_time = first_child.get_value()['data']['startedTime']
+        mean_time = data['task']['meanElapsedTime']
+
+    # the task icon
+    ret = get_task_icon(
+        data['state'],
+        is_held=data['isHeld'],
+        is_queued=data['isQueued'],
+        is_runahead=data['isRunahead'],
+        start_time=start_time,
+        mean_time=mean_time
+    )
+
+    # the most recent job status
+    ret.append(' ')
+    if first_child:
+        state = first_child.get_value()['data']['state']
+        ret += [(f'job_{state}', f'{JOB_ICON}'), ' ']
+
+    # the task name
+    ret.append(f'{data["name"]}')
+    return ret
+
+
+def _render_family(node, data):
+    return [
+        get_task_icon(
+            data['state'],
+            is_held=data['isHeld'],
+            is_queued=data['isQueued'],
+            is_runahead=data['isRunahead']
         ),
-        ' - ',
-        (
-            f'workflow_{status}',
-            status
-        )
+        ' ',
+        Tokens(data['id']).pop_token()[1]
     ]
 
 
+def _render_unknown(node, data):
+    try:
+        state_totals = get_task_status_summary(data)
+        status = data['status']
+        status_msg = [
+            (
+                'title',
+                _display_workflow_id(data),
+            ),
+            ' - ',
+            (
+                f'workflow_{status}',
+                status
+            )
+        ]
+    except KeyError:
+        return Tokens(data['id']).pop_token()[1]
+
+    return [*status_msg, *state_totals]
+
+
+def _display_workflow_id(data):
+    return data['name']
+
+
+RENDER_FUNCTIONS = {
+    'user': _render_user,
+    'root': _render_user,
+    'job_info': _render_job_info,
+    'job': _render_job,
+    'task': _render_task,
+    'cycle': _render_family,
+    'family': _render_family,
+}
+
+
 def render_node(node, data, type_):
     """Render a tree node as text.
 
@@ -387,68 +513,7 @@ def render_node(node, data, type_):
             The node type (e.g. `task`, `job`, `family`).
 
     """
-    if type_ == 'job_info':
-        key_len = max(len(key) for key in data)
-        ret = [
-            f'{key} {" " * (key_len - len(key))} {value}\n'
-            for key, value in data.items()
-        ]
-        ret[-1] = ret[-1][:-1]  # strip trailing newline
-        return ret
-
-    if type_ == 'job':
-        return [
-            f'#{data["submitNum"]:02d} ',
-            get_job_icon(data['state'])
-        ]
-
-    if type_ == 'task':
-        start_time = None
-        mean_time = None
-        try:
-            # due to sorting this is the most recent job
-            first_child = node.get_child_node(0)
-        except IndexError:
-            first_child = None
-
-        # progress information
-        if data['state'] == TASK_STATUS_RUNNING and first_child:
-            start_time = first_child.get_value()['data']['startedTime']
-            mean_time = data['task']['meanElapsedTime']
-
-        # the task icon
-        ret = get_task_icon(
-            data['state'],
-            is_held=data['isHeld'],
-            is_queued=data['isQueued'],
-            is_runahead=data['isRunahead'],
-            start_time=start_time,
-            mean_time=mean_time
-        )
-
-        # the most recent job status
-        ret.append(' ')
-        if first_child:
-            state = first_child.get_value()['data']['state']
-            ret += [(f'job_{state}', f'{JOB_ICON}'), ' ']
-
-        # the task name
-        ret.append(f'{data["name"]}')
-        return ret
-
-    if type_ in ['family', 'cycle']:
-        return [
-            get_task_icon(
-                data['state'],
-                is_held=data['isHeld'],
-                is_queued=data['isQueued'],
-                is_runahead=data['isRunahead']
-            ),
-            ' ',
-            Tokens(data['id']).pop_token()[1]
-        ]
-
-    return Tokens(data['id']).pop_token()[1]
+    return RENDER_FUNCTIONS.get(type_, _render_unknown)(node, data)
 
 
 PARTS = [
@@ -476,9 +541,18 @@ def extract_context(selection):
         {'user': ['a'], 'workflow': ['b'], 'cycle': ['c'],
         'task': ['d'], 'job': ['e']}
 
+        >>> list(extract_context(['root']).keys())
+        ['user']
+
     """
     ret = {}
     for item in selection:
+        if item == 'root':
+            # special handling for the Tui "root" node
+            # (this represents the workflow owner which is always the same as
+            # user for Tui)
+            ret['user'] = ME
+            continue
         tokens = Tokens(item)
         for key, value in tokens.items():
             if (
diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py
new file mode 100644
index 00000000000..8c33a43761d
--- /dev/null
+++ b/tests/integration/tui/conftest.py
@@ -0,0 +1,266 @@
+from contextlib import contextmanager
+from difflib import unified_diff
+import os
+from pathlib import Path
+import re
+from time import sleep
+from uuid import uuid1
+
+import pytest
+
+from cylc.flow.id import Tokens
+from cylc.flow.tui.app import TuiApp
+from cylc.flow.scripts.tui import configure_screenshot
+from cylc.flow.tui.overlay import _get_display_id
+
+
+SCREENSHOT_DIR = Path(__file__).parent / 'screenshots'
+
+
+def format_test_failure(expected, got, description):
+    diff = '\n'.join(unified_diff(
+        expected.splitlines(),
+        got.splitlines(),
+        fromfile='expected',
+        tofile='got',
+    ))
+    return f'''
+        <html>
+            <head></head>
+            <h2>{description}</h2>
+            <table>
+                <tr>
+                    <td><b>Expected</b></td>
+                    <td><b>Got</b></td>
+                </tr>
+                <tr>
+                    <td>{expected}</td>
+                    <td>{got}</td>
+                </tr>
+            </table>
+            <br />
+            <h4>Diff:</h4>
+            <pre>{ diff }</pre>
+        </html>
+    '''
+
+
+class RaikuraSession:
+    """Convenience class for accessing Raikura functionality."""
+
+    def __init__(self, app, html_fragment, test_dir, test_name):
+        self.app = app
+        self.html_fragment = html_fragment
+        self.test_dir = test_dir
+        self.test_name = test_name
+
+    def user_input(self, *keys):
+        """Simulate a user pressing keys.
+
+        Each "key" is a keyboard button e.g. "x" or "enter".
+
+        If you provide more than one key, each one will be pressed, one
+        after another.
+
+        You can combine keys in a single string, e.g. "ctrl d".
+        """
+        return self.app.loop.process_input(keys)
+
+    def compare_screenshot(
+        self,
+        name,
+        description,
+        retries=3,
+        delay=0.1,
+        force_update=True,
+    ):
+        """Take a screenshot and compare it to one taken previously.
+
+        To update the screenshot, set the environment variable
+        "CYLC_UPDATE_SCREENSHOTS" to "true".
+
+        Note, if the comparison fails, "force_update" is called and the
+        test is retried.
+
+        Arguments:
+            name:
+                The name to use for the screenshot, this is used in the
+                filename for the generated HTML fragment.
+            description:
+                A description of the test to be used on test failure.
+            retries:
+                The maximum number of retries for this test before failing.
+            delay:
+                The delay between retries. This helps overcome timing issues
+                with data provision.
+
+        Raises:
+            Exception:
+                If the screenshot does not match the reference.
+
+        """
+        filename = SCREENSHOT_DIR / f'{self.test_name}.{name}.html'
+
+        exc = None
+        for _try in range(retries):
+            # load the expected result
+            expected = '<screenshot not found>'
+            if filename.exists():
+                with open(filename, 'r') as expected_file:
+                    expected = expected_file.read()
+            # update to pick up latest data
+            if force_update:
+                self.force_update()
+            # force urwid to draw the screen
+            # (the main loop isn't runing so this doesn't happen automatically)
+            self.app.loop.draw_screen()
+            # take a screenshot
+            screenshot = self.html_fragment.screenshot_collect()[-1]
+
+            try:
+                if expected != screenshot:
+                    # screenshot does not match
+                    # => write an html file with the visual diff
+                    out = self.test_dir / filename.name
+                    with open(out, 'w+') as out_file:
+                        out_file.write(
+                            format_test_failure(
+                                expected,
+                                screenshot,
+                                description,
+                            )
+                        )
+                    raise Exception(
+                        'Screenshot differs:'
+                        '\n* Set "CYLC_UPDATE_SCREENSHOTS=true" to update'
+                        f'\n* To debug see: file:////{out}'
+                    )
+
+                break
+            except Exception as exc_:
+                exc = exc_
+                # wait a while to allow the updater to do its job
+                sleep(delay)
+        else:
+            if os.environ.get('CYLC_UPDATE_SCREENSHOTS', '').lower() == 'true':
+                with open(filename, 'w+') as expected_file:
+                    expected_file.write(screenshot)
+            else:
+                raise exc
+
+    def force_update(self):
+        """Run Tui's update method.
+
+        This is done automatically by compare_screenshot but you may want to
+        call it in a test, e.g. before pressing navigation keys.
+
+        With Raikura, the Tui event loop is not running so the data is never
+        refreshed.
+
+        You do NOT need to call this method for key presses, but you do need to
+        call this if the data has changed (e.g. if you've changed a task state)
+        OR if you've changed any filters (because filters are handled by the
+        update code).
+        """
+        # flush any prior updates
+        self.app.get_update()
+        # wait for the next update
+        while not self.app.update():
+            pass
+
+    def wait_until_loaded(self, *ids, retries=20):
+        """Wait until the given ID appears in the Tui tree, then expand them.
+
+        Useful for waiting whilst Tui loads a workflow.
+
+        Note, this is a blocking wait with no timeout!
+        """
+        ids = self.app.wait_until_loaded(*ids, retries=retries)
+        if ids:
+            self.compare_screenshot(
+                f'fail-{uuid1()}',
+                (
+                    'Requested nodes did not appear in Tui after'
+                    f' {retries} retries: '
+                    + ', '.join(ids)
+                ),
+                1,
+            )
+
+
+@pytest.fixture
+def raikura(test_dir, request, monkeypatch):
+    """Visual regression test framework for Urwid apps.
+
+    Like Cypress but for Tui so named after a NZ island with lots of Tuis.
+
+    When called this yields a RaikuraSession object loaded with test
+    utilities. All tests have default retries to avoid flaky tests.
+
+    Similar to the "start" fixture, which starts a Scheduler without running
+    the main loop, raikura starts Tui without running the main loop.
+
+    Arguments:
+        workflow_id:
+            The "WORKFLOW" argument of the "cylc tui" command line.
+        size:
+            The virtual terminal size for screenshots as a comma
+            separated string e.g. "80,50" for 80 cols wide by 50 rows tall.
+
+    Returns:
+        A RaikuraSession context manager which provides useful utilities for
+        testing.
+
+    """
+    # make the workflow and scan update intervals match (more reliable)
+    # and speed things up a little whilst we're at it
+    monkeypatch.setattr(
+        'cylc.flow.tui.updater.Updater.BASE_UPDATE_INTERVAL',
+        0.1,
+    )
+    monkeypatch.setattr(
+        'cylc.flow.tui.updater.Updater.BASE_SCAN_INTERVAL',
+        0.1,
+    )
+
+    # the user name and the prefix of workflow IDs are both variable
+    # so we patch the render functions to make test output stable
+    def get_display_id(id_):
+        tokens = Tokens(id_)
+        return _get_display_id(
+            tokens.duplicate(
+                user='cylc',
+                workflow=tokens.get('workflow', '').rsplit('/', 1)[-1],
+            ).id
+        )
+    monkeypatch.setattr('cylc.flow.tui.util.ME', 'cylc')
+    monkeypatch.setattr(
+        'cylc.flow.tui.util._display_workflow_id',
+        lambda data: data['name'].rsplit('/', 1)[-1]
+    )
+    monkeypatch.setattr(
+        'cylc.flow.tui.overlay._get_display_id',
+        get_display_id,
+    )
+
+    # filter Tui so that only workflows created within our test show up
+    id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser()))
+    workflow_filter = re.escape(id_base) + r'/.*'
+
+    @contextmanager
+    def _raikura(workflow_id=None, size='80,50'):
+        screen, html_fragment = configure_screenshot(size)
+        app = TuiApp(screen=screen)
+        with app.main(
+            workflow_id,
+            id_filter=workflow_filter,
+            interactive=False,
+        ):
+            yield RaikuraSession(
+                app,
+                html_fragment,
+                test_dir,
+                request.function.__name__,
+            )
+
+    return _raikura
diff --git a/tests/integration/tui/test_app.py b/tests/integration/tui/test_app.py
new file mode 100644
index 00000000000..d7dc26457b3
--- /dev/null
+++ b/tests/integration/tui/test_app.py
@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+import urwid
+
+from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.task_state import (
+#     TASK_STATUS_RUNNING,
+    TASK_STATUS_SUCCEEDED,
+#     TASK_STATUS_FAILED,
+#     TASK_STATUS_WAITING,
+)
+from cylc.flow.workflow_status import StopMode
+
+
+def set_task_state(schd, task_states):
+    """Force tasks into the desired states.
+
+    Task states should be of the format (cycle, task, state, is_held).
+    """
+    for cycle, task, state, is_held in task_states:
+        itask = schd.pool.get_task(cycle, task)
+        if not itask:
+            itask = schd.pool.spawn_task(task, cycle, {1})
+        itask.state_reset(state, is_held=is_held)
+        schd.data_store_mgr.delta_task_state(itask)
+        schd.data_store_mgr.increment_graph_window(
+            itask.tokens,
+            cycle,
+            {1},
+        )
+
+
+async def test_tui_basics(raikura):
+    """Test basic Tui interaction with no workflows."""
+    with raikura(size='80,40') as rk:
+        # the app should open
+        rk.compare_screenshot('test-raikura', 'the app should have loaded')
+
+        # "h" should bring up the onscreen help
+        rk.user_input('h')
+        rk.compare_screenshot(
+            'test-raikura-help',
+            'the help screen should be visible'
+        )
+
+        # "q" should close the popup
+        rk.user_input('q')
+        rk.compare_screenshot(
+            'test-raikura',
+            'the help screen should have closed',
+        )
+
+        # "enter" should bring up the context menu
+        rk.user_input('enter')
+        rk.compare_screenshot(
+            'test-raikura-enter',
+            'the context menu should have opened',
+        )
+
+        # "enter" again should close it via the "cancel" button
+        rk.user_input('enter')
+        rk.compare_screenshot(
+            'test-raikura',
+            'the context menu should have closed',
+        )
+
+        # "ctrl d" should exit Tui
+        with pytest.raises(urwid.ExitMainLoop):
+            rk.user_input('ctrl d')
+
+        # "q" should exit Tui
+        with pytest.raises(urwid.ExitMainLoop):
+            rk.user_input('q')
+
+
+async def test_subscribe_unsubscribe(one_conf, flow, scheduler, start, raikura):
+    """Test a simple workflow with one task."""
+    id_ = flow(one_conf, name='one')
+    schd = scheduler(id_)
+    async with start(schd):
+        await schd.update_data_structure()
+        with raikura(size='80,15') as rk:
+            rk.compare_screenshot(
+                'unsubscribed',
+                'the workflow should be collapsed'
+                ' (no subscription no state totals)',
+            )
+
+            # expand the workflow to subscribe to it
+            rk.user_input('down', 'right')
+            rk.wait_until_loaded()
+            rk.compare_screenshot(
+                'subscribed',
+                'the workflow should be expanded',
+            )
+
+            # collapse the workflow to unsubscribe from it
+            rk.user_input('left', 'up')
+            rk.force_update()
+            rk.compare_screenshot(
+                'unsubscribed',
+                'the workflow should be collapsed'
+                ' (no subscription no state totals)',
+            )
+
+
+async def test_workflow_states(one_conf, flow, scheduler, start, raikura):
+    """Test viewing multiple workflows in different states."""
+    # one => stopping
+    id_1 = flow(one_conf, name='one')
+    schd_1 = scheduler(id_1)
+    # two => paused
+    id_2 = flow(one_conf, name='two')
+    schd_2 = scheduler(id_2)
+    # tre => stopped
+    flow(one_conf, name='tre')
+
+    async with start(schd_1):
+        schd_1.stop_mode = StopMode.AUTO  # make it look like we're stopping
+        await schd_1.update_data_structure()
+
+        async with start(schd_2):
+            await schd_2.update_data_structure()
+            with raikura(size='80,15') as rk:
+                rk.compare_screenshot(
+                    'unfiltered',
+                    'All workflows should be visible (one, two, tree)',
+                )
+
+                # filter for active workflows (i.e. paused, running, stopping)
+                rk.user_input('p')
+                rk.compare_screenshot(
+                    'filter-active',
+                    'Only active workflows should be visible (one, two)'
+                )
+
+                # invert the filter so we are filtering for stopped workflows
+                rk.user_input('W', 'enter', 'q')
+                rk.compare_screenshot(
+                    'filter-stopped',
+                    'Only stopped workflow should be visible (tre)'
+                )
+
+                # filter in paused workflows
+                rk.user_input('W', 'down', 'enter', 'q')
+                rk.force_update()
+                rk.compare_screenshot(
+                    'filter-stopped-or-paused',
+                    'Only stopped or paused workflows should be visible'
+                    ' (two, tre)',
+                )
+
+                # reset the state filters
+                rk.user_input('W', 'down', 'down', 'enter', 'down', 'enter')
+
+                # scroll to the id filter text box
+                rk.user_input('down', 'down', 'down', 'down')
+
+                # scroll to the end of the ID
+                rk.user_input(*['right'] * (
+                    len(schd_1.tokens['workflow'].rsplit('/', 1)[0]) + 1)
+                )
+
+                # type the letter "t"
+                # (this should filter for workflows starting with "t")
+                rk.user_input('t')
+                rk.force_update()  # this is required for the tests
+                rk.user_input('page up', 'q')  # close the dialogue
+
+                rk.compare_screenshot(
+                    'filter-starts-with-t',
+                    'Only workflows starting with the letter "t" should be'
+                    ' visible (two, tre)',
+                )
+
+
+# TODO: Task state filtering is currently broken
+# see: https://github.com/cylc/cylc-flow/issues/5716
+#
+# async def test_task_states(flow, scheduler, start, raikura):
+#     id_ = flow({
+#         'scheduler': {
+#             'allow implicit tasks': 'true',
+#         },
+#         'scheduling': {
+#             'initial cycle point': '1',
+#             'cycling mode': 'integer',
+#             'runahead limit': 'P1',
+#             'graph': {
+#                 'P1': '''
+#                     a => b => c
+#                     b[-P1] => b
+#                 '''
+#             }
+#         }
+#     }, name='test_task_states')
+#     schd = scheduler(id_)
+#     async with start(schd):
+#         set_task_state(
+#             schd,
+#             [
+#                 (IntegerPoint('1'), 'a', TASK_STATUS_SUCCEEDED, False),
+#                 # (IntegerPoint('1'), 'b', TASK_STATUS_FAILED, False),
+#                 (IntegerPoint('1'), 'c', TASK_STATUS_RUNNING, False),
+#                 # (IntegerPoint('2'), 'a', TASK_STATUS_RUNNING, False),
+#                 (IntegerPoint('2'), 'b', TASK_STATUS_WAITING, True),
+#             ]
+#         )
+#         await schd.update_data_structure()
+#
+#         with raikura(schd.tokens.id, size='80,20') as rk:
+#             rk.compare_screenshot('unfiltered')
+#
+#             # filter out waiting tasks
+#             rk.user_input('T', 'down', 'enter', 'q')
+#             rk.compare_screenshot('filter-not-waiting')
+
+
+async def test_navigation(flow, scheduler, start, raikura):
+    """Test navigating with the arrow keys."""
+    id_ = flow({
+        'scheduling': {
+            'graph': {
+                'R1': 'A & B1 & B2',
+            }
+        },
+        'runtime': {
+            'A': {},
+            'B': {},
+            'B1': {'inherit': 'B'},
+            'B2': {'inherit': 'B'},
+            'a1': {'inherit': 'A'},
+            'a2': {'inherit': 'A'},
+            'b11': {'inherit': 'B1'},
+            'b12': {'inherit': 'B1'},
+            'b21': {'inherit': 'B2'},
+            'b22': {'inherit': 'B2'},
+        }
+    }, name='one')
+    schd = scheduler(id_)
+    async with start(schd):
+        await schd.update_data_structure()
+
+        with raikura(size='80,30') as rk:
+            rk.compare_screenshot(
+                'on-load',
+                'the workflow should be collapsed when Tui is loaded',
+            )
+
+            # pressing "right" should connect to the workflow
+            # and expand it once the data arrives
+            rk.user_input('down', 'right')
+            rk.wait_until_loaded(schd.tokens.id)
+            rk.compare_screenshot(
+                'workflow-expanded',
+                'the workflow should be expanded',
+            )
+
+            # pressing "left" should collapse the node
+            rk.user_input('down', 'down', 'left')
+            rk.compare_screenshot(
+                'family-A-collapsed',
+                'the family "1/A" should be collapsed',
+            )
+
+            # the "page up" and "page down" buttons should navigate to the top
+            # and bottom of the screen
+            rk.user_input('page down')
+            rk.compare_screenshot(
+                'cursor-at-bottom-of-screen',
+                'the cursor should be at the bottom of the screen',
+            )
+
+
+async def test_auto_expansion(flow, scheduler, start, raikura):
+    """It should automatically expand cycles and top-level families.
+
+    When a workflow is expanded, Tui should auto expand cycles and top-level
+    families. Any new cycles and top-level families should be auto-expanded
+    when added.
+    """
+    id_ = flow({
+        'scheduling': {
+            'runahead limit': 'P1',
+            'initial cycle point': '1',
+            'cycling mode': 'integer',
+            'graph': {
+                'P1': 'b[-P1] => a => b'
+            },
+        },
+        'runtime': {
+            'A': {},
+            'a': {'inherit': 'A'},
+            'b': {},
+        },
+    }, name='one')
+    schd = scheduler(id_)
+    with raikura(size='80,20') as rk:
+        async with start(schd):
+            await schd.update_data_structure()
+
+            # open the workflow
+            rk.force_update()
+            rk.user_input('down', 'right')
+            rk.wait_until_loaded(schd.tokens.id)
+
+            rk.compare_screenshot(
+                'on-load',
+                'cycle "1" and top-level family "1/A" should be expanded',
+            )
+
+            for task in ('a', 'b'):
+                itask = schd.pool.get_task(IntegerPoint('1'), task)
+                itask.state_reset(TASK_STATUS_SUCCEEDED)
+                schd.pool.spawn_on_output(itask, TASK_STATUS_SUCCEEDED)
+            await schd.update_data_structure()
+
+            rk.compare_screenshot(
+                'later-time',
+                'cycle "2" and top-level family "2/A" should be expanded',
+            )
+
+
+async def test_restart_reconnect(one_conf, flow, scheduler, start, raikura):
+    """It should handle workflow shutdown and restart.
+
+    The Cylc client can raise exceptions e.g. WorkflowStopped. Any text written
+    to stdout/err will mess with Tui. The purpose of this test is to ensure Tui
+    can handle shutdown / restart without any errors occuring and any spurious
+    text appearing on the screen.
+    """
+    with raikura(size='80,20') as rk:
+        schd = scheduler(flow(one_conf, name='one'))
+
+        # 1- start the workflow
+        async with start(schd):
+            await schd.update_data_structure()
+            rk.force_update()
+            rk.user_input('down', 'right')
+            rk.wait_until_loaded(schd.tokens.id)
+            rk.compare_screenshot(
+                '1-workflow-running',
+                'the workflow should appear in tui and be expanded on load',
+            )
+
+        # 2 - stop the worlflow
+        rk.compare_screenshot(
+            '2-workflow-stopped',
+            'the stopped workflow should be collapsed with a message saying'
+            ' workflow stopped',
+        )
+
+        # 3- restart the workflow
+        schd = scheduler(flow(one_conf, name='one'))
+        async with start(schd):
+            await schd.update_data_structure()
+            rk.wait_until_loaded(schd.tokens.id)
+            # rk.user_input('down', 'right')
+            rk.compare_screenshot(
+                '3-workflow-restarted',
+                'the restarted workflow should be expanded',
+            )
diff --git a/tests/integration/tui/test_mutations.py b/tests/integration/tui/test_mutations.py
new file mode 100644
index 00000000000..659c74ac1d0
--- /dev/null
+++ b/tests/integration/tui/test_mutations.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+
+import pytest
+
+from cylc.flow.exceptions import ClientError
+
+
+async def gen_commands(schd):
+    """Yield commands from the scheduler's command queue."""
+    while True:
+        await asyncio.sleep(0.1)
+        if not schd.command_queue.empty():
+            yield schd.command_queue.get()
+
+
+async def test_online_mutation(
+    one_conf,
+    flow,
+    scheduler,
+    start,
+    raikura,
+    monkeypatch,
+):
+    """Test a simple workflow with one task."""
+    id_ = flow(one_conf, name='one')
+    schd = scheduler(id_)
+    with raikura(size='80,15') as rk:
+        async with start(schd):
+            await schd.update_data_structure()
+            assert schd.command_queue.empty()
+
+            # open the workflow
+            rk.force_update()
+            rk.user_input('down', 'right')
+            rk.wait_until_loaded(schd.tokens.id)
+
+            # focus on a task
+            rk.user_input('down', 'right', 'down', 'right')
+            rk.compare_screenshot(
+                # take a screenshot to ensure we have focused on the task
+                # successfully
+                'task-selected',
+                'the cursor should be on the task 1/foo',
+            )
+
+            # focus on the hold mutation for a task
+            rk.user_input('enter', 'down')
+            rk.compare_screenshot(
+                # take a screenshot to ensure we have focused on the mutation
+                # successfully
+                'hold-mutation-selected',
+                'the cursor should be on the "hold" mutation',
+            )
+
+            # run the hold mutation
+            rk.user_input('enter')
+
+            # the mutation should be in the scheduler's command_queue
+            command = None
+            async for command in gen_commands(schd):
+                break
+            assert command == ('hold', (['1/one'],), {})
+
+        # close the dialogue and re-run the hold mutation
+        rk.user_input('q', 'q', 'enter')
+        rk.compare_screenshot(
+            'command-failed-workflow-stopped',
+            'an error should be visible explaining that the operation'
+            ' cannot be performed on a stopped workflow',
+            # NOTE: don't update so Tui still thinks the workflow is running
+            force_update=False,
+        )
+
+        # force mutations to raise ClientError
+        def _get_client(*args, **kwargs):
+            raise ClientError('mock error')
+        monkeypatch.setattr(
+            'cylc.flow.tui.data.get_client',
+            _get_client,
+        )
+
+        # close the dialogue and re-run the hold mutation
+        rk.user_input('q', 'q', 'enter')
+        rk.compare_screenshot(
+            'command-failed-client-error',
+            'an error should be visible explaining that the operation'
+            ' failed due to a client error',
+            # NOTE: don't update so Tui still thinks the workflow is running
+            force_update=False,
+        )
+
+
+@pytest.fixture
+def standardise_cli_cmds(monkeypatch):
+    """This remove the variable bit of the workflow ID from CLI commands.
+
+    The workflow ID changes from run to run. In order to make screenshots
+    stable, this 
+    """
+    from cylc.flow.tui.data import extract_context
+    def _extract_context(selection):
+        context = extract_context(selection)
+        if 'workflow' in context:
+            context['workflow'] = [
+                workflow.rsplit('/', 1)[-1]
+                for workflow in context.get('workflow', [])
+            ]
+        return context
+    monkeypatch.setattr(
+        'cylc.flow.tui.data.extract_context',
+        _extract_context,
+    )
+
+@pytest.fixture
+def capture_commands(monkeypatch):
+    ret = []
+    returncode = [0]
+
+    class _Popen:
+        def __init__(self, *args, **kwargs):
+            nonlocal ret
+            ret.append(args)
+
+        def communicate(self):
+            return 'mock-stdout', 'mock-stderr'
+
+        @property
+        def returncode(self):
+            nonlocal returncode
+            return returncode[0]
+
+    monkeypatch.setattr(
+        'cylc.flow.tui.data.Popen',
+        _Popen,
+    )
+
+    return ret, returncode
+
+
+async def test_offline_mutation(
+    one_conf,
+    flow,
+    raikura,
+    capture_commands,
+    standardise_cli_cmds,
+):
+    id_ = flow(one_conf, name='one')
+    commands, returncode = capture_commands
+
+    with raikura(size='80,15') as rk:
+        # run the stop-all mutation
+        rk.wait_until_loaded('root')
+        rk.user_input('enter', 'down')
+        rk.compare_screenshot(
+            # take a screenshot to ensure we have focused on the task
+            # successfully
+            'stop-all-mutation-selected',
+            'the stop-all mutation should be selected',
+        )
+        rk.user_input('enter')
+
+        # the command "cylc stop '*'" should have been run
+        assert commands == [(['cylc', 'stop', '*'],)]
+        commands.clear()
+
+        # run the clean command on the workflow
+        rk.user_input('down', 'enter', 'down')
+        rk.compare_screenshot(
+            # take a screenshot to ensure we have focused on the mutation
+            # successfully
+            'clean-mutation-selected',
+            'the clean mutation should be selected',
+        )
+        rk.user_input('enter')
+
+        # the command "cylc clean <id>" should have been run
+        assert commands == [(['cylc', 'clean', '--yes', 'one'],)]
+        commands.clear()
+
+        # make commands fail
+        returncode[:] = [1]
+        rk.user_input('enter', 'down')
+        rk.compare_screenshot(
+            # take a screenshot to ensure we have focused on the mutation
+            # successfully
+            'clean-mutation-selected',
+            'the clean mutation should be selected',
+        )
+        rk.user_input('enter')
+
+        assert commands == [(['cylc', 'clean', '--yes', 'one'],)]
+
+        rk.compare_screenshot(
+            # take a screenshot to ensure we have focused on the mutation
+            # successfully
+            'clean-command-error',
+            'there should be a box displaying the error containing the stderr'
+            ' returned by the command',
+        )
diff --git a/tests/integration/tui/test_updater.py b/tests/integration/tui/test_updater.py
new file mode 100644
index 00000000000..39a9a606eff
--- /dev/null
+++ b/tests/integration/tui/test_updater.py
@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+from copy import deepcopy
+from pathlib import Path
+import re
+
+from async_timeout import timeout
+
+from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.id import Tokens
+from cylc.flow.tui.updater import (
+    Updater,
+    get_default_filters,
+)
+from cylc.flow.workflow_status import WorkflowStatus
+
+
+async def await_update(updater):
+    """Wait for the latest update from the Updater.
+
+    Note:
+        If multiple updates are waiting, this returns the most recent.Q
+
+    Returns:
+        The latest update from the Updater's "update_queue".
+
+    """
+    while updater.update_queue.empty():
+        await asyncio.sleep(0.1)
+    while not updater.update_queue.empty():
+        # flush out any older updates to avoid race conditions in tests
+        update = updater.update_queue.get()
+    return update
+
+
+async def wait_workflow_connected(updater, tokens, connected=True, time=3):
+    """Wait for the Updater to connect to a workflow.
+
+    This will return once the updater has connected to a workflow and returned
+    the first data from it.
+
+    Arugments:
+        tokens:
+            The tokens of the workflow you're waiting for.
+        connected:
+            If True this waits for the updater to connect to the workflow, if
+            False it waits for the updater to disconnect from it.
+        time:
+            The maximum time to wait for this to happen.
+
+    Returns:
+        The first update from the Updater which contains the workflow data.
+
+    """
+    async with timeout(time):
+        while True:
+            root_node = await await_update(updater)
+            workflow_node = root_node['children'][0]
+            for workflow_node in root_node['children']:
+                if (
+                    workflow_node['id_'] == tokens.id
+                    and (
+                        workflow_node['children'][0]['id_'] != '#spring'
+                    ) == connected
+                ):
+                    # if the spring node is still there then we haven't
+                    # recieved the first update from the workflow yet
+                    return root_node
+
+
+def get_child_tokens(root_node, types, relative=False):
+    """Return all ID of the specified types contained within the provided tree.
+
+    Args:
+        root_node:
+            The Tui tree you want to look for IDs in.
+        types:
+            The Tui types (e.g. 'workflow' or 'task') you want to extract.
+        relative:
+            If True, the relative IDs will be returned.
+
+    """
+    ret = set()
+    stack = [root_node]
+    while stack:
+        node = stack.pop()
+        stack.extend(node['children'])
+        if node['type_'] in types:
+
+            tokens = Tokens(node['id_'])
+            if relative:
+                ret.add(tokens.relative_id)
+            else:
+                ret.add(tokens.id)
+    return ret
+
+
+async def test_subscribe(one_conf, flow, scheduler, run, test_dir):
+    """It should subscribe and unsubscribe from workflows."""
+    id_ = flow(one_conf)
+    schd = scheduler(id_)
+
+    updater = Updater()
+
+    async def the_test():
+        nonlocal updater
+
+        try:
+            # wait for the first update
+            root_node = await await_update(updater)
+
+            # there should be a root root_node
+            assert root_node['id_'] == 'root'
+            # a single root_node representing the workflow
+            assert root_node['children'][0]['id_'] == schd.tokens.id
+            # and a "spring" root_node used to active the subscription
+            # mechanism
+            assert root_node['children'][0]['children'][0]['id_'] == '#spring'
+
+            # subscribe to the workflow
+            updater.subscribe(schd.tokens.id)
+
+            # wait for it to connect to the workflow
+            root_node = await wait_workflow_connected(updater, schd.tokens)
+
+            # check the workflow contains one cycle with one task in it
+            workflow_node = root_node['children'][0]
+            assert len(workflow_node['children']) == 1
+            cycle_node = workflow_node['children'][0]
+            assert Tokens(cycle_node['id_']).relative_id == '1'  # cycle ID
+            assert len(cycle_node['children']) == 1
+            task_node = cycle_node['children'][0]
+            assert Tokens(task_node['id_']).relative_id == '1/one'  # task ID
+
+            # unsubscribe from the workflow
+            updater.unsubscribe(schd.tokens.id)
+
+            # wait for it to disconnect from the workflow
+            root_node = await wait_workflow_connected(
+                updater,
+                schd.tokens,
+                connected=False,
+            )
+
+        finally:
+            # shut down the updater
+            updater.terminate()
+
+    async with run(schd):
+        filters = get_default_filters()
+        filters['workflows']['id'] = f'{re.escape(str(test_dir.relative_to(Path("~/cylc-run").expanduser())))}/.*'
+
+        # run the updater and the test
+        async with timeout(10):
+            await asyncio.gather(
+                asyncio.create_task(updater.run(filters)),
+                asyncio.create_task(the_test()),
+            )
+
+
+async def test_filters(one_conf, flow, scheduler, run, test_dir):
+    """It should filter workflow and task states.
+
+    Note:
+        The workflow ID filter is not explicitly tested here, but it is
+        indirectly tested, otherwise other workflows would show up in the
+        updater results.
+
+    """
+    one = scheduler(flow({
+        'scheduler': {
+            'allow implicit tasks': 'True',
+        },
+        'scheduling': {
+            'graph': {
+                'R1': 'a & b & c',
+            }
+        }
+    }, name='one'), paused_start=True)
+    two = scheduler(flow(one_conf, name='two'))
+    tre = scheduler(flow(one_conf, name='tre'))
+
+    filters = get_default_filters()
+    id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser()))
+    filters['workflows']['id'] = f'^{re.escape(id_base)}/.*'
+
+    updater = Updater()
+
+    async def the_test():
+        nonlocal filters
+        try:
+            root_node = await await_update(updater)
+            assert {child['id_'] for child in root_node['children']} == {
+                one.tokens.id,
+                two.tokens.id,
+                tre.tokens.id,
+            }
+
+            # filter out paused workflows
+            filters = deepcopy(filters)
+            filters['workflows'][WorkflowStatus.STOPPED.value] = True
+            filters['workflows'][WorkflowStatus.PAUSED.value] = False
+            updater.update_filters(filters)
+
+            # "one" and "two" should now be filtered out
+            root_node = await await_update(updater)
+            assert {child['id_'] for child in root_node['children']} == {
+                tre.tokens.id,
+            }
+
+            # filter out stopped workflows
+            filters = deepcopy(filters)
+            filters['workflows'][WorkflowStatus.STOPPED.value] = False
+            filters['workflows'][WorkflowStatus.PAUSED.value] = True
+            updater.update_filters(filters)
+
+            # "tre" should now be filtered out
+            root_node = await await_update(updater)
+            assert {child['id_'] for child in root_node['children']} == {
+                one.tokens.id,
+                two.tokens.id,
+            }
+
+            # subscribe to "one"
+            updater.subscribe(one.tokens.id)
+            root_node = await wait_workflow_connected(updater, one.tokens)
+            assert get_child_tokens(
+                root_node, types={'task'}, relative=True
+            ) == {
+                '1/a',
+                '1/b',
+                '1/c',
+            }
+
+            # filter out running tasks
+            # TODO: see https://github.com/cylc/cylc-flow/issues/5716
+            # filters = deepcopy(filters)
+            # filters['tasks'][TASK_STATUS_RUNNING] = False
+            # updater.update_filters(filters)
+
+            # root_node = await await_update(updater)
+            # assert get_child_tokens(
+            #   root_node,
+            #   types={'task'},
+            #   relative=True
+            # ) == {
+            #     '1/b',
+            #     '1/c',
+            # }
+
+        finally:
+            # shut down the updater
+            updater.terminate()
+
+    # start workflow "one"
+    async with run(one):
+        # mark "1/a" as running and "1/b" as succeeded
+        one_a = one.pool.get_task(IntegerPoint('1'), 'a')
+        one_a.state_reset('running')
+        one.data_store_mgr.delta_task_state(one_a)
+        one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded')
+
+        # start workflow "two"
+        async with run(two):
+            # run the updater and the test
+            async with timeout(10):
+                await asyncio.gather(
+                    asyncio.create_task(the_test()),
+                    asyncio.create_task(updater.run(filters)),
+                )
diff --git a/tests/unit/tui/test_data.py b/tests/unit/tui/test_data.py
index a2d17bf2e76..85805a5d1ea 100644
--- a/tests/unit/tui/test_data.py
+++ b/tests/unit/tui/test_data.py
@@ -28,7 +28,7 @@ def test_generate_mutation(monkeypatch):
     monkeypatch.setattr(cylc.flow.tui.data, 'ARGUMENT_TYPES', arg_types)
     assert generate_mutation(
         'my_mutation',
-        ['foo', 'bar']
+        {'foo': 'foo', 'bar': 'bar', 'user': 'user'}
     ) == '''
         mutation($foo: String!, $bar: [Int]) {
             my_mutation (foos: $foo, bars: $bar) {
diff --git a/tests/unit/tui/test_overlay.py b/tests/unit/tui/test_overlay.py
index 42334aac009..013e8480c21 100644
--- a/tests/unit/tui/test_overlay.py
+++ b/tests/unit/tui/test_overlay.py
@@ -21,7 +21,9 @@
 import pytest
 import urwid
 
+from cylc.flow.tui.app import BINDINGS
 import cylc.flow.tui.overlay
+from cylc.flow.workflow_status import WorkflowStatus
 
 
 @pytest.fixture
@@ -39,6 +41,7 @@ def overlay_functions():
         getattr(cylc.flow.tui.overlay, obj.name)
         for obj in tree.body
         if isinstance(obj, ast.FunctionDef)
+        and not obj.name.startswith('_')
     ]
 
 
@@ -47,14 +50,21 @@ def test_interface(overlay_functions):
     for function in overlay_functions:
         # mock up an app object to keep things working
         app = Mock(
-            filter_states={},
+            filters={'tasks': {}, 'workflows': {'id': '.*'}},
+            bindings=BINDINGS,
             tree_walker=Mock(
                 get_focus=Mock(
                     return_value=[
                         Mock(
                             get_node=Mock(
                                 return_value=Mock(
-                                    get_value=lambda: {'id_': 'a'}
+                                    get_value=lambda: {
+                                        'id_': '~u/a',
+                                        'type_': 'workflow',
+                                        'data': {
+                                            'status': WorkflowStatus.RUNNING,
+                                        },
+                                    }
                                 )
                             )
                         )
diff --git a/tests/unit/tui/test_util.py b/tests/unit/tui/test_util.py
index 00ac9fa95be..2b3231e0f7e 100644
--- a/tests/unit/tui/test_util.py
+++ b/tests/unit/tui/test_util.py
@@ -189,77 +189,87 @@ def test_compute_tree():
 
     """
     tree = compute_tree({
-        'id': 'workflow id',
-        'cyclePoints': [
-            {
-                'id': '1/family-suffix',
-                'cyclePoint': '1'
-            }
-        ],
-        'familyProxies': [
-            {  # top level family
-                'name': 'FOO',
-                'id': '1/FOO',
-                'cyclePoint': '1',
-                'firstParent': {'name': 'root', 'id': '1/root'}
-            },
-            {  # nested family
-                'name': 'FOOT',
-                'id': '1/FOOT',
-                'cyclePoint': '1',
-                'firstParent': {'name': 'FOO', 'id': '1/FOO'}
-            },
-        ],
-        'taskProxies': [
-            {  # top level task
-                'name': 'pub',
-                'id': '1/pub',
-                'firstParent': {'name': 'root', 'id': '1/root'},
-                'cyclePoint': '1',
-                'jobs': []
-            },
-            {  # child task (belongs to family)
-                'name': 'fan',
-                'id': '1/fan',
-                'firstParent': {'name': 'fan', 'id': '1/fan'},
-                'cyclePoint': '1',
-                'jobs': []
-            },
-            {  # nested child task (belongs to incestuous family)
-                'name': 'fool',
-                'id': '1/fool',
-                'firstParent': {'name': 'FOOT', 'id': '1/FOOT'},
-                'cyclePoint': '1',
-                'jobs': []
-            },
-            {  # a task which has jobs
-                'name': 'worker',
-                'id': '1/worker',
-                'firstParent': {'name': 'root', 'id': '1/root'},
-                'cyclePoint': '1',
-                'jobs': [
-                    {'id': '1/worker/03', 'submitNum': '3'},
-                    {'id': '1/worker/02', 'submitNum': '2'},
-                    {'id': '1/worker/01', 'submitNum': '1'}
-                ]
-            }
-        ]
+        'workflows': [{
+            'id': 'workflow id',
+            'port': 1234,
+            'cyclePoints': [
+                {
+                    'id': '1/family-suffix',
+                    'cyclePoint': '1'
+                }
+            ],
+            'familyProxies': [
+                {  # top level family
+                    'name': 'FOO',
+                    'id': '1/FOO',
+                    'cyclePoint': '1',
+                    'firstParent': {'name': 'root', 'id': '1/root'}
+                },
+                {  # nested family
+                    'name': 'FOOT',
+                    'id': '1/FOOT',
+                    'cyclePoint': '1',
+                    'firstParent': {'name': 'FOO', 'id': '1/FOO'}
+                },
+            ],
+            'taskProxies': [
+                {  # top level task
+                    'name': 'pub',
+                    'id': '1/pub',
+                    'firstParent': {'name': 'root', 'id': '1/root'},
+                    'cyclePoint': '1',
+                    'jobs': []
+                },
+                {  # child task (belongs to family)
+                    'name': 'fan',
+                    'id': '1/fan',
+                    'firstParent': {'name': 'fan', 'id': '1/fan'},
+                    'cyclePoint': '1',
+                    'jobs': []
+                },
+                {  # nested child task (belongs to incestuous family)
+                    'name': 'fool',
+                    'id': '1/fool',
+                    'firstParent': {'name': 'FOOT', 'id': '1/FOOT'},
+                    'cyclePoint': '1',
+                    'jobs': []
+                },
+                {  # a task which has jobs
+                    'name': 'worker',
+                    'id': '1/worker',
+                    'firstParent': {'name': 'root', 'id': '1/root'},
+                    'cyclePoint': '1',
+                    'jobs': [
+                        {'id': '1/worker/03', 'submitNum': '3'},
+                        {'id': '1/worker/02', 'submitNum': '2'},
+                        {'id': '1/worker/01', 'submitNum': '1'}
+                    ]
+                }
+            ]
+        }]
     })
 
+    # the root node
+    assert tree['type_'] == 'root'
+    assert tree['id_'] == 'root'
+    assert len(tree['children']) == 1
+
     # the workflow node
-    assert tree['type_'] == 'workflow'
-    assert tree['id_'] == 'workflow id'
-    assert list(tree['data']) == [
+    workflow = tree['children'][0]
+    assert workflow['type_'] == 'workflow'
+    assert workflow['id_'] == 'workflow id'
+    assert set(workflow['data']) == {
         # whatever if present on the node should end up in data
-        'id',
         'cyclePoints',
         'familyProxies',
+        'id',
+        'port',
         'taskProxies'
-    ]
-    assert len(tree['children']) == 1
+    }
+    assert len(workflow['children']) == 1
 
     # the cycle point node
-    cycle = tree['children'][0]
+    cycle = workflow['children'][0]
     assert cycle['type_'] == 'cycle'
     assert cycle['id_'] == '//1'
     assert list(cycle['data']) == [

From b1752185575e506d278133a62009a0920fe818da Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 14 Sep 2023 17:48:55 +0100
Subject: [PATCH 020/110] tui: update screenshots

---
 .../test_auto_expansion.later-time.html       | 21 ++++++++++
 .../test_auto_expansion.on-load.html          | 21 ++++++++++
 ...navigation.cursor-at-bottom-of-screen.html | 31 ++++++++++++++
 .../test_navigation.family-A-collapsed.html   | 31 ++++++++++++++
 .../screenshots/test_navigation.on-load.html  | 31 ++++++++++++++
 .../test_navigation.workflow-expanded.html    | 31 ++++++++++++++
 ..._offline_mutation.clean-command-error.html | 16 ++++++++
 ...line_mutation.clean-mutation-selected.html | 16 ++++++++
 ...e_mutation.stop-all-mutation-selected.html | 16 ++++++++
 ..._mutation.command-failed-client-error.html | 16 ++++++++
 ...ation.command-failed-workflow-stopped.html | 16 ++++++++
 .../test_online_mutation.command-failed.html  | 16 ++++++++
 ...nline_mutation.hold-mutation-selected.html | 16 ++++++++
 .../test_online_mutation.task-selected.html   | 16 ++++++++
 ..._restart_reconnect.1-workflow-running.html | 21 ++++++++++
 ..._restart_reconnect.2-workflow-stopped.html | 21 ++++++++++
 ...estart_reconnect.3-workflow-restarted.html | 21 ++++++++++
 ...test_subscribe_unsubscribe.subscribed.html | 16 ++++++++
 ...st_subscribe_unsubscribe.unsubscribed.html | 16 ++++++++
 .../test_tui_basics.test-raikura-enter.html   | 41 +++++++++++++++++++
 .../test_tui_basics.test-raikura-help.html    | 41 +++++++++++++++++++
 .../test_tui_basics.test-raikura.html         | 41 +++++++++++++++++++
 .../test_workflow_states.filter-active.html   | 16 ++++++++
 ..._workflow_states.filter-starts-with-t.html | 16 ++++++++
 ...kflow_states.filter-stopped-or-paused.html | 16 ++++++++
 .../test_workflow_states.filter-stopped.html  | 16 ++++++++
 .../test_workflow_states.unfiltered.html      | 16 ++++++++
 27 files changed, 592 insertions(+)
 create mode 100644 tests/integration/tui/screenshots/test_auto_expansion.later-time.html
 create mode 100644 tests/integration/tui/screenshots/test_auto_expansion.on-load.html
 create mode 100644 tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
 create mode 100644 tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
 create mode 100644 tests/integration/tui/screenshots/test_navigation.on-load.html
 create mode 100644 tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
 create mode 100644 tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html
 create mode 100644 tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
 create mode 100644 tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
 create mode 100644 tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html
 create mode 100644 tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html
 create mode 100644 tests/integration/tui/screenshots/test_online_mutation.command-failed.html
 create mode 100644 tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
 create mode 100644 tests/integration/tui/screenshots/test_online_mutation.task-selected.html
 create mode 100644 tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
 create mode 100644 tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
 create mode 100644 tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
 create mode 100644 tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
 create mode 100644 tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
 create mode 100644 tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html
 create mode 100644 tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html
 create mode 100644 tests/integration/tui/screenshots/test_tui_basics.test-raikura.html
 create mode 100644 tests/integration/tui/screenshots/test_workflow_states.filter-active.html
 create mode 100644 tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
 create mode 100644 tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
 create mode 100644 tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
 create mode 100644 tests/integration/tui/screenshots/test_workflow_states.unfiltered.html

diff --git a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html
new file mode 100644
index 00000000000..f5a19fd428d
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html
@@ -0,0 +1,21 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 2                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ a                                                               </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">○ b                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
new file mode 100644
index 00000000000..df3c9f5c41b
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
@@ -0,0 +1,21 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ a                                                               </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">○ b                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
new file mode 100644
index 00000000000..bf5e3812008
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B1                                                              </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b11                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b12                                                          </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B2                                                              </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b21                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#e5e5e5;background:#000000"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b22                                                          </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
new file mode 100644
index 00000000000..cefab5264f4
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#e5e5e5;background:#000000">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B1                                                              </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b11                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b12                                                          </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B2                                                              </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b21                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b22                                                          </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_navigation.on-load.html b/tests/integration/tui/screenshots/test_navigation.on-load.html
new file mode 100644
index 00000000000..a0bd107742b
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_navigation.on-load.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
new file mode 100644
index 00000000000..6b26ced563e
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ a1                                                              </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ a2                                                              </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B1                                                              </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b11                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b12                                                          </span>
+<span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B2                                                              </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b21                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b22                                                          </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html
new file mode 100644
index 00000000000..b930bb4d776
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stop</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Ac</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error in command cylc clean --yes one       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">mock-stderr                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
new file mode 100644
index 00000000000..b343a46ea47
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id: ~cylc/one                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stop</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">lean                                   </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">play                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">reinstall-reload                        </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
new file mode 100644
index 00000000000..c2355597f78
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id: ~cylc/root                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stop</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">s</span><span style="color:#000000;background:#e5e5e5">top-all                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html
new file mode 100644
index 00000000000..11cdcfd67f9
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paus</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Ac</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error connecting to workflow: mock error    </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ on</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html
new file mode 100644
index 00000000000..fae4a429cc6
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paus</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Ac</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Cannot peform command hold on a stopped     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">workflow                                    </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ on</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed.html
new file mode 100644
index 00000000000..fae4a429cc6
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paus</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Ac</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Cannot peform command hold on a stopped     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">workflow                                    </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ on</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
new file mode 100644
index 00000000000..9cda0e3e150
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   work</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id: 1/one                                   </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc        </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paus</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ on</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">h</span><span style="color:#000000;background:#e5e5e5">old                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">kill                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">poll                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">release                                 </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">trigger                                 </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
new file mode 100644
index 00000000000..7d94d5e43dd
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#e5e5e5;background:#000000"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
new file mode 100644
index 00000000000..74c02508239
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
@@ -0,0 +1,21 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
new file mode 100644
index 00000000000..09c3bbd7fb0
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
@@ -0,0 +1,21 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">Workflow is not running                                                 </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
new file mode 100644
index 00000000000..74c02508239
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
@@ -0,0 +1,21 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
new file mode 100644
index 00000000000..019184ec897
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
new file mode 100644
index 00000000000..8fa0f4329a1
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html
new file mode 100644
index 00000000000..d54d9538d26
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html
@@ -0,0 +1,41 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id: ~cylc/root                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">(</span><span style="color:#000000;background:#e5e5e5">cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">stop-all                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html
new file mode 100644
index 00000000000..1795c586d9a
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html
@@ -0,0 +1,41 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">─────────────────────────────</span><span style="color:#000000;background:#e5e5e5">─────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   _        _         _               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                  | |      | |       (_)              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">         ___ _   _| | ___  | |_ _   _ _               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        / __| | | | |/ __| | __| | | | |              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">       | (__| |_| | | (__  | |_| |_| | |              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        \___|\__, |_|\___|  \__|\__,_|_|              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              __/ |                                   </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">             |___/                                    </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">          ( scroll using arrow keys )                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                           _,@@@@@@.                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                         &lt;=@@@, `@@@@@.               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                            `-@@@@@@@@@@@'            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                               :@@@@@@@@@@.           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                              (.@@@@@@@@@@@           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                             ( '@@@@@@@@@@@@.         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                            ;.@@@@@@@@@@@@@@@         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                          '@@@@@@@@@@@@@@@@@@,        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                        ,@@@@@@@@@@@@@@@@@@@@'        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                      :.@@@@@@@@@@@@@@@@@@@@@.        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                    .@@@@@@@@@@@@@@@@@@@@@@@@.        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                  '@@@@@@@@@@@@@@@@@@@@@@@@@.         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                ;@@@@@@@@@@@@@@@@@@@@@@@@@@@          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">               .@@@@@@@@@@@@@@@@@@@@@@@@@@.           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              .@@@@@@@@@@@@@@@@@@@@@@@@@@,            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">             .@@@@@@@@@@@@@@@@@@@@@@@@@'              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">            .@@@@@@@@@@@@@@@@@@@@@@@@'     ,          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">          :@@@@@@@@@@@@@@@@@@@@@..''';,,,;::-         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">         '@@@@@@@@@@@@@@@@@@@.        `.   `          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        .@@@@@@.: ,.@@@@@@@.            `             </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">      :@@@@@@@,         ;.@,                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">     '@@@@@@.              `@'                        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  h</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tas</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html b/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html
new file mode 100644
index 00000000000..7f80031804b
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html
@@ -0,0 +1,41 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
new file mode 100644
index 00000000000..282f76735ed
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff00ff;background:#e5e5e5">stopping</span><span style="color:#000000;background:#e5e5e5">                                                             </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
new file mode 100644
index 00000000000..8c26ce6ccc9
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
new file mode 100644
index 00000000000..8c26ce6ccc9
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
new file mode 100644
index 00000000000..1ff602df101
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
new file mode 100644
index 00000000000..0651eedec30
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
@@ -0,0 +1,16 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff00ff;background:#e5e5e5">stopping</span><span style="color:#000000;background:#e5e5e5">                                                             </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file

From 6f3f1701c1106ecd779b11b08ad4293186678da0 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 28 Sep 2023 16:55:10 +0100
Subject: [PATCH 021/110] tui: fix an obscure freezing issue

* Fixes an obscure issue where the applications main loop becomes
  suspended as the result of attempting to register an `on_press`
  callback which takes as an argument a Cylc client.
* Even though this callback is not actually called at this point,
  just passing the reference to it appears to be enough to cause
  thread deadlocking because Cylc clients are async based and
  not thread safe.
---
 cylc/flow/tui/data.py    | 34 ++++++++++++++++++++++++----------
 cylc/flow/tui/overlay.py | 34 ++++++++++++++++------------------
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index 3a3ed4ee378..4d00753cf3f 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -17,7 +17,12 @@
 from functools import partial
 from subprocess import Popen, PIPE
 
-from cylc.flow.exceptions import ClientError
+from cylc.flow.exceptions import (
+    ClientError,
+    ClientTimeout,
+    WorkflowStopped,
+)
+from cylc.flow.network.client_factory import get_client
 from cylc.flow.id import Tokens
 from cylc.flow.tui.util import (
     extract_context
@@ -187,11 +192,25 @@ def generate_mutation(mutation, arguments):
     '''
 
 
-def list_mutations(client, selection):
+def list_mutations(selection, is_running=True):
+    """List mutations relevant to the provided selection.
+
+    Args:
+        selection:
+            The user selection.
+        is_running:
+            If False, then mutations which require the scheduler to be
+            running will be omitted.
+
+            Note, this is only relevant for workflow nodes because if a
+            workflow is stopped, then any tasks within it will be removed
+            anyway.
+
+    """
     context = extract_context(selection)
     selection_type = list(context)[-1]
     ret = []
-    if client:
+    if is_running:
         # add the online mutations
         ret.extend(MUTATIONS.get(selection_type, []))
     # add the offline mutations
@@ -232,20 +251,15 @@ def context_to_variables(context):
     return variables
 
 
-def mutate(client, mutation, selection):
+def mutate(mutation, selection):
     if mutation in {
         _mutation
         for section in OFFLINE_MUTATIONS.values()
         for _mutation in section
     }:
         offline_mutate(mutation, selection)
-    elif client:
-        online_mutate(client, mutation, selection)
     else:
-        raise Exception(
-            f'Cannot peform command {mutation} on a stopped workflow'
-            ' or invalid command.'
-        )
+        online_mutate(mutation, selection)
 
 
 def online_mutate(mutation, selection):
diff --git a/cylc/flow/tui/overlay.py b/cylc/flow/tui/overlay.py
index f4d5ca02f94..87b544b0604 100644
--- a/cylc/flow/tui/overlay.py
+++ b/cylc/flow/tui/overlay.py
@@ -37,20 +37,13 @@
 
 """
 
-from contextlib import suppress
 from functools import partial
 import re
 import sys
 
 import urwid
 
-from cylc.flow.exceptions import (
-    ClientError,
-    ClientTimeout,
-    WorkflowStopped,
-)
 from cylc.flow.id import Tokens
-from cylc.flow.network.client_factory import get_client
 from cylc.flow.task_state import (
     TASK_STATUSES_ORDERED,
     TASK_STATUS_WAITING
@@ -61,7 +54,6 @@
     TUI
 )
 from cylc.flow.tui.data import (
-    extract_context,
     list_mutations,
     mutate,
 )
@@ -296,20 +288,23 @@ def context(app):
     """An overlay for context menus."""
     value = app.tree_walker.get_focus()[0].get_node().get_value()
     selection = [value['id_']]  # single selection ATM
-    context = extract_context(selection)
 
-    client = None
-    if 'workflow' in context:
-        w_id = context['workflow'][0]
-        with suppress(WorkflowStopped, ClientError, ClientTimeout):
-            client = get_client(w_id)
+    is_running = True
+    if (
+        value['type_'] == 'workflow'
+        and value['data']['status'] not in {'running', 'paused'}
+    ):
+        # this is a stopped workflow
+        # => don't display mutations only valid for a running workflow
+        is_running = False
 
     def _mutate(mutation, _):
-        nonlocal app, client
+        nonlocal app, selection
+
         app.open_overlay(partial(progress, text='Running Command'))
         try:
-            mutate(client, mutation, selection)
-        except ClientError as exc:
+            mutate(mutation, selection)
+        except Exception as exc:
             app.open_overlay(partial(error, text=str(exc)))
         else:
             app.close_topmost()
@@ -334,7 +329,10 @@ def _mutate(mutation, _):
                     mutation,
                     on_press=partial(_mutate, mutation)
                 )
-                for mutation in list_mutations(client, selection)
+                for mutation in list_mutations(
+                    selection,
+                    is_running,
+                )
             ]
         )
     )

From 653317f987db34cb151de2f5a91bd7ce4d5b5868 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 17 Oct 2023 16:21:23 +0100
Subject: [PATCH 022/110] graphql: remove extraLogs field (#5672)

Remove residual code leftover from the obsolete `extra log files` runtime configuration option.
---
 cylc/flow/data_messages.proto                 |   2 +-
 cylc/flow/data_messages_pb2.py                | 121 +++++++++---------
 cylc/flow/data_store_mgr.py                   |   1 -
 .../flow/job_runner_handlers/documentation.py |   1 -
 cylc/flow/network/schema.py                   |   5 -
 cylc/flow/scripts/dump.py                     |   1 -
 cylc/flow/task_job_mgr.py                     |  16 +--
 cylc/flow/task_proxy.py                       |   3 -
 tests/integration/test_data_store_mgr.py      |   1 -
 tests/integration/test_graphql.py             |   1 -
 10 files changed, 66 insertions(+), 86 deletions(-)

diff --git a/cylc/flow/data_messages.proto b/cylc/flow/data_messages.proto
index ced0cad7bef..cc56fba2e62 100644
--- a/cylc/flow/data_messages.proto
+++ b/cylc/flow/data_messages.proto
@@ -131,6 +131,7 @@ message PbRuntime {
 
 // Nodes
 message PbJob {
+    reserved 29;  /* see https://github.com/cylc/cylc-flow/pull/5672 */
     optional string stamp = 1;
     optional string id = 2;
     optional int32 submit_num = 3;
@@ -144,7 +145,6 @@ message PbJob {
     optional float execution_time_limit = 14;
     optional string platform = 15;
     optional string job_log_dir = 17;
-    repeated string extra_logs = 29;
     optional string name = 30; /* filter item */
     optional string cycle_point = 31; /* filter item */
     repeated string messages = 32;
diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py
index 437cc941fdf..1bf44b36dae 100644
--- a/cylc/flow/data_messages_pb2.py
+++ b/cylc/flow/data_messages_pb2.py
@@ -14,26 +14,25 @@
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xa2\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18  \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updated\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\xab\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x12\n\nextra_logs\x18\x1d \x03(\t\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18  \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtime\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xe7\x07\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtime\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\x84\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtime\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xa2\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18  \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updated\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18  \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xe7\x07\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtime\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\x84\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtime\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
   DESCRIPTOR._options = None
-  _PBWORKFLOW_STATETOTALSENTRY._options = None
-  _PBWORKFLOW_STATETOTALSENTRY._serialized_options = b'8\001'
-  _PBWORKFLOW_LATESTSTATETASKSENTRY._options = None
-  _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_options = b'8\001'
-  _PBTASKPROXY_OUTPUTSENTRY._options = None
-  _PBTASKPROXY_OUTPUTSENTRY._serialized_options = b'8\001'
-  _PBTASKPROXY_EXTERNALTRIGGERSENTRY._options = None
-  _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_options = b'8\001'
-  _PBTASKPROXY_XTRIGGERSENTRY._options = None
-  _PBTASKPROXY_XTRIGGERSENTRY._serialized_options = b'8\001'
-  _PBFAMILYPROXY_STATETOTALSENTRY._options = None
-  _PBFAMILYPROXY_STATETOTALSENTRY._serialized_options = b'8\001'
+  _globals['_PBWORKFLOW_STATETOTALSENTRY']._options = None
+  _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_options = b'8\001'
+  _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._options = None
+  _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_options = b'8\001'
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._options = None
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._options = None
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_options = b'8\001'
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._options = None
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_options = b'8\001'
+  _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._options = None
+  _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_options = b'8\001'
   _globals['_PBMETA']._serialized_start=24
   _globals['_PBMETA']._serialized_end=174
   _globals['_PBTIMEZONE']._serialized_start=177
@@ -49,53 +48,53 @@
   _globals['_PBRUNTIME']._serialized_start=1964
   _globals['_PBRUNTIME']._serialized_end=2789
   _globals['_PBJOB']._serialized_start=2792
-  _globals['_PBJOB']._serialized_end=3475
-  _globals['_PBTASK']._serialized_start=3478
-  _globals['_PBTASK']._serialized_end=3832
-  _globals['_PBPOLLTASK']._serialized_start=3835
-  _globals['_PBPOLLTASK']._serialized_end=4051
-  _globals['_PBCONDITION']._serialized_start=4054
-  _globals['_PBCONDITION']._serialized_end=4257
-  _globals['_PBPREREQUISITE']._serialized_start=4260
-  _globals['_PBPREREQUISITE']._serialized_end=4410
-  _globals['_PBOUTPUT']._serialized_start=4413
-  _globals['_PBOUTPUT']._serialized_end=4553
-  _globals['_PBTRIGGER']._serialized_start=4556
-  _globals['_PBTRIGGER']._serialized_end=4721
-  _globals['_PBTASKPROXY']._serialized_start=4724
-  _globals['_PBTASKPROXY']._serialized_end=5723
-  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5349
-  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5406
-  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5408
-  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5475
-  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5477
-  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5537
-  _globals['_PBFAMILY']._serialized_start=5726
-  _globals['_PBFAMILY']._serialized_end=6054
-  _globals['_PBFAMILYPROXY']._serialized_start=6057
-  _globals['_PBFAMILYPROXY']._serialized_end=6829
+  _globals['_PBJOB']._serialized_end=3461
+  _globals['_PBTASK']._serialized_start=3464
+  _globals['_PBTASK']._serialized_end=3818
+  _globals['_PBPOLLTASK']._serialized_start=3821
+  _globals['_PBPOLLTASK']._serialized_end=4037
+  _globals['_PBCONDITION']._serialized_start=4040
+  _globals['_PBCONDITION']._serialized_end=4243
+  _globals['_PBPREREQUISITE']._serialized_start=4246
+  _globals['_PBPREREQUISITE']._serialized_end=4396
+  _globals['_PBOUTPUT']._serialized_start=4399
+  _globals['_PBOUTPUT']._serialized_end=4539
+  _globals['_PBTRIGGER']._serialized_start=4542
+  _globals['_PBTRIGGER']._serialized_end=4707
+  _globals['_PBTASKPROXY']._serialized_start=4710
+  _globals['_PBTASKPROXY']._serialized_end=5709
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5335
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5392
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5394
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5461
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5463
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5523
+  _globals['_PBFAMILY']._serialized_start=5712
+  _globals['_PBFAMILY']._serialized_end=6040
+  _globals['_PBFAMILYPROXY']._serialized_start=6043
+  _globals['_PBFAMILYPROXY']._serialized_end=6815
   _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1411
   _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1461
-  _globals['_PBEDGE']._serialized_start=6832
-  _globals['_PBEDGE']._serialized_end=7020
-  _globals['_PBEDGES']._serialized_start=7022
-  _globals['_PBEDGES']._serialized_end=7145
-  _globals['_PBENTIREWORKFLOW']._serialized_start=7148
-  _globals['_PBENTIREWORKFLOW']._serialized_end=7390
-  _globals['_EDELTAS']._serialized_start=7393
-  _globals['_EDELTAS']._serialized_end=7568
-  _globals['_FDELTAS']._serialized_start=7571
-  _globals['_FDELTAS']._serialized_end=7750
-  _globals['_FPDELTAS']._serialized_start=7753
-  _globals['_FPDELTAS']._serialized_end=7943
-  _globals['_JDELTAS']._serialized_start=7946
-  _globals['_JDELTAS']._serialized_end=8119
-  _globals['_TDELTAS']._serialized_start=8122
-  _globals['_TDELTAS']._serialized_end=8297
-  _globals['_TPDELTAS']._serialized_start=8300
-  _globals['_TPDELTAS']._serialized_end=8486
-  _globals['_WDELTAS']._serialized_start=8489
-  _globals['_WDELTAS']._serialized_end=8684
-  _globals['_ALLDELTAS']._serialized_start=8687
-  _globals['_ALLDELTAS']._serialized_end=8896
+  _globals['_PBEDGE']._serialized_start=6818
+  _globals['_PBEDGE']._serialized_end=7006
+  _globals['_PBEDGES']._serialized_start=7008
+  _globals['_PBEDGES']._serialized_end=7131
+  _globals['_PBENTIREWORKFLOW']._serialized_start=7134
+  _globals['_PBENTIREWORKFLOW']._serialized_end=7376
+  _globals['_EDELTAS']._serialized_start=7379
+  _globals['_EDELTAS']._serialized_end=7554
+  _globals['_FDELTAS']._serialized_start=7557
+  _globals['_FDELTAS']._serialized_end=7736
+  _globals['_FPDELTAS']._serialized_start=7739
+  _globals['_FPDELTAS']._serialized_end=7929
+  _globals['_JDELTAS']._serialized_start=7932
+  _globals['_JDELTAS']._serialized_end=8105
+  _globals['_TDELTAS']._serialized_start=8108
+  _globals['_TDELTAS']._serialized_end=8283
+  _globals['_TPDELTAS']._serialized_start=8286
+  _globals['_TPDELTAS']._serialized_end=8472
+  _globals['_WDELTAS']._serialized_start=8475
+  _globals['_WDELTAS']._serialized_end=8670
+  _globals['_ALLDELTAS']._serialized_start=8673
+  _globals['_ALLDELTAS']._serialized_end=8882
 # @@protoc_insertion_point(module_scope)
diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py
index becd21f50e9..28f9a2f874e 100644
--- a/cylc/flow/data_store_mgr.py
+++ b/cylc/flow/data_store_mgr.py
@@ -1396,7 +1396,6 @@ def insert_job(self, name, cycle_point, status, job_conf):
         # Add in log files.
         j_buf.job_log_dir = get_task_job_log(
             self.schd.workflow, tproxy.cycle_point, tproxy.name, sub_num)
-        j_buf.extra_logs.extend(job_conf.get('logfiles', []))
 
         self.added[JOBS][j_id] = j_buf
         getattr(self.updated[WORKFLOW], JOBS).append(j_id)
diff --git a/cylc/flow/job_runner_handlers/documentation.py b/cylc/flow/job_runner_handlers/documentation.py
index 735b1801b16..7f4a0d289e4 100644
--- a/cylc/flow/job_runner_handlers/documentation.py
+++ b/cylc/flow/job_runner_handlers/documentation.py
@@ -110,7 +110,6 @@ class MyHandler():
        * ``job_file_path``
        * ``job_runner_command_template``
        * ``job_runner_name``
-       * ``logfiles``
        * ``namespace_hierarchy``
        * ``param_var``
        * ``platform``
diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 50b92c99445..63d9f236d7a 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -887,11 +887,6 @@ class Meta:
     job_log_dir = String(
         description="The path to the job's log directory.",
     )
-    extra_logs = graphene.List(
-        # TODO: remove. see https://github.com/cylc/cylc-flow/issues/5610
-        String,
-        description='Obsolete, do not use.',
-    )
     messages = graphene.List(
         String,
         description='The list of task messages generated by this job.',
diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py
index 636f0c81947..c62bf6719aa 100755
--- a/cylc/flow/scripts/dump.py
+++ b/cylc/flow/scripts/dump.py
@@ -75,7 +75,6 @@
     startedTime
     finishedTime
     jobLogDir
-    extraLogs
     platform
     executionTimeLimit
     jobRunnerName
diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py
index 20ee7379d27..412ba23b97a 100644
--- a/cylc/flow/task_job_mgr.py
+++ b/cylc/flow/task_job_mgr.py
@@ -27,7 +27,6 @@
 from contextlib import suppress
 import json
 import os
-from copy import deepcopy
 from logging import (
     CRITICAL,
     DEBUG,
@@ -1190,12 +1189,11 @@ def _prep_submit_task_job(
                 self._set_retry_timers(itask, rtconfig)
 
         try:
-            job_conf = {
-                **self._prep_submit_task_job_impl(
-                    workflow, itask, rtconfig
-                ),
-                'logfiles': deepcopy(itask.summary['logfiles']),
-            }
+            job_conf = self._prep_submit_task_job_impl(
+                workflow,
+                itask,
+                rtconfig,
+            )
             itask.jobs.append(job_conf)
 
             local_job_file_path = get_task_job_job_log(
@@ -1326,8 +1324,6 @@ def get_job_conf(
             'try_num': itask.get_try_num(),
             'uuid_str': self.task_events_mgr.uuid_str,
             'work_d': rtconfig['work sub-directory'],
-            # this field is populated retrospectively for regular job subs
-            'logfiles': [],
         }
 
     def get_simulation_job_conf(self, itask, workflow):
@@ -1359,6 +1355,4 @@ def get_simulation_job_conf(self, itask, workflow):
             'try_num': itask.get_try_num(),
             'uuid_str': self.task_events_mgr.uuid_str,
             'work_d': 'SIMULATION',
-            # this field is populated retrospectively for regular job subs
-            'logfiles': [],
         }
diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py
index 271049126fd..05531a2553a 100644
--- a/cylc/flow/task_proxy.py
+++ b/cylc/flow/task_proxy.py
@@ -100,8 +100,6 @@ class TaskProxy:
                 Jobs' platform by submit number.
             label (str):
                 The .point attribute as string.
-            logfiles (list):
-                List of names of (extra) known job log files.
             name (str):
                 Same as the .tdef.name attribute.
             started_time (float):
@@ -225,7 +223,6 @@ def __init__(
             'started_time_string': None,
             'finished_time': None,
             'finished_time_string': None,
-            'logfiles': [],
             'platforms_used': {},
             'execution_time_limit': None,
             'job_runner_name': None,
diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py
index 6dbae4b6fd5..21b95280443 100644
--- a/tests/integration/test_data_store_mgr.py
+++ b/tests/integration/test_data_store_mgr.py
@@ -58,7 +58,6 @@ def job_config(schd):
         'directives': {},
         'environment': {},
         'param_var': {},
-        'logfiles': [],
         'platform': {'name': 'platform'},
     }
 
diff --git a/tests/integration/test_graphql.py b/tests/integration/test_graphql.py
index ddad4907660..b0a51c0b6b8 100644
--- a/tests/integration/test_graphql.py
+++ b/tests/integration/test_graphql.py
@@ -48,7 +48,6 @@ def job_config(schd):
         'directives': {},
         'environment': {},
         'param_var': {},
-        'logfiles': [],
         'platform': {'name': 'platform'},
     }
 

From cd409a25a082f5a0e61f65e1a4cf241b75a2a0cc Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 17 Oct 2023 16:33:41 +0100
Subject: [PATCH 023/110] speed up import time (#5770)

* Don't import modules for type checking purposes.
* Move some light weight functions out of a heavy weight module to
  reduce the import penalty.
* Make a couple of heavy weight imports dynamic to avoid importing them
  when not necessary.
* This reduces the import time of cylc.flow.scripts.message by ~66%.
* This reduces the task runtime (`script=true`) by ~1s.
---
 cylc/flow/config.py                           |   2 +-
 cylc/flow/exceptions.py                       |   7 +-
 cylc/flow/flow_mgr.py                         |   7 +-
 cylc/flow/id_cli.py                           |  11 +-
 cylc/flow/job_file.py                         |   2 +-
 .../flow/job_runner_handlers/documentation.py |   9 +-
 cylc/flow/log_level.py                        | 116 ++++++++++++++++++
 cylc/flow/network/server.py                   |   4 +-
 cylc/flow/option_parsers.py                   |  99 +--------------
 cylc/flow/parsec/fileparse.py                 |   4 +-
 cylc/flow/remote.py                           |   2 +-
 cylc/flow/scheduler.py                        |  20 +--
 cylc/flow/scripts/cat_log.py                  |   2 +-
 cylc/flow/scripts/kill.py                     |   8 +-
 cylc/flow/scripts/lint.py                     |   6 +-
 cylc/flow/scripts/set_outputs.py              |   5 +-
 cylc/flow/scripts/set_verbosity.py            |   6 +-
 cylc/flow/scripts/show.py                     |   7 +-
 cylc/flow/scripts/validate.py                 |   5 +-
 cylc/flow/task_events_mgr.py                  |   4 +-
 cylc/flow/task_id.py                          |  11 +-
 cylc/flow/task_proxy.py                       |   2 +-
 cylc/flow/task_queues/__init__.py             |  13 +-
 cylc/flow/task_queues/independent.py          |  28 +++--
 cylc/flow/task_remote_mgr.py                  |  18 ++-
 cylc/flow/templatevars.py                     |  11 +-
 cylc/flow/terminal.py                         |   8 +-
 cylc/flow/workflow_files.py                   |   7 +-
 cylc/flow/xtrigger_mgr.py                     |  47 ++++---
 setup.cfg                                     |   1 +
 tests/unit/test_id_cli.py                     |   2 +-
 31 files changed, 278 insertions(+), 196 deletions(-)
 create mode 100644 cylc/flow/log_level.py

diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index 79d305da57a..0096d924c8b 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -68,7 +68,7 @@
 import cylc.flow.flags
 from cylc.flow.graph_parser import GraphParser
 from cylc.flow.listify import listify
-from cylc.flow.option_parsers import verbosity_to_env
+from cylc.flow.log_level import verbosity_to_env
 from cylc.flow.graphnode import GraphNodeParser
 from cylc.flow.param_expand import NameExpander
 from cylc.flow.parsec.exceptions import ItemNotFoundError
diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py
index 79a726d7bbe..d1a459f996b 100644
--- a/cylc/flow/exceptions.py
+++ b/cylc/flow/exceptions.py
@@ -26,11 +26,14 @@
     Tuple,
     Type,
     Union,
+    TYPE_CHECKING,
 )
 
-from cylc.flow.subprocctx import SubFuncContext
 from cylc.flow.util import format_cmd
 
+if TYPE_CHECKING:
+    from cylc.flow.subprocctx import SubFuncContext
+
 
 class CylcError(Exception):
     """Generic exception for Cylc errors.
@@ -198,7 +201,7 @@ def __init__(
         message: str,
         platform_name: str,
         *,
-        ctx: Optional[SubFuncContext] = None,
+        ctx: 'Optional[SubFuncContext]' = None,
         cmd: Optional[Union[str, Iterable]] = None,
         ret_code: Optional[int] = None,
         out: Optional[str] = None,
diff --git a/cylc/flow/flow_mgr.py b/cylc/flow/flow_mgr.py
index 148adb8213f..41aec947a80 100644
--- a/cylc/flow/flow_mgr.py
+++ b/cylc/flow/flow_mgr.py
@@ -16,11 +16,14 @@
 
 """Manage flow counter and flow metadata."""
 
-from typing import Dict, Set, Optional
+from typing import Dict, Set, Optional, TYPE_CHECKING
 import datetime
 
 from cylc.flow import LOG
-from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
+
+
+if TYPE_CHECKING:
+    from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
 
 
 FlowNums = Set[int]
diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py
index 96cb438fded..803c8506439 100644
--- a/cylc/flow/id_cli.py
+++ b/cylc/flow/id_cli.py
@@ -31,11 +31,6 @@
     upgrade_legacy_ids,
 )
 from cylc.flow.pathutil import EXPLICIT_RELATIVE_PATH_REGEX
-from cylc.flow.network.scan import (
-    filter_name,
-    is_active,
-    scan,
-)
 from cylc.flow.workflow_files import (
     check_flow_file,
     detect_both_flow_and_suite,
@@ -487,6 +482,12 @@ async def _expand_workflow_tokens_impl(tokens, match_active=True):
             'currently supported.'
         )
 
+    # import only when needed to avoid slowing CLI unnecessarily
+    from cylc.flow.network.scan import (
+        filter_name,
+        is_active,
+        scan,
+    )
     # construct the pipe
     pipe = scan | filter_name(fnmatch.translate(tokens['workflow']))
     if match_active is not None:
diff --git a/cylc/flow/job_file.py b/cylc/flow/job_file.py
index cc8576dad37..86e65e15a8e 100644
--- a/cylc/flow/job_file.py
+++ b/cylc/flow/job_file.py
@@ -25,7 +25,7 @@
 from cylc.flow import __version__ as CYLC_VERSION
 from cylc.flow.job_runner_mgr import JobRunnerManager
 import cylc.flow.flags
-from cylc.flow.option_parsers import verbosity_to_env
+from cylc.flow.log_level import verbosity_to_env
 from cylc.flow.config import interpolate_template, ParamExpandError
 
 # the maximum number of task dependencies which Cylc will list before
diff --git a/cylc/flow/job_runner_handlers/documentation.py b/cylc/flow/job_runner_handlers/documentation.py
index 7f4a0d289e4..a6f6af983be 100644
--- a/cylc/flow/job_runner_handlers/documentation.py
+++ b/cylc/flow/job_runner_handlers/documentation.py
@@ -22,13 +22,16 @@
 not intended to be subclassed.
 """
 
-import re
 from typing import (
     Iterable,
     List,
     Tuple,
+    TYPE_CHECKING,
 )
 
+if TYPE_CHECKING:
+    import re
+
 
 class ExampleHandler():
     """Documentation for writing job runner handlers.
@@ -257,7 +260,7 @@ class QSUBHandler(PBSHandler):
 
     """
 
-    REC_ID_FROM_SUBMIT_OUT: re.Pattern
+    REC_ID_FROM_SUBMIT_OUT: 're.Pattern'
     """Regular expression to extract job IDs from submission stderr.
 
     A regular expression (compiled) to extract the job "id" from the standard
@@ -265,7 +268,7 @@ class QSUBHandler(PBSHandler):
 
     """
 
-    REC_ID_FROM_SUBMIT_ERR: re.Pattern
+    REC_ID_FROM_SUBMIT_ERR: 're.Pattern'
     """Regular expression to extract job IDs from submission stderr.
 
     See :py:attr:`ExampleHandler.REC_ID_FROM_SUBMIT_OUT`.
diff --git a/cylc/flow/log_level.py b/cylc/flow/log_level.py
new file mode 100644
index 00000000000..4b091154893
--- /dev/null
+++ b/cylc/flow/log_level.py
@@ -0,0 +1,116 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Utilities for configuring logging level via the CLI."""
+
+import logging
+from typing import List, Dict, Union, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import os
+
+
+def verbosity_to_log_level(verb: int) -> int:
+    """Convert Cylc verbosity to log severity level."""
+    if verb < 0:
+        return logging.WARNING
+    if verb > 0:
+        return logging.DEBUG
+    return logging.INFO
+
+
+def log_level_to_verbosity(lvl: int) -> int:
+    """Convert log severity level to Cylc verbosity.
+
+    Examples:
+        >>> log_level_to_verbosity(logging.NOTSET)
+        2
+        >>> log_level_to_verbosity(logging.DEBUG)
+        1
+        >>> log_level_to_verbosity(logging.INFO)
+        0
+        >>> log_level_to_verbosity(logging.WARNING)
+        -1
+        >>> log_level_to_verbosity(logging.ERROR)
+        -1
+    """
+    if lvl < logging.DEBUG:
+        return 2
+    if lvl < logging.INFO:
+        return 1
+    if lvl == logging.INFO:
+        return 0
+    return -1
+
+
+def verbosity_to_opts(verb: int) -> List[str]:
+    """Convert Cylc verbosity to the CLI opts required to replicate it.
+
+    Examples:
+        >>> verbosity_to_opts(0)
+        []
+        >>> verbosity_to_opts(-2)
+        ['-q', '-q']
+        >>> verbosity_to_opts(2)
+        ['-v', '-v']
+
+    """
+    return [
+        '-q'
+        for _ in range(verb, 0)
+    ] + [
+        '-v'
+        for _ in range(0, verb)
+    ]
+
+
+def verbosity_to_env(verb: int) -> Dict[str, str]:
+    """Convert Cylc verbosity to the env vars required to replicate it.
+
+    Examples:
+        >>> verbosity_to_env(0)
+        {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'}
+        >>> verbosity_to_env(1)
+        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'}
+        >>> verbosity_to_env(2)
+        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'}
+
+    """
+    return {
+        'CYLC_VERBOSE': str((verb > 0)).lower(),
+        'CYLC_DEBUG': str((verb > 1)).lower(),
+    }
+
+
+def env_to_verbosity(env: 'Union[Dict, os._Environ]') -> int:
+    """Extract verbosity from environment variables.
+
+    Examples:
+        >>> env_to_verbosity({})
+        0
+        >>> env_to_verbosity({'CYLC_VERBOSE': 'true'})
+        1
+        >>> env_to_verbosity({'CYLC_DEBUG': 'true'})
+        2
+        >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'})
+        2
+
+    """
+    return (
+        2 if env.get('CYLC_DEBUG', '').lower() == 'true'
+        else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true'
+        else 0
+    )
diff --git a/cylc/flow/network/server.py b/cylc/flow/network/server.py
index f6847dcef59..5c070472025 100644
--- a/cylc/flow/network/server.py
+++ b/cylc/flow/network/server.py
@@ -21,7 +21,6 @@
 from time import sleep
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 
-from graphql.execution import ExecutionResult
 from graphql.execution.executors.asyncio import AsyncioExecutor
 import zmq
 from zmq.auth.thread import ThreadAuthenticator
@@ -41,6 +40,7 @@
 
 if TYPE_CHECKING:
     from cylc.flow.scheduler import Scheduler
+    from graphql.execution import ExecutionResult
 
 
 # maps server methods to the protobuf message (for client/UIS import)
@@ -368,7 +368,7 @@ def graphql(
             object: Execution result, or a list with errors.
         """
         try:
-            executed: ExecutionResult = schema.execute(
+            executed: 'ExecutionResult' = schema.execute(
                 request_string,
                 variable_values=variables,
                 context_value={
diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py
index cb21f5e8bcf..ee84a477da7 100644
--- a/cylc/flow/option_parsers.py
+++ b/cylc/flow/option_parsers.py
@@ -33,7 +33,7 @@
 
 import sys
 from textwrap import dedent
-from typing import Any, Dict, Optional, List, Tuple, Union
+from typing import Any, Dict, Optional, List, Tuple
 
 from cylc.flow import LOG
 from cylc.flow.terminal import supports_color, DIM
@@ -42,6 +42,10 @@
     CylcLogFormatter,
     setup_segregated_log_streams,
 )
+from cylc.flow.log_level import (
+    env_to_verbosity,
+    verbosity_to_log_level
+)
 
 WORKFLOW_ID_ARG_DOC = ('WORKFLOW', 'Workflow ID')
 WORKFLOW_ID_MULTI_ARG_DOC = ('WORKFLOW ...', 'Workflow ID(s)')
@@ -177,99 +181,6 @@ def format_help_headings(string):
     )
 
 
-def verbosity_to_log_level(verb: int) -> int:
-    """Convert Cylc verbosity to log severity level."""
-    if verb < 0:
-        return logging.WARNING
-    if verb > 0:
-        return logging.DEBUG
-    return logging.INFO
-
-
-def log_level_to_verbosity(lvl: int) -> int:
-    """Convert log severity level to Cylc verbosity.
-
-    Examples:
-        >>> log_level_to_verbosity(logging.NOTSET)
-        2
-        >>> log_level_to_verbosity(logging.DEBUG)
-        1
-        >>> log_level_to_verbosity(logging.INFO)
-        0
-        >>> log_level_to_verbosity(logging.WARNING)
-        -1
-        >>> log_level_to_verbosity(logging.ERROR)
-        -1
-    """
-    if lvl < logging.DEBUG:
-        return 2
-    if lvl < logging.INFO:
-        return 1
-    if lvl == logging.INFO:
-        return 0
-    return -1
-
-
-def verbosity_to_opts(verb: int) -> List[str]:
-    """Convert Cylc verbosity to the CLI opts required to replicate it.
-
-    Examples:
-        >>> verbosity_to_opts(0)
-        []
-        >>> verbosity_to_opts(-2)
-        ['-q', '-q']
-        >>> verbosity_to_opts(2)
-        ['-v', '-v']
-
-    """
-    return [
-        '-q'
-        for _ in range(verb, 0)
-    ] + [
-        '-v'
-        for _ in range(0, verb)
-    ]
-
-
-def verbosity_to_env(verb: int) -> Dict[str, str]:
-    """Convert Cylc verbosity to the env vars required to replicate it.
-
-    Examples:
-        >>> verbosity_to_env(0)
-        {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'}
-        >>> verbosity_to_env(1)
-        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'}
-        >>> verbosity_to_env(2)
-        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'}
-
-    """
-    return {
-        'CYLC_VERBOSE': str((verb > 0)).lower(),
-        'CYLC_DEBUG': str((verb > 1)).lower(),
-    }
-
-
-def env_to_verbosity(env: Union[Dict, os._Environ]) -> int:
-    """Extract verbosity from environment variables.
-
-    Examples:
-        >>> env_to_verbosity({})
-        0
-        >>> env_to_verbosity({'CYLC_VERBOSE': 'true'})
-        1
-        >>> env_to_verbosity({'CYLC_DEBUG': 'true'})
-        2
-        >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'})
-        2
-
-    """
-    return (
-        2 if env.get('CYLC_DEBUG', '').lower() == 'true'
-        else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true'
-        else 0
-    )
-
-
 class CylcOption(Option):
     """Optparse option which adds a decrement action."""
 
diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py
index 8ac0a20a327..f4f8d723f0b 100644
--- a/cylc/flow/parsec/fileparse.py
+++ b/cylc/flow/parsec/fileparse.py
@@ -31,7 +31,6 @@
 """
 
 import os
-from optparse import Values
 from pathlib import Path
 import re
 import sys
@@ -50,6 +49,9 @@
 from cylc.flow.workflow_files import (
     get_workflow_source_dir, check_flow_file)
 
+if t.TYPE_CHECKING:
+    from optparse import Values
+
 
 # heading/sections can contain commas (namespace name lists) and any
 # regex pattern characters (this was for pre cylc-6 satellite tasks).
diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py
index 62cd77fd9af..fd51934af11 100644
--- a/cylc/flow/remote.py
+++ b/cylc/flow/remote.py
@@ -32,7 +32,7 @@
 
 import cylc.flow.flags
 from cylc.flow import __version__ as CYLC_VERSION, LOG
-from cylc.flow.option_parsers import verbosity_to_opts
+from cylc.flow.log_level import verbosity_to_opts
 from cylc.flow.platforms import get_platform, get_host_from_platform
 from cylc.flow.util import format_cmd
 
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 7449e7d274a..e5c3fc7c01a 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -18,7 +18,6 @@
 import asyncio
 from contextlib import suppress
 from collections import deque
-from optparse import Values
 import os
 import inspect
 from pathlib import Path
@@ -84,16 +83,15 @@
     patch_log_level
 )
 from cylc.flow.timer import Timer
-from cylc.flow.network import API
-from cylc.flow.network.authentication import key_housekeeping
-from cylc.flow.network.resolvers import TaskMsg
-from cylc.flow.network.schema import WorkflowStopMode
-from cylc.flow.network.server import WorkflowRuntimeServer
-from cylc.flow.option_parsers import (
+from cylc.flow.log_level import (
     log_level_to_verbosity,
     verbosity_to_env,
     verbosity_to_opts,
 )
+from cylc.flow.network import API
+from cylc.flow.network.authentication import key_housekeeping
+from cylc.flow.network.schema import WorkflowStopMode
+from cylc.flow.network.server import WorkflowRuntimeServer
 from cylc.flow.parsec.exceptions import ParsecError
 from cylc.flow.parsec.OrderedDict import DictTree
 from cylc.flow.parsec.validate import DurationFloat
@@ -155,6 +153,8 @@
     # FROM: Python 3.7
     # TO: Python 3.8
     from typing_extensions import Literal
+    from optparse import Values
+    from cylc.flow.network.resolvers import TaskMsg
 
 
 class SchedulerStop(CylcError):
@@ -228,7 +228,7 @@ class Scheduler:
 
     # configuration
     config: WorkflowConfig  # flow config
-    options: Values
+    options: 'Values'
     cylc_config: DictTree  # [scheduler] config
     template_vars: Dict[str, Any]
 
@@ -271,7 +271,7 @@ class Scheduler:
 
     time_next_kill: Optional[float] = None
 
-    def __init__(self, id_: str, options: Values) -> None:
+    def __init__(self, id_: str, options: 'Values') -> None:
         # flow information
         self.workflow = id_
         self.workflow_name = get_workflow_name_from_id(self.workflow)
@@ -862,7 +862,7 @@ def _load_task_run_times(self, row_idx, row):
 
     def process_queued_task_messages(self) -> None:
         """Handle incoming task messages for each task proxy."""
-        messages: Dict[str, List[Tuple[Optional[int], TaskMsg]]] = {}
+        messages: 'Dict[str, List[Tuple[Optional[int], TaskMsg]]]' = {}
         while self.message_queue.qsize():
             try:
                 task_msg = self.message_queue.get(block=False)
diff --git a/cylc/flow/scripts/cat_log.py b/cylc/flow/scripts/cat_log.py
index e8003d66da9..59142beb897 100755
--- a/cylc/flow/scripts/cat_log.py
+++ b/cylc/flow/scripts/cat_log.py
@@ -74,10 +74,10 @@
 import cylc.flow.flags
 from cylc.flow.hostuserutil import is_remote_platform
 from cylc.flow.id_cli import parse_id
+from cylc.flow.log_level import verbosity_to_opts
 from cylc.flow.option_parsers import (
     ID_MULTI_ARG_DOC,
     CylcOptionParser as COP,
-    verbosity_to_opts,
 )
 from cylc.flow.pathutil import (
     expand_path,
diff --git a/cylc/flow/scripts/kill.py b/cylc/flow/scripts/kill.py
index 40f0efd9b9b..2af4c34200e 100755
--- a/cylc/flow/scripts/kill.py
+++ b/cylc/flow/scripts/kill.py
@@ -34,7 +34,6 @@
 from functools import partial
 from typing import TYPE_CHECKING
 
-from cylc.flow.id import Tokens
 from cylc.flow.network.client_factory import get_client
 from cylc.flow.network.multi import call_multi
 from cylc.flow.option_parsers import (
@@ -45,6 +44,7 @@
 
 if TYPE_CHECKING:
     from optparse import Values
+    from cylc.flow.id import Tokens
 
 
 MUTATION = '''
@@ -62,7 +62,7 @@
 '''
 
 
-def get_option_parser() -> COP:
+def get_option_parser() -> 'COP':
     parser = COP(
         __doc__,
         comms=True,
@@ -74,7 +74,7 @@ def get_option_parser() -> COP:
     return parser
 
 
-async def run(options: 'Values', workflow_id: str, *tokens_list: Tokens):
+async def run(options: 'Values', workflow_id: str, *tokens_list: 'Tokens'):
     pclient = get_client(workflow_id, timeout=options.comms_timeout)
 
     mutation_kwargs = {
@@ -92,7 +92,7 @@ async def run(options: 'Values', workflow_id: str, *tokens_list: Tokens):
 
 
 @cli_function(get_option_parser)
-def main(parser: COP, options: 'Values', *ids: str):
+def main(parser: 'COP', options: 'Values', *ids: str):
     """CLI of "cylc kill"."""
     call_multi(
         partial(run, options),
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index 681870dac76..951e6aa3643 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -39,7 +39,6 @@
 """
 from colorama import Fore
 import functools
-from optparse import Values
 from pathlib import Path
 import re
 import sys
@@ -59,7 +58,7 @@
         loads as toml_loads,
         TOMLDecodeError,
     )
-from typing import Callable, Dict, Iterator, List, Union
+from typing import Callable, Dict, Iterator, List, Union, TYPE_CHECKING
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
@@ -73,6 +72,9 @@
 from cylc.flow.scripts.cylc import DEAD_ENDS
 from cylc.flow.terminal import cli_function
 
+if TYPE_CHECKING:
+    from optparse import Values
+
 DEPRECATED_ENV_VARS = {
     'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST',
     'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER',
diff --git a/cylc/flow/scripts/set_outputs.py b/cylc/flow/scripts/set_outputs.py
index 9ca3b38a7ba..b45ddbff268 100755
--- a/cylc/flow/scripts/set_outputs.py
+++ b/cylc/flow/scripts/set_outputs.py
@@ -48,7 +48,7 @@
 """
 
 from functools import partial
-from optparse import Values
+from typing import TYPE_CHECKING
 
 from cylc.flow.network.client_factory import get_client
 from cylc.flow.network.multi import call_multi
@@ -58,6 +58,9 @@
 )
 from cylc.flow.terminal import cli_function
 
+if TYPE_CHECKING:
+    from optparse import Values
+
 MUTATION = '''
 mutation (
   $wFlows: [WorkflowID]!,
diff --git a/cylc/flow/scripts/set_verbosity.py b/cylc/flow/scripts/set_verbosity.py
index a87655d2267..aa40d01345c 100755
--- a/cylc/flow/scripts/set_verbosity.py
+++ b/cylc/flow/scripts/set_verbosity.py
@@ -26,7 +26,7 @@
 """
 
 from functools import partial
-from optparse import Values
+from typing import TYPE_CHECKING
 
 from cylc.flow import LOG_LEVELS
 from cylc.flow.exceptions import InputError
@@ -38,6 +38,10 @@
 )
 from cylc.flow.terminal import cli_function
 
+if TYPE_CHECKING:
+    from optparse import Values
+
+
 MUTATION = '''
 mutation (
   $wFlows: [WorkflowID]!,
diff --git a/cylc/flow/scripts/show.py b/cylc/flow/scripts/show.py
index 2ff427d12fb..5a36eaf8e0d 100755
--- a/cylc/flow/scripts/show.py
+++ b/cylc/flow/scripts/show.py
@@ -38,9 +38,8 @@
 
 import asyncio
 import json
-from optparse import Values
 import sys
-from typing import Dict
+from typing import Dict, TYPE_CHECKING
 
 from ansimarkup import ansiprint
 
@@ -59,6 +58,10 @@
 from cylc.flow.terminal import cli_function
 
 
+if TYPE_CHECKING:
+    from optparse import Values
+
+
 WORKFLOW_META_QUERY = '''
 query ($wFlows: [ID]!) {
   workflows (ids: $wFlows, stripNull: false) {
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index 47a29897d60..7cfd42c2138 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -28,8 +28,8 @@
 import asyncio
 from ansimarkup import parse as cparse
 from copy import deepcopy
-from optparse import Values
 import sys
+from typing import TYPE_CHECKING
 
 from cylc.flow import LOG, __version__ as CYLC_VERSION
 from cylc.flow.config import WorkflowConfig
@@ -56,6 +56,9 @@
 from cylc.flow.terminal import cli_function
 from cylc.flow.scheduler_cli import RUN_MODE
 
+if TYPE_CHECKING:
+    from cylc.flow.option_parsers import Values
+
 
 VALIDATE_RUN_MODE = deepcopy(RUN_MODE)
 VALIDATE_RUN_MODE.sources = {'validate'}
diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py
index 9a8a19ba18a..d4d9130ee27 100644
--- a/cylc/flow/task_events_mgr.py
+++ b/cylc/flow/task_events_mgr.py
@@ -581,7 +581,7 @@ def process_message(
         if submit_num is None:
             submit_num = itask.submit_num
         if isinstance(severity, int):
-            severity = cast(str, getLevelName(severity))
+            severity = cast('str', getLevelName(severity))
         lseverity = str(severity).lower()
 
         # Any message represents activity.
@@ -811,7 +811,7 @@ def _process_message_check(
                 )
             return False
 
-        severity = cast(int, LOG_LEVELS.get(severity, INFO))
+        severity = cast('int', LOG_LEVELS.get(severity, INFO))
         # Demote log level to DEBUG if this is a message that duplicates what
         # gets logged by itask state change anyway (and not manual poll)
         if severity > DEBUG and flag != self.FLAG_POLLED and message in {
diff --git a/cylc/flow/task_id.py b/cylc/flow/task_id.py
index 821d637c2ff..4a17a59548d 100644
--- a/cylc/flow/task_id.py
+++ b/cylc/flow/task_id.py
@@ -17,13 +17,15 @@
 
 
 import re
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
-from cylc.flow.cycling import PointBase
 from cylc.flow.cycling.loader import get_point, standardise_point_string
 from cylc.flow.exceptions import PointParsingError
 from cylc.flow.id import Tokens
 
+if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
+
 
 _TASK_NAME_PREFIX = r'\w'
 _TASK_NAME_CHARACTERS = [r'\w', r'\-', '+', '%', '@']
@@ -94,7 +96,10 @@ def get_standardised_point_string(cls, point_string):
         return point_string
 
     @classmethod
-    def get_standardised_point(cls, point_string: str) -> Optional[PointBase]:
+    def get_standardised_point(
+        cls,
+        point_string: str,
+    ) -> 'Optional[PointBase]':
         """Return a standardised point."""
         return get_point(cls.get_standardised_point_string(point_string))
 
diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py
index 05531a2553a..9d51794ec6c 100644
--- a/cylc/flow/task_proxy.py
+++ b/cylc/flow/task_proxy.py
@@ -26,7 +26,6 @@
 from metomi.isodatetime.timezone import get_local_time_zone
 
 from cylc.flow import LOG
-from cylc.flow.id import Tokens
 from cylc.flow.platforms import get_platform
 from cylc.flow.task_action_timer import TimerFlags
 from cylc.flow.task_state import TaskState, TASK_STATUS_WAITING
@@ -42,6 +41,7 @@
     from cylc.flow.cycling import PointBase
     from cylc.flow.task_action_timer import TaskActionTimer
     from cylc.flow.taskdef import TaskDef
+    from cylc.flow.id import Tokens
 
 
 class TaskProxy:
diff --git a/cylc/flow/task_queues/__init__.py b/cylc/flow/task_queues/__init__.py
index 0e0c8bce90b..32983789632 100644
--- a/cylc/flow/task_queues/__init__.py
+++ b/cylc/flow/task_queues/__init__.py
@@ -16,10 +16,11 @@
 
 """Define the Cylc task queue management API."""
 
-from typing import List, Dict, Counter, Any
+from typing import List, Dict, Counter, Any, TYPE_CHECKING
 from abc import ABCMeta, abstractmethod
 
-from cylc.flow.task_proxy import TaskProxy
+if TYPE_CHECKING:
+    from cylc.flow.task_proxy import TaskProxy
 
 
 class TaskQueueManagerBase(metaclass=ABCMeta):
@@ -42,22 +43,22 @@ def __init__(self,
         pass
 
     @abstractmethod
-    def push_task(self, itask: TaskProxy) -> None:
+    def push_task(self, itask: 'TaskProxy') -> None:
         """Queue the given task."""
         pass
 
     @abstractmethod
-    def release_tasks(self, active: Counter[str]) -> List[TaskProxy]:
+    def release_tasks(self, active: Counter[str]) -> 'List[TaskProxy]':
         """Release tasks, given current active task counts."""
         pass
 
     @abstractmethod
-    def remove_task(self, itask: TaskProxy) -> None:
+    def remove_task(self, itask: 'TaskProxy') -> None:
         """Remove a task from the queueing system."""
         pass
 
     @abstractmethod
-    def force_release_task(self, itask: TaskProxy) -> None:
+    def force_release_task(self, itask: 'TaskProxy') -> None:
         """Remove a task from whichever queue it belongs to.
 
         To be returned when release_tasks() is next called.
diff --git a/cylc/flow/task_queues/independent.py b/cylc/flow/task_queues/independent.py
index d423a5f1702..d0e82995898 100644
--- a/cylc/flow/task_queues/independent.py
+++ b/cylc/flow/task_queues/independent.py
@@ -18,11 +18,13 @@
 
 from collections import deque
 from contextlib import suppress
-from typing import List, Set, Dict, Counter, Any
+from typing import List, Set, Dict, Counter, Any, TYPE_CHECKING
 
-from cylc.flow.task_proxy import TaskProxy
 from cylc.flow.task_queues import TaskQueueManagerBase
 
+if TYPE_CHECKING:
+    from cylc.flow.task_proxy import TaskProxy
+
 
 class LimitedTaskQueue:
     """One task queue with group members and active limit."""
@@ -33,16 +35,16 @@ def __init__(self, limit: int, members: Set[str]) -> None:
         self.members = members  # member task names
         self.deque: deque = deque()
 
-    def push_task(self, itask: TaskProxy) -> None:
+    def push_task(self, itask: 'TaskProxy') -> None:
         """Queue task if in my membership list."""
         if itask.tdef.name in self.members:
             self.deque.appendleft(itask)
 
-    def release(self, active: Counter[str]) -> List[TaskProxy]:
+    def release(self, active: Counter[str]) -> 'List[TaskProxy]':
         """Release tasks if below the active limit."""
         # The "active" argument counts active tasks by name.
-        released: List[TaskProxy] = []
-        held: List[TaskProxy] = []
+        released: 'List[TaskProxy]' = []
+        held: 'List[TaskProxy]' = []
         n_active: int = 0
         for mem in self.members:
             n_active += active[mem]
@@ -62,7 +64,7 @@ def release(self, active: Counter[str]) -> List[TaskProxy]:
             self.deque.appendleft(itask)
         return released
 
-    def remove(self, itask: TaskProxy) -> bool:
+    def remove(self, itask: 'TaskProxy') -> bool:
         """Remove a single task from queue, return True if removed."""
         try:
             self.deque.remove(itask)
@@ -111,16 +113,16 @@ def __init__(self,
                 config["limit"], config["members"]
             )
 
-        self.force_released: Set[TaskProxy] = set()
+        self.force_released: 'Set[TaskProxy]' = set()
 
-    def push_task(self, itask: TaskProxy) -> None:
+    def push_task(self, itask: 'TaskProxy') -> None:
         """Push a task to the appropriate queue."""
         for queue in self.queues.values():
             queue.push_task(itask)
 
-    def release_tasks(self, active: Counter[str]) -> List[TaskProxy]:
+    def release_tasks(self, active: Counter[str]) -> 'List[TaskProxy]':
         """Release tasks up to the queue limits."""
-        released: List[TaskProxy] = []
+        released: 'List[TaskProxy]' = []
         for queue in self.queues.values():
             released += queue.release(active)
         if self.force_released:
@@ -128,13 +130,13 @@ def release_tasks(self, active: Counter[str]) -> List[TaskProxy]:
             self.force_released = set()
         return released
 
-    def remove_task(self, itask: TaskProxy) -> None:
+    def remove_task(self, itask: 'TaskProxy') -> None:
         """Remove a task from whichever queue it belongs to."""
         for queue in self.queues.values():
             if queue.remove(itask):
                 break
 
-    def force_release_task(self, itask: TaskProxy) -> None:
+    def force_release_task(self, itask: 'TaskProxy') -> None:
         """Remove a task from whichever queue it belongs to.
 
         To be returned when release_tasks() is next called.
diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py
index 541bc1265a0..9ef4d465bf2 100644
--- a/cylc/flow/task_remote_mgr.py
+++ b/cylc/flow/task_remote_mgr.py
@@ -25,7 +25,6 @@
 from collections import deque
 from contextlib import suppress
 from pathlib import Path
-from cylc.flow.option_parsers import verbosity_to_opts
 import os
 from shlex import quote
 import re
@@ -33,8 +32,15 @@
 import tarfile
 from time import sleep, time
 from typing import (
-    Any, Deque, Dict, TYPE_CHECKING, List,
-    NamedTuple, Optional, Set, Tuple
+    Any,
+    Deque,
+    Dict,
+    List,
+    NamedTuple,
+    Optional,
+    Set,
+    TYPE_CHECKING,
+    Tuple,
 )
 
 from cylc.flow import LOG
@@ -42,6 +48,9 @@
     PlatformError, PlatformLookupError, NoHostsError, NoPlatformsError
 )
 import cylc.flow.flags
+from cylc.flow.hostuserutil import is_remote_host
+from cylc.flow.log_level import verbosity_to_opts
+from cylc.flow.loggingutil import get_next_log_number, get_sorted_logs_by_time
 from cylc.flow.network.client_factory import CommsMeth
 from cylc.flow.pathutil import (
     get_dirs_to_symlink,
@@ -70,9 +79,6 @@
     get_workflow_srv_dir,
 )
 
-from cylc.flow.loggingutil import get_next_log_number, get_sorted_logs_by_time
-from cylc.flow.hostuserutil import is_remote_host
-
 if TYPE_CHECKING:
     from zmq.auth.thread import ThreadAuthenticator
 
diff --git a/cylc/flow/templatevars.py b/cylc/flow/templatevars.py
index 7474761e226..e3d0d8f8e02 100644
--- a/cylc/flow/templatevars.py
+++ b/cylc/flow/templatevars.py
@@ -16,8 +16,7 @@
 """Load custom variables for template processor."""
 
 from ast import literal_eval
-from optparse import Values
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import InputError
@@ -25,7 +24,9 @@
 from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
 from cylc.flow.workflow_files import WorkflowFiles
 
-from pathlib import Path
+if TYPE_CHECKING:
+    from optparse import Values
+    from pathlib import Path
 
 
 OVERWRITE_WARNING = (
@@ -111,7 +112,7 @@ def parse_string_list(stringlist: str) -> List:
 
 def load_template_vars(
     template_vars: Optional[List[str]] = None,
-    template_vars_file: Union[Path, str, None] = None,
+    template_vars_file: 'Union[Path, str, None]' = None,
     templatevars_lists: Optional[List[str]] = None
 ) -> Dict[str, Any]:
     """Load template variables from key=value strings."""
@@ -182,7 +183,7 @@ def load_template_vars(
     return res
 
 
-def get_template_vars(options: Values) -> Dict[str, Any]:
+def get_template_vars(options: 'Values') -> Dict[str, Any]:
     """Convienence wrapper for ``load_template_vars``.
 
     Args:
diff --git a/cylc/flow/terminal.py b/cylc/flow/terminal.py
index 48e95f2df00..bc45f6d65d3 100644
--- a/cylc/flow/terminal.py
+++ b/cylc/flow/terminal.py
@@ -20,12 +20,11 @@
 import inspect
 import json
 import logging
-from optparse import OptionParser
 import os
 from subprocess import PIPE, Popen  # nosec
 import sys
 from textwrap import wrap
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Optional, TYPE_CHECKING
 
 from ansimarkup import parse as cparse
 from colorama import init as color_init
@@ -36,6 +35,9 @@
 from cylc.flow.loggingutil import CylcLogFormatter
 from cylc.flow.parsec.exceptions import ParsecError
 
+if TYPE_CHECKING:
+    from optparse import OptionParser
+
 
 # CLI exception message format
 EXC_EXIT = cparse('<red><bold>{name}: </bold>{exc}</red>')
@@ -184,7 +186,7 @@ def parse_dirty_json(stdout):
 
 
 def cli_function(
-    parser_function: Optional[Callable[..., OptionParser]] = None,
+    parser_function: 'Optional[Callable[..., OptionParser]]' = None,
     **parser_kwargs: Any
 ):
     """Decorator for CLI entry points.
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 3f1d011a9da..de30bc1e7d2 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -60,10 +60,6 @@
     get_workflow_run_dir,
     make_localhost_symlinks,
 )
-from cylc.flow.remote import (
-    construct_cylc_server_ssh_cmd,
-)
-from cylc.flow.terminal import parse_dirty_json
 from cylc.flow.unicode_rules import WorkflowNameValidator
 from cylc.flow.util import cli_format
 
@@ -382,6 +378,9 @@ def _is_process_running(
         False
 
     """
+    from cylc.flow.remote import construct_cylc_server_ssh_cmd
+    from cylc.flow.terminal import parse_dirty_json
+
     # See if the process is still running or not.
     metric = f'[["Process", {pid}]]'
     if is_remote_host(host):
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index 4512d97b7c8..c89113641ef 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -20,20 +20,23 @@
 import re
 from copy import deepcopy
 from time import time
-from typing import Any, Dict, List, Optional, Tuple, Callable
+from typing import Any, Dict, List, Optional, Tuple, Callable, TYPE_CHECKING
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import XtriggerConfigError
 import cylc.flow.flags
 from cylc.flow.hostuserutil import get_user
 from cylc.flow.xtriggers.wall_clock import wall_clock
+from cylc.flow.subprocpool import (
+    SubProcPool,
+    get_func,
+)
 
-from cylc.flow.subprocctx import SubFuncContext
-from cylc.flow.broadcast_mgr import BroadcastMgr
-from cylc.flow.data_store_mgr import DataStoreMgr
-from cylc.flow.subprocpool import SubProcPool
-from cylc.flow.task_proxy import TaskProxy
-from cylc.flow.subprocpool import get_func
+if TYPE_CHECKING:
+    from cylc.flow.broadcast_mgr import BroadcastMgr
+    from cylc.flow.data_store_mgr import DataStoreMgr
+    from cylc.flow.subprocctx import SubFuncContext
+    from cylc.flow.task_proxy import TaskProxy
 
 
 class TemplateVariables(Enum):
@@ -195,15 +198,15 @@ class XtriggerManager:
     def __init__(
         self,
         workflow: str,
-        broadcast_mgr: BroadcastMgr,
-        data_store_mgr: DataStoreMgr,
+        broadcast_mgr: 'BroadcastMgr',
+        data_store_mgr: 'DataStoreMgr',
         proc_pool: SubProcPool,
         user: Optional[str] = None,
         workflow_run_dir: Optional[str] = None,
         workflow_share_dir: Optional[str] = None,
     ):
         # Workflow function and clock triggers by label.
-        self.functx_map: Dict[str, SubFuncContext] = {}
+        self.functx_map: 'Dict[str, SubFuncContext]' = {}
         # When next to call a function, by signature.
         self.t_next_call: dict = {}
         # Satisfied triggers and their function results, by signature.
@@ -234,7 +237,11 @@ def __init__(
         self.data_store_mgr = data_store_mgr
 
     @staticmethod
-    def validate_xtrigger(label: str, fctx: SubFuncContext, fdir: str) -> None:
+    def validate_xtrigger(
+        label: str,
+        fctx: 'SubFuncContext',
+        fdir: str,
+    ) -> None:
         """Validate an Xtrigger function.
 
         Args:
@@ -305,7 +312,7 @@ def validate_xtrigger(label: str, fctx: SubFuncContext, fdir: str) -> None:
                 f' {", ".join(t.value for t in deprecated_variables)}'
             )
 
-    def add_trig(self, label: str, fctx: SubFuncContext, fdir: str) -> None:
+    def add_trig(self, label: str, fctx: 'SubFuncContext', fdir: str) -> None:
         """Add a new xtrigger function.
 
         Check the xtrigger function exists here (e.g. during validation).
@@ -334,7 +341,7 @@ def load_xtrigger_for_restart(self, row_idx: int, row: Tuple[str, str]):
         sig, results = row
         self.sat_xtrig[sig] = json.loads(results)
 
-    def _get_xtrigs(self, itask: TaskProxy, unsat_only: bool = False,
+    def _get_xtrigs(self, itask: 'TaskProxy', unsat_only: bool = False,
                     sigs_only: bool = False):
         """(Internal helper method.)
 
@@ -361,7 +368,11 @@ def _get_xtrigs(self, itask: TaskProxy, unsat_only: bool = False,
                 res.append((label, sig, ctx, satisfied))
         return res
 
-    def get_xtrig_ctx(self, itask: TaskProxy, label: str) -> SubFuncContext:
+    def get_xtrig_ctx(
+        self,
+        itask: 'TaskProxy',
+        label: str,
+    ) -> 'SubFuncContext':
         """Get a real function context from the template.
 
         Args:
@@ -411,7 +422,7 @@ def get_xtrig_ctx(self, itask: TaskProxy, label: str) -> SubFuncContext:
         ctx.update_command(self.workflow_run_dir)
         return ctx
 
-    def call_xtriggers_async(self, itask: TaskProxy):
+    def call_xtriggers_async(self, itask: 'TaskProxy'):
         """Call itask's xtrigger functions via the process pool...
 
         ...if previous call not still in-process and retry period is up.
@@ -456,7 +467,7 @@ def call_xtriggers_async(self, itask: TaskProxy):
             self.active.append(sig)
             self.proc_pool.put_command(ctx, callback=self.callback)
 
-    def housekeep(self, itasks: List[TaskProxy]):
+    def housekeep(self, itasks: 'List[TaskProxy]'):
         """Delete satisfied xtriggers no longer needed by any task.
 
         Args:
@@ -469,7 +480,7 @@ def housekeep(self, itasks: List[TaskProxy]):
             if sig not in all_xtrig:
                 del self.sat_xtrig[sig]
 
-    def callback(self, ctx: SubFuncContext):
+    def callback(self, ctx: 'SubFuncContext'):
         """Callback for asynchronous xtrigger functions.
 
         Record satisfaction status and function results dict.
@@ -494,7 +505,7 @@ def callback(self, ctx: SubFuncContext):
 
     def check_xtriggers(
             self,
-            itask: TaskProxy,
+            itask: 'TaskProxy',
             db_update_func: Callable[[dict], None]) -> bool:
         """Check if all of itasks' xtriggers have become satisfied.
 
diff --git a/setup.cfg b/setup.cfg
index fbac4552f57..6c17d2091ef 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -113,6 +113,7 @@ tests =
     flake8-debugger>=4.0.0
     flake8-mutable>=1.2.0
     flake8-simplify>=0.14.0
+    flake8-type-checking; python_version > "3.7"
     flake8>=3.0.0
     mypy>=0.910
     pytest-asyncio>=0.17
diff --git a/tests/unit/test_id_cli.py b/tests/unit/test_id_cli.py
index 4f1099b164b..149990a9bb0 100644
--- a/tests/unit/test_id_cli.py
+++ b/tests/unit/test_id_cli.py
@@ -570,7 +570,7 @@ async def _scan():
         # something that looks like scan but doesn't do anything
         yield
 
-    monkeypatch.setattr('cylc.flow.id_cli.scan', _scan)
+    monkeypatch.setattr('cylc.flow.network.scan.scan', _scan)
 
 
 async def test_expand_workflow_tokens_impl_selector(no_scan):

From af0b92f4eed4a1629d1ab8a9c22af84aaf25ae05 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 21 Sep 2023 16:26:06 +0100
Subject: [PATCH 024/110] Remove PYTHONPATH items from sys.path Add
 CYLC_PYTHONPATH items to sys.path

---
 changes.d/5727.break.md                       |  2 ++
 .../etc/python-job.settings                   |  4 +--
 cylc/flow/scripts/cylc.py                     | 29 +++++++++++++++++--
 tests/unit/scripts/test_cylc.py               | 26 ++++++++++++++++-
 4 files changed, 56 insertions(+), 5 deletions(-)
 create mode 100644 changes.d/5727.break.md

diff --git a/changes.d/5727.break.md b/changes.d/5727.break.md
new file mode 100644
index 00000000000..8d0a1cd2314
--- /dev/null
+++ b/changes.d/5727.break.md
@@ -0,0 +1,2 @@
+Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value.
+If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.
\ No newline at end of file
diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
index 15de1f8ea13..a1366917d8f 100644
--- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
+++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
@@ -7,6 +7,6 @@
         [[[environment]]]
             # These environment variables ensure that tasks can
             # run in the same environment as the workflow:
-            {% from "sys" import path, executable %}
-            PYTHONPATH = {{':'.join(path)}}
+            {% from "sys" import executable %}
+            PYTHONPATH = {{executable}}/../lib/python:$PYTHONPATH
             PATH = $(dirname {{executable}}):$PATH
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 4f13dc59496..c65b4c8c9ba 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -16,10 +16,35 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """cylc main entry point"""
 
-import argparse
-from contextlib import contextmanager
 import os
 import sys
+
+
+def pythonpath_manip():
+    """Stop PYTHONPATH contaminating the Cylc Environment
+
+    * Remove PYTHONPATH items from sys.path to prevent PYTHONPATH
+      contaminating the Cylc Environment.
+    * Re-add items from CYLC_PYTHONPATH to sys.path.
+
+    See Also:
+        https://github.com/cylc/cylc-flow/issues/5124
+    """
+    if 'CYLC_PYTHONPATH' in os.environ:
+        for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep):
+            abspath = os.path.abspath(item)
+            sys.path.insert(0, abspath)
+    elif 'PYTHONPATH' in os.environ:
+        for item in os.environ['PYTHONPATH'].split(os.pathsep):
+            abspath = os.path.abspath(item)
+            if abspath in sys.path:
+                sys.path.remove(abspath)
+
+
+pythonpath_manip()
+
+import argparse
+from contextlib import contextmanager
 from typing import Iterator, NoReturn, Optional, Tuple
 
 from ansimarkup import parse as cparse
diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py
index f1483e9ee4a..9f461367377 100644
--- a/tests/unit/scripts/test_cylc.py
+++ b/tests/unit/scripts/test_cylc.py
@@ -15,14 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import pkg_resources
+import sys
 from types import SimpleNamespace
 from typing import Callable
 from unittest.mock import Mock
 
 import pytest
 
-from cylc.flow.scripts.cylc import iter_commands
+from cylc.flow.scripts.cylc import iter_commands, pythonpath_manip
 
 from ..conftest import MonkeyMock
 
@@ -136,3 +138,25 @@ def test_execute_cmd(
     # the "bad" entry point should raise an exception
     with pytest.raises(ModuleNotFoundError):
         execute_cmd('bad')
+
+
+def test_pythonpath_manip(monkeypatch):
+    """pythonpath_manip removes items in PYTHONPATH from sys.path
+
+    and adds items from CYLC_PYTHONPATH
+    """
+    # If PYTHONPATH is set...
+    monkeypatch.setenv('PYTHONPATH', '/remove-from-sys.path')
+    monkeypatch.setattr('sys.path', ['/leave-alone', '/remove-from-sys.path'])
+    pythonpath_manip()
+    # ... we don't change PYTHONPATH
+    assert os.environ['PYTHONPATH'] == '/remove-from-sys.path'
+    # ... but we do remove PYTHONPATH items from sys.path, and don't remove
+    # items there not in PYTHONPATH
+    assert sys.path == ['/leave-alone']
+
+    # If CYLC_PYTHONPATH is set we retrieve its contents and
+    # add them to the sys.path:
+    monkeypatch.setenv('CYLC_PYTHONPATH', '/add-to-sys.path')
+    pythonpath_manip()
+    assert sys.path == ['/add-to-sys.path', '/leave-alone']

From 093cb13cd130e97d8b88925ba9ace199da62ca49 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 25 Oct 2023 09:53:52 +0100
Subject: [PATCH 025/110] Apply suggestions from code review

Co-authored-by: Hilary James Oliver <hilary.j.oliver@gmail.com>
---
 cylc/flow/scripts/cylc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index c65b4c8c9ba..60abd1913b1 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -25,7 +25,7 @@ def pythonpath_manip():
 
     * Remove PYTHONPATH items from sys.path to prevent PYTHONPATH
       contaminating the Cylc Environment.
-    * Re-add items from CYLC_PYTHONPATH to sys.path.
+    * Add items from CYLC_PYTHONPATH to sys.path.
 
     See Also:
         https://github.com/cylc/cylc-flow/issues/5124
@@ -34,7 +34,7 @@ def pythonpath_manip():
         for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep):
             abspath = os.path.abspath(item)
             sys.path.insert(0, abspath)
-    elif 'PYTHONPATH' in os.environ:
+    if 'PYTHONPATH' in os.environ:
         for item in os.environ['PYTHONPATH'].split(os.pathsep):
             abspath = os.path.abspath(item)
             if abspath in sys.path:

From 02a8e7357d8e420c4317405e3a788a7f5b431d4c Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 25 Oct 2023 11:26:45 +0100
Subject: [PATCH 026/110] Update changes.d/5727.break.md

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 changes.d/5727.break.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/changes.d/5727.break.md b/changes.d/5727.break.md
index 8d0a1cd2314..06cb196216d 100644
--- a/changes.d/5727.break.md
+++ b/changes.d/5727.break.md
@@ -1,2 +1 @@
-Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value.
-If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.
\ No newline at end of file
+Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.
\ No newline at end of file

From 8d75c0448d4f3fef11b031de8c6b67800a182b03 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 25 Oct 2023 11:27:54 +0100
Subject: [PATCH 027/110] Update
 cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 .../tutorial/cylc-forecasting-workflow/etc/python-job.settings   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
index a1366917d8f..53b4aa2c17a 100644
--- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
+++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
@@ -8,5 +8,4 @@
             # These environment variables ensure that tasks can
             # run in the same environment as the workflow:
             {% from "sys" import executable %}
-            PYTHONPATH = {{executable}}/../lib/python:$PYTHONPATH
             PATH = $(dirname {{executable}}):$PATH

From 8606f93ea8c8260fee19fa2dac14da9245999c20 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 26 Oct 2023 10:38:04 +0100
Subject: [PATCH 028/110] Test the simulation mode code. (#5712)

* Test the simulation mode code.
- Refactor simulation mode functions.
- Improve documentation of sim mode settings.
- Ensure 100% test coverage of sim mode.
---
 cylc/flow/cfgspec/workflow.py                 |   7 +
 cylc/flow/config.py                           |  67 +-----
 cylc/flow/scheduler.py                        |   7 +-
 cylc/flow/simulation.py                       | 219 ++++++++++++++++++
 cylc/flow/task_events_mgr.py                  |  33 ++-
 cylc/flow/task_job_mgr.py                     |   8 +-
 cylc/flow/task_pool.py                        |  41 ----
 tests/functional/modes/03-simulation.t        |  32 +++
 .../functional/modes/03-simulation/flow.cylc  |  16 ++
 .../modes/03-simulation/reference.log         |   2 +
 tests/integration/test_simulation.py          | 125 ++++++++++
 tests/unit/test_config.py                     |  32 ++-
 tests/unit/test_simulation.py                 | 166 +++++++++++++
 13 files changed, 628 insertions(+), 127 deletions(-)
 create mode 100644 cylc/flow/simulation.py
 create mode 100644 tests/functional/modes/03-simulation.t
 create mode 100644 tests/functional/modes/03-simulation/flow.cylc
 create mode 100644 tests/functional/modes/03-simulation/reference.log
 create mode 100644 tests/integration/test_simulation.py
 create mode 100644 tests/unit/test_simulation.py

diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
index a6d16bbe76d..f521614c73a 100644
--- a/cylc/flow/cfgspec/workflow.py
+++ b/cylc/flow/cfgspec/workflow.py
@@ -1260,10 +1260,17 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                     - ``all`` - all instance of the task will fail
                     - ``2017-08-12T06, 2017-08-12T18`` - these instances of
                       the task will fail
+
+                    If you set :cylc:conf:`[..][..]execution retry delays`
+                    the second attempt will succeed unless you set
+                    :cylc:conf:`[..]fail try 1 only = False`.
                 ''')
                 Conf('fail try 1 only', VDR.V_BOOLEAN, True, desc='''
                     If ``True`` only the first run of the task
                     instance will fail, otherwise retries will fail too.
+
+                    Task instances must be set to fail by
+                    :cylc:conf:`[..]fail cycle points`.
                 ''')
                 Conf('disable task event handlers', VDR.V_BOOLEAN, True,
                      desc='''
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index 0096d924c8b..d80456266bf 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -79,8 +79,8 @@
     get_cylc_run_dir,
     is_relative_to,
 )
-from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM
 from cylc.flow.print_tree import print_tree
+from cylc.flow.simulation import configure_sim_modes
 from cylc.flow.subprocctx import SubFuncContext
 from cylc.flow.task_events_mgr import (
     EventData,
@@ -521,7 +521,8 @@ def __init__(
         self.process_runahead_limit()
 
         if self.run_mode('simulation', 'dummy'):
-            self.configure_sim_modes()
+            configure_sim_modes(
+                self.taskdefs.values(), self.run_mode())
 
         self.configure_workflow_state_polling_tasks()
 
@@ -1340,68 +1341,6 @@ def configure_workflow_state_polling_tasks(self):
             script = "echo " + comstr + "\n" + comstr
             rtc['script'] = script
 
-    def configure_sim_modes(self):
-        """Adjust task defs for simulation and dummy mode."""
-        for tdef in self.taskdefs.values():
-            # Compute simulated run time by scaling the execution limit.
-            rtc = tdef.rtconfig
-            limit = rtc['execution time limit']
-            speedup = rtc['simulation']['speedup factor']
-            if limit and speedup:
-                sleep_sec = (DurationParser().parse(
-                    str(limit)).get_seconds() / speedup)
-            else:
-                sleep_sec = DurationParser().parse(
-                    str(rtc['simulation']['default run length'])
-                ).get_seconds()
-            rtc['execution time limit'] = (
-                sleep_sec + DurationParser().parse(str(
-                    rtc['simulation']['time limit buffer'])).get_seconds()
-            )
-            rtc['job']['simulated run length'] = sleep_sec
-
-            # Generate dummy scripting.
-            rtc['init-script'] = ""
-            rtc['env-script'] = ""
-            rtc['pre-script'] = ""
-            rtc['post-script'] = ""
-            scr = "sleep %d" % sleep_sec
-            # Dummy message outputs.
-            for msg in rtc['outputs'].values():
-                scr += "\ncylc message '%s'" % msg
-            if rtc['simulation']['fail try 1 only']:
-                arg1 = "true"
-            else:
-                arg1 = "false"
-            arg2 = " ".join(rtc['simulation']['fail cycle points'])
-            scr += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2)
-            rtc['script'] = scr
-
-            # Dummy mode jobs should run on platform localhost
-            # All Cylc 7 config items which conflict with platform are removed.
-            for section, keys in FORBIDDEN_WITH_PLATFORM.items():
-                if section in rtc:
-                    for key in keys:
-                        if key in rtc[section]:
-                            rtc[section][key] = None
-
-            rtc['platform'] = 'localhost'
-
-            # Disable environment, in case it depends on env-script.
-            rtc['environment'] = {}
-
-            # Simulation mode tasks should fail in which cycle points?
-            f_pts = []
-            f_pts_orig = rtc['simulation']['fail cycle points']
-            if 'all' in f_pts_orig:
-                # None for "fail all points".
-                f_pts = None
-            else:
-                # (And [] for "fail no points".)
-                for point_str in f_pts_orig:
-                    f_pts.append(get_point(point_str).standardise())
-            rtc['simulation']['fail cycle points'] = f_pts
-
     def get_parent_lists(self):
         return self.runtime['parents']
 
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 1d9fc5c21dd..effe02e910d 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -113,6 +113,7 @@
 )
 from cylc.flow.profiler import Profiler
 from cylc.flow.resources import get_resources
+from cylc.flow.simulation import sim_time_check
 from cylc.flow.subprocpool import SubProcPool
 from cylc.flow.templatevars import eval_var
 from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
@@ -1740,7 +1741,11 @@ async def main_loop(self) -> None:
             self.pool.set_expired_tasks()
             self.release_queued_tasks()
 
-            if self.pool.sim_time_check(self.message_queue):
+            if (
+                self.pool.config.run_mode('simulation')
+                and sim_time_check(
+                    self.message_queue, self.pool.get_tasks())
+            ):
                 # A simulated task state change occurred.
                 self.reset_inactivity_timer()
 
diff --git a/cylc/flow/simulation.py b/cylc/flow/simulation.py
new file mode 100644
index 00000000000..15314f8e3e7
--- /dev/null
+++ b/cylc/flow/simulation.py
@@ -0,0 +1,219 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Utilities supporting simulation and skip modes
+"""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from time import time
+
+from cylc.flow.cycling.loader import get_point
+from cylc.flow.network.resolvers import TaskMsg
+from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM
+from cylc.flow.task_state import (
+    TASK_STATUS_RUNNING,
+    TASK_STATUS_FAILED,
+    TASK_STATUS_SUCCEEDED,
+)
+from cylc.flow.wallclock import get_current_time_string
+
+from metomi.isodatetime.parsers import DurationParser
+
+if TYPE_CHECKING:
+    from queue import Queue
+    from cylc.flow.cycling import PointBase
+    from cylc.flow.task_proxy import TaskProxy
+
+
+def configure_sim_modes(taskdefs, sim_mode):
+    """Adjust task defs for simulation and dummy mode.
+
+    """
+    dummy_mode = bool(sim_mode == 'dummy')
+
+    for tdef in taskdefs:
+        # Compute simulated run time by scaling the execution limit.
+        rtc = tdef.rtconfig
+        sleep_sec = get_simulated_run_len(rtc)
+
+        rtc['execution time limit'] = (
+            sleep_sec + DurationParser().parse(str(
+                rtc['simulation']['time limit buffer'])).get_seconds()
+        )
+
+        rtc['simulation']['simulated run length'] = sleep_sec
+        rtc['submission retry delays'] = [1]
+
+        # Generate dummy scripting.
+        rtc['init-script'] = ""
+        rtc['env-script'] = ""
+        rtc['pre-script'] = ""
+        rtc['post-script'] = ""
+        rtc['script'] = build_dummy_script(
+            rtc, sleep_sec) if dummy_mode else ""
+
+        disable_platforms(rtc)
+
+        # Disable environment, in case it depends on env-script.
+        rtc['environment'] = {}
+
+        rtc["simulation"][
+            "fail cycle points"
+        ] = parse_fail_cycle_points(
+            rtc["simulation"]["fail cycle points"]
+        )
+
+
+def get_simulated_run_len(rtc: Dict[str, Any]) -> int:
+    """Get simulated run time.
+
+    rtc = run time config
+    """
+    limit = rtc['execution time limit']
+    speedup = rtc['simulation']['speedup factor']
+    if limit and speedup:
+        sleep_sec = (DurationParser().parse(
+            str(limit)).get_seconds() / speedup)
+    else:
+        sleep_sec = DurationParser().parse(
+            str(rtc['simulation']['default run length'])
+        ).get_seconds()
+
+    return sleep_sec
+
+
+def build_dummy_script(rtc: Dict[str, Any], sleep_sec: int) -> str:
+    """Create fake scripting for dummy mode.
+
+    This is for Dummy mode only.
+    """
+    script = "sleep %d" % sleep_sec
+    # Dummy message outputs.
+    for msg in rtc['outputs'].values():
+        script += "\ncylc message '%s'" % msg
+    if rtc['simulation']['fail try 1 only']:
+        arg1 = "true"
+    else:
+        arg1 = "false"
+    arg2 = " ".join(rtc['simulation']['fail cycle points'])
+    script += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2)
+    return script
+
+
+def disable_platforms(
+    rtc: Dict[str, Any]
+) -> None:
+    """Force platform = localhost
+
+    Remove legacy sections [job] and [remote], which would conflict
+    with setting platforms.
+
+    This can be simplified when support for the FORBIDDEN_WITH_PLATFORM
+    configurations is dropped.
+    """
+    for section, keys in FORBIDDEN_WITH_PLATFORM.items():
+        if section in rtc:
+            for key in keys:
+                if key in rtc[section]:
+                    rtc[section][key] = None
+    rtc['platform'] = 'localhost'
+
+
+def parse_fail_cycle_points(
+    f_pts_orig: List[str]
+) -> 'Union[None, List[PointBase]]':
+    """Parse `[simulation][fail cycle points]`.
+
+    - None for "fail all points".
+    - Else a list of cycle point objects.
+
+    Examples:
+        >>> this = parse_fail_cycle_points
+        >>> this(['all']) is None
+        True
+        >>> this([])
+        []
+    """
+    f_pts: 'Optional[List[PointBase]]'
+    if 'all' in f_pts_orig:
+        f_pts = None
+    else:
+        f_pts = []
+        for point_str in f_pts_orig:
+            f_pts.append(get_point(point_str).standardise())
+    return f_pts
+
+
+def sim_time_check(
+    message_queue: 'Queue[TaskMsg]', itasks: 'List[TaskProxy]'
+) -> bool:
+    """Check if sim tasks have been "running" for as long as required.
+
+    If they have change the task state.
+
+    Returns:
+        True if _any_ simulated task state has changed.
+    """
+    sim_task_state_changed = False
+    now = time()
+    for itask in itasks:
+        if itask.state.status != TASK_STATUS_RUNNING:
+            continue
+        # Started time is not set on restart
+        if itask.summary['started_time'] is None:
+            itask.summary['started_time'] = now
+        timeout = (
+            itask.summary['started_time'] +
+            itask.tdef.rtconfig['simulation']['simulated run length']
+        )
+        if now > timeout:
+            job_d = itask.tokens.duplicate(job=str(itask.submit_num))
+            now_str = get_current_time_string()
+            if sim_task_failed(
+                itask.tdef.rtconfig['simulation'],
+                itask.point,
+                itask.get_try_num()
+            ):
+                message_queue.put(
+                    TaskMsg(job_d, now_str, 'CRITICAL', TASK_STATUS_FAILED)
+                )
+            else:
+                # Simulate message outputs.
+                for msg in itask.tdef.rtconfig['outputs'].values():
+                    message_queue.put(
+                        TaskMsg(job_d, now_str, 'DEBUG', msg)
+                    )
+                message_queue.put(
+                    TaskMsg(job_d, now_str, 'DEBUG', TASK_STATUS_SUCCEEDED)
+                )
+            sim_task_state_changed = True
+    return sim_task_state_changed
+
+
+def sim_task_failed(
+        sim_conf: Dict[str, Any],
+        point: 'PointBase',
+        try_num: int,
+) -> bool:
+    """Encapsulate logic for deciding whether a sim task has failed.
+
+    Allows Unit testing.
+    """
+    return (
+        sim_conf['fail cycle points'] is None  # i.e. "all"
+        or point in sim_conf['fail cycle points']
+    ) and (
+        try_num == 1 or not sim_conf['fail try 1 only']
+    )
diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py
index d4d9130ee27..ec5e467b8aa 100644
--- a/cylc/flow/task_events_mgr.py
+++ b/cylc/flow/task_events_mgr.py
@@ -575,6 +575,7 @@ def process_message(
             True: if polling is required to confirm a reversal of status.
 
         """
+
         # Log messages
         if event_time is None:
             event_time = get_current_time_string()
@@ -617,7 +618,6 @@ def process_message(
             self.setup_event_handlers(
                 itask, self.EVENT_STARTED, f'job {self.EVENT_STARTED}')
             self.spawn_func(itask, TASK_OUTPUT_STARTED)
-
         if message == self.EVENT_STARTED:
             if (
                     flag == self.FLAG_RECEIVED
@@ -777,24 +777,23 @@ def _process_message_check(
             return False
 
         if (
-                itask.state(TASK_STATUS_WAITING)
-                and
+            itask.state(TASK_STATUS_WAITING)
+            and itask.tdef.run_mode == 'live'   # Polling in live mode only.
+            and (
                 (
-                    (
-                        # task has a submit-retry lined up
-                        TimerFlags.SUBMISSION_RETRY in itask.try_timers
-                        and itask.try_timers[
-                            TimerFlags.SUBMISSION_RETRY].num > 0
-                    )
-                    or
-                    (
-                        # task has an execution-retry lined up
-                        TimerFlags.EXECUTION_RETRY in itask.try_timers
-                        and itask.try_timers[
-                            TimerFlags.EXECUTION_RETRY].num > 0
-                    )
+                    # task has a submit-retry lined up
+                    TimerFlags.SUBMISSION_RETRY in itask.try_timers
+                    and itask.try_timers[
+                        TimerFlags.SUBMISSION_RETRY].num > 0
                 )
-
+                or
+                (
+                    # task has an execution-retry lined up
+                    TimerFlags.EXECUTION_RETRY in itask.try_timers
+                    and itask.try_timers[
+                        TimerFlags.EXECUTION_RETRY].num > 0
+                )
+            )
         ):
             # Ignore messages if task has a retry lined up
             # (caused by polling overlapping with task failure)
diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py
index 412ba23b97a..d6bc59fb255 100644
--- a/cylc/flow/task_job_mgr.py
+++ b/cylc/flow/task_job_mgr.py
@@ -260,12 +260,13 @@ def submit_task_jobs(self, workflow, itasks, curve_auth,
 
         Return (list): list of tasks that attempted submission.
         """
-
         if is_simulation:
             return self._simulation_submit_task_jobs(itasks, workflow)
+
         # Prepare tasks for job submission
         prepared_tasks, bad_tasks = self.prep_submit_task_jobs(
             workflow, itasks)
+
         # Reset consumed host selection results
         self.task_remote_mgr.subshell_eval_reset()
 
@@ -999,16 +1000,17 @@ def _simulation_submit_task_jobs(self, itasks, workflow):
             itask.waiting_on_job_prep = False
             itask.submit_num += 1
             self._set_retry_timers(itask)
+
             itask.platform = {'name': 'SIMULATION'}
             itask.summary['job_runner_name'] = 'SIMULATION'
             itask.summary[self.KEY_EXECUTE_TIME_LIMIT] = (
-                itask.tdef.rtconfig['job']['simulated run length']
+                itask.tdef.rtconfig['simulation']['simulated run length']
             )
             itask.jobs.append(
                 self.get_simulation_job_conf(itask, workflow)
             )
             self.task_events_mgr.process_message(
-                itask, INFO, TASK_OUTPUT_SUBMITTED
+                itask, INFO, TASK_OUTPUT_SUBMITTED,
             )
 
         return itasks
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 2e8d8a634ac..f9cc94deaaf 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -40,7 +40,6 @@
 from cylc.flow.id import Tokens, detokenise
 from cylc.flow.id_cli import contains_fnmatch
 from cylc.flow.id_match import filter_ids
-from cylc.flow.network.resolvers import TaskMsg
 from cylc.flow.workflow_status import StopMode
 from cylc.flow.task_action_timer import TaskActionTimer, TimerFlags
 from cylc.flow.task_events_mgr import (
@@ -74,7 +73,6 @@
 from cylc.flow.flow_mgr import FLOW_ALL, FLOW_NONE, FLOW_NEW
 
 if TYPE_CHECKING:
-    from queue import Queue
     from cylc.flow.config import WorkflowConfig
     from cylc.flow.cycling import IntervalBase, PointBase
     from cylc.flow.data_store_mgr import DataStoreMgr
@@ -1756,45 +1754,6 @@ def force_trigger_tasks(
 
         return len(unmatched)
 
-    def sim_time_check(self, message_queue: 'Queue[TaskMsg]') -> bool:
-        """Simulation mode: simulate task run times and set states."""
-        if not self.config.run_mode('simulation'):
-            return False
-        sim_task_state_changed = False
-        now = time()
-        for itask in self.get_tasks():
-            if itask.state.status != TASK_STATUS_RUNNING:
-                continue
-            # Started time is not set on restart
-            if itask.summary['started_time'] is None:
-                itask.summary['started_time'] = now
-            timeout = (itask.summary['started_time'] +
-                       itask.tdef.rtconfig['job']['simulated run length'])
-            if now > timeout:
-                conf = itask.tdef.rtconfig['simulation']
-                job_d = itask.tokens.duplicate(job=str(itask.submit_num))
-                now_str = get_current_time_string()
-                if (
-                    conf['fail cycle points'] is None  # i.e. "all"
-                    or itask.point in conf['fail cycle points']
-                ) and (
-                    itask.get_try_num() == 1 or not conf['fail try 1 only']
-                ):
-                    message_queue.put(
-                        TaskMsg(job_d, now_str, 'CRITICAL', TASK_STATUS_FAILED)
-                    )
-                else:
-                    # Simulate message outputs.
-                    for msg in itask.tdef.rtconfig['outputs'].values():
-                        message_queue.put(
-                            TaskMsg(job_d, now_str, 'DEBUG', msg)
-                        )
-                    message_queue.put(
-                        TaskMsg(job_d, now_str, 'DEBUG', TASK_STATUS_SUCCEEDED)
-                    )
-                sim_task_state_changed = True
-        return sim_task_state_changed
-
     def set_expired_tasks(self):
         res = False
         for itask in self.get_tasks():
diff --git a/tests/functional/modes/03-simulation.t b/tests/functional/modes/03-simulation.t
new file mode 100644
index 00000000000..87a7ca37a9b
--- /dev/null
+++ b/tests/functional/modes/03-simulation.t
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test that simulation mode runs, and reruns a failed task successfully
+# when execution retry delays is configured.
+
+. "$(dirname "$0")/test_header"
+set_test_number 2
+
+install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}"
+workflow_run_ok "${TEST_NAME_BASE}-run" \
+    cylc play \
+        --no-detach \
+        --mode=simulation \
+        --reference-test "${WORKFLOW_NAME}"
+purge
+exit
diff --git a/tests/functional/modes/03-simulation/flow.cylc b/tests/functional/modes/03-simulation/flow.cylc
new file mode 100644
index 00000000000..49300212d39
--- /dev/null
+++ b/tests/functional/modes/03-simulation/flow.cylc
@@ -0,0 +1,16 @@
+[scheduler]
+    [[events]]
+        workflow timeout = PT30S
+
+[scheduling]
+    initial cycle point = 2359
+    [[graph]]
+        R1 = get_observations
+
+[runtime]
+    [[get_observations]]
+        execution retry delays = PT2S
+        [[[simulation]]]
+            fail cycle points = all
+            fail try 1 only = True
+
diff --git a/tests/functional/modes/03-simulation/reference.log b/tests/functional/modes/03-simulation/reference.log
new file mode 100644
index 00000000000..2d14bc201fb
--- /dev/null
+++ b/tests/functional/modes/03-simulation/reference.log
@@ -0,0 +1,2 @@
+23590101T0000Z/get_observations -triggered off [] in flow 1
+23590101T0000Z/get_observations -triggered off [] in flow 1
diff --git a/tests/integration/test_simulation.py b/tests/integration/test_simulation.py
new file mode 100644
index 00000000000..e0ada2c3e49
--- /dev/null
+++ b/tests/integration/test_simulation.py
@@ -0,0 +1,125 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+from queue import Queue
+from types import SimpleNamespace
+
+from cylc.flow.cycling.iso8601 import ISO8601Point
+from cylc.flow.simulation import sim_time_check
+
+
+def get_msg_queue_item(queue, id_):
+    for item in queue.queue:
+        if id_ in str(item.job_id):
+            return item
+
+
+@pytest.fixture(scope='module')
+async def sim_time_check_setup(
+    mod_flow, mod_scheduler, mod_start, mod_one_conf
+):
+    schd = mod_scheduler(mod_flow({
+        'scheduler': {'cycle point format': '%Y'},
+        'scheduling': {
+            'initial cycle point': '1066',
+            'graph': {
+                'R1': 'one & fail_all & fast_forward'
+            }
+        },
+        'runtime': {
+            'one': {},
+            'fail_all': {
+                'simulation': {'fail cycle points': 'all'},
+                'outputs': {'foo': 'bar'}
+            },
+            # This task ought not be finished quickly, but for the speed up
+            'fast_forward': {
+                'execution time limit': 'PT1M',
+                'simulation': {'speedup factor': 2}
+            }
+        }
+    }))
+    msg_q = Queue()
+    async with mod_start(schd):
+        itasks = schd.pool.get_tasks()
+        for i in itasks:
+            i.try_timers = {'execution-retry': SimpleNamespace(num=0)}
+        yield schd, itasks, msg_q
+
+
+def test_false_if_not_running(sim_time_check_setup, monkeypatch):
+    schd, itasks, msg_q = sim_time_check_setup
+
+    # False if task status not running:
+    assert sim_time_check(msg_q, itasks) is False
+
+
+def test_sim_time_check_sets_started_time(sim_time_check_setup):
+    """But sim_time_check still returns False"""
+    schd, _, msg_q = sim_time_check_setup
+    one_1066 = schd.pool.get_task(ISO8601Point('1066'), 'one')
+    one_1066.state.status = 'running'
+    assert one_1066.summary['started_time'] is None
+    assert sim_time_check(msg_q, [one_1066]) is False
+    assert one_1066.summary['started_time'] is not None
+
+
+def test_task_finishes(sim_time_check_setup, monkeypatch):
+    """...and an appropriate message sent.
+
+    Checks all possible outcomes in sim_time_check where elapsed time is
+    greater than the simulation time.
+
+    Does NOT check every possible cause on an outcome - this is done
+    in unit tests.
+    """
+    schd, _, msg_q = sim_time_check_setup
+    monkeypatch.setattr('cylc.flow.simulation.time', lambda: 0)
+
+    # Setup a task to fail
+    fail_all_1066 = schd.pool.get_task(ISO8601Point('1066'), 'fail_all')
+    fail_all_1066.state.status = 'running'
+    fail_all_1066.try_timers = {'execution-retry': SimpleNamespace(num=0)}
+
+    # Before simulation time is up:
+    assert sim_time_check(msg_q, [fail_all_1066]) is False
+
+    # After simulation time is up:
+    monkeypatch.setattr('cylc.flow.simulation.time', lambda: 12)
+    assert sim_time_check(msg_q, [fail_all_1066]) is True
+    assert get_msg_queue_item(msg_q, '1066/fail_all').message == 'failed'
+
+    # Succeeds and records messages for all outputs:
+    fail_all_1066.try_timers = {'execution-retry': SimpleNamespace(num=1)}
+    msg_q = Queue()
+    assert sim_time_check(msg_q, [fail_all_1066]) is True
+    assert sorted(i.message for i in msg_q.queue) == ['bar', 'succeeded']
+
+
+def test_task_sped_up(sim_time_check_setup, monkeypatch):
+    """Task will speed up by a factor set in config."""
+    schd, _, msg_q = sim_time_check_setup
+    fast_forward_1066 = schd.pool.get_task(
+        ISO8601Point('1066'), 'fast_forward')
+    fast_forward_1066.state.status = 'running'
+
+    monkeypatch.setattr('cylc.flow.simulation.time', lambda: 0)
+    assert sim_time_check(msg_q, [fast_forward_1066]) is False
+    monkeypatch.setattr('cylc.flow.simulation.time', lambda: 29)
+    assert sim_time_check(msg_q, [fast_forward_1066]) is False
+    monkeypatch.setattr('cylc.flow.simulation.time', lambda: 31)
+    assert sim_time_check(msg_q, [fast_forward_1066]) is True
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 34c596a770c..bb55cbf295e 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from copy import deepcopy
 import os
 import sys
 from optparse import Values
@@ -38,12 +39,13 @@
 from cylc.flow.parsec.exceptions import Jinja2Error, EmPyError
 from cylc.flow.scheduler_cli import RunOptions
 from cylc.flow.scripts.validate import ValidateOptions
+from cylc.flow.simulation import configure_sim_modes
 from cylc.flow.workflow_files import WorkflowFiles
 from cylc.flow.wallclock import get_utc_mode, set_utc_mode
 from cylc.flow.xtrigger_mgr import XtriggerManager
 from cylc.flow.task_outputs import (
     TASK_OUTPUT_SUBMITTED,
-    TASK_OUTPUT_SUCCEEDED
+    TASK_OUTPUT_SUCCEEDED,
 )
 
 from cylc.flow.cycling.iso8601 import ISO8601Point
@@ -1741,3 +1743,31 @@ def test_cylc_env_at_parsing(
             assert var in cylc_env
         else:
             assert var not in cylc_env
+
+
+def test_configure_sim_mode(caplog):
+    job_section = {}
+    sim_section = {
+        'speedup factor': '',
+        'default run length': 'PT10S',
+        'time limit buffer': 'PT0S',
+        'fail try 1 only': False,
+        'fail cycle points': '',
+    }
+    rtconfig_1 = {
+        'execution time limit': '',
+        'simulation': sim_section,
+        'job': job_section,
+        'outputs': {},
+    }
+    rtconfig_2 = deepcopy(rtconfig_1)
+    rtconfig_2['simulation']['default run length'] = 'PT2S'
+
+    taskdefs = [
+        SimpleNamespace(rtconfig=rtconfig_1),
+        SimpleNamespace(rtconfig=rtconfig_2),
+    ]
+    configure_sim_modes(taskdefs, 'simulation')
+    results = [
+        i.rtconfig['simulation']['simulated run length'] for i in taskdefs]
+    assert results == [10.0, 2.0]
diff --git a/tests/unit/test_simulation.py b/tests/unit/test_simulation.py
new file mode 100644
index 00000000000..1c490f35c16
--- /dev/null
+++ b/tests/unit/test_simulation.py
@@ -0,0 +1,166 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Tests for utilities supporting simulation and skip modes
+"""
+import pytest
+from pytest import param
+
+from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.cycling.iso8601 import ISO8601Point
+from cylc.flow.simulation import (
+    parse_fail_cycle_points,
+    build_dummy_script,
+    disable_platforms,
+    get_simulated_run_len,
+    sim_task_failed,
+)
+
+
+@pytest.mark.parametrize(
+    'execution_time_limit, speedup_factor, default_run_length',
+    (
+        param(None, None, 'PT1H', id='default-run-length'),
+        param(None, 10, 'PT1H', id='speedup-factor-alone'),
+        param('PT1H', None, 'PT1H', id='execution-time-limit-alone'),
+        param('P1D', 24, 'PT1M', id='speed-up-and-execution-tl'),
+    )
+)
+def test_get_simulated_run_len(
+    execution_time_limit, speedup_factor, default_run_length
+):
+    """Test the logic of the presence or absence of config items.
+
+    Avoid testing the correct workign of DurationParser.
+    """
+    rtc = {
+        'execution time limit': execution_time_limit,
+        'simulation': {
+            'speedup factor': speedup_factor,
+            'default run length': default_run_length
+        }
+    }
+    assert get_simulated_run_len(rtc) == 3600
+
+
+@pytest.mark.parametrize(
+    'fail_one_time_only', (True, False)
+)
+def test_set_simulation_script(fail_one_time_only):
+    rtc = {
+        'outputs': {'foo': '1', 'bar': '2'},
+        'simulation': {
+            'fail try 1 only': fail_one_time_only,
+            'fail cycle points': '1',
+        }
+    }
+    result = build_dummy_script(rtc, 60)
+    assert result.split('\n') == [
+        'sleep 60',
+        "cylc message '1'",
+        "cylc message '2'",
+        f"cylc__job__dummy_result {str(fail_one_time_only).lower()}"
+        " 1 || exit 1"
+    ]
+
+
+@pytest.mark.parametrize(
+    'rtc, expect', (
+        ({'platform': 'skarloey'}, 'localhost'),
+        ({'remote': {'host': 'rheneas'}}, 'localhost'),
+        ({'job': {'batch system': 'loaf'}}, 'localhost'),
+    )
+)
+def test_disable_platforms(rtc, expect):
+    """A sampling of items FORBIDDEN_WITH_PLATFORMS are removed from a
+    config passed to this method.
+    """
+    disable_platforms(rtc)
+    assert rtc['platform'] == expect
+    subdicts = [v for v in rtc.values() if isinstance(v, dict)]
+    for subdict in subdicts:
+        for k, val in subdict.items():
+            if k != 'platform':
+                assert val is None
+
+
+def test_parse_fail_cycle_points(set_cycling_type):
+    before = ['2', '4']
+    set_cycling_type()
+    assert parse_fail_cycle_points(before) == [
+        IntegerPoint(i) for i in before
+    ]
+
+
+@pytest.mark.parametrize(
+    'conf, point, try_, expect',
+    (
+        param(
+            {'fail cycle points': [], 'fail try 1 only': True},
+            ISO8601Point('1'),
+            1,
+            False,
+            id='defaults'
+        ),
+        param(
+            {'fail cycle points': None, 'fail try 1 only': True},
+            ISO8601Point('1066'),
+            1,
+            True,
+            id='fail-all'
+        ),
+        param(
+            {
+                'fail cycle points': [
+                    ISO8601Point('1066'), ISO8601Point('1067')],
+                'fail try 1 only': True
+            },
+            ISO8601Point('1067'),
+            1,
+            True,
+            id='point-in-failCP'
+        ),
+        param(
+            {
+                'fail cycle points': [
+                    ISO8601Point('1066'), ISO8601Point('1067')],
+                'fail try 1 only': True
+            },
+            ISO8601Point('1000'),
+            1,
+            False,
+            id='point-notin-failCP'
+        ),
+        param(
+            {'fail cycle points': None, 'fail try 1 only': True},
+            ISO8601Point('1066'),
+            2,
+            False,
+            id='succeed-attempt2'
+        ),
+        param(
+            {'fail cycle points': None, 'fail try 1 only': False},
+            ISO8601Point('1066'),
+            7,
+            True,
+            id='fail-attempt7'
+        ),
+    )
+)
+def test_sim_task_failed(
+    conf, point, try_, expect, set_cycling_type
+):
+    set_cycling_type('iso8601')
+    assert sim_task_failed(conf, point, try_) == expect

From ac071c6fb976f8e059ca196df9dad092e5eecf71 Mon Sep 17 00:00:00 2001
From: Scott Wales <scott.wales@bom.gov.au>
Date: Fri, 27 Oct 2023 14:22:43 +1100
Subject: [PATCH 029/110] Set forward variables by platform

---
 cylc/flow/cfgspec/globalcfg.py       | 16 ++++++++--------
 cylc/flow/remote.py                  |  5 +----
 tests/unit/cfgspec/test_globalcfg.py | 10 ++++++++++
 tests/unit/test_remote.py            | 12 ++----------
 4 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index d42415d2a9c..e07db4c140b 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -784,14 +784,6 @@ def default_for(
 
                    {REPLACES}``[suite servers][run host select]rank``.
             ''')
-            Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
-                 desc='''
-                A list containing the names of the environment variables to
-                forward with SSH connections to the workflow host from
-                the host running 'cylc play'
-
-                .. versionadded:: 8.3.0
-            ''')
 
         with Conf('host self-identification', desc=f'''
             How Cylc determines and shares the identity of the workflow host.
@@ -1640,6 +1632,14 @@ def default_for(
 
                 .. versionadded:: 8.0.0
             ''')
+            Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
+                 desc='''
+                A list containing the names of the environment variables to
+                forward with SSH connections to the workflow host from
+                the host running 'cylc play'
+
+                .. versionadded:: 8.3.0
+            ''')
             with Conf('selection', desc='''
                 How to select a host from the list of platform hosts.
 
diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py
index 5d9ed7c3937..8a72d56b6d6 100644
--- a/cylc/flow/remote.py
+++ b/cylc/flow/remote.py
@@ -35,7 +35,6 @@
 from cylc.flow.option_parsers import verbosity_to_opts
 from cylc.flow.platforms import get_platform, get_host_from_platform
 from cylc.flow.util import format_cmd
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 
 
 def get_proc_ancestors():
@@ -300,9 +299,7 @@ def construct_ssh_cmd(
         'CYLC_COVERAGE',
         'CLIENT_COMMS_METH',
         'CYLC_ENV_NAME',
-        *(glbl_cfg().get(['scheduler'])
-            ['run hosts']
-            ['ssh forward environment variables']),
+        *platform['ssh forward environment variables'],
     ]:
         if envvar in os.environ:
             command.append(
diff --git a/tests/unit/cfgspec/test_globalcfg.py b/tests/unit/cfgspec/test_globalcfg.py
index 2becda1caad..6db8d76dcf8 100644
--- a/tests/unit/cfgspec/test_globalcfg.py
+++ b/tests/unit/cfgspec/test_globalcfg.py
@@ -148,3 +148,13 @@ def test_source_dir_validation(
         assert "must be an absolute path" in str(excinfo.value)
     else:
         glblcfg.load()
+
+def test_platform_ssh_forward_variables(mock_global_config):
+
+    glblcfg: GlobalConfig = mock_global_config('''
+    [platforms]
+        [[foo]]
+            ssh forward environment variables = "FOO", "BAR"
+    ''')
+
+    assert glblcfg.get(['platforms','foo','ssh forward environment variables']) == ["FOO", "BAR"]
diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py
index c3c0c1ca9ff..8d24fc0bb3a 100644
--- a/tests/unit/test_remote.py
+++ b/tests/unit/test_remote.py
@@ -90,25 +90,17 @@ def test_construct_rsync_over_ssh_cmd():
     ]
 
 
-def test_construct_ssh_cmd_forward_env(mock_glbl_cfg):
+def test_construct_ssh_cmd_forward_env():
     """ Test for 'ssh forward environment variables'
     """
     import os
 
-    mock_glbl_cfg(
-        'cylc.flow.remote.glbl_cfg',
-        '''
-        [scheduler]
-            [[run hosts]]
-                ssh forward environment variables = FOO, BAR
-        '''
-    )
-
     host = 'example.com'
     config = {
         'ssh command': 'ssh',
         'use login shell': None,
         'cylc path': None,
+        'ssh forward environment variables': ['FOO', 'BAZ'],
         }
 
     # Variable isn't set, no change to command

From 36a6ce9fe1ff8030caeb93824959d46616b1ffae Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 31 Oct 2023 11:01:45 +0000
Subject: [PATCH 030/110] Fix test polluted by env vars (#5796)

---
 tests/unit/test_remote.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py
index 8d24fc0bb3a..5982fb746f1 100644
--- a/tests/unit/test_remote.py
+++ b/tests/unit/test_remote.py
@@ -15,8 +15,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Test the cylc.flow.remote module."""
 
-from cylc.flow.remote import run_cmd, construct_rsync_over_ssh_cmd, construct_ssh_cmd
+import os
 from unittest import mock
+
+import pytest
+
+from cylc.flow.remote import (
+    run_cmd, construct_rsync_over_ssh_cmd, construct_ssh_cmd
+)
 import cylc.flow
 
 
@@ -90,10 +96,13 @@ def test_construct_rsync_over_ssh_cmd():
     ]
 
 
-def test_construct_ssh_cmd_forward_env():
+def test_construct_ssh_cmd_forward_env(monkeypatch: pytest.MonkeyPatch):
     """ Test for 'ssh forward environment variables'
     """
-    import os
+    # Clear CYLC_* env vars as these will show up in the command
+    for env_var in os.environ:
+        if env_var.startswith('CYLC'):
+            monkeypatch.delenv(env_var)
 
     host = 'example.com'
     config = {
@@ -101,7 +110,7 @@ def test_construct_ssh_cmd_forward_env():
         'use login shell': None,
         'cylc path': None,
         'ssh forward environment variables': ['FOO', 'BAZ'],
-        }
+    }
 
     # Variable isn't set, no change to command
     expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'cylc', 'play']
@@ -109,7 +118,7 @@ def test_construct_ssh_cmd_forward_env():
     assert cmd == expect
 
     # Variable is set, appears in `env` list
-    with mock.patch.dict(os.environ, {'FOO': 'BAR'}):
-        expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'FOO=BAR', 'cylc', 'play']
-        cmd = construct_ssh_cmd(['play'], config, host)
-        assert cmd == expect
+    monkeypatch.setenv('FOO', 'BAR')
+    expect = ['ssh', host, 'env', f'CYLC_VERSION={cylc.flow.__version__}', 'FOO=BAR', 'cylc', 'play']
+    cmd = construct_ssh_cmd(['play'], config, host)
+    assert cmd == expect

From 034157e3e006c145eae522010bbfa6380b1acfe9 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 24 Oct 2023 18:17:08 +0100
Subject: [PATCH 031/110] replace pkg_resources with importlib_metadata and
 packaging

* Closes #5775
* Replace the pkg_resources library (deprecated) with newer alternatives.
* This resolves an efficiency issue due to the high import time of
  pkg_resources. This import time hits every single Cylc command
  including `cylc message`.
---
 conda-environment.yml                        |   4 +-
 cylc/flow/__init__.py                        |  18 ++-
 cylc/flow/cfgspec/globalcfg.py               |   5 +-
 cylc/flow/network/scan.py                    |  16 +-
 cylc/flow/parsec/fileparse.py                |   2 +-
 cylc/flow/scheduler_cli.py                   |   2 +-
 cylc/flow/scripts/completion_server.py       |  26 ++--
 cylc/flow/scripts/cylc.py                    | 147 ++++++++++---------
 cylc/flow/scripts/install.py                 |   4 +-
 cylc/flow/scripts/reinstall.py               |   4 +-
 cylc/flow/terminal.py                        |  42 ++++++
 cylc/flow/workflow_db_mgr.py                 |   3 +-
 setup.cfg                                    |   6 +-
 tests/functional/cli/01-help.t               |   6 +-
 tests/integration/test_install.py            |   2 +-
 tests/unit/plugins/test_pre_configure.py     |   2 +-
 tests/unit/scripts/test_completion_server.py |  25 ++--
 tests/unit/scripts/test_cylc.py              |  52 +++----
 18 files changed, 201 insertions(+), 165 deletions(-)

diff --git a/conda-environment.yml b/conda-environment.yml
index 248e4dd21c0..6ff6edf1637 100644
--- a/conda-environment.yml
+++ b/conda-environment.yml
@@ -10,13 +10,13 @@ dependencies:
   # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat
   - jinja2 >=3.0,<3.1
   - metomi-isodatetime >=1!3.0.0, <1!3.2.0
+  - packaging
   # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
   - protobuf >=4.21.2,<4.22.0
   - psutil >=5.6.0
   - python
   - pyzmq >=22
-  - setuptools >=49,!=67.*
-  - importlib_metadata # [py<3.8]
+  - importlib_metadata >=5.0 # [py<3.12]
   - urwid >=2,<3
   - tomli >=2 # [py<3.11]
 
diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py
index bc9f24aa573..d4192f72766 100644
--- a/cylc/flow/__init__.py
+++ b/cylc/flow/__init__.py
@@ -58,11 +58,19 @@ def environ_init():
 
 def iter_entry_points(entry_point_name):
     """Iterate over Cylc entry points."""
-    import pkg_resources
+    import sys
+    if sys.version_info[:2] > (3, 11):
+        from importlib.metadata import entry_points
+    else:
+        # BACK COMPAT: importlib_metadata
+        #   importlib.metadata was added in Python 3.8. The required interfaces
+        #   were completed by 3.12. For lower versions we must use the
+        #   importlib_metadata backport.
+        # FROM: Python 3.7
+        # TO: Python: 3.12
+        from importlib_metadata import entry_points
     yield from (
         entry_point
-        for entry_point in pkg_resources.iter_entry_points(entry_point_name)
-        # Filter out the cylc namespace as it should be empty.
-        # All cylc packages should take the form cylc-<name>
-        if entry_point.dist.key != 'cylc'
+        # for entry_point in entry_points()[entry_point_name]
+        for entry_point in entry_points().select(group=entry_point_name)
     )
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 5251c0ffad3..b404a42fdbe 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -22,7 +22,7 @@
 from typing import List, Optional, Tuple, Any, Union
 
 from contextlib import suppress
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version, Version
 
 from cylc.flow import LOG
 from cylc.flow import __version__ as CYLC_VERSION
@@ -1866,8 +1866,7 @@ def get_version_hierarchy(version: str) -> List[str]:
         ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev']
 
     """
-    smart_ver: Any = parse_version(version)
-    # (No type anno. yet for Version in pkg_resources.extern.packaging.version)
+    smart_ver: Version = parse_version(version)
     base = [str(i) for i in smart_ver.release]
     hierarchy = ['']
     hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)]
diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py
index 54222a510c7..c2202f3f31e 100644
--- a/cylc/flow/network/scan.py
+++ b/cylc/flow/network/scan.py
@@ -50,10 +50,8 @@
 import re
 from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union
 
-from pkg_resources import (
-    parse_requirements,
-    parse_version
-)
+from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
 
 from cylc.flow import LOG
 from cylc.flow.async_util import (
@@ -354,11 +352,7 @@ async def validate_contact_info(flow):
 
 def parse_requirement(requirement_string):
     """Parse a requirement from a requirement string."""
-    # we have to give the requirement a name but what we call it doesn't
-    # actually matter
-    for req in parse_requirements(f'x {requirement_string}'):
-        # there should only be one requirement
-        return (req,), {}
+    return (SpecifierSet(requirement_string),), {}
 
 
 @pipe(preproc=parse_requirement)
@@ -373,7 +367,7 @@ async def cylc_version(flow, requirement):
         flow (dict):
             Flow information dictionary, provided by scan through the pipe.
         requirement (str):
-            Requirement specifier in pkg_resources format e.g. ``> 8, < 9``
+            Requirement specifier in PEP 440 format e.g. ``> 8, < 9``
 
     """
     return parse_version(flow[ContactFileFields.VERSION]) in requirement
@@ -391,7 +385,7 @@ async def api_version(flow, requirement):
         flow (dict):
             Flow information dictionary, provided by scan through the pipe.
         requirement (str):
-            Requirement specifier in pkg_resources format e.g. ``> 8, < 9``
+            Requirement specifier in PEP 440 format e.g. ``> 8, < 9``
 
     """
     return parse_version(flow[ContactFileFields.API]) in requirement
diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py
index f4f8d723f0b..42e8a4aa40b 100644
--- a/cylc/flow/parsec/fileparse.py
+++ b/cylc/flow/parsec/fileparse.py
@@ -277,7 +277,7 @@ def process_plugins(fpath, opts):
         try:
             # If you want it to work on sourcedirs you need to get the options
             # to here.
-            plugin_result = entry_point.resolve()(
+            plugin_result = entry_point.load()(
                 srcdir=fpath, opts=opts
             )
         except Exception as exc:
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index 5c58b1eddba..e2f2d98f46a 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -25,7 +25,7 @@
 import sys
 from typing import TYPE_CHECKING
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 
 from cylc.flow import LOG, __version__
 from cylc.flow.exceptions import (
diff --git a/cylc/flow/scripts/completion_server.py b/cylc/flow/scripts/completion_server.py
index 2ac87f87b1b..ead311955da 100644
--- a/cylc/flow/scripts/completion_server.py
+++ b/cylc/flow/scripts/completion_server.py
@@ -46,10 +46,8 @@
 import sys
 import typing as t
 
-from pkg_resources import (
-    parse_requirements,
-    parse_version
-)
+from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
 
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.id import tokenise, IDTokens, Tokens
@@ -74,7 +72,7 @@
 # set the compatibility range for completion scripts with this server
 # I.E. if we change the server interface, change this compatibility range.
 # User's will be presented with an upgrade notice if this happens.
-REQUIRED_SCRIPT_VERSION = 'completion-script >=1.0.0, <2.0.0'
+REQUIRED_SCRIPT_VERSION = '>=1.0.0, <2.0.0'
 
 # register the psudo "help" and "version" commands
 COMMAND_LIST = list(COMMANDS) + ['help', 'version']
@@ -317,7 +315,7 @@ def list_options(command: str) -> t.List[str]:
     if command in COMMAND_OPTION_MAP:
         return COMMAND_OPTION_MAP[command]
     try:
-        entry_point = COMMANDS[command].resolve()
+        entry_point = COMMANDS[command].load()
     except KeyError:
         return []
     parser = entry_point.parser_function()
@@ -637,15 +635,13 @@ def check_completion_script_compatibility(
 
     # check that the installed completion script is compabile with this
     # completion server version
-    for requirement in parse_requirements(REQUIRED_SCRIPT_VERSION):
-        # NOTE: there's only one requirement but we have to iterate to get it
-        if installed_version not in requirement:
-            is_compatible = False
-            print(
-                f'The Cylc {completion_lang} script needs to be updated to'
-                ' work with this version of Cylc.',
-                file=sys.stderr,
-            )
+    if installed_version not in SpecifierSet(REQUIRED_SCRIPT_VERSION):
+        is_compatible = False
+        print(
+            f'The Cylc {completion_lang} script needs to be updated to'
+            ' work with this version of Cylc.',
+            file=sys.stderr,
+        )
 
     # check for completion script updates
     if installed_version < current_version:
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 60abd1913b1..46c8127585a 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -43,12 +43,28 @@ def pythonpath_manip():
 
 pythonpath_manip()
 
+if sys.version_info[:2] > (3, 11):
+    from importlib.metadata import (
+        entry_points,
+        files,
+    )
+else:
+    # BACK COMPAT: importlib_metadata
+    #   importlib.metadata was added in Python 3.8. The required interfaces
+    #   were completed by 3.12. For lower versions we must use the
+    #   importlib_metadata backport.
+    # FROM: Python 3.7
+    # TO: Python: 3.12
+    from importlib_metadata import (
+        entry_points,
+        files,
+    )
+
 import argparse
 from contextlib import contextmanager
 from typing import Iterator, NoReturn, Optional, Tuple
 
 from ansimarkup import parse as cparse
-import pkg_resources
 
 from cylc.flow import __version__, iter_entry_points
 from cylc.flow.option_parsers import (
@@ -306,9 +322,9 @@ def execute_cmd(cmd: str, *args: str) -> NoReturn:
         args: Command line arguments to pass to that command.
 
     """
-    entry_point: pkg_resources.EntryPoint = COMMANDS[cmd]
+    entry_point = COMMANDS[cmd]
     try:
-        entry_point.resolve()(*args)
+        entry_point.load()(*args)
     except ModuleNotFoundError as exc:
         msg = handle_missing_dependency(entry_point, exc)
         print(msg, file=sys.stderr)
@@ -401,7 +417,7 @@ def iter_commands() -> Iterator[Tuple[str, Optional[str], Optional[str]]]:
     """
     for cmd, entry_point in sorted(COMMANDS.items()):
         try:
-            module = __import__(entry_point.module_name, fromlist=[''])
+            module = __import__(entry_point.module, fromlist=[''])
         except ModuleNotFoundError as exc:
             handle_missing_dependency(entry_point, exc)
             continue
@@ -417,18 +433,10 @@ def print_id_help():
 
 
 def print_license() -> None:
-    try:
-        from importlib.metadata import files
-    except ImportError:
-        # BACK COMPAT: importlib_metadata
-        #   importlib.metadata was added in Python 3.8
-        # FROM: Python 3.7
-        # TO: Python: 3.8
-        from importlib_metadata import files  # type: ignore[no-redef]
-    license_file = next(filter(
-        lambda f: f.name == 'COPYING', files('cylc-flow')
-    ))
-    print(license_file.read_text())
+    for file in files('cylc-flow') or []:
+        if file.name == 'COPYING':
+            print(file.read_text())
+            return
 
 
 def print_command_list(commands=None, indent=0):
@@ -467,54 +475,57 @@ def cli_version(long_fmt=False):
     """Wrapper for get_version."""
     print(get_version(long_fmt))
     if long_fmt:
-        print(list_plugins())
+        print(cparse(list_plugins()))
     sys.exit(0)
 
 
 def list_plugins():
-    entry_point_names = [
-        entry_point_name
-        for entry_point_name
-        in pkg_resources.get_entry_map('cylc-flow').keys()
-        if entry_point_name.startswith('cylc.')
-    ]
-
-    entry_point_groups = {
-        entry_point_name: [
-            entry_point
-            for entry_point
-            in iter_entry_points(entry_point_name)
-            if not entry_point.module_name.startswith('cylc.flow')
-        ]
-        for entry_point_name in entry_point_names
-    }
-
-    dists = {
-        entry_point.dist
-        for entry_points in entry_point_groups.values()
-        for entry_point in entry_points
-    }
-
-    lines = []
-    if dists:
-        lines.append('\nPlugins:')
-        maxlen1 = max(len(dist.project_name) for dist in dists) + 2
-        maxlen2 = max(len(dist.version) for dist in dists) + 2
-        for dist in dists:
-            lines.append(
-                f'  {dist.project_name.ljust(maxlen1)}'
-                f' {dist.version.ljust(maxlen2)}'
-                f' {dist.module_path}'
-            )
-
-        lines.append('\nEntry Points:')
-        for entry_point_name, entry_points in entry_point_groups.items():
-            if entry_points:
-                lines.append(f'  {entry_point_name}:')
-                for entry_point in entry_points:
-                    lines.append(f'    {entry_point}')
-
-    return '\n'.join(lines)
+    from cylc.flow.terminal import DIM, format_grid
+    # go through all Cylc entry points
+    _dists = set()
+    __entry_points = {}
+    for entry_point in entry_points():
+        if (
+            # all Cylc entry points are under the "cylc" namespace
+            entry_point.group.startswith('cylc.')
+            # don't list cylc-flow entry-points (i.e. built-ins)
+            and not entry_point.value.startswith('cylc.flow')
+        ):
+            _dists.add(entry_point.dist)
+            __entry_points.setdefault(
+                entry_point.group,
+                [],
+            ).append(entry_point)
+
+    # list all the distriutions which provide Cylc entry points
+    _plugins = []
+    for dist in _dists:
+        _plugins.append((
+            '',
+            f'<light-blue>{dist.name}</light-blue>',
+            dist.version,
+            f'<{DIM}>{dist.locate_file("__init__.py").parent}</{DIM}>',
+        ))
+
+    # list all of the entry points by "group" (e.g. "cylc.command")
+    _entry_points = []
+    for group, points in sorted(__entry_points.items()):
+        _entry_points.append((f'  {group}:', '', ''))
+        for entry_point in points:
+            _entry_points.append((
+                f'    {entry_point.name}',
+                f'<light-blue>{entry_point.dist.name}</light-blue>',
+                f'<{DIM}>{entry_point.value}</{DIM}>',
+            ))
+
+    return '\n'.join((
+        '\n<bold>Plugins:</bold>',
+        *format_grid(_plugins),
+        '\n<bold>Entry Points:</bold>',
+        *format_grid(
+            _entry_points
+        ),
+    ))
 
 
 @contextmanager
@@ -686,7 +697,7 @@ def main():
 
 
 def handle_missing_dependency(
-    entry_point: pkg_resources.EntryPoint,
+    entry_point,
     err: ModuleNotFoundError
 ) -> str:
     """Return a suitable error message for a missing optional dependency.
@@ -698,12 +709,8 @@ def handle_missing_dependency(
 
     Re-raises the given ModuleNotFoundError if it is unexpected.
     """
-    try:
-        # Check for missing optional dependencies
-        entry_point.require()
-    except pkg_resources.DistributionNotFound as exc:
-        # Confirmed missing optional dependencies
-        return f"cylc {entry_point.name}: {exc}"
-    else:
-        # Error not due to missing optional dependencies; this is unexpected
-        raise err
+    msg = f'"cylc {entry_point.name}" requires "{entry_point.dist.name}'
+    if entry_point.extras:
+        msg += f'[{",".join(entry_point.extras)}]'
+    msg += f'"\n\n{err.__class__.__name__}: {err}'
+    return msg
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index 45010e4c714..be9583137e9 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -301,7 +301,7 @@ def install(
         'cylc.pre_configure'
     ):
         try:
-            entry_point.resolve()(srcdir=source, opts=opts)
+            entry_point.load()(srcdir=source, opts=opts)
         except Exception as exc:
             # NOTE: except Exception (purposefully vague)
             # this is to separate plugin from core Cylc errors
@@ -329,7 +329,7 @@ def install(
         'cylc.post_install'
     ):
         try:
-            entry_point.resolve()(
+            entry_point.load()(
                 srcdir=source_dir,
                 opts=opts,
                 rundir=str(rundir)
diff --git a/cylc/flow/scripts/reinstall.py b/cylc/flow/scripts/reinstall.py
index c3b48b23c74..5e1c75056aa 100644
--- a/cylc/flow/scripts/reinstall.py
+++ b/cylc/flow/scripts/reinstall.py
@@ -338,7 +338,7 @@ def pre_configure(opts: 'Values', src_dir: Path) -> None:
         'cylc.pre_configure'
     ):
         try:
-            entry_point.resolve()(srcdir=src_dir, opts=opts)
+            entry_point.load()(srcdir=src_dir, opts=opts)
         except Exception as exc:
             # NOTE: except Exception (purposefully vague)
             # this is to separate plugin from core Cylc errors
@@ -355,7 +355,7 @@ def post_install(opts: 'Values', src_dir: Path, run_dir: Path) -> None:
         'cylc.post_install'
     ):
         try:
-            entry_point.resolve()(
+            entry_point.load()(
                 srcdir=src_dir,
                 opts=opts,
                 rundir=str(run_dir)
diff --git a/cylc/flow/terminal.py b/cylc/flow/terminal.py
index bc45f6d65d3..e848691252e 100644
--- a/cylc/flow/terminal.py
+++ b/cylc/flow/terminal.py
@@ -94,6 +94,48 @@ def print_contents(contents, padding=5, char='.', indent=0):
             print(f'{indent}  {" " * title_width}{" " * padding}{line}')
 
 
+def format_grid(rows, gutter=2):
+    """Format gridded text.
+
+    This takes a 2D table of text and formats it to the maximum width of each
+    column and adds a bit of space between them.
+
+    Args:
+        rows:
+            2D list containing the text to format.
+        gutter:
+            The width of the gutter between columns.
+
+    Examples:
+        >>> format_grid([
+        ...     ['a', 'b', 'ccccc'],
+        ...     ['ddddd', 'e', 'f'],
+        ... ])
+        ['a      b  ccccc  ',
+         'ddddd  e  f      ']
+
+        >>> format_grid([])
+        []
+
+    """
+    if not rows:
+        return rows
+    templ = [
+        '{col:%d}' % (max(
+            len(row[ind])
+            for row in rows
+        ) + gutter)
+        for ind in range(len(rows[0]))
+    ]
+    lines = []
+    for row in rows:
+        ret = ''
+        for ind, col in enumerate(row):
+            ret += templ[ind].format(col=col)
+        lines.append(ret)
+    return lines
+
+
 def supports_color():
     """Determine if running in a terminal which supports color.
 
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index 091d1712429..e9402ba058b 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -25,7 +25,6 @@
 
 import json
 import os
-from pkg_resources import parse_version
 from shutil import copy, rmtree
 from sqlite3 import OperationalError
 from tempfile import mkstemp
@@ -33,6 +32,8 @@
     Any, AnyStr, Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Union
 )
 
+from packaging.version import parse as parse_version
+
 from cylc.flow import LOG
 from cylc.flow.broadcast_report import get_broadcast_change_iter
 from cylc.flow.rundb import CylcWorkflowDAO
diff --git a/setup.cfg b/setup.cfg
index 6581f341571..355204b481c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -69,12 +69,11 @@ install_requires =
     jinja2==3.0.*
     metomi-isodatetime>=1!3.0.0,<1!3.2.0
     # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
+    packaging
     protobuf>=4.21.2,<4.22.0
     psutil>=5.6.0
     pyzmq>=22
-    # https://github.com/pypa/setuptools/issues/3802
-    setuptools>=49,!=67.*
-    importlib_metadata; python_version < "3.8"
+    importlib_metadata>=5.0; python_version < "3.12"
     urwid==2.*
     # unpinned transient dependencies used for type checking
     rx
@@ -127,7 +126,6 @@ tests =
     # Type annotation stubs
     # http://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html
     types-Jinja2>=0.1.3
-    types-pkg_resources>=0.1.2
     types-protobuf>=0.1.10
     types-six>=0.1.6
     typing-extensions>=4
diff --git a/tests/functional/cli/01-help.t b/tests/functional/cli/01-help.t
index 3f78ada0dbf..935864e1873 100755
--- a/tests/functional/cli/01-help.t
+++ b/tests/functional/cli/01-help.t
@@ -19,7 +19,7 @@
 
 . "$(dirname "$0")/test_header"
 # Number of tests depends on the number of 'cylc' commands.
-set_test_number 26
+set_test_number 28
 
 # Top help
 run_ok "${TEST_NAME_BASE}-0" cylc
@@ -73,6 +73,10 @@ run_ok "${TEST_NAME_BASE}-version.stdout" \
 cmp_ok "${TEST_NAME_BASE}-version.stdout" "${TEST_NAME_BASE}---version.stdout"
 cmp_ok "${TEST_NAME_BASE}-version.stdout" "${TEST_NAME_BASE}-V.stdout"
 
+# Supplementary help
+run_ok "${TEST_NAME_BASE}-all" cylc help all
+run_ok "${TEST_NAME_BASE}-id" cylc help id
+
 # Check "cylc version --long" output is correct.
 cylc version --long | head -n 1 > long1
 WHICH="$(command -v cylc)"
diff --git a/tests/integration/test_install.py b/tests/integration/test_install.py
index cbac55c5361..f681c5a2a01 100644
--- a/tests/integration/test_install.py
+++ b/tests/integration/test_install.py
@@ -162,7 +162,7 @@ def test_install_gets_back_compat_mode_for_plugins(
     class failIfDeprecated:
         """A fake Cylc Plugin entry point"""
         @staticmethod
-        def resolve():
+        def load():
             return failIfDeprecated.raiser
 
         @staticmethod
diff --git a/tests/unit/plugins/test_pre_configure.py b/tests/unit/plugins/test_pre_configure.py
index 717648ea594..e9f91054e12 100644
--- a/tests/unit/plugins/test_pre_configure.py
+++ b/tests/unit/plugins/test_pre_configure.py
@@ -31,7 +31,7 @@ def __init__(self, fcn):
         self.name = fcn.__name__
         self.fcn = fcn
 
-    def resolve(self):
+    def load(self):
         return self.fcn
 
 
diff --git a/tests/unit/scripts/test_completion_server.py b/tests/unit/scripts/test_completion_server.py
index d50e46b7039..186e13b7272 100644
--- a/tests/unit/scripts/test_completion_server.py
+++ b/tests/unit/scripts/test_completion_server.py
@@ -398,18 +398,17 @@ def test_list_options(monkeypatch):
     assert list_options('zz9+za') == []
 
     # patch the logic to turn off the auto_add behaviour of CylcOptionParser
-    def _resolve():
-        def _parser_function():
-            parser = get_option_parser()
-            del parser.auto_add
-            return parser
-
-        return SimpleNamespace(parser_function=_parser_function)
-
-    monkeypatch.setattr(
-        COMMANDS['trigger'],
-        'resolve',
-        _resolve
+    class EntryPoint:
+        def load(self):
+            def _parser_function():
+                parser = get_option_parser()
+                del parser.auto_add
+                return parser
+            return SimpleNamespace(parser_function=_parser_function)
+    monkeypatch.setitem(
+        COMMANDS,
+        'trigger',
+        EntryPoint(),
     )
 
     # with auto_add turned off the --color option should be absent
@@ -674,7 +673,7 @@ def _get_current_completion_script_version(_script, lang):
     # set the completion script compatibility range to >=1.0.0, <2.0.0
     monkeypatch.setattr(
         'cylc.flow.scripts.completion_server.REQUIRED_SCRIPT_VERSION',
-        'completion-script >=1.0.0, <2.0.0',
+        '>=1.0.0, <2.0.0',
     )
     monkeypatch.setattr(
         'cylc.flow.scripts.completion_server'
diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py
index 9f461367377..4b844dafdc7 100644
--- a/tests/unit/scripts/test_cylc.py
+++ b/tests/unit/scripts/test_cylc.py
@@ -16,7 +16,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import pkg_resources
 import sys
 from types import SimpleNamespace
 from typing import Callable
@@ -32,12 +31,9 @@
 @pytest.fixture
 def mock_entry_points(monkeypatch: pytest.MonkeyPatch):
     """Mock a range of entry points."""
-    def _resolve_fail(*args, **kwargs):
+    def _load_fail(*args, **kwargs):
         raise ModuleNotFoundError('foo')
 
-    def _require_fail(*args, **kwargs):
-        raise pkg_resources.DistributionNotFound('foo', ['my_extras'])
-
     def _resolve_ok(*args, **kwargs):
         return Mock()
 
@@ -50,23 +46,17 @@ def _mocked_entry_points(include_bad: bool = False):
             'good': SimpleNamespace(
                 name='good',
                 module_name='os.path',
-                resolve=_resolve_ok,
-                require=_require_ok,
+                load=_resolve_ok,
+                extras=[],
+                dist=SimpleNamespace(name='a'),
             ),
             # an entry point with optional dependencies missing:
             'missing': SimpleNamespace(
                 name='missing',
                 module_name='not.a.python.module',  # force an import error
-                resolve=_resolve_fail,
-                require=_require_fail,
-            ),
-            # an entry point with optional dependencies missing, but they
-            # are not needed for the core functionality of the entry point:
-            'partial': SimpleNamespace(
-                name='partial',
-                module_name='os.path',
-                resolve=_resolve_ok,
-                require=_require_fail,
+                load=_load_fail,
+                extras=[],
+                dist=SimpleNamespace(name='foo'),
             ),
         }
         if include_bad:
@@ -75,8 +65,10 @@ def _mocked_entry_points(include_bad: bool = False):
             commands['bad'] = SimpleNamespace(
                 name='bad',
                 module_name='not.a.python.module',
-                resolve=_resolve_fail,
+                load=_load_fail,
                 require=_require_ok,
+                extras=[],
+                dist=SimpleNamespace(name='d'),
             )
         monkeypatch.setattr('cylc.flow.scripts.cylc.COMMANDS', commands)
 
@@ -90,14 +82,13 @@ def test_iter_commands(mock_entry_points):
     """
     mock_entry_points()
     commands = list(iter_commands())
-    assert [i[0] for i in commands] == ['good', 'partial']
+    assert [i[0] for i in commands] == ['good']
 
 
 def test_iter_commands_bad(mock_entry_points):
-    """Test listing commands fails if there is an unexpected import error."""
+    """Test listing commands doesn't fail on import error."""
     mock_entry_points(include_bad=True)
-    with pytest.raises(ModuleNotFoundError):
-        list(iter_commands())
+    list(iter_commands())
 
 
 def test_execute_cmd(
@@ -125,19 +116,16 @@ def test_execute_cmd(
     execute_cmd('missing')
     capexit.assert_any_call(1)
     assert capsys.readouterr().err.strip() == (
-        "cylc missing: The 'foo' distribution was not found and is"
-        " required by my_extras"
+        '"cylc missing" requires "foo"\n\nModuleNotFoundError: foo'
     )
 
-    # the "partial" entry point should exit 0
-    capexit.reset_mock()
-    execute_cmd('partial')
-    capexit.assert_called_once_with()
-    assert capsys.readouterr().err == ''
+    # the "bad" entry point should log an error
+    execute_cmd('bad')
+    capexit.assert_any_call(1)
 
-    # the "bad" entry point should raise an exception
-    with pytest.raises(ModuleNotFoundError):
-        execute_cmd('bad')
+    stderr = capsys.readouterr().err.strip()
+    assert '"cylc bad" requires "d"' in stderr
+    assert 'ModuleNotFoundError: foo' in stderr
 
 
 def test_pythonpath_manip(monkeypatch):

From a351f091761e3b49fe55894cc55b23502ba0f1dd Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 26 Oct 2023 10:54:01 +0100
Subject: [PATCH 032/110] actions: move lint checks into their own job

---
 .github/workflows/test_fast.yml | 77 +++++++++++++++++++--------------
 tests/unit/scripts/test_cylc.py |  6 +--
 2 files changed, 47 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index d0bff50326d..8a21de93ae8 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
-      fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
+      fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
@@ -39,7 +39,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo apt-get update
-          sudo apt-get install -y shellcheck sqlite3
+          sudo apt-get install -y sqlite3
 
       - name: Install
         run: |
@@ -48,37 +48,10 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: Check changelog
-        if: startsWith(matrix.os, 'ubuntu')
-        run: towncrier build --draft
-
-      - name: Style
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          flake8
-          etc/bin/shellchecker
-
-      # note: exclude python 3.10+ from mypy checks as these produce false
-      # positives in installed libraries for python 3.7
-      - name: Typing
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9)
-        run: mypy
-
-      - name: Doctests
-        timeout-minutes: 4
-        run: |
-          pytest cylc/flow
-
       - name: Unit Tests
-        timeout-minutes: 4
-        run: |
-          pytest tests/unit
-
-      - name: Bandit
-        if: ${{ matrix.python-version == '3.7' }}
-        # https://github.com/PyCQA/bandit/issues/658
+        timeout-minutes: 5
         run: |
-          bandit -r --ini .bandit cylc/flow
+          pytest cylc/flow tests/unit
 
       - name: Integration Tests
         timeout-minutes: 6
@@ -104,9 +77,47 @@ jobs:
           path: coverage.xml
           retention-days: 7
 
+  lint:
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 10
+    steps:
+      - name: Apt-Get Install
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # note: exclude python 3.10+ from mypy checks as these produce false
+      # positives in installed libraries for python 3.7
+      - name: Configure Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install
+        run: |
+          pip install -e ."[tests]"
+
+      - name: Flake8
+        run: flake8
+
+      - name: Bandit
+        run: |
+          bandit -r --ini .bandit cylc/flow
+
+      - name: Shellchecker
+        run: etc/bin/shellchecker
+
+      - name: MyPy
+        run: mypy
+
+      - name: Towncrier
+        run: towncrier build --draft
+
       - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+        run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py
 
   codecov:
     needs: test
diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py
index 4b844dafdc7..819583a296c 100644
--- a/tests/unit/scripts/test_cylc.py
+++ b/tests/unit/scripts/test_cylc.py
@@ -45,7 +45,7 @@ def _mocked_entry_points(include_bad: bool = False):
             # an entry point with all dependencies installed:
             'good': SimpleNamespace(
                 name='good',
-                module_name='os.path',
+                module='os.path',
                 load=_resolve_ok,
                 extras=[],
                 dist=SimpleNamespace(name='a'),
@@ -53,7 +53,7 @@ def _mocked_entry_points(include_bad: bool = False):
             # an entry point with optional dependencies missing:
             'missing': SimpleNamespace(
                 name='missing',
-                module_name='not.a.python.module',  # force an import error
+                module='not.a.python.module',  # force an import error
                 load=_load_fail,
                 extras=[],
                 dist=SimpleNamespace(name='foo'),
@@ -64,7 +64,7 @@ def _mocked_entry_points(include_bad: bool = False):
             # missing:
             commands['bad'] = SimpleNamespace(
                 name='bad',
-                module_name='not.a.python.module',
+                module='not.a.python.module',
                 load=_load_fail,
                 require=_require_ok,
                 extras=[],

From dff33b487da9fa08588bec139f82a0e716a0e516 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 2 Nov 2023 15:58:34 +0000
Subject: [PATCH 033/110] Regenerate protobuf pb2 post-merge

Using libprotoc 24.0
---
 cylc/flow/data_messages_pb2.py | 139 ++++++++++++++++-----------------
 1 file changed, 69 insertions(+), 70 deletions(-)

diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py
index 311a84f8958..9894ea55ebc 100644
--- a/cylc/flow/data_messages_pb2.py
+++ b/cylc/flow/data_messages_pb2.py
@@ -1,12 +1,11 @@
-# type: ignore
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: data_messages.proto
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -14,12 +13,12 @@
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18  \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\xab\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x12\n\nextra_logs\x18\x1d \x03(\t\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18  \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtime\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18  \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xb9\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputs\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18  \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
   DESCRIPTOR._options = None
   _PBWORKFLOW_STATETOTALSENTRY._options = None
   _PBWORKFLOW_STATETOTALSENTRY._serialized_options = b'8\001'
@@ -33,68 +32,68 @@
   _PBTASKPROXY_XTRIGGERSENTRY._serialized_options = b'8\001'
   _PBFAMILYPROXY_STATETOTALSENTRY._options = None
   _PBFAMILYPROXY_STATETOTALSENTRY._serialized_options = b'8\001'
-  _PBMETA._serialized_start=24
-  _PBMETA._serialized_end=174
-  _PBTIMEZONE._serialized_start=177
-  _PBTIMEZONE._serialized_end=347
-  _PBTASKPROXYREFS._serialized_start=349
-  _PBTASKPROXYREFS._serialized_end=388
-  _PBWORKFLOW._serialized_start=391
-  _PBWORKFLOW._serialized_end=2011
-  _PBWORKFLOW_STATETOTALSENTRY._serialized_start=1441
-  _PBWORKFLOW_STATETOTALSENTRY._serialized_end=1491
-  _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_start=1493
-  _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_end=1566
-  _PBRUNTIME._serialized_start=2014
-  _PBRUNTIME._serialized_end=2839
-  _PBJOB._serialized_start=2842
-  _PBJOB._serialized_end=3525
-  _PBTASK._serialized_start=3528
-  _PBTASK._serialized_end=3882
-  _PBPOLLTASK._serialized_start=3885
-  _PBPOLLTASK._serialized_end=4101
-  _PBCONDITION._serialized_start=4104
-  _PBCONDITION._serialized_end=4307
-  _PBPREREQUISITE._serialized_start=4310
-  _PBPREREQUISITE._serialized_end=4460
-  _PBOUTPUT._serialized_start=4463
-  _PBOUTPUT._serialized_end=4603
-  _PBTRIGGER._serialized_start=4606
-  _PBTRIGGER._serialized_end=4771
-  _PBTASKPROXY._serialized_start=4774
-  _PBTASKPROXY._serialized_end=5815
-  _PBTASKPROXY_OUTPUTSENTRY._serialized_start=5425
-  _PBTASKPROXY_OUTPUTSENTRY._serialized_end=5482
-  _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_start=5484
-  _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_end=5551
-  _PBTASKPROXY_XTRIGGERSENTRY._serialized_start=5553
-  _PBTASKPROXY_XTRIGGERSENTRY._serialized_end=5613
-  _PBFAMILY._serialized_start=5818
-  _PBFAMILY._serialized_end=6146
-  _PBFAMILYPROXY._serialized_start=6149
-  _PBFAMILYPROXY._serialized_end=6963
-  _PBFAMILYPROXY_STATETOTALSENTRY._serialized_start=1441
-  _PBFAMILYPROXY_STATETOTALSENTRY._serialized_end=1491
-  _PBEDGE._serialized_start=6966
-  _PBEDGE._serialized_end=7154
-  _PBEDGES._serialized_start=7156
-  _PBEDGES._serialized_end=7279
-  _PBENTIREWORKFLOW._serialized_start=7282
-  _PBENTIREWORKFLOW._serialized_end=7524
-  _EDELTAS._serialized_start=7527
-  _EDELTAS._serialized_end=7702
-  _FDELTAS._serialized_start=7705
-  _FDELTAS._serialized_end=7884
-  _FPDELTAS._serialized_start=7887
-  _FPDELTAS._serialized_end=8077
-  _JDELTAS._serialized_start=8080
-  _JDELTAS._serialized_end=8253
-  _TDELTAS._serialized_start=8256
-  _TDELTAS._serialized_end=8431
-  _TPDELTAS._serialized_start=8434
-  _TPDELTAS._serialized_end=8620
-  _WDELTAS._serialized_start=8623
-  _WDELTAS._serialized_end=8818
-  _ALLDELTAS._serialized_start=8821
-  _ALLDELTAS._serialized_end=9030
+  _globals['_PBMETA']._serialized_start=24
+  _globals['_PBMETA']._serialized_end=174
+  _globals['_PBTIMEZONE']._serialized_start=177
+  _globals['_PBTIMEZONE']._serialized_end=347
+  _globals['_PBTASKPROXYREFS']._serialized_start=349
+  _globals['_PBTASKPROXYREFS']._serialized_end=388
+  _globals['_PBWORKFLOW']._serialized_start=391
+  _globals['_PBWORKFLOW']._serialized_end=2011
+  _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_start=1441
+  _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_end=1491
+  _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=1493
+  _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=1566
+  _globals['_PBRUNTIME']._serialized_start=2014
+  _globals['_PBRUNTIME']._serialized_end=2839
+  _globals['_PBJOB']._serialized_start=2842
+  _globals['_PBJOB']._serialized_end=3511
+  _globals['_PBTASK']._serialized_start=3514
+  _globals['_PBTASK']._serialized_end=3868
+  _globals['_PBPOLLTASK']._serialized_start=3871
+  _globals['_PBPOLLTASK']._serialized_end=4087
+  _globals['_PBCONDITION']._serialized_start=4090
+  _globals['_PBCONDITION']._serialized_end=4293
+  _globals['_PBPREREQUISITE']._serialized_start=4296
+  _globals['_PBPREREQUISITE']._serialized_end=4446
+  _globals['_PBOUTPUT']._serialized_start=4449
+  _globals['_PBOUTPUT']._serialized_end=4589
+  _globals['_PBTRIGGER']._serialized_start=4592
+  _globals['_PBTRIGGER']._serialized_end=4757
+  _globals['_PBTASKPROXY']._serialized_start=4760
+  _globals['_PBTASKPROXY']._serialized_end=5801
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5411
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5468
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5470
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5537
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5539
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5599
+  _globals['_PBFAMILY']._serialized_start=5804
+  _globals['_PBFAMILY']._serialized_end=6132
+  _globals['_PBFAMILYPROXY']._serialized_start=6135
+  _globals['_PBFAMILYPROXY']._serialized_end=6949
+  _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1441
+  _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1491
+  _globals['_PBEDGE']._serialized_start=6952
+  _globals['_PBEDGE']._serialized_end=7140
+  _globals['_PBEDGES']._serialized_start=7142
+  _globals['_PBEDGES']._serialized_end=7265
+  _globals['_PBENTIREWORKFLOW']._serialized_start=7268
+  _globals['_PBENTIREWORKFLOW']._serialized_end=7510
+  _globals['_EDELTAS']._serialized_start=7513
+  _globals['_EDELTAS']._serialized_end=7688
+  _globals['_FDELTAS']._serialized_start=7691
+  _globals['_FDELTAS']._serialized_end=7870
+  _globals['_FPDELTAS']._serialized_start=7873
+  _globals['_FPDELTAS']._serialized_end=8063
+  _globals['_JDELTAS']._serialized_start=8066
+  _globals['_JDELTAS']._serialized_end=8239
+  _globals['_TDELTAS']._serialized_start=8242
+  _globals['_TDELTAS']._serialized_end=8417
+  _globals['_TPDELTAS']._serialized_start=8420
+  _globals['_TPDELTAS']._serialized_end=8606
+  _globals['_WDELTAS']._serialized_start=8609
+  _globals['_WDELTAS']._serialized_end=8804
+  _globals['_ALLDELTAS']._serialized_start=8807
+  _globals['_ALLDELTAS']._serialized_end=9016
 # @@protoc_insertion_point(module_scope)

From 7b2058c14e77847be5744f13b9c768e84f14976e Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 13 Oct 2023 13:02:09 +0100
Subject: [PATCH 034/110] tui: tidy dangling interfaces

* Post-refactor clean up of dangling code.
---
 cylc/flow/scripts/tui.py          | 12 ---------
 cylc/flow/tui/__init__.py         | 29 +++++++++++++++++++++
 cylc/flow/tui/app.py              | 12 +++++++--
 cylc/flow/tui/data.py             | 42 ++++++++++++++++++-------------
 cylc/flow/tui/util.py             |  9 -------
 tests/integration/tui/conftest.py | 25 +++++++++++++++++-
 6 files changed, 88 insertions(+), 41 deletions(-)

diff --git a/cylc/flow/scripts/tui.py b/cylc/flow/scripts/tui.py
index 2185efbf7e4..f8f0879a1e6 100644
--- a/cylc/flow/scripts/tui.py
+++ b/cylc/flow/scripts/tui.py
@@ -32,7 +32,6 @@
 from getpass import getuser
 from textwrap import indent
 from typing import TYPE_CHECKING, Optional
-from urwid import html_fragment
 
 from cylc.flow.id import Tokens
 from cylc.flow.id_cli import parse_id
@@ -65,17 +64,6 @@ def get_option_parser() -> COP:
     return parser
 
 
-def configure_screenshot(v_term_size):
-    screen = html_fragment.HtmlGenerator()
-    screen.set_terminal_properties(256)
-    screen.register_palette(TuiApp.palette)
-    html_fragment.screenshot_init(
-        [tuple(map(int, v_term_size.split(',')))],
-        []
-    )
-    return screen, html_fragment
-
-
 @cli_function(get_option_parser)
 def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None:
     # get workflow ID if specified
diff --git a/cylc/flow/tui/__init__.py b/cylc/flow/tui/__init__.py
index 78ce0f36d62..6beaeae059e 100644
--- a/cylc/flow/tui/__init__.py
+++ b/cylc/flow/tui/__init__.py
@@ -106,12 +106,26 @@
 
 
 class Bindings:
+    """Represets key bindings for the Tui app."""
 
     def __init__(self):
         self.bindings = []
         self.groups = {}
 
     def bind(self, keys, group, desc, callback):
+        """Register a key binding.
+
+        Args:
+            keys:
+                The keys to bind.
+            group:
+                The group to which this binding should belong.
+            desc:
+                Description for this binding, used to generate help.
+            callback:
+                The thing to call when this binding is pressed.
+
+        """
         if group not in self.groups:
             raise ValueError(f'Group {group} not registered.')
         binding = {
@@ -124,6 +138,15 @@ def bind(self, keys, group, desc, callback):
         self.groups[group]['bindings'].append(binding)
 
     def add_group(self, group, desc):
+        """Add a new binding group.
+
+        Args:
+            group:
+                The name of the group.
+            desc:
+                A description of the group, used to generate help.
+
+        """
         self.groups[group] = {
             'name': group,
             'desc': desc,
@@ -134,6 +157,12 @@ def __iter__(self):
         return iter(self.bindings)
 
     def list_groups(self):
+        """List groups and the bindings in them.
+
+        Yields:
+            (group_name, [binding, ...])
+
+        """
         for name, group in self.groups.items():
             yield (
                 group,
diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py
index 86b5f0dcf98..f030cdd7b28 100644
--- a/cylc/flow/tui/app.py
+++ b/cylc/flow/tui/app.py
@@ -56,6 +56,8 @@
 )
 
 
+# default workflow / task filters
+# (i.e. show everything)
 DEFAULT_FILTERS = get_default_filters()
 
 
@@ -126,8 +128,8 @@ def keypress(self, size, key):
         return key
 
     def get_indented_widget(self):
+        """Override the Urwid method to handle leaf nodes differently."""
         if self.is_leaf:
-
             self._innerwidget = urwid.Columns(
                 [
                     ('fixed', 1, self.unexpandable_icon),
@@ -193,7 +195,13 @@ def load_child_node(self, key):
 
 @contextmanager
 def updater_subproc(filters):
-    """Runs the Updater in its own process."""
+    """Runs the Updater in its own process.
+
+    The updater provides the data for Tui to render. Running the updater
+    in its own process removes its CPU load from the Tui app allowing
+    it to remain responsive whilst updates are being gathered as well as
+    decoupling the application update logic from the data update logic.
+    """
     # start the updater
     updater = Updater()
     p = Process(target=updater.start, args=(filters,))
diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index 4d00753cf3f..f176f34ed89 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -29,6 +29,8 @@
 )
 
 
+# the GraphQL query which Tui runs against each of the workflows
+# is is subscribed to
 QUERY = '''
   query cli($taskStates: [String]){
     workflows {
@@ -88,6 +90,7 @@
   }
 '''
 
+# the list of mutations we can call on a running scheduler
 MUTATIONS = {
     'workflow': [
         'pause',
@@ -113,28 +116,14 @@
     ]
 }
 
+# mapping of Tui's node types (e.g. workflow) onto GraphQL argument types
+# (e.g. WorkflowID)
 ARGUMENT_TYPES = {
+    # <tui-node-type>: <graphql-argument-type>
     'workflow': '[WorkflowID]!',
     'task': '[NamespaceIDGlob]!',
 }
 
-MUTATION_TEMPLATES = {
-    'workflow': '''
-        mutation($workflow: [WorkflowID]!) {
-            pause (workflows: $workflow) {
-            result
-          }
-        }
-    ''',
-    'task': '''
-        mutation($workflow: [WorkflowID]!, $task: [NamespaceIDGlob]!) {
-          trigger (workflows: $workflow, tasks: $task) {
-            result
-          }
-        }
-    '''
-}
-
 
 def cli_cmd(*cmd):
     """Issue a CLI command.
@@ -160,6 +149,7 @@ def cli_cmd(*cmd):
         raise ClientError(f'Error in command cylc {" ".join(cmd)}\n{err}')
 
 
+# the mutations we have to go through the CLI to perform
 OFFLINE_MUTATIONS = {
     'user': {
         'stop-all': partial(cli_cmd, 'stop', '*'),
@@ -173,6 +163,15 @@ def cli_cmd(*cmd):
 
 
 def generate_mutation(mutation, arguments):
+    """Return a GraphQL mutation string.
+
+    Args:
+        mutation:
+            The mutation name.
+        Arguments:
+            The arguments to provide to it.
+
+    """
     arguments.pop('user')
     graphql_args = ', '.join([
         f'${argument}: {ARGUMENT_TYPES[argument]}'
@@ -252,6 +251,15 @@ def context_to_variables(context):
 
 
 def mutate(mutation, selection):
+    """Call a mutation.
+
+    Args:
+        mutation:
+            The mutation name (e.g. stop).
+        selection:
+            The Tui selection (i.e. the row(s) selected in Tui).
+
+    """
     if mutation in {
         _mutation
         for section in OFFLINE_MUTATIONS.values()
diff --git a/cylc/flow/tui/util.py b/cylc/flow/tui/util.py
index 6587cd4be1d..72cb4c4d80f 100644
--- a/cylc/flow/tui/util.py
+++ b/cylc/flow/tui/util.py
@@ -516,15 +516,6 @@ def render_node(node, data, type_):
     return RENDER_FUNCTIONS.get(type_, _render_unknown)(node, data)
 
 
-PARTS = [
-    'user',
-    'workflow',
-    'cycle',
-    'task',
-    'job'
-]
-
-
 def extract_context(selection):
     """Return a dictionary of all component types in the selection.
 
diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py
index 8c33a43761d..a16c15820f3 100644
--- a/tests/integration/tui/conftest.py
+++ b/tests/integration/tui/conftest.py
@@ -7,17 +7,40 @@
 from uuid import uuid1
 
 import pytest
+from urwid import html_fragment
 
 from cylc.flow.id import Tokens
 from cylc.flow.tui.app import TuiApp
-from cylc.flow.scripts.tui import configure_screenshot
 from cylc.flow.tui.overlay import _get_display_id
 
 
 SCREENSHOT_DIR = Path(__file__).parent / 'screenshots'
 
 
+def configure_screenshot(v_term_size):
+    """Configure Urwid HTML screenshots."""
+    screen = html_fragment.HtmlGenerator()
+    screen.set_terminal_properties(256)
+    screen.register_palette(TuiApp.palette)
+    html_fragment.screenshot_init(
+        [tuple(map(int, v_term_size.split(',')))],
+        []
+    )
+    return screen, html_fragment
+
+
 def format_test_failure(expected, got, description):
+    """Return HTML to represent a screenshot test failure.
+
+    Args:
+        expected:
+            HTML fragment for the expected screenshot.
+        got:
+            HTML fragment for the test screenshot.
+        description:
+            Test description.
+
+    """
     diff = '\n'.join(unified_diff(
         expected.splitlines(),
         got.splitlines(),

From de237662ed2731ddc64e524631ab6b432b62ae3e Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 2 Nov 2023 17:02:58 +0000
Subject: [PATCH 035/110] tui: log view

* Add a log view to tui and add it to the context menu for workflows,
  tasks and jobs.
* Allow raikura sessions to see module-scoped fixtures when appropriate.
* Fix a bug (which only surfaced in tests for some reason) where a
  colour was specified as an empty string which urwid doesn't appear to
  like.
---
 cylc/flow/data_store_mgr.py                   |   2 +-
 cylc/flow/tui/app.py                          |   2 +-
 cylc/flow/tui/data.py                         | 143 ++++++-
 cylc/flow/tui/overlay.py                      |  95 ++++-
 cylc/flow/tui/util.py                         |   2 +-
 tests/conftest.py                             |   9 +
 tests/integration/tui/conftest.py             |  15 +
 .../screenshots/test_errors.list-error.html   |  31 ++
 .../screenshots/test_errors.open-error.html   |  31 ++
 .../screenshots/test_job_logs.01-job.out.html |  31 ++
 .../screenshots/test_job_logs.02-job.out.html |  31 ++
 ..._offline_mutation.clean-command-error.html |   2 +-
 ...line_mutation.clean-mutation-selected.html |   2 +-
 ..._mutation.command-failed-client-error.html |   2 +-
 ...ation.command-failed-workflow-stopped.html |   2 +-
 ...nline_mutation.hold-mutation-selected.html |   2 +-
 ...est_scheduler_logs.log-file-selection.html |  31 ++
 ...est_scheduler_logs.scheduler-log-file.html |  31 ++
 ...uler_logs.workflow-configuration-file.html |  31 ++
 .../test_task_logs.latest-job.err.html        |  31 ++
 .../test_task_logs.latest-job.out.html        |  31 ++
 tests/integration/tui/test_logs.py            | 351 ++++++++++++++++++
 22 files changed, 890 insertions(+), 18 deletions(-)
 create mode 100644 tests/integration/tui/screenshots/test_errors.list-error.html
 create mode 100644 tests/integration/tui/screenshots/test_errors.open-error.html
 create mode 100644 tests/integration/tui/screenshots/test_job_logs.01-job.out.html
 create mode 100644 tests/integration/tui/screenshots/test_job_logs.02-job.out.html
 create mode 100644 tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
 create mode 100644 tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
 create mode 100644 tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
 create mode 100644 tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
 create mode 100644 tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
 create mode 100644 tests/integration/tui/test_logs.py

diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py
index becd21f50e9..b4a15d3f9a2 100644
--- a/cylc/flow/data_store_mgr.py
+++ b/cylc/flow/data_store_mgr.py
@@ -1380,7 +1380,7 @@ def insert_job(self, name, cycle_point, status, job_conf):
             name=tproxy.name,
             cycle_point=tproxy.cycle_point,
             execution_time_limit=job_conf.get('execution_time_limit'),
-            platform=job_conf.get('platform')['name'],
+            platform=job_conf['platform']['name'],
             job_runner_name=job_conf.get('job_runner_name'),
         )
         # Not all fields are populated with some submit-failures,
diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py
index f030cdd7b28..63e7fb36803 100644
--- a/cylc/flow/tui/app.py
+++ b/cylc/flow/tui/app.py
@@ -546,7 +546,7 @@ def open_overlay(self, fcn):
             fcn: A function which returns an urwid widget to overlay.
 
         """
-        self.create_overlay(*fcn(self))
+        self.create_overlay(*fcn(app=self))
 
     def create_overlay(self, widget, kwargs):
         """Open an overlay over the monitor.
diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index f176f34ed89..8f9d72f6763 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -125,12 +125,14 @@
 }
 
 
-def cli_cmd(*cmd):
+def cli_cmd(*cmd, ret=False):
     """Issue a CLI command.
 
     Args:
         cmd:
             The command without the 'cylc' prefix'.
+        ret:
+            If True, the stdout will be returned.
 
     Rasies:
         ClientError:
@@ -147,6 +149,89 @@ def cli_cmd(*cmd):
     _out, err = proc.communicate()
     if proc.returncode != 0:
         raise ClientError(f'Error in command cylc {" ".join(cmd)}\n{err}')
+    if ret:
+        return _out
+
+
+def _log(id_):
+    """Special mutation to open the log view."""
+    # dynamic import to avoid circular import issues
+    from cylc.flow.tui.overlay import log
+    return partial(
+        log,
+        id_=id_,
+        list_files=partial(_list_log_files, id_),
+        get_log=partial(_get_log, id_),
+    )
+
+
+def _parse_log_header(contents):
+    """Parse the cat-log header.
+
+    The "--prepend-path" option to "cat-log" adds a line containing the host
+    and path to the file being viewed in the form:
+
+        # <host>:<path>
+
+    Args:
+        contents:
+            The raw log file contents as returned by "cat-log".
+
+    Returns:
+        tuple - (host, path, text)
+
+        host:
+            The host where the file was retrieved from.
+        path:
+            The absolute path to the log file.
+        text:
+            The log file contents with the header removed.
+
+    """
+    contents, text = contents.split('\n', 1)
+    contents = contents.replace('# ', '')
+    host, path = contents.split(':')
+    return host, path, text
+
+
+def _get_log(id_, filename=None):
+    """Retrieve the contents of a log file.
+
+    Args:
+        id_:
+            The Cylc universal ID of the thing you want to fetch the log file
+            for.
+        filename:
+            The file name to retrieve (note name not path).
+            If "None", then the default log file will be retrieved.
+
+    """
+    cmd = [
+        'cat-log',
+        '--mode=cat',
+        '--prepend-path',
+    ]
+    if filename:
+        cmd.append(f'--file={filename}')
+    text = cli_cmd(
+        *cmd,
+        id_,
+        ret=True,
+    )
+    return _parse_log_header(text)
+
+
+def _list_log_files(id_):
+    """Return a list of available log files.
+
+    Args:
+        id_:
+            The Cylc universal ID of the thing you want to fetch the log file
+            for.
+
+    """
+    text = cli_cmd('cat-log', '--mode=list-dir', id_, ret=True)
+    return text.splitlines()
 
 
 # the mutations we have to go through the CLI to perform
@@ -158,7 +243,14 @@ def cli_cmd(*cmd):
         'play': partial(cli_cmd, 'play'),
         'clean': partial(cli_cmd, 'clean', '--yes'),
         'reinstall-reload': partial(cli_cmd, 'vr', '--yes'),
-    }
+        'log': _log,
+    },
+    'task': {
+        'log': _log,
+    },
+    'job': {
+        'log': _log,
+    },
 }
 
 
@@ -217,9 +309,12 @@ def list_mutations(selection, is_running=True):
     return sorted(ret)
 
 
-def context_to_variables(context):
+def context_to_variables(context, jobs=False):
     """Derive multiple selection out of single selection.
 
+    Note, this interface exists with the aim of facilitating the addition of
+    multiple selection at a later date.
+
     Examples:
         >>> context_to_variables(extract_context(['~a/b//c/d']))
         {'user': ['a'], 'workflow': ['b'], 'task': ['c/d']}
@@ -230,13 +325,31 @@ def context_to_variables(context):
         >>> context_to_variables(extract_context(['~a/b']))
         {'user': ['a'], 'workflow': ['b']}
 
+        # Note, jobs are omitted by default
+        >>> context_to_variables(extract_context(['~a/b//c/d/01']))
+        {'user': ['a'], 'workflow': ['b'], 'task': ['c/d']}
+
+        # This is because Cylc commands cannot generally operate on jobs only
+        # tasks.
+        # To let jobs slide through:
+        >>> context_to_variables(extract_context(['~a/b//c/d/01']), jobs=True)
+        {'user': ['a'], 'workflow': ['b'], 'job': ['c/d/01']}
+
     """
     # context_to_variables because it can only handle single-selection ATM
     variables = {'user': context['user']}
 
     if 'workflow' in context:
         variables['workflow'] = context['workflow']
-    if 'task' in context:
+    if jobs and 'job' in context:
+        variables['job'] = [
+            Tokens(
+                cycle=context['cycle'][0],
+                task=context['task'][0],
+                job=context['job'][0],
+            ).relative_id
+        ]
+    elif 'task' in context:
         variables['task'] = [
             Tokens(
                 cycle=context['cycle'][0],
@@ -265,9 +378,10 @@ def mutate(mutation, selection):
         for section in OFFLINE_MUTATIONS.values()
         for _mutation in section
     }:
-        offline_mutate(mutation, selection)
+        return offline_mutate(mutation, selection)
     else:
         online_mutate(mutation, selection)
+    return None
 
 
 def online_mutate(mutation, selection):
@@ -301,10 +415,21 @@ def online_mutate(mutation, selection):
 def offline_mutate(mutation, selection):
     """Issue a mutation over the CLI or other offline interface."""
     context = extract_context(selection)
-    variables = context_to_variables(context)
+    variables = context_to_variables(context, jobs=True)
+    if 'job' in variables:
+        for job in variables['job']:
+            id_ = Tokens(job, relative=True).duplicate(
+                workflow=variables['workflow'][0]
+            )
+            return OFFLINE_MUTATIONS['workflow'][mutation](id_.id)
+    if 'task' in variables:
+        for task in variables['task']:
+            id_ = Tokens(task, relative=True).duplicate(
+                workflow=variables['workflow'][0]
+            )
+            return OFFLINE_MUTATIONS['workflow'][mutation](id_.id)
     if 'workflow' in variables:
         for workflow in variables['workflow']:
-            # NOTE: this currently only supports workflow mutations
-            OFFLINE_MUTATIONS['workflow'][mutation](workflow)
+            return OFFLINE_MUTATIONS['workflow'][mutation](workflow)
     else:
-        OFFLINE_MUTATIONS['user'][mutation]()
+        return OFFLINE_MUTATIONS['user'][mutation]()
diff --git a/cylc/flow/tui/overlay.py b/cylc/flow/tui/overlay.py
index 87b544b0604..b310ab7ee45 100644
--- a/cylc/flow/tui/overlay.py
+++ b/cylc/flow/tui/overlay.py
@@ -302,13 +302,16 @@ def _mutate(mutation, _):
         nonlocal app, selection
 
         app.open_overlay(partial(progress, text='Running Command'))
+        overlay_fcn = None
         try:
-            mutate(mutation, selection)
+            overlay_fcn = mutate(mutation, selection)
         except Exception as exc:
             app.open_overlay(partial(error, text=str(exc)))
         else:
             app.close_topmost()
             app.close_topmost()
+        if overlay_fcn:
+            app.open_overlay(overlay_fcn)
 
     # determine the ID to display for the context menu
     display_id = _get_display_id(value['id_'])
@@ -363,3 +366,93 @@ def progress(app, text='Working'):
         ]),
         {'width': 30, 'height': 10}
     )
+
+
+def log(app, id_=None, list_files=None, get_log=None):
+    """An overlay for displaying log files."""
+    # display the host name where the file is coming from
+    host_widget = urwid.Text('loading...')
+    # display the log filepath
+    file_widget = urwid.Text('')
+    # display the actual log file itself
+    text_widget = urwid.Text('')
+
+    def open_menu(*_args, **_kwargs):
+        """Open an overlay for selecting a log file."""
+        nonlocal app, id_
+        app.open_overlay(select_log)
+
+    def select_log(*_args, **_kwargs):
+        """Create an overlay for selecting a log file."""
+        nonlocal list_files, id_
+        try:
+            files = list_files()
+        except Exception as exc:
+            return error(app, text=str(exc))
+        return (
+            urwid.ListBox([
+                *[
+                    urwid.Text('Select File'),
+                    urwid.Divider(),
+                ],
+                *[
+                    urwid.Button(
+                        filename,
+                        on_press=partial(
+                            open_log,
+                            filename=filename,
+                            close=True,
+                        ),
+                    )
+                    for filename in files
+                ],
+            ]),
+            # NOTE: the "+6" avoids the need for scrolling
+            {'width': 40, 'height': len(files) + 6}
+        )
+
+    def open_log(*_, filename=None, close=False):
+        """View the provided log file.
+
+        Args:
+            filename:
+                The name of the file to open (note name not path).
+            close:
+                If True, then the topmost overlay will be closed when a file is
+                selected. Use this to close the "select_log" overlay.
+
+        """
+
+        nonlocal host_widget, file_widget, text_widget
+        try:
+            host, path, text = get_log(filename)
+        except Exception as exc:
+            host_widget.set_text(f'Error: {exc}')
+            file_widget.set_text('')
+            text_widget.set_text('')
+        else:
+            host_widget.set_text(f'Host: {host}')
+            file_widget.set_text(f'Path: {path}')
+            text_widget.set_text(text)
+            if close:
+                app.close_topmost()
+
+    # load the default log file
+    if id_:
+        # NOTE: the kwargs are not provided in the overlay unit tests
+        open_log()
+
+    return (
+        urwid.ListBox([
+            host_widget,
+            file_widget,
+            urwid.Button(
+                'Select File',
+                on_press=open_menu,
+            ),
+            urwid.Divider(),
+            text_widget,
+        ]),
+        # open full screen
+        {'width': 9999, 'height': 9999}
+    )
diff --git a/cylc/flow/tui/util.py b/cylc/flow/tui/util.py
index 72cb4c4d80f..9f590889b03 100644
--- a/cylc/flow/tui/util.py
+++ b/cylc/flow/tui/util.py
@@ -373,7 +373,7 @@ def get_task_status_summary(flow):
     state_totals = flow['stateTotals']
     return [
         [
-            ('', ' '),
+            ' ',
             (f'job_{state}', str(state_totals[state])),
             (f'job_{state}', JOB_ICON)
         ]
diff --git a/tests/conftest.py b/tests/conftest.py
index 8f07a4f3441..8ed3e210fc6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -27,6 +27,15 @@
 from cylc.flow.parsec.validate import cylc_config_validate
 
 
+@pytest.fixture(scope='module')
+def mod_monkeypatch():
+    """A module-scoped version of the monkeypatch fixture."""
+    from _pytest.monkeypatch import MonkeyPatch
+    mpatch = MonkeyPatch()
+    yield mpatch
+    mpatch.undo()
+
+
 @pytest.fixture
 def mock_glbl_cfg(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
     """A Pytest fixture for fiddling global config values.
diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py
index a16c15820f3..5b9ddcae61b 100644
--- a/tests/integration/tui/conftest.py
+++ b/tests/integration/tui/conftest.py
@@ -235,6 +235,21 @@ def raikura(test_dir, request, monkeypatch):
         testing.
 
     """
+    return _raikura(test_dir, request, monkeypatch)
+
+
+@pytest.fixture
+def mod_raikura(test_dir, request, monkeypatch):
+    """Same as raikura but configured to view module-scoped workflows.
+
+    Note: This is *not* a module-scoped fixture (no need, creating Tui sessions
+    is not especially slow), it is configured to display module-scoped
+    "scheduler" fixtures (which may be more expensive to create/destroy).
+    """
+    return _raikura(test_dir.parent, request, monkeypatch)
+
+
+def _raikura(test_dir, request, monkeypatch):
     # make the workflow and scan update intervals match (more reliable)
     # and speed things up a little whilst we're at it
     monkeypatch.setattr(
diff --git a/tests/integration/tui/screenshots/test_errors.list-error.html b/tests/integration/tui/screenshots/test_errors.list-error.html
new file mode 100644
index 00000000000..02448aa0267
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_errors.list-error.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error: Somethi</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">Select File </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Something went wrong :(                     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">        </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#000000;background:#e5e5e5">────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_errors.open-error.html b/tests/integration/tui/screenshots/test_errors.open-error.html
new file mode 100644
index 00000000000..142d0d88c72
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_errors.open-error.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error: Something went wrong :(                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
new file mode 100644
index 00000000000..c1c767b98cf
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/01                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job log                                                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
new file mode 100644
index 00000000000..0eb94051201
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/02                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job log                                                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html
index b930bb4d776..88defab9486 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-command-error.html
@@ -7,7 +7,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
index b343a46ea47..f28cced0714 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
@@ -5,11 +5,11 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">lean                                   </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">log                                     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">play                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">reinstall-reload                        </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
 <span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">               </span>
diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html
index 11cdcfd67f9..895856c6ea2 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed-client-error.html
@@ -9,7 +9,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
 <span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
diff --git a/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html b/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html
index fae4a429cc6..6f9954926ef 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.command-failed-workflow-stopped.html
@@ -9,7 +9,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
 <span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
index 9cda0e3e150..cc3941d6e28 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -6,10 +6,10 @@
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ on</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">h</span><span style="color:#000000;background:#e5e5e5">old                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">kill                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">log                                     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">poll                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">release                                 </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">trigger                                 </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
 <span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">               </span>
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
new file mode 100644
index 00000000000..f88e1b0124d
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">Select File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is the                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">scheduler log file                                                        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Select File                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">onfig/01-start-01.cylc       </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">config/flow-processed.cylc    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">scheduler/01-start-01.log     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                      </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                           </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
new file mode 100644
index 00000000000..68dbcc10f9c
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is the                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">scheduler log file                                                        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
new file mode 100644
index 00000000000..e3fcdfbab22
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[scheduling]                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[graph]]                                                             </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        R1 = a                                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[runtime]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[a]]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
new file mode 100644
index 00000000000..4814892df7a
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/02                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job error                                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
new file mode 100644
index 00000000000..0eb94051201
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
@@ -0,0 +1,31 @@
+<pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/02                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job log                                                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                                              </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                                                   </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/test_logs.py b/tests/integration/tui/test_logs.py
new file mode 100644
index 00000000000..aca3edf9169
--- /dev/null
+++ b/tests/integration/tui/test_logs.py
@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.exceptions import ClientError
+from cylc.flow.task_job_logs import get_task_job_log
+from cylc.flow.task_state import (
+    TASK_STATUS_FAILED,
+    TASK_STATUS_SUCCEEDED,
+)
+from cylc.flow.tui.data import _get_log
+
+import pytest
+
+if TYPE_CHECKING:
+    from cylc.flow.id import Tokens
+
+
+def get_job_log(tokens: 'Tokens', suffix: str) -> Path:
+    """Return the path to a job log file.
+
+    Args:
+        tokens: Job tokens.
+        suffix: Filename.
+
+    """
+    return Path(get_task_job_log(
+        tokens['workflow'],
+        tokens['cycle'],
+        tokens['task'],
+        tokens['job'],
+        suffix=suffix,
+    ))
+
+
+@pytest.fixture(scope='module')
+def standarise_host_and_path(mod_monkeypatch):
+    """Replace variable content in the log view.
+
+    The log view displays the "Host" and "Path" of the log file. These will
+    differer from user to user, so we mock away the difference to produce
+    stable results.
+    """
+    def _parse_log_header(contents):
+        _header, text = contents.split('\n', 1)
+        return 'myhost', 'mypath', text
+
+    mod_monkeypatch.setattr(
+        'cylc.flow.tui.data._parse_log_header',
+        _parse_log_header,
+    )
+
+
+@pytest.fixture
+def wait_log_loaded(monkeypatch):
+    """Wait for Tui to successfully open a log file."""
+    # previous log open count
+    before = 0
+    # live log open count
+    count = 0
+
+    # wrap the Tui "_get_log" method to count the number of times it has
+    # returned
+    def __get_log(*args, **kwargs):
+        nonlocal count
+        try:
+            ret = _get_log(*args, **kwargs)
+        except ClientError as exc:
+            count += 1
+            raise exc
+        count += 1
+        return ret
+    monkeypatch.setattr(
+        'cylc.flow.tui.data._get_log',
+        __get_log,
+    )
+
+    async def _wait_log_loaded(tries: int = 25, delay: float = 0.1):
+        """Wait for the log file to be loaded.
+
+        Args:
+            tries: The number of (re)tries to attempt before failing.
+            delay: The delay between retries.
+
+        """
+        nonlocal before, count
+        for _try in range(tries):
+            if count > before:
+                await asyncio.sleep(0)
+                before += 1
+                return
+            await asyncio.sleep(delay)
+        raise Exception(f'Log file was not loaded within {delay * tries}s')
+
+    return _wait_log_loaded
+
+
+@pytest.fixture(scope='module')
+async def workflow(mod_flow, mod_scheduler, mod_start, standarise_host_and_path):
+    """Test fixture providing a workflow with some log files to poke at."""
+    id_ = mod_flow({
+        'scheduling': {
+            'graph': {
+                'R1': 'a',
+            }
+        },
+        'runtime': {
+            'a': {},
+        }
+    }, name='one')
+    schd = mod_scheduler(id_)
+    async with mod_start(schd):
+        # create some log files for tests to inspect
+
+        # create a scheduler log
+        # (note the scheduler log doesn't get created in integration tests)
+        scheduler_log = Path(schd.workflow_log_dir, '01-start-01.log')
+        with open(scheduler_log, 'w+') as logfile:
+            logfile.write('this is the\nscheduler log file')
+
+        # task 1/a
+        itask = schd.pool.get_task(IntegerPoint('1'), 'a')
+        itask.submit_num = 2
+
+        # mark 1/a/01 as failed
+        job_1 = schd.tokens.duplicate(cycle='1', task='a', job='01')
+        schd.data_store_mgr.insert_job(
+            'a',
+            IntegerPoint('1'),
+            TASK_STATUS_SUCCEEDED,
+            {'submit_num': 1, 'platform': {'name': 'x'}}
+        )
+        schd.data_store_mgr.delta_job_state(job_1, TASK_STATUS_FAILED)
+
+        # mark 1/a/02 as succeeded
+        job_2 = schd.tokens.duplicate(cycle='1', task='a', job='02')
+        schd.data_store_mgr.insert_job(
+            'a',
+            IntegerPoint('1'),
+            TASK_STATUS_SUCCEEDED,
+            {'submit_num': 2, 'platform': {'name': 'x'}}
+        )
+        schd.data_store_mgr.delta_job_state(job_1, TASK_STATUS_SUCCEEDED)
+        schd.data_store_mgr.delta_task_state(itask)
+
+        # mark 1/a as succeeded
+        itask.state_reset(TASK_STATUS_SUCCEEDED)
+        schd.data_store_mgr.delta_task_state(itask)
+
+        # 1/a/01 - job.out
+        job_1_out = get_job_log(job_1, 'job.out')
+        job_1_out.parent.mkdir(parents=True)
+        with open(job_1_out, 'w+') as log:
+            log.write(f'job: {job_1.relative_id}\nthis is a job log\n')
+
+        # 1/a/02 - job.out
+        job_2_out = get_job_log(job_2, 'job.out')
+        job_2_out.parent.mkdir(parents=True)
+        with open(job_2_out, 'w+') as log:
+            log.write(f'job: {job_2.relative_id}\nthis is a job log\n')
+
+        # 1/a/02 - job.err
+        job_2_err = get_job_log(job_2, 'job.err')
+        with open(job_2_err, 'w+') as log:
+            log.write(f'job: {job_2.relative_id}\nthis is a job error\n')
+        
+        # 1/a/NN -> 1/a/02
+        (job_2_out.parent.parent / 'NN').symlink_to(
+            (job_2_out.parent.parent / '02'),
+            target_is_directory=True,
+        )
+
+        # populate the data store
+        await schd.update_data_structure()
+
+        yield schd
+
+
+async def test_scheduler_logs(
+    workflow,
+    mod_raikura,
+    wait_log_loaded,
+):
+    """Test viewing the scheduler log files."""
+    with mod_raikura(size='80,30') as rk:
+        # open the workflow in Tui
+        rk.user_input('down', 'right')
+        rk.wait_until_loaded(workflow.tokens.id)
+
+        # open the log view for the workflow
+        rk.user_input('enter')
+        rk.user_input('down', 'down', 'enter')
+
+        # wait for the default log file to load
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            'scheduler-log-file',
+            'the scheduler log file should be open',
+        )
+
+        # open the list of log files
+        rk.user_input('enter')
+        rk.compare_screenshot(
+            'log-file-selection',
+            'the list of available log files should be displayed'
+        )
+
+        # select the processed workflow configuration file
+        rk.user_input('down', 'enter')
+
+        # wait for the file to load
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            'workflow-configuration-file',
+            'the workflow configuration file should be open'
+        )
+
+
+async def test_task_logs(
+    workflow,
+    mod_raikura,
+    wait_log_loaded,
+):
+    """Test viewing task log files.
+
+    I.E. Test viewing job log files by opening the log view on a task.
+    """
+    with mod_raikura(size='80,30') as rk:
+        # open the workflow in Tui
+        rk.user_input('down', 'right')
+        rk.wait_until_loaded(workflow.tokens.id)
+
+        # open the context menu for the task 1/a
+        rk.user_input('down', 'down', 'enter')
+
+        # open the log view for the task 1/a
+        rk.user_input('down', 'down', 'down', 'enter')
+
+        # wait for the default log file to load
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            'latest-job.out',
+            'the job.out file for the second job should be open',
+        )
+
+        rk.user_input('enter')
+        rk.user_input('enter')
+
+        # wait for the job.err file to load
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            'latest-job.err',
+            'the job.out file for the second job should be open',
+        )
+
+
+async def test_job_logs(
+    workflow,
+    mod_raikura,
+    wait_log_loaded,
+):
+    """Test viewing the job log files.
+
+    I.E. Test viewing job log files by opening the log view on a job.
+    """
+    with mod_raikura(size='80,30') as rk:
+        # open the workflow in Tui
+        rk.user_input('down', 'right')
+        rk.wait_until_loaded(workflow.tokens.id)
+
+        # open the context menu for the job 1/a/02
+        rk.user_input('down', 'down', 'right', 'down', 'enter')
+
+        # open the log view for the job 1/a/02
+        rk.user_input('down', 'down', 'down', 'enter')
+
+        # wait for the default log file to load
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            '02-job.out',
+            'the job.out file for the *second* job should be open',
+        )
+
+        # close log view
+        rk.user_input('q')
+
+        # open the log view for the job 1/a/01
+        rk.user_input('down', 'enter')
+        rk.user_input('down', 'down', 'down', 'enter')
+
+        # wait for the default log file to load
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            '01-job.out',
+            'the job.out file for the *first* job should be open',
+        )
+
+
+async def test_errors(
+    workflow,
+    mod_raikura,
+    wait_log_loaded,
+    monkeypatch,
+):
+    """Test error handing of cat-log commands."""
+    # make it look like cat-log commands are failing
+    def cli_cmd_fail(*args, **kwargs):
+        raise ClientError('Something went wrong :(')
+
+    monkeypatch.setattr(
+        'cylc.flow.tui.data.cli_cmd',
+        cli_cmd_fail,
+    )
+
+    with mod_raikura(size='80,30') as rk:
+        # open the log view on scheduler
+        rk.user_input('down', 'enter', 'down', 'down', 'enter')
+
+        # it will fail to open
+        await wait_log_loaded()
+        rk.compare_screenshot(
+            'open-error',
+            'the error message should be displayed in the log view header',
+        )
+
+        # open the file selector
+        rk.user_input('enter')
+
+        # it will fail to list avialable log files
+        rk.compare_screenshot(
+            'list-error',
+            'the error message should be displayed in a pop up',
+        )

From c303aa9a620ff8868a781730cf883ba6fb8f4baa Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 8 Nov 2023 10:46:59 +0000
Subject: [PATCH 036/110] tui: show view

* Add `cylc show` support for tasks in Tui.
---
 cylc/flow/tui/data.py                         | 15 +++-
 cylc/flow/tui/overlay.py                      | 13 ++++
 cylc/flow/tui/util.py                         | 22 ++++++
 tests/integration/tui/__init__.py             |  0
 ...nline_mutation.hold-mutation-selected.html |  2 +-
 .../tui/screenshots/test_show.fail.html       | 41 +++++++++++
 .../tui/screenshots/test_show.success.html    | 41 +++++++++++
 tests/integration/tui/test_logs.py            |  2 +-
 tests/integration/tui/test_show.py            | 70 +++++++++++++++++++
 9 files changed, 202 insertions(+), 4 deletions(-)
 create mode 100644 tests/integration/tui/__init__.py
 create mode 100644 tests/integration/tui/screenshots/test_show.fail.html
 create mode 100644 tests/integration/tui/screenshots/test_show.success.html
 create mode 100644 tests/integration/tui/test_show.py

diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index 8f9d72f6763..04c3d7220b1 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -153,6 +153,16 @@ def cli_cmd(*cmd, ret=False):
         return _out
 
 
+def _show(id_):
+    """Special mutation to display cylc show output."""
+    # dynamic import to avoid circular import issues
+    from cylc.flow.tui.overlay import text_box
+    return partial(
+        text_box,
+        text=cli_cmd('show', id_, '--color=never', ret=True),
+    )
+
+
 def _log(id_):
     """Special mutation to open the log view."""
     # dynamic import to avoid circular import issues
@@ -247,6 +257,7 @@ def _list_log_files(id_):
     },
     'task': {
         'log': _log,
+        'show': _show,
     },
     'job': {
         'log': _log,
@@ -421,13 +432,13 @@ def offline_mutate(mutation, selection):
             id_ = Tokens(job, relative=True).duplicate(
                 workflow=variables['workflow'][0]
             )
-            return OFFLINE_MUTATIONS['workflow'][mutation](id_.id)
+            return OFFLINE_MUTATIONS['job'][mutation](id_.id)
     if 'task' in variables:
         for task in variables['task']:
             id_ = Tokens(task, relative=True).duplicate(
                 workflow=variables['workflow'][0]
             )
-            return OFFLINE_MUTATIONS['workflow'][mutation](id_.id)
+            return OFFLINE_MUTATIONS['task'][mutation](id_.id)
     if 'workflow' in variables:
         for workflow in variables['workflow']:
             return OFFLINE_MUTATIONS['workflow'][mutation](workflow)
diff --git a/cylc/flow/tui/overlay.py b/cylc/flow/tui/overlay.py
index b310ab7ee45..94c1a8b2ef5 100644
--- a/cylc/flow/tui/overlay.py
+++ b/cylc/flow/tui/overlay.py
@@ -59,6 +59,7 @@
 )
 from cylc.flow.tui.util import (
     get_task_icon,
+    get_text_dimensions,
 )
 
 
@@ -456,3 +457,15 @@ def open_log(*_, filename=None, close=False):
         # open full screen
         {'width': 9999, 'height': 9999}
     )
+
+
+def text_box(app, text=''):
+    """A simple text box overlay."""
+    width, height = get_text_dimensions(text)
+    return (
+        urwid.ListBox([
+            urwid.Text(text),
+        ]),
+        # NOTE: those fudge factors account for the overlay border & padding
+        {'width': width + 4, 'height': height + 6}
+    )
diff --git a/cylc/flow/tui/util.py b/cylc/flow/tui/util.py
index 9f590889b03..88e960e249a 100644
--- a/cylc/flow/tui/util.py
+++ b/cylc/flow/tui/util.py
@@ -21,6 +21,7 @@
 from itertools import zip_longest
 import re
 from time import time
+from typing import Tuple
 
 from cylc.flow import LOG
 from cylc.flow.id import Tokens
@@ -554,3 +555,24 @@ def extract_context(selection):
                 if value not in lst:
                     lst.append(value)
     return ret
+
+
+def get_text_dimensions(text: str) -> Tuple[int, int]:
+    """Return the monospace size of a block of multiline text.
+
+    Examples:
+        >>> get_text_dimensions('foo')
+        (3, 1)
+
+        >>> get_text_dimensions('''
+        ...     foo bar
+        ...     baz
+        ... ''')
+        (11, 3)
+
+        >>> get_text_dimensions('')
+        (0, 0)
+
+    """
+    lines = text.splitlines()
+    return max((0, *(len(line) for line in lines))), len(lines)
diff --git a/tests/integration/tui/__init__.py b/tests/integration/tui/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
index cc3941d6e28..34be2ffa0ce 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -9,7 +9,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">log                                     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">poll                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">release                                 </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">trigger                                 </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">show                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
 <span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">               </span>
diff --git a/tests/integration/tui/screenshots/test_show.fail.html b/tests/integration/tui/screenshots/test_show.fail.html
new file mode 100644
index 00000000000..f788e5b3a55
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_show.fail.html
@@ -0,0 +1,41 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows</span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc             </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">:(                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1         </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ foo    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Ac</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q t</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">                    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  co</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s </span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_show.success.html b/tests/integration/tui/screenshots/test_show.success.html
new file mode 100644
index 00000000000..afdcd1a73b4
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_show.success.html
@@ -0,0 +1,41 @@
+<pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ foo                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">title: Foo                                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">description: The first metasyntactic        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">variable.                                   </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">URL: (not given)                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">state: waiting                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">prerequisites: (None)                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">outputs: ('-': not completed)               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo expired                           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo submitted                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo submit-failed                     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo started                           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo succeeded                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo failed                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                     </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#000000;background:#e5e5e5">                                                                                </span>
+<span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  help: </span><span style="color:#00ffff;background:#0000ee">h</span><span style="color:#ffffff;background:#0000ee">  context: </span><span style="color:#00ffff;background:#0000ee">enter</span><span style="color:#ffffff;background:#0000ee">  tree: </span><span style="color:#00ffff;background:#0000ee">- ← + → </span><span style="color:#ffffff;background:#0000ee"> navigation: </span><span style="color:#00ffff;background:#0000ee">↑ ↓ ↥ ↧ Home End </span><span style="color:#ffffff;background:#0000ee">  </span>
+<span style="color:#ffffff;background:#0000ee">filter tasks: </span><span style="color:#00ffff;background:#0000ee">T f s r R </span><span style="color:#ffffff;background:#0000ee"> filter workflows: </span><span style="color:#00ffff;background:#0000ee">W E p </span><span style="color:#ffffff;background:#0000ee">                               </span>
+</pre>
\ No newline at end of file
diff --git a/tests/integration/tui/test_logs.py b/tests/integration/tui/test_logs.py
index aca3edf9169..7b734030bb7 100644
--- a/tests/integration/tui/test_logs.py
+++ b/tests/integration/tui/test_logs.py
@@ -181,7 +181,7 @@ async def workflow(mod_flow, mod_scheduler, mod_start, standarise_host_and_path)
         job_2_err = get_job_log(job_2, 'job.err')
         with open(job_2_err, 'w+') as log:
             log.write(f'job: {job_2.relative_id}\nthis is a job error\n')
-        
+
         # 1/a/NN -> 1/a/02
         (job_2_out.parent.parent / 'NN').symlink_to(
             (job_2_out.parent.parent / '02'),
diff --git a/tests/integration/tui/test_show.py b/tests/integration/tui/test_show.py
new file mode 100644
index 00000000000..ac6aa8532a6
--- /dev/null
+++ b/tests/integration/tui/test_show.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cylc.flow.exceptions import ClientError
+from cylc.flow.tui.data import _show
+
+
+async def test_show(flow, scheduler, start, raikura, monkeypatch):
+    """Test "cylc show" support in Tui."""
+    id_ = flow({
+        'scheduling': {
+            'graph': {
+                'R1': 'foo'
+            },
+        },
+        'runtime': {
+            'foo': {
+                'meta': {
+                    'title': 'Foo',
+                    'description': 'The first metasyntactic variable.'
+                },
+            },
+        },
+    }, name='one')
+    schd = scheduler(id_)
+    async with start(schd):
+        await schd.update_data_structure()
+
+        with raikura(size='80,40') as rk:
+            rk.user_input('down', 'right')
+            rk.wait_until_loaded(schd.tokens.id)
+
+            # select a task
+            rk.user_input('down', 'down', 'enter')
+
+            # select the "show" context option
+            rk.user_input(*(['down'] * 6), 'enter')
+            rk.compare_screenshot(
+                'success',
+                'the show output should be displayed',
+            )
+
+            # make it look like "cylc show" failed
+            def cli_cmd_fail(*args, **kwargs):
+                raise ClientError(':(')
+            monkeypatch.setattr(
+                'cylc.flow.tui.data.cli_cmd',
+                cli_cmd_fail,
+            )
+
+            # select the "show" context option
+            rk.user_input('q', 'enter', *(['down'] * 6), 'enter')
+            rk.compare_screenshot(
+                'fail',
+                'the error should be displayed',
+            )

From e5339e4cd0860e704d3af9f9d36a96b4709f6f2e Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 10 Nov 2023 15:06:32 +0000
Subject: [PATCH 037/110] Fix mypy config

---
 mypy.ini | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mypy.ini b/mypy.ini
index 4c35cf1d53e..fe03c3bbe12 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -19,3 +19,7 @@ exclude= cylc/flow/etc/tutorial/.*
 # Suppress the following messages:
 # By default the bodies of untyped functions are not checked, consider using --check-untyped-defs
 disable_error_code = annotation-unchecked
+
+# For some reason, couldn't exclude this with the exclude directive above
+[mypy-cylc.flow.data_messages_pb2]
+ignore_errors = True

From a85b652ee845b022f2c5b5964fd6cea82876e107 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Sat, 11 Nov 2023 23:12:21 +1300
Subject: [PATCH 038/110] Add Oliver's comments to integration test docs. [skip
 ci]

---
 tests/integration/README.md | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index 71bd6bfb922..1a776232a2b 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -12,7 +12,9 @@ $ pytest tests/i --dist=no -n0  # turn off xdist (allows --pdb etc)
 
 ## What Are Integration Tests
 
-These tests are intended to test the interaction of different modules.
+Integration tests aren't end-to-end tests. They focus on targeted interactions
+of multiple modules and may do a bit of monkeypatching to achieve that result.
+
 With Cylc this typically involves running workflows.
 
 The general approach is:
@@ -21,8 +23,34 @@ The general approach is:
 2) Put it in a funny state.
 3) Test how components interract to handle this state.
 
-Integration tests aren't end-to-end tests, they focus on targeted interactions
-and behaviour and may do a bit of monkeypatching to achieve that result.
+I.e., the integration test framework runs the scheduler. The only thing it's
+really cutting out is the CLI.
+
+You can do everything, up to and including reference tests with it if so
+inclined, although that would really be a functional test implemented in Python:
+
+async with run(schd) as log:
+    # run the workflow with a timeout of 60 seconds
+    await asyncio.sleep(60)
+assert reftest(log) == '''
+1/b triggered off [1/a]
+1/c triggered off [1/b]
+'''
+
+For a more integration'y approach to reftests we can do something like this
+which is essentially just another way of getting the "triggered off" information
+without having to run the main loop and bring race conditions into play:
+
+async with start(schd):
+    assert set(schd.pool.get_tasks()) == {'1/a'}
+
+    # setting a:succeeded should spawn b
+    schd.command_reset('1/a', 'succeeded')
+    assert set(schd.pool.get_tasks()) == {'1/b'}
+    
+    # setting b:x should spawn c
+    schd.command_reset('1/b', 'x')
+    assert set(schd.pool.get_tasks()) == {'1/b', '1/c'}
 
 ## Guidelines
 

From 99fa0f5ce6a04e85dbba574d688803f49138fd16 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Mon, 13 Nov 2023 12:51:29 +0000
Subject: [PATCH 039/110] Support Python 3.12 (#5794)

* add generic Python 3 (i.e. A bleeding edge version) to tests
* remove report timings from Python 3.12
* fix flake8 issues
* Commented out broken test lines (see #5812)
* replace smtpd with aiosmtpd in the test_header file
* fix tests

---------

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 .github/workflows/test_fast.yml               |  2 +-
 .github/workflows/test_functional.yml         |  4 +-
 changes.d/5794.break.md                       |  1 +
 cylc/flow/cfgspec/globalcfg.py                | 17 ++++----
 cylc/flow/cfgspec/workflow.py                 | 10 ++---
 cylc/flow/id.py                               |  2 +-
 cylc/flow/scripts/report_timings.py           | 13 +++++-
 setup.cfg                                     |  3 +-
 tests/functional/events/09-task-event-mail.t  | 11 ++---
 .../events/18-workflow-event-mail.t           | 10 ++---
 .../functional/events/29-task-event-mail-1.t  |  8 ++--
 .../functional/events/30-task-event-mail-2.t  | 40 ++++++++++---------
 tests/functional/lib/bash/test_header         |  7 +++-
 tests/unit/test_indep_task_queues.py          |  7 ++--
 14 files changed, 80 insertions(+), 55 deletions(-)
 create mode 100644 changes.d/5794.break.md

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 8a21de93ae8..2f303e36ee7 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -20,7 +20,7 @@ jobs:
       fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3']
         include:
           - os: 'macos-latest'
             python-version: '3.7'
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index 75dac06a964..baaad6b61f3 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -46,9 +46,9 @@ jobs:
         # NOTE: includes must define ALL of the matrix values
         include:
           # latest python
-          - name: 'py-3.11'
+          - name: 'py-3-latest'
             os: 'ubuntu-latest'
-            python-version: '3.11'
+            python-version: '3'
             test-base: 'tests/f'
             chunk: '1/4'
             platform: '_local_background*'
diff --git a/changes.d/5794.break.md b/changes.d/5794.break.md
new file mode 100644
index 00000000000..53c5315b013
--- /dev/null
+++ b/changes.d/5794.break.md
@@ -0,0 +1 @@
+Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12.
\ No newline at end of file
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index b404a42fdbe..573c9b86b29 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -1222,6 +1222,9 @@ def default_for(
 
                    {PLATFORM_REPLACES.format("[job]batch system")}
             ''')
+            replaces = PLATFORM_REPLACES.format(
+                "[job]batch submit command template"
+            )
             Conf('job runner command template', VDR.V_STRING, desc=f'''
                 Set the command used by the chosen job runner.
 
@@ -1230,9 +1233,7 @@ def default_for(
 
                 .. versionadded:: 8.0.0
 
-                   {PLATFORM_REPLACES.format(
-                       "[job]batch submit command template"
-                    )}
+                   {replaces}
             ''')
             Conf('shell', VDR.V_STRING, '/bin/bash', desc='''
 
@@ -1465,6 +1466,8 @@ def default_for(
                    {REPLACES}``global.rc[hosts][<host>]retrieve job logs
                    command``.
             ''')
+            replaces = PLATFORM_REPLACES.format(
+                "[remote]retrieve job logs max size")
             Conf('retrieve job logs max size', VDR.V_STRING, desc=f'''
                 {LOG_RETR_SETTINGS['retrieve job logs max size']}
 
@@ -1472,9 +1475,10 @@ def default_for(
 
                    {REPLACES}``global.rc[hosts][<host>]retrieve job logs
                    max size``.
-                   {PLATFORM_REPLACES.format(
-                       "[remote]retrieve job logs max size")}
+                   {replaces}
             ''')
+            replaces = PLATFORM_REPLACES.format(
+                "[remote]retrieve job logs retry delays")
             Conf('retrieve job logs retry delays', VDR.V_INTERVAL_LIST,
                  desc=f'''
                 {LOG_RETR_SETTINGS['retrieve job logs retry delays']}
@@ -1483,8 +1487,7 @@ def default_for(
 
                    {REPLACES}``global.rc[hosts][<host>]retrieve job logs
                    retry delays``.
-                   {PLATFORM_REPLACES.format(
-                       "[remote]retrieve job logs retry delays")}
+                   {replaces}
             ''')
             Conf('tail command template',
                  VDR.V_STRING, 'tail -n +1 --follow=name %(filename)s',
diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
index f521614c73a..d75013fd621 100644
--- a/cylc/flow/cfgspec/workflow.py
+++ b/cylc/flow/cfgspec/workflow.py
@@ -401,7 +401,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                     # differentiate between not set vs set to empty
                     default = None
                 elif item.endswith("handlers"):
-                    desc = desc + '\n\n' + dedent(rf'''
+                    desc = desc + '\n\n' + dedent(f'''
                         Examples:
 
                         .. code-block:: cylc
@@ -413,9 +413,9 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                            {item} = echo %(workflow)s
 
                            # configure multiple event handlers
-                           {item} = \
-                               'echo %(workflow)s, %(event)s', \
-                               'my_exe %(event)s %(message)s' \
+                           {item} = \\
+                               'echo %(workflow)s, %(event)s', \\
+                               'my_exe %(event)s %(message)s' \\
                                'curl -X PUT -d event=%(event)s host:port'
                     ''')
                 elif item.startswith("abort on"):
@@ -1856,7 +1856,7 @@ def upg(cfg, descr):
         ['scheduling', 'max active cycle points'],
         ['scheduling', 'runahead limit'],
         cvtr=converter(
-            lambda x: f'P{int(x)-1}' if x != '' else '',
+            lambda x: f'P{int(x) - 1}' if x != '' else '',
             '"{old}" -> "{new}"'
         ),
         silent=cylc.flow.flags.cylc7_back_compat,
diff --git a/cylc/flow/id.py b/cylc/flow/id.py
index b8f34fb217f..222d16f82b6 100644
--- a/cylc/flow/id.py
+++ b/cylc/flow/id.py
@@ -497,7 +497,7 @@ def duplicate(
             )?
             (?:
                 # cycle/task/job
-                { RELATIVE_PATTERN }
+                {RELATIVE_PATTERN}
             )?
           )?
         )?
diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py
index 8d05ed2748d..b6d796ad732 100755
--- a/cylc/flow/scripts/report_timings.py
+++ b/cylc/flow/scripts/report_timings.py
@@ -54,6 +54,8 @@
 import sys
 from typing import TYPE_CHECKING
 
+
+from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 from cylc.flow.id_cli import parse_id
 from cylc.flow.option_parsers import (
@@ -123,6 +125,12 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None:
         constraint='workflows',
     )
 
+    LOG.warning(
+        "cylc report-timings is deprecated."
+        " The analysis view in the GUI provides"
+        " similar functionality."
+    )
+
     output_options = [
         options.show_raw, options.show_summary, options.html_summary
     ]
@@ -246,7 +254,10 @@ def _check_imports(self):
         try:
             import pandas
         except ImportError:
-            raise CylcError('Cannot import pandas - summary unavailable.')
+            raise CylcError(
+                'Cannot import pandas - summary unavailable.'
+                ' try: pip install cylc-flow[report-timings]'
+            )
         else:
             del pandas
 
diff --git a/setup.cfg b/setup.cfg
index e7b2ad2742d..e8e7f46b2d2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -52,6 +52,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: Implementation :: CPython
     Topic :: Scientific/Engineering :: Atmospheric Science
@@ -102,6 +103,7 @@ report-timings =
     pandas==1.*
     matplotlib
 tests =
+    aiosmtpd
     async_generator
     bandit>=1.7.0
     coverage>=5.0.0,<7.3.1
@@ -139,7 +141,6 @@ all =
     %(main_loop-log_db)s
     %(main_loop-log_main_loop)s
     %(main_loop-log_memory)s
-    %(report-timings)s
     %(tests)s
     %(tutorials)s
 
diff --git a/tests/functional/events/09-task-event-mail.t b/tests/functional/events/09-task-event-mail.t
index fb95c228a68..a689268ea1d 100755
--- a/tests/functional/events/09-task-event-mail.t
+++ b/tests/functional/events/09-task-event-mail.t
@@ -50,14 +50,15 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'retry: 1/t1/01'
-b'succeeded: 1/t1/02'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+retry: 1/t1/01
+succeeded: 1/t1/02
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[1/t1/02 succeeded\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/02 succeeded\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill
diff --git a/tests/functional/events/18-workflow-event-mail.t b/tests/functional/events/18-workflow-event-mail.t
index fcde6da0bd7..eae45962db0 100755
--- a/tests/functional/events/18-workflow-event-mail.t
+++ b/tests/functional/events/18-workflow-event-mail.t
@@ -49,11 +49,11 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'event: startup'
-b'message: workflow starting'
-b'event: shutdown'
-b'message: AUTOMATIC'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+event: startup
+message: workflow starting
+event: shutdown
+message: AUTOMATIC
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
 
 purge
diff --git a/tests/functional/events/29-task-event-mail-1.t b/tests/functional/events/29-task-event-mail-1.t
index b6669ea8741..bef890c9c71 100755
--- a/tests/functional/events/29-task-event-mail-1.t
+++ b/tests/functional/events/29-task-event-mail-1.t
@@ -37,12 +37,14 @@ run_ok "${TEST_NAME_BASE}-validate" \
 workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME"
 
+
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'retry: 1/t1/01'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+retry: 1/t1/01
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill
diff --git a/tests/functional/events/30-task-event-mail-2.t b/tests/functional/events/30-task-event-mail-2.t
index 6e8ff8a1e86..10ef0119892 100755
--- a/tests/functional/events/30-task-event-mail-2.t
+++ b/tests/functional/events/30-task-event-mail-2.t
@@ -50,27 +50,29 @@ workflow_run_fail "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'retry: 1/t1/01'
-b'retry: 1/t2/01'
-b'retry: 1/t3/01'
-b'retry: 1/t4/01'
-b'retry: 1/t5/01'
-b'retry: 1/t1/02'
-b'retry: 1/t2/02'
-b'retry: 1/t3/02'
-b'retry: 1/t4/02'
-b'retry: 1/t5/02'
-b'failed: 1/t1/03'
-b'failed: 1/t2/03'
-b'failed: 1/t3/03'
-b'failed: 1/t4/03'
-b'failed: 1/t5/03'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+retry: 1/t1/01
+retry: 1/t2/01
+retry: 1/t3/01
+retry: 1/t4/01
+retry: 1/t5/01
+retry: 1/t1/02
+retry: 1/t2/02
+retry: 1/t3/02
+retry: 1/t4/02
+retry: 1/t5/02
+failed: 1/t1/03
+failed: 1/t2/03
+failed: 1/t3/03
+failed: 1/t4/03
+failed: 1/t5/03
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[. tasks retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[. tasks retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[. tasks failed\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
-purge
+    grep -qPizo "Subject: \[. tasks failed\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+
+    purge
 mock_smtpd_kill
 exit
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
index 458a7dc908c..c4a07603126 100644
--- a/tests/functional/lib/bash/test_header
+++ b/tests/functional/lib/bash/test_header
@@ -807,10 +807,13 @@ mock_smtpd_init() {  # Logic borrowed from Rose
         local SMTPD_LOG="${TEST_DIR}/smtpd.log"
         local SMTPD_HOST="localhost:${SMTPD_PORT}"
         # Set up fake SMTP server to catch outgoing mail & redirect to log:
-        python3 -u -m 'smtpd' -c 'DebuggingServer' -d -n "${SMTPD_HOST}" \
+        python3 -u -m 'aiosmtpd' \
+            --class aiosmtpd.handlers.Debugging stdout \
+            --debug --nosetuid \
+            --listen "${SMTPD_HOST}" \
             1>"${SMTPD_LOG}" 2>&1 &  # Runs in background
         local SMTPD_PID="$!"
-        while ! grep -q 'DebuggingServer started' "${SMTPD_LOG}" 2>'/dev/null'
+        while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null'
         do
             if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then
                 sleep 1  # Still waiting for fake server to start
diff --git a/tests/unit/test_indep_task_queues.py b/tests/unit/test_indep_task_queues.py
index a0a1894cece..bb11fbce463 100644
--- a/tests/unit/test_indep_task_queues.py
+++ b/tests/unit/test_indep_task_queues.py
@@ -92,9 +92,10 @@ def test_queue_and_release(
     assert sorted([r.tdef.name for r in released]) == sorted(expected_released)
 
     # check released tasks change state to "preparing", and not is_queued
-    for r in released:
-        assert r.state.reset.called_with(TASK_STATUS_PREPARING)
-        assert r.state.reset.called_with(is_queued=False)
+    # Commented out pending https://github.com/cylc/cylc-flow/issues/5812
+    # for r in released:
+    #     assert r.state.reset.called_with(TASK_STATUS_PREPARING)
+    #     assert r.state.reset.called_with(is_queued=False)
 
     # check that adopted orphans end up in the default queue
     orphans = ["orphan1", "orphan2"]

From 6b1da953e8d2e43a097992a3c2013eb37e2167f6 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Mon, 13 Nov 2023 12:55:50 +0000
Subject: [PATCH 040/110] Address Mypy & Flake8 identified problems

---
 cylc/flow/cfgspec/globalcfg.py | 4 ++--
 cylc/flow/scheduler_cli.py     | 4 ++--
 cylc/flow/task_proxy.py        | 1 -
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index b404a42fdbe..59c5d26d62c 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -22,7 +22,7 @@
 from typing import List, Optional, Tuple, Any, Union
 
 from contextlib import suppress
-from packaging.version import parse as parse_version, Version
+from packaging.version import Version
 
 from cylc.flow import LOG
 from cylc.flow import __version__ as CYLC_VERSION
@@ -1866,7 +1866,7 @@ def get_version_hierarchy(version: str) -> List[str]:
         ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev']
 
     """
-    smart_ver: Version = parse_version(version)
+    smart_ver = Version(version)
     base = [str(i) for i in smart_ver.release]
     hierarchy = ['']
     hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)]
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index e2f2d98f46a..b1d69770914 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -25,7 +25,7 @@
 import sys
 from typing import TYPE_CHECKING
 
-from packaging.version import parse as parse_version
+from packaging.version import Version
 
 from cylc.flow import LOG, __version__
 from cylc.flow.exceptions import (
@@ -468,7 +468,7 @@ def _version_check(
     if not db_file.is_file():
         # not a restart
         return True
-    this_version = parse_version(__version__)
+    this_version = Version(__version__)
     last_run_version = WorkflowDatabaseManager.check_db_compatibility(db_file)
 
     for itt, (this, that) in enumerate(zip_longest(
diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py
index ef5203cbeae..3db5c731ec7 100644
--- a/cylc/flow/task_proxy.py
+++ b/cylc/flow/task_proxy.py
@@ -42,7 +42,6 @@
     from cylc.flow.cycling import PointBase
     from cylc.flow.task_action_timer import TaskActionTimer
     from cylc.flow.taskdef import TaskDef
-    from cylc.flow.id import Tokens
 
 
 class TaskProxy:

From 1f9c03a5ae0d03471ae33d3aedc9135869d96105 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 15 Nov 2023 15:15:18 +0000
Subject: [PATCH 041/110] Fix broken tests (#5819)

---
 .github/workflows/test_functional.yml          | 5 ++++-
 tests/functional/events/09-task-event-mail.t   | 5 +++--
 tests/functional/events/29-task-event-mail-1.t | 3 +--
 tests/functional/events/30-task-event-mail-2.t | 4 ++--
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index baaad6b61f3..d3a6a38dad3 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -108,7 +108,7 @@ jobs:
         run: |
           # install system deps
           brew update
-          brew install bash coreutils gnu-sed
+          brew install bash coreutils gnu-sed grep
 
           # add GNU coreutils and sed to the user PATH
           # (see instructions in brew install output)
@@ -118,6 +118,9 @@ jobs:
           echo \
             "/usr/local/opt/gnu-sed/libexec/gnubin" \
             >> "${GITHUB_PATH}"
+          echo \
+            "/usr/local/opt/grep/libexec/gnubin" \
+            >> "${GITHUB_PATH}"
 
           # add coreutils to the bashrc too (for jobs)
           cat >> "${HOME}/.bashrc" <<__HERE__
diff --git a/tests/functional/events/09-task-event-mail.t b/tests/functional/events/09-task-event-mail.t
index a689268ea1d..9f6195feb20 100755
--- a/tests/functional/events/09-task-event-mail.t
+++ b/tests/functional/events/09-task-event-mail.t
@@ -55,10 +55,11 @@ succeeded: 1/t1/02
 see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
 
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/01 retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[1/t1/02 succeeded\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/02 succeeded\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill
diff --git a/tests/functional/events/29-task-event-mail-1.t b/tests/functional/events/29-task-event-mail-1.t
index bef890c9c71..7ca6e76a7c2 100755
--- a/tests/functional/events/29-task-event-mail-1.t
+++ b/tests/functional/events/29-task-event-mail-1.t
@@ -37,14 +37,13 @@ run_ok "${TEST_NAME_BASE}-validate" \
 workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME"
 
-
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
 retry: 1/t1/01
 see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
 
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/01 retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill
diff --git a/tests/functional/events/30-task-event-mail-2.t b/tests/functional/events/30-task-event-mail-2.t
index 10ef0119892..85f6b301654 100755
--- a/tests/functional/events/30-task-event-mail-2.t
+++ b/tests/functional/events/30-task-event-mail-2.t
@@ -69,9 +69,9 @@ see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
 
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[. tasks retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[. tasks retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[. tasks failed\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[. tasks failed\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
     purge
 mock_smtpd_kill

From 8140c6f6a2fa4c7ac44ded0f60bdcdf3bd145bb2 Mon Sep 17 00:00:00 2001
From: Mark Dawson <mark.dawson@metoffice.gov.ukpython>
Date: Wed, 1 Nov 2023 17:00:39 +0000
Subject: [PATCH 042/110] made reinstall work on multiple workflows

flake8 clean up

fixed unit tests

fixed cylc-combination/01-vr-reload functional test

fixed cylc-reinstall/00-simple.t functional test

updated change log

review amends
---
 changes.d/5803.feat.md                      |  1 +
 cylc/flow/scheduler_cli.py                  | 46 +++++++++++--------
 cylc/flow/scripts/reinstall.py              | 35 +++++++++------
 cylc/flow/scripts/validate_reinstall.py     | 25 ++++++-----
 tests/functional/cylc-reinstall/00-simple.t |  1 +
 tests/integration/test_reinstall.py         | 49 +++++++++++----------
 6 files changed, 91 insertions(+), 66 deletions(-)
 create mode 100644 changes.d/5803.feat.md

diff --git a/changes.d/5803.feat.md b/changes.d/5803.feat.md
new file mode 100644
index 00000000000..a4bc0f1b898
--- /dev/null
+++ b/changes.d/5803.feat.md
@@ -0,0 +1 @@
+Updated 'reinstall' functionality to support multiple workflows
\ No newline at end of file
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index b1d69770914..61fd408089b 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -37,7 +37,7 @@
 from cylc.flow.id import upgrade_legacy_ids
 from cylc.flow.host_select import select_workflow_host
 from cylc.flow.hostuserutil import is_remote_host
-from cylc.flow.id_cli import parse_ids
+from cylc.flow.id_cli import parse_ids_async
 from cylc.flow.loggingutil import (
     close_log,
     RotatingLogFileHandler,
@@ -354,7 +354,11 @@ def _open_logs(id_: str, no_detach: bool, restart_num: int) -> None:
     )
 
 
-def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None:
+async def scheduler_cli(
+    options: 'Values',
+    workflow_id_raw: str,
+    parse_workflow_id: bool = True
+) -> None:
     """Run the workflow.
 
     This function should contain all of the command line facing
@@ -368,15 +372,18 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None:
     # Parse workflow name but delay Cylc 7 suite.rc deprecation warning
     # until after the start-up splash is printed.
     # TODO: singleton
-    (workflow_id,), _ = parse_ids(
-        workflow_id_raw,
-        constraint='workflows',
-        max_workflows=1,
-        # warn_depr=False,  # TODO
-    )
+    if parse_workflow_id:
+        (workflow_id,), _ = await parse_ids_async(
+            workflow_id_raw,
+            constraint='workflows',
+            max_workflows=1,
+            # warn_depr=False,  # TODO
+        )
+    else:
+        workflow_id = workflow_id_raw
 
     # resume the workflow if it is already running
-    _resume(workflow_id, options)
+    await _resume(workflow_id, options)
 
     # check the workflow can be safely restarted with this version of Cylc
     db_file = Path(get_workflow_srv_dir(workflow_id), 'db')
@@ -400,9 +407,7 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None:
     # NOTE: asyncio.run opens an event loop, runs your coro,
     #       then shutdown async generators and closes the event loop
     scheduler = Scheduler(workflow_id, options)
-    asyncio.run(
-        _setup(scheduler)
-    )
+    await _setup(scheduler)
 
     # daemonize if requested
     # NOTE: asyncio event loops cannot persist across daemonization
@@ -419,9 +424,14 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None:
     )
 
     # run the workflow
-    ret = asyncio.run(
-        _run(scheduler)
-    )
+    if options.no_detach:
+        ret = await _run(scheduler)
+    else:
+        # Note: The daemonization messes with asyncio so we have to start a
+        # new event loop if detaching
+        ret = asyncio.run(
+            _run(scheduler)
+        )
 
     # exit
     # NOTE: we must clean up all asyncio / threading stuff before exiting
@@ -432,7 +442,7 @@ def scheduler_cli(options: 'Values', workflow_id_raw: str) -> None:
     sys.exit(ret)
 
 
-def _resume(workflow_id, options):
+async def _resume(workflow_id, options):
     """Resume the workflow if it is already running."""
     try:
         detect_old_contact_file(workflow_id)
@@ -448,7 +458,7 @@ def _resume(workflow_id, options):
                 'wFlows': [workflow_id]
             }
         }
-        pclient('graphql', mutation_kwargs)
+        await pclient.async_request('graphql', mutation_kwargs)
         sys.exit(0)
     except CylcError as exc:
         LOG.error(exc)
@@ -651,4 +661,4 @@ def _play(parser: COP, options: 'Values', id_: str):
             *options.starttask,
             relative=True,
         )
-    return scheduler_cli(options, id_)
+    return asyncio.run(scheduler_cli(options, id_))
diff --git a/cylc/flow/scripts/reinstall.py b/cylc/flow/scripts/reinstall.py
index 5e1c75056aa..a2acbc7f771 100644
--- a/cylc/flow/scripts/reinstall.py
+++ b/cylc/flow/scripts/reinstall.py
@@ -71,6 +71,7 @@
 from pathlib import Path
 import sys
 from typing import Optional, TYPE_CHECKING, List, Callable
+from functools import partial
 
 from ansimarkup import parse as cparse
 
@@ -83,12 +84,13 @@
 from cylc.flow.install import (
     reinstall_workflow,
 )
-from cylc.flow.id_cli import parse_id
+from cylc.flow.network.multi import call_multi
 from cylc.flow.option_parsers import (
     CylcOptionParser as COP,
     OptionSettings,
-    WORKFLOW_ID_ARG_DOC,
+    ID_MULTI_ARG_DOC
 )
+
 from cylc.flow.pathutil import get_workflow_run_dir
 from cylc.flow.workflow_files import (
     get_workflow_source_dir,
@@ -101,7 +103,6 @@
 
 _input = input  # to enable testing
 
-
 REINSTALL_CYLC_ROSE_OPTIONS = [
     OptionSettings(
         ['--clear-rose-install-options'],
@@ -127,7 +128,12 @@
 
 def get_option_parser() -> COP:
     parser = COP(
-        __doc__, comms=True, argdoc=[WORKFLOW_ID_ARG_DOC]
+        __doc__,
+        comms=True,
+        multiworkflow=True,
+        argdoc=[
+            ID_MULTI_ARG_DOC,
+        ],
     )
 
     try:
@@ -149,27 +155,28 @@ def get_option_parser() -> COP:
 def main(
     _parser: COP,
     opts: 'Values',
-    args: Optional[str] = None
+    *ids: str
 ) -> None:
     """CLI wrapper."""
-    reinstall_cli(opts, args)
+    call_multi(
+        partial(reinstall_cli, opts),
+        *ids,
+        constraint='workflows',
+        report=lambda x: print('Done')
+    )
 
 
-def reinstall_cli(
+async def reinstall_cli(
     opts: 'Values',
-    args: Optional[str] = None,
-    print_reload_tip: bool = True,
+    workflow_id,
+    *tokens_list,
+    print_reload_tip: bool = True
 ) -> bool:
     """Implement cylc reinstall.
 
     This is the bit which contains all the CLI logic.
     """
     run_dir: Optional[Path]
-    workflow_id: str
-    workflow_id, *_ = parse_id(
-        args,
-        constraint='workflows',
-    )
     run_dir = Path(get_workflow_run_dir(workflow_id))
     if not run_dir.is_dir():
         raise WorkflowFilesError(
diff --git a/cylc/flow/scripts/validate_reinstall.py b/cylc/flow/scripts/validate_reinstall.py
index 7a4472901ef..3fb665cf999 100644
--- a/cylc/flow/scripts/validate_reinstall.py
+++ b/cylc/flow/scripts/validate_reinstall.py
@@ -50,7 +50,7 @@
     ContactFileExists,
     CylcError,
 )
-from cylc.flow.id_cli import parse_id
+from cylc.flow.id_cli import parse_id_async
 from cylc.flow.loggingutil import set_timestamps
 from cylc.flow.option_parsers import (
     WORKFLOW_ID_ARG_DOC,
@@ -62,7 +62,7 @@
 from cylc.flow.scheduler_cli import PLAY_OPTIONS, scheduler_cli
 from cylc.flow.scripts.validate import (
     VALIDATE_OPTIONS,
-    _main as cylc_validate
+    wrapped_main as cylc_validate
 )
 from cylc.flow.scripts.reinstall import (
     REINSTALL_CYLC_ROSE_OPTIONS,
@@ -70,11 +70,12 @@
     reinstall_cli as cylc_reinstall,
 )
 from cylc.flow.scripts.reload import (
-    reload_cli as cylc_reload
+    run as cylc_reload
 )
 from cylc.flow.terminal import cli_function
 from cylc.flow.workflow_files import detect_old_contact_file
 
+import asyncio
 
 CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options()
 VR_OPTIONS = combine_options(
@@ -124,16 +125,16 @@ def check_tvars_and_workflow_stopped(
 
 @cli_function(get_option_parser)
 def main(parser: COP, options: 'Values', workflow_id: str):
-    sys.exit(vro_cli(parser, options, workflow_id))
+    sys.exit(asyncio.run(vr_cli(parser, options, workflow_id)))
 
 
-def vro_cli(parser: COP, options: 'Values', workflow_id: str):
+async def vr_cli(parser: COP, options: 'Values', workflow_id: str):
     """Run Cylc (re)validate - reinstall - reload in sequence."""
     # Attempt to work out whether the workflow is running.
     # We are trying to avoid reinstalling then subsequently being
     # unable to play or reload because we cannot identify workflow state.
     unparsed_wid = workflow_id
-    workflow_id, *_ = parse_id(
+    workflow_id, *_ = await parse_id_async(
         workflow_id,
         constraint='workflows',
     )
@@ -166,10 +167,14 @@ def vro_cli(parser: COP, options: 'Values', workflow_id: str):
     # Force on the against_source option:
     options.against_source = True   # Make validate check against source.
     log_subcommand('validate --against-source', workflow_id)
-    cylc_validate(parser, options, workflow_id)
+    await cylc_validate(parser, options, workflow_id)
 
     log_subcommand('reinstall', workflow_id)
-    reinstall_ok = cylc_reinstall(options, workflow_id, print_reload_tip=False)
+    reinstall_ok = await cylc_reinstall(
+        options, workflow_id,
+        [],
+        print_reload_tip=False
+    )
     if not reinstall_ok:
         LOG.warning(
             'No changes to source: No reinstall or'
@@ -180,7 +185,7 @@ def vro_cli(parser: COP, options: 'Values', workflow_id: str):
     # Run reload if workflow is running or paused:
     if workflow_running:
         log_subcommand('reload', workflow_id)
-        cylc_reload(options, workflow_id)
+        await cylc_reload(options, workflow_id)
 
     # run play anyway, to play a stopped workflow:
     else:
@@ -197,4 +202,4 @@ def vro_cli(parser: COP, options: 'Values', workflow_id: str):
             source='',  # Intentionally blank
         )
         log_subcommand(*sys.argv[1:])
-        scheduler_cli(options, workflow_id)
+        await scheduler_cli(options, workflow_id, parse_workflow_id=False)
diff --git a/tests/functional/cylc-reinstall/00-simple.t b/tests/functional/cylc-reinstall/00-simple.t
index ef880416bdf..d494f7dbc12 100644
--- a/tests/functional/cylc-reinstall/00-simple.t
+++ b/tests/functional/cylc-reinstall/00-simple.t
@@ -48,6 +48,7 @@ run_ok "${TEST_NAME}-reinstall" cylc reinstall "${RND_WORKFLOW_NAME}/run1"
 cmp_ok "${TEST_NAME}-reinstall.stdout" <<__OUT__
 REINSTALLED $RND_WORKFLOW_NAME/run1 from ${RND_WORKFLOW_SOURCE}
 Successfully reinstalled.
+Done
 __OUT__
 popd || exit 1
 purge_rnd_workflow
diff --git a/tests/integration/test_reinstall.py b/tests/integration/test_reinstall.py
index 3dadc4b7df0..1b965f18b8e 100644
--- a/tests/integration/test_reinstall.py
+++ b/tests/integration/test_reinstall.py
@@ -19,6 +19,7 @@
 from pathlib import Path
 from types import SimpleNamespace
 from uuid import uuid1
+from functools import partial
 
 import pytest
 
@@ -37,7 +38,7 @@
 from cylc.flow.workflow_files import (
     WorkflowFiles,
 )
-
+from cylc.flow.network.multi import call_multi
 
 ReInstallOptions = Options(reinstall_gop())
 
@@ -90,14 +91,14 @@ def one_run(one_src, test_dir, run_dir):
     )
 
 
-def test_rejects_random_workflows(one):
+async def test_rejects_random_workflows(one, one_run):
     """It should only work with workflows installed by cylc install."""
     with pytest.raises(WorkflowFilesError) as exc_ctx:
-        reinstall_cli(opts=ReInstallOptions(), args=one.workflow)
+        await reinstall_cli(opts=ReInstallOptions(), workflow_id=one.workflow)
     assert 'was not installed with cylc install' in str(exc_ctx.value)
 
 
-def test_invalid_source_dir(one_src, one_run):
+async def test_invalid_source_dir(one_src, one_run):
     """It should detect & fail for an invalid source symlink"""
     source_link = Path(
         one_run.path,
@@ -108,22 +109,22 @@ def test_invalid_source_dir(one_src, one_run):
     source_link.symlink_to(one_src.path / 'flow.cylc')
 
     with pytest.raises(WorkflowFilesError) as exc_ctx:
-        reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+        await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert 'Workflow source dir is not accessible' in str(exc_ctx.value)
 
 
-def test_no_changes_needed(one_src, one_run, capsys, interactive):
+async def test_no_changes_needed(one_src, one_run, capsys, interactive):
     """It should not reinstall if no changes are needed.
 
     This is not a hard requirement, in practice rsync output may differ
     from expectation so this is a nice-to-have, not expected to work 100%
     of the time.
     """
-    assert not reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    assert not await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert 'up to date with' in capsys.readouterr().out
 
 
-def test_non_interactive(one_src, one_run, capsys, capcall, non_interactive):
+async def test_non_interactive(one_src, one_run, capsys, capcall, non_interactive):
     """It should not perform a dry-run or prompt in non-interactive mode."""
     # capture reinstall calls
     reinstall_calls = capcall(
@@ -133,13 +134,13 @@ def test_non_interactive(one_src, one_run, capsys, capcall, non_interactive):
     # give it something to reinstall
     (one_src.path / 'a').touch()
     # reinstall
-    assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     # only one rsync call should have been made (i.e. no --dry-run)
     assert len(reinstall_calls) == 1
     assert 'Successfully reinstalled' in capsys.readouterr().out
 
 
-def test_interactive(
+async def test_interactive(
     one_src,
     one_run,
     capsys,
@@ -161,7 +162,7 @@ def test_interactive(
         'cylc.flow.scripts.reinstall._input',
         lambda x: 'n'
     )
-    assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id) is False
+    assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id) is False
 
     # only one rsync call should have been made (i.e. the --dry-run)
     assert [call[1].get('dry_run') for call in reinstall_calls] == [True]
@@ -173,7 +174,7 @@ def test_interactive(
         'cylc.flow.scripts.reinstall._input',
         lambda x: 'y'
     )
-    assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
 
     # two rsync calls should have been made (i.e. the --dry-run and the real)
     assert [call[1].get('dry_run') for call in reinstall_calls] == [
@@ -182,7 +183,7 @@ def test_interactive(
     assert 'Successfully reinstalled' in capsys.readouterr().out
 
 
-def test_workflow_running(
+async def test_workflow_running(
     one_src,
     one_run,
     monkeypatch,
@@ -194,7 +195,7 @@ def test_workflow_running(
     reload_message = f'Run "cylc reload {one_run.id}"'
 
     # reinstall with a stopped workflow (reload message shouldn't show)
-    assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert reload_message not in capsys.readouterr().out
 
     # reinstall with a running workflow (reload message should show)
@@ -203,11 +204,11 @@ def test_workflow_running(
         'cylc.flow.scripts.reinstall.load_contact_file',
         lambda x: None,
     )
-    assert reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    assert await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert reload_message in capsys.readouterr().out
 
 
-def test_rsync_stuff(one_src, one_run, capsys, non_interactive):
+async def test_rsync_stuff(one_src, one_run, capsys, non_interactive):
     """Make sure rsync is working correctly."""
     # src contains files: a, b
     (one_src.path / 'a').touch()
@@ -219,7 +220,7 @@ def test_rsync_stuff(one_src, one_run, capsys, non_interactive):
     (one_run.path / 'b').touch()
     (one_run.path / 'c').touch()
 
-    reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
 
     # a should have been left
     assert (one_run.path / 'a').exists()
@@ -231,7 +232,7 @@ def test_rsync_stuff(one_src, one_run, capsys, non_interactive):
     assert not (one_run.path / 'c').exists()
 
 
-def test_rose_warning(one_src, one_run, capsys, interactive, monkeypatch):
+async def test_rose_warning(one_src, one_run, capsys, interactive, monkeypatch):
     """It should warn that Rose installed files will be deleted.
 
     See https://github.com/cylc/cylc-rose/issues/149
@@ -249,16 +250,16 @@ def test_rose_warning(one_src, one_run, capsys, interactive, monkeypatch):
     (one_src.path / 'a').touch()  # give it something to install
 
     # reinstall (with rose-suite.conf file)
-    reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert rose_message in capsys.readouterr().err
 
     # reinstall (no rose-suite.conf file)
     (one_src.path / 'rose-suite.conf').unlink()
-    reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert rose_message not in capsys.readouterr().err
 
 
-def test_keyboard_interrupt(
+async def test_keyboard_interrupt(
     one_src,
     one_run,
     interactive,
@@ -279,11 +280,11 @@ def raise_keyboard_interrupt():
         raise_keyboard_interrupt,
     )
 
-    reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+    await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert 'Reinstall canceled, no changes made' in capsys.readouterr().out
 
 
-def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive):
+async def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive):
     """It should raise an error on rsync failure."""
     mock_glbl_cfg(
         'cylc.flow.install.glbl_cfg',
@@ -296,5 +297,5 @@ def test_rsync_fail(one_src, one_run, mock_glbl_cfg, non_interactive):
 
     (one_src.path / 'a').touch()  # give it something to install
     with pytest.raises(WorkflowFilesError) as exc_ctx:
-        reinstall_cli(opts=ReInstallOptions(), args=one_run.id)
+        await reinstall_cli(opts=ReInstallOptions(), workflow_id=one_run.id)
     assert 'An error occurred reinstalling' in str(exc_ctx.value)

From 1e3fe2840c937b0391399e82f4dc8ee88c9c3aa1 Mon Sep 17 00:00:00 2001
From: David Sutherland <davidwollow@gmail.com>
Date: Thu, 23 Nov 2023 06:47:18 +1300
Subject: [PATCH 043/110] protobuf 4.24.4 upgrade (#5828)

undefined
---
 conda-environment.yml          |  2 +-
 cylc/flow/data_messages_pb2.py | 25 +++++++++++++------------
 setup.cfg                      |  2 +-
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/conda-environment.yml b/conda-environment.yml
index 6ff6edf1637..93f5cda68fc 100644
--- a/conda-environment.yml
+++ b/conda-environment.yml
@@ -12,7 +12,7 @@ dependencies:
   - metomi-isodatetime >=1!3.0.0, <1!3.2.0
   - packaging
   # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
-  - protobuf >=4.21.2,<4.22.0
+  - protobuf >=4.24.4,<4.25.0
   - psutil >=5.6.0
   - python
   - pyzmq >=22
diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py
index 9894ea55ebc..82c620bcacf 100644
--- a/cylc/flow/data_messages_pb2.py
+++ b/cylc/flow/data_messages_pb2.py
@@ -1,3 +1,4 @@
+# type: ignore
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: data_messages.proto
@@ -20,18 +21,18 @@
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'data_messages_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
   DESCRIPTOR._options = None
-  _PBWORKFLOW_STATETOTALSENTRY._options = None
-  _PBWORKFLOW_STATETOTALSENTRY._serialized_options = b'8\001'
-  _PBWORKFLOW_LATESTSTATETASKSENTRY._options = None
-  _PBWORKFLOW_LATESTSTATETASKSENTRY._serialized_options = b'8\001'
-  _PBTASKPROXY_OUTPUTSENTRY._options = None
-  _PBTASKPROXY_OUTPUTSENTRY._serialized_options = b'8\001'
-  _PBTASKPROXY_EXTERNALTRIGGERSENTRY._options = None
-  _PBTASKPROXY_EXTERNALTRIGGERSENTRY._serialized_options = b'8\001'
-  _PBTASKPROXY_XTRIGGERSENTRY._options = None
-  _PBTASKPROXY_XTRIGGERSENTRY._serialized_options = b'8\001'
-  _PBFAMILYPROXY_STATETOTALSENTRY._options = None
-  _PBFAMILYPROXY_STATETOTALSENTRY._serialized_options = b'8\001'
+  _globals['_PBWORKFLOW_STATETOTALSENTRY']._options = None
+  _globals['_PBWORKFLOW_STATETOTALSENTRY']._serialized_options = b'8\001'
+  _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._options = None
+  _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_options = b'8\001'
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._options = None
+  _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._options = None
+  _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_options = b'8\001'
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._options = None
+  _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_options = b'8\001'
+  _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._options = None
+  _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_options = b'8\001'
   _globals['_PBMETA']._serialized_start=24
   _globals['_PBMETA']._serialized_end=174
   _globals['_PBTIMEZONE']._serialized_start=177
diff --git a/setup.cfg b/setup.cfg
index e8e7f46b2d2..6b0ae3e1cc8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -71,7 +71,7 @@ install_requires =
     metomi-isodatetime>=1!3.0.0,<1!3.2.0
     # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
     packaging
-    protobuf>=4.21.2,<4.22.0
+    protobuf>=4.24.4,<4.25.0
     psutil>=5.6.0
     pyzmq>=22
     importlib_metadata>=5.0; python_version < "3.12"

From afa988ff760ca32c8c5a4f9603c1282fee710914 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 23 Nov 2023 10:34:28 +0000
Subject: [PATCH 044/110] Refactor.lint (#5718)

* Refactor cylc lint

Refactored get_reference_rst and get_reference_text into a single method.

Actually use the URL in rst reference mode

Added urls to CLI linter listing

move extraneous stuff away from main func

added type hinting and removed toml merging routines from the top level functions

Move the logic for checking if the target is a Cylc 8 workflow
to its own function.

We want the target to be the top level folder, not the flow.cylc file

* Response to review: Don't replicate url management code.
---
 cylc/flow/scripts/lint.py       | 229 +++++++++++++++++---------------
 tests/unit/scripts/test_lint.py |  49 ++++---
 2 files changed, 149 insertions(+), 129 deletions(-)

diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index ef7634ce885..c215436387d 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -66,7 +66,8 @@
         loads as toml_loads,
         TOMLDecodeError,
     )
-from typing import TYPE_CHECKING, Callable, Dict, Iterator, List, Union
+from typing import (
+    TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Union)
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
@@ -676,10 +677,43 @@ def get_pyproject_toml(dir_):
     return tomldata
 
 
-def merge_cli_with_tomldata(
-    clidata, tomldata,
-    override_cli_default_rules=False
-):
+def merge_cli_with_tomldata(target: Path, options: 'Values') -> Dict[str, Any]:
+    """Get a list of checks based on the checking options
+
+    Args:
+        target: Location being linted, in which we might find a
+            pyproject.toml file.
+        options: Cli Options
+
+    This has not been merged with merged with the logic in
+    _merge_cli_with_tomldata to keep the testing of file-system touching
+    and pure logic separate.
+    """
+    ruleset_default = False
+    if options.linter == 'all':
+        options.linter = ['728', 'style']
+    elif options.linter == '':
+        options.linter = ['728', 'style']
+        ruleset_default = True
+    else:
+        options.linter = [options.linter]
+    tomlopts = get_pyproject_toml(target)
+    return _merge_cli_with_tomldata(
+        {
+            'exclude': [],
+            'ignore': options.ignores,
+            'rulesets': options.linter
+        },
+        tomlopts,
+        ruleset_default
+    )
+
+
+def _merge_cli_with_tomldata(
+    clidata: Dict[str, Any],
+    tomldata: Dict[str, Any],
+    override_cli_default_rules: bool = False
+) -> Dict[str, Any]:
     """Merge options set by pyproject.toml with CLI options.
 
     rulesets: CLI should override toml.
@@ -693,7 +727,7 @@ def merge_cli_with_tomldata(
             default, but only if we ask for it on the CLI.
 
     Examples:
-    >>> result = merge_cli_with_tomldata(
+    >>> result = _merge_cli_with_tomldata(
     ... {'rulesets': ['foo'], 'ignore': ['R101'], 'exclude': []},
     ... {'rulesets': ['bar'], 'ignore': ['R100'], 'exclude': ['*.bk']})
     >>> result['ignore']
@@ -1025,88 +1059,100 @@ def get_cylc_files(
                 yield path
 
 
-def get_reference_rst(checks):
-    """Print a reference for checks to be carried out.
+REFERENCE_TEMPLATES = {
+    'section heading': '\n{title}\n{underline}\n',
+    'issue heading': {
+        'text': '\n{check}:\n    {summary}\n    {url}\n\n',
+        'rst': '\n`{check} <{url}>`_\n{underline}\n{summary}\n\n',
+    },
+    'auto gen message': (
+        'U998 and U999 represent automatically generated'
+        ' sets of deprecations and upgrades.'
+    ),
+}
+
 
-    Returns:
-        RST compatible text.
+def get_reference(linter, output_type):
+    """Fill out a template with all the issues Cylc Lint looks for.
     """
+    if linter in {'all', ''}:
+        rulesets = ['728', 'style']
+    else:
+        rulesets = [linter]
+    checks = parse_checks(rulesets, reference=True)
+
+    issue_heading_template = REFERENCE_TEMPLATES['issue heading'][output_type]
     output = ''
     current_checkset = ''
     for index, meta in checks.items():
         # Check if the purpose has changed - if so create a new
-        # section title:
+        # section heading:
         if meta['purpose'] != current_checkset:
             current_checkset = meta['purpose']
             title = CHECKS_DESC[meta["purpose"]]
-            output += f'\n{title}\n{"-" * len(title)}\n\n'
+            output += REFERENCE_TEMPLATES['section heading'].format(
+                title=title, underline="-" * len(title))
 
             if current_checkset == 'A':
-                output += (
-                    '\n.. note::\n'
-                    '\n   U998 and U999 represent automatically generated '
-                    'sets of deprecations and upgrades.\n\n'
-                )
+                output += REFERENCE_TEMPLATES['auto gen message']
 
-        if current_checkset == 'A':
+        # Fill a template with info about the issue.
+        if output_type == 'rst':
             summary = meta.get("rst", meta['short'])
-            output += '\n- ' + summary
+        elif output_type == 'text':
+            summary = meta.get("short").replace('``', '')
+
+        if current_checkset == 'A':
+            # Condensed check summary for auto-generated lint items.
+            if output_type == 'rst':
+                output += '\n'
+            output += '\n* ' + summary
         else:
-            # Fill a template with info about the issue.
-            template = (
-                '{check}\n^^^^\n{summary}\n\n'
-            )
+            template = issue_heading_template
             url = get_url(meta)
-            summary = meta.get("rst", meta['short'])
             msg = template.format(
+                title=index,
                 check=get_index_str(meta, index),
                 summary=summary,
                 url=url,
+                underline=(
+                    len(get_index_str(meta, index)) + len(url) + 6
+                ) * '^'
             )
             output += msg
     output += '\n'
     return output
 
 
-def get_reference_text(checks):
-    """Print a reference for checks to be carried out.
-
-    Returns:
-        RST compatible text.
+def target_version_check(
+    target: Path,
+    quiet: 'Values',
+    mergedopts: Dict[str, Any]
+) -> List:
     """
-    output = ''
-    current_checkset = ''
-    for index, meta in checks.items():
-        # Check if the purpose has changed - if so create a new
-        # section title:
-        if meta['purpose'] != current_checkset:
-            current_checkset = meta['purpose']
-            title = CHECKS_DESC[meta["purpose"]]
-            output += f'\n{title}\n{"-" * len(title)}\n\n'
+    Check whether target is an upgraded Cylc 8 workflow.
 
-            if current_checkset == 'A':
-                output += (
-                    'U998 and U999 represent automatically generated'
-                    ' sets of deprecations and upgrades.'
-                )
-        # Fill a template with info about the issue.
-        if current_checkset == 'A':
-            summary = meta.get("rst", meta['short']).replace('``', '')
-            output += '\n* ' + summary
-        else:
-            template = (
-                '{check}:\n    {summary}\n\n'
-            )
-            url = get_url(meta)
-            msg = template.format(
-                title=index,
-                check=get_index_str(meta, index),
-                summary=meta['short'],
-                url=url,
-            )
-            output += msg
-    output += '\n'
-    return output
+    If it isn't then we shouldn't run the 7-to-8 checks upon
+    it.
+
+    If it isn't and the only ruleset requested by the user is '728'
+    we should exit with an error code unless the user has specifically
+    disabled thatr with --exit-zero.
+    """
+    cylc8 = (target / 'flow.cylc').exists()
+    if not cylc8 and mergedopts['rulesets'] == ['728']:
+        LOG.error(
+            f'{target} not a Cylc 8 workflow: '
+            'Lint after renaming '
+            '"suite.rc" to "flow.cylc"'
+        )
+        sys.exit(not quiet)
+    elif not cylc8 and '728' in mergedopts['rulesets']:
+        check_names = mergedopts['rulesets']
+        check_names.remove('728')
+    else:
+        check_names = mergedopts['rulesets']
+    return check_names
 
 
 def get_option_parser() -> COP:
@@ -1169,72 +1215,37 @@ def get_option_parser() -> COP:
 @cli_function(get_option_parser)
 def main(parser: COP, options: 'Values', target=None) -> None:
     if options.ref_mode:
-        if options.linter in {'all', ''}:
-            rulesets = ['728', 'style']
-        else:
-            rulesets = [options.linter]
-        print(get_reference_text(parse_checks(rulesets, reference=True)))
+        print(get_reference(options.linter, 'text'))
         sys.exit(0)
 
-    # If target not given assume we are looking at PWD
+    # If target not given assume we are looking at PWD:
     if target is None:
         target = str(Path.cwd())
 
-    # make sure the target is a src/run directories
+    # make sure the target is a src/run directory:
     _, _, target = parse_id(
         target,
         src=True,
         constraint='workflows',
     )
 
-    # Get a list of checks bas ed on the checking options:
-    # Allow us to check any number of folders at once
+    # We want target to be the containing folder, not the flow.cylc
+    # file identified by parse_id:
     target = target.parent
-    ruleset_default = False
-    if options.linter == 'all':
-        options.linter = ['728', 'style']
-    elif options.linter == '':
-        options.linter = ['728', 'style']
-        ruleset_default = True
-    else:
-        options.linter = [options.linter]
-    tomlopts = get_pyproject_toml(target)
-    mergedopts = merge_cli_with_tomldata(
-        {
-            'exclude': [],
-            'ignore': options.ignores,
-            'rulesets': options.linter
-        },
-        tomlopts,
-        ruleset_default
-    )
 
-    # Check whether target is an upgraded Cylc 8 workflow.
-    # If it isn't then we shouldn't run the 7-to-8 checks upon
-    # it:
-    cylc8 = (target / 'flow.cylc').exists()
-    if not cylc8 and mergedopts['rulesets'] == ['728']:
-        LOG.error(
-            f'{target} not a Cylc 8 workflow: '
-            'Lint after renaming '
-            '"suite.rc" to "flow.cylc"'
-        )
-        # Exit with an error code if --exit-zero was not set.
-        # Return codes: sys.exit(True) == 1, sys.exit(False) == 0
-        sys.exit(not options.exit_zero)
-    elif not cylc8 and '728' in mergedopts['rulesets']:
-        check_names = mergedopts['rulesets']
-        check_names.remove('728')
-    else:
-        check_names = mergedopts['rulesets']
+    mergedopts = merge_cli_with_tomldata(target, options)
+
+    check_names = target_version_check(
+        target=target, quiet=options.exit_zero, mergedopts=mergedopts)
 
-    # Check each file:
+    # Get the checks object.
     checks = parse_checks(
         check_names,
         ignores=mergedopts['ignore'],
         max_line_len=mergedopts['max-line-length']
     )
 
+    # Check each file matching a pattern:
     counter: Dict[str, int] = {}
     for file in get_cylc_files(target, mergedopts['exclude']):
         LOG.debug(f'Checking {file}')
@@ -1271,6 +1282,4 @@ def main(parser: COP, options: 'Values', target=None) -> None:
 
 # NOTE: use += so that this works with __import__
 # (docstring needed for `cylc help all` output)
-__doc__ += TOMLDOC.format(
-    '\n\n.. code-block:: toml\n', str(LINT_SECTIONS)) + get_reference_rst(
-    parse_checks(['728', 'style'], reference=True))
+__doc__ += get_reference('all', 'rst')
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 0aa108f8826..d33fca7efc1 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -28,11 +28,10 @@
     MANUAL_DEPRECATIONS,
     get_cylc_files,
     get_pyproject_toml,
-    get_reference_rst,
-    get_reference_text,
+    get_reference,
     get_upgrader_info,
     lint,
-    merge_cli_with_tomldata,
+    _merge_cli_with_tomldata,
     parse_checks,
     validate_toml_items
 )
@@ -361,35 +360,47 @@ def test_get_cylc_files_get_all_rcs(tmp_path):
     assert sorted(result) == sorted(expect)
 
 
-MOCK_CHECKS = {
-    'U042': {
-        'short': 'section `[vizualization]` has been removed.',
-        'url': 'some url or other',
-        'purpose': 'U',
-        'rst': 'section ``[vizualization]`` has been removed.',
-        'function': re.compile('not a regex')
-    },
-}
+def mock_parse_checks(*args, **kwargs):
+    return {
+        'U042': {
+            'short': 'section `[vizualization]` has been removed.',
+            'url': 'some url or other',
+            'purpose': 'U',
+            'rst': 'section ``[vizualization]`` has been removed.',
+            'function': re.compile('not a regex')
+        },
+    }
 
 
-def test_get_reference_rst():
+def test_get_reference_rst(monkeypatch):
     """It produces a reference file for our linting."""
-    ref = get_reference_rst(MOCK_CHECKS)
+    monkeypatch.setattr(
+        'cylc.flow.scripts.lint.parse_checks', mock_parse_checks
+    )
+    ref = get_reference('all', 'rst')
     expect = (
         '\n7 to 8 upgrades\n---------------\n\n'
-        'U042\n^^^^\nsection ``[vizualization]`` has been '
+        '`U042 <https://cylc.github.io/cylc-doc/stable'
+        '/html/7-to-8/some url or other>`_'
+        f'\n{ "^" * 78 }'
+        '\nsection ``[vizualization]`` has been '
         'removed.\n\n\n'
     )
     assert ref == expect
 
 
-def test_get_reference_text():
+def test_get_reference_text(monkeypatch):
     """It produces a reference file for our linting."""
-    ref = get_reference_text(MOCK_CHECKS)
+    monkeypatch.setattr(
+        'cylc.flow.scripts.lint.parse_checks', mock_parse_checks
+    )
+    ref = get_reference('all', 'text')
     expect = (
         '\n7 to 8 upgrades\n---------------\n\n'
         'U042:\n    section `[vizualization]` has been '
-        'removed.\n\n\n'
+        'removed.'
+        '\n    https://cylc.github.io/cylc-doc/stable/html/7-to-8/some'
+        ' url or other\n\n\n'
     )
     assert ref == expect
 
@@ -558,7 +569,7 @@ def test_validate_toml_items(input_, error):
 )
 def test_merge_cli_with_tomldata(clidata, tomldata, expect):
     """It merges each of the three sections correctly: see function.__doc__"""
-    assert merge_cli_with_tomldata(clidata, tomldata) == expect
+    assert _merge_cli_with_tomldata(clidata, tomldata) == expect
 
 
 def test_invalid_tomlfile(tmp_path):

From b1342972b1d77257cb09dea7fb436ac75f0fddf1 Mon Sep 17 00:00:00 2001
From: Mark Dawson <mark.dawson@metoffice.gov.uk>
Date: Thu, 23 Nov 2023 12:15:21 +0000
Subject: [PATCH 045/110] remove cylc task dependencies env var (#5836)

---
 changes.d/5836.break.md     |  1 +
 cylc/flow/job_file.py       | 19 ----------------
 tests/unit/test_job_file.py | 45 -------------------------------------
 3 files changed, 1 insertion(+), 64 deletions(-)
 create mode 100644 changes.d/5836.break.md

diff --git a/changes.d/5836.break.md b/changes.d/5836.break.md
new file mode 100644
index 00000000000..8c14b101f63
--- /dev/null
+++ b/changes.d/5836.break.md
@@ -0,0 +1 @@
+Removed the 'CYLC_TASK_DEPENDENCIES' environment variable
\ No newline at end of file
diff --git a/cylc/flow/job_file.py b/cylc/flow/job_file.py
index 86e65e15a8e..930331dc5a4 100644
--- a/cylc/flow/job_file.py
+++ b/cylc/flow/job_file.py
@@ -28,13 +28,6 @@
 from cylc.flow.log_level import verbosity_to_env
 from cylc.flow.config import interpolate_template, ParamExpandError
 
-# the maximum number of task dependencies which Cylc will list before
-# omitting the CYLC_TASK_DEPENDENCIES environment variable
-# see: https://github.com/cylc/cylc-flow/issues/5551
-# NOTE: please update `src/reference/job-script-vars/var-list.txt`
-#       in cylc-doc if changing this value
-MAX_CYLC_TASK_DEPENDENCIES_LEN = 50
-
 
 class JobFileWriter:
 
@@ -227,18 +220,6 @@ def _write_task_environment(self, handle, job_conf):
         handle.write(
             '\n    export CYLC_TASK_NAMESPACE_HIERARCHY="%s"' %
             ' '.join(job_conf['namespace_hierarchy']))
-        if len(job_conf['dependencies']) <= MAX_CYLC_TASK_DEPENDENCIES_LEN:
-            handle.write(
-                '\n    export CYLC_TASK_DEPENDENCIES="%s"' %
-                ' '.join(job_conf['dependencies']))
-        else:
-            # redact the CYLC_TASK_DEPENDENCIES variable but leave a note
-            # explaining why
-            # see: https://github.com/cylc/cylc-flow/issues/5551
-            handle.write(
-                '\n    # CYLC_TASK_DEPENDENCIES=disabled'
-                f' (more than {MAX_CYLC_TASK_DEPENDENCIES_LEN} dependencies)'
-            )
         handle.write(
             '\n    export CYLC_TASK_TRY_NUMBER=%s' % job_conf['try_num'])
         handle.write(
diff --git a/tests/unit/test_job_file.py b/tests/unit/test_job_file.py
index 92be2d167c1..9dbfa58f3a1 100644
--- a/tests/unit/test_job_file.py
+++ b/tests/unit/test_job_file.py
@@ -31,7 +31,6 @@
 )
 from cylc.flow.job_file import (
     JobFileWriter,
-    MAX_CYLC_TASK_DEPENDENCIES_LEN,
 )
 from cylc.flow.platforms import platform_from_name
 
@@ -398,7 +397,6 @@ def test_write_task_environment():
                 'export CYLC_TASK_COMMS_METHOD=ssh\n    '
                 'export CYLC_TASK_JOB="1/moo/01"\n    export '
                 'CYLC_TASK_NAMESPACE_HIERARCHY="baa moo"\n    export '
-                'CYLC_TASK_DEPENDENCIES="moo neigh quack"\n    export '
                 'CYLC_TASK_TRY_NUMBER=1\n    export '
                 'CYLC_TASK_FLOW_NUMBERS=1\n    export '
                 'CYLC_TASK_PARAM_duck="quack"\n    export '
@@ -537,46 +535,3 @@ def test_homeless_platform(fixture_get_platform):
         if 'HOME' in job_sh_txt:
             raise Exception('$HOME found in job.sh\n{job_sh_txt}')
 
-
-def test_cylc_task_dependencies_length():
-    f"""Test CYLC_TASK_DEPENDENCIES variable toggle.
-
-    The CYLC_TASK_DEPENDENCIES variriable should only be exported if there are
-    { MAX_CYLC_TASK_DEPENDENCIES_LEN } or fewer dependencies.
-
-    See: https://github.com/cylc/cylc-flow/issues/5551
-
-    """
-    job_conf = {
-        'platform': {'communication method': 'zmq'},
-        'job_d': 'a/b/c',
-        'namespace_hierarchy': ['a', 'b'],
-        # the maximum permitted number of dependencies before the environment
-        # variable is omitted
-        'dependencies': ['a'] * (MAX_CYLC_TASK_DEPENDENCIES_LEN),
-        'try_num': 1,
-        'flow_nums': {1},
-        'param_var': {},
-        'work_d': 'b/c/d',
-    }
-
-    # write the job environment
-    with io.StringIO() as fake_file:
-        JobFileWriter()._write_task_environment(fake_file, job_conf)
-        output = fake_file.getvalue()
-
-    # assert the env var is exported
-    lines = [line.strip().split('=')[0] for line in output.splitlines()]
-    assert 'export CYLC_TASK_DEPENDENCIES' in lines
-
-    # add an extra dependency to push it over the limit
-    job_conf['dependencies'] += ['b']
-
-    # write the job environment
-    with io.StringIO() as fake_file:
-        JobFileWriter()._write_task_environment(fake_file, job_conf)
-        output = fake_file.getvalue()
-
-    # assert the env var is redacted
-    lines = [line.strip().split('=')[0] for line in output.splitlines()]
-    assert '# CYLC_TASK_DEPENDENCIES' in lines  # var should be commented out

From 4cf753281849b667b4ed2d2c208a1c97da1918df Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 28 Nov 2023 17:02:35 +0000
Subject: [PATCH 046/110] tests/i: fix flaky tui/test_updater tests

* These tests were trying to be a bit too clever, testing the updater
  whilst it was running.
* This wasn't really needed for what the tests were tring to assert,
  there are other tests which run Tui for real (live updater and all),
  those tests didn't exist when these ones were written.
* This commit freezes the updater so we can run it in lock-step with
  the tests.
---
 cylc/flow/tui/updater.py              |  49 ++++----
 tests/integration/tui/test_updater.py | 171 +++++++++-----------------
 2 files changed, 83 insertions(+), 137 deletions(-)

diff --git a/cylc/flow/tui/updater.py b/cylc/flow/tui/updater.py
index 5321a4d6c1f..f9bd0cf2c00 100644
--- a/cylc/flow/tui/updater.py
+++ b/cylc/flow/tui/updater.py
@@ -155,7 +155,11 @@ async def run(self, filters):
         """
         with suppress_logging():
             self._update_filters(filters)
-            await self._update()
+            while True:
+                ret = await self._update()
+                if ret == self.SIGNAL_TERMINATE:
+                    break
+                self.update_queue.put(ret)
 
     def _subscribe(self, w_id):
         if w_id not in self._clients:
@@ -183,27 +187,30 @@ def _update_filters(self, filters):
         self.filters = filters
 
     async def _update(self):
-        last_scan_time = 0
-        while True:
-            # process any pending commands
-            if not self._command_queue.empty():
-                (command, payload) = self._command_queue.get()
-                if command == self.SIGNAL_TERMINATE:
-                    break
-                getattr(self, command)(payload)
-                continue
+        """Run one iteration of the updater.
 
-            # do a workflow scan if it's due
-            update_start_time = time()
-            if update_start_time - last_scan_time > self.BASE_SCAN_INTERVAL:
-                data = await self._scan()
-
-            # get the next snapshot from workflows we are subscribed to
-            self.update_queue.put(await self._run_update(data))
-
-            # schedule the next update
-            update_time = time() - update_start_time
-            await sleep(self.BASE_UPDATE_INTERVAL - update_time)
+        Either returns the next update or "self.SIGNAL_TERMINATE".
+        """
+        last_scan_time = 0
+        # process any pending commands
+        while not self._command_queue.empty():
+            (command, payload) = self._command_queue.get()
+            if command == self.SIGNAL_TERMINATE:
+                return command
+            getattr(self, command)(payload)
+
+        # do a workflow scan if it's due
+        update_start_time = time()
+        if update_start_time - last_scan_time > self.BASE_SCAN_INTERVAL:
+            data = await self._scan()
+
+        # get the next snapshot from workflows we are subscribed to
+        update = await self._run_update(data)
+
+        # schedule the next update
+        update_time = time() - update_start_time
+        await sleep(self.BASE_UPDATE_INTERVAL - update_time)
+        return update
 
     async def _run_update(self, data):
         # copy the scanned data so it can be reused for future updates
diff --git a/tests/integration/tui/test_updater.py b/tests/integration/tui/test_updater.py
index 39a9a606eff..b3daac5a328 100644
--- a/tests/integration/tui/test_updater.py
+++ b/tests/integration/tui/test_updater.py
@@ -15,12 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import asyncio
 from copy import deepcopy
 from pathlib import Path
+from queue import Queue
 import re
 
 from async_timeout import timeout
+import pytest
 
 from cylc.flow.cycling.integer import IntegerPoint
 from cylc.flow.id import Tokens
@@ -31,57 +32,35 @@
 from cylc.flow.workflow_status import WorkflowStatus
 
 
-async def await_update(updater):
-    """Wait for the latest update from the Updater.
-
-    Note:
-        If multiple updates are waiting, this returns the most recent.Q
-
-    Returns:
-        The latest update from the Updater's "update_queue".
-
-    """
-    while updater.update_queue.empty():
-        await asyncio.sleep(0.1)
-    while not updater.update_queue.empty():
-        # flush out any older updates to avoid race conditions in tests
-        update = updater.update_queue.get()
-    return update
-
-
-async def wait_workflow_connected(updater, tokens, connected=True, time=3):
-    """Wait for the Updater to connect to a workflow.
-
-    This will return once the updater has connected to a workflow and returned
-    the first data from it.
+@pytest.fixture
+def updater(monkeypatch, test_dir):
+    """Return an updater ready for testing."""
+    # patch the update intervals so that everything runs for every update
+    monkeypatch.setattr(
+        'cylc.flow.tui.updater.Updater.BASE_UPDATE_INTERVAL',
+        0,
+    )
+    monkeypatch.setattr(
+        'cylc.flow.tui.updater.Updater.BASE_SCAN_INTERVAL',
+        0,
+    )
+
+    # create the updater
+    updater = Updater()
 
-    Arugments:
-        tokens:
-            The tokens of the workflow you're waiting for.
-        connected:
-            If True this waits for the updater to connect to the workflow, if
-            False it waits for the updater to disconnect from it.
-        time:
-            The maximum time to wait for this to happen.
+    # swap multiprocessing.Queue for queue.Queue
+    # (this means queued operations are instant making tests more stable)
+    updater.update_queue = Queue()
+    updater._command_queue = Queue()
 
-    Returns:
-        The first update from the Updater which contains the workflow data.
+    # set up the filters
+    # (these filter for the workflows created in this test only)
+    filters = get_default_filters()
+    id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser()))
+    filters['workflows']['id'] = f'^{re.escape(id_base)}/.*'
+    updater._update_filters(filters)
 
-    """
-    async with timeout(time):
-        while True:
-            root_node = await await_update(updater)
-            workflow_node = root_node['children'][0]
-            for workflow_node in root_node['children']:
-                if (
-                    workflow_node['id_'] == tokens.id
-                    and (
-                        workflow_node['children'][0]['id_'] != '#spring'
-                    ) == connected
-                ):
-                    # if the spring node is still there then we haven't
-                    # recieved the first update from the workflow yet
-                    return root_node
+    return updater
 
 
 def get_child_tokens(root_node, types, relative=False):
@@ -111,19 +90,16 @@ def get_child_tokens(root_node, types, relative=False):
     return ret
 
 
-async def test_subscribe(one_conf, flow, scheduler, run, test_dir):
+async def test_subscribe(one_conf, flow, scheduler, run, updater):
     """It should subscribe and unsubscribe from workflows."""
     id_ = flow(one_conf)
     schd = scheduler(id_)
 
-    updater = Updater()
-
-    async def the_test():
-        nonlocal updater
-
-        try:
+    async with run(schd):
+        # run the updater and the test
+        async with timeout(10):
             # wait for the first update
-            root_node = await await_update(updater)
+            root_node = await updater._update()
 
             # there should be a root root_node
             assert root_node['id_'] == 'root'
@@ -135,9 +111,7 @@ async def the_test():
 
             # subscribe to the workflow
             updater.subscribe(schd.tokens.id)
-
-            # wait for it to connect to the workflow
-            root_node = await wait_workflow_connected(updater, schd.tokens)
+            root_node = await updater._update()
 
             # check the workflow contains one cycle with one task in it
             workflow_node = root_node['children'][0]
@@ -150,31 +124,13 @@ async def the_test():
 
             # unsubscribe from the workflow
             updater.unsubscribe(schd.tokens.id)
+            root_node = await updater._update()
 
-            # wait for it to disconnect from the workflow
-            root_node = await wait_workflow_connected(
-                updater,
-                schd.tokens,
-                connected=False,
-            )
-
-        finally:
-            # shut down the updater
-            updater.terminate()
-
-    async with run(schd):
-        filters = get_default_filters()
-        filters['workflows']['id'] = f'{re.escape(str(test_dir.relative_to(Path("~/cylc-run").expanduser())))}/.*'
-
-        # run the updater and the test
-        async with timeout(10):
-            await asyncio.gather(
-                asyncio.create_task(updater.run(filters)),
-                asyncio.create_task(the_test()),
-            )
+            # the workflow should be replaced by a "spring" node again
+            assert root_node['children'][0]['children'][0]['id_'] == '#spring'
 
 
-async def test_filters(one_conf, flow, scheduler, run, test_dir):
+async def test_filters(one_conf, flow, scheduler, run, updater):
     """It should filter workflow and task states.
 
     Note:
@@ -196,16 +152,20 @@ async def test_filters(one_conf, flow, scheduler, run, test_dir):
     two = scheduler(flow(one_conf, name='two'))
     tre = scheduler(flow(one_conf, name='tre'))
 
-    filters = get_default_filters()
-    id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser()))
-    filters['workflows']['id'] = f'^{re.escape(id_base)}/.*'
+    # start workflow "one"
+    async with run(one):
+        # mark "1/a" as running and "1/b" as succeeded
+        one_a = one.pool.get_task(IntegerPoint('1'), 'a')
+        one_a.state_reset('running')
+        one.data_store_mgr.delta_task_state(one_a)
+        one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded')
 
-    updater = Updater()
+        # start workflow "two"
+        async with run(two):
+            # run the updater and the test
+            filters = deepcopy(updater.filters)
 
-    async def the_test():
-        nonlocal filters
-        try:
-            root_node = await await_update(updater)
+            root_node = await updater._update()
             assert {child['id_'] for child in root_node['children']} == {
                 one.tokens.id,
                 two.tokens.id,
@@ -219,7 +179,7 @@ async def the_test():
             updater.update_filters(filters)
 
             # "one" and "two" should now be filtered out
-            root_node = await await_update(updater)
+            root_node = await updater._update()
             assert {child['id_'] for child in root_node['children']} == {
                 tre.tokens.id,
             }
@@ -231,15 +191,15 @@ async def the_test():
             updater.update_filters(filters)
 
             # "tre" should now be filtered out
-            root_node = await await_update(updater)
+            root_node = await updater._update()
             assert {child['id_'] for child in root_node['children']} == {
                 one.tokens.id,
                 two.tokens.id,
             }
 
             # subscribe to "one"
-            updater.subscribe(one.tokens.id)
-            root_node = await wait_workflow_connected(updater, one.tokens)
+            updater._subscribe(one.tokens.id)
+            root_node = await updater._update()
             assert get_child_tokens(
                 root_node, types={'task'}, relative=True
             ) == {
@@ -254,7 +214,7 @@ async def the_test():
             # filters['tasks'][TASK_STATUS_RUNNING] = False
             # updater.update_filters(filters)
 
-            # root_node = await await_update(updater)
+            # root_node = await updater._update()
             # assert get_child_tokens(
             #   root_node,
             #   types={'task'},
@@ -263,24 +223,3 @@ async def the_test():
             #     '1/b',
             #     '1/c',
             # }
-
-        finally:
-            # shut down the updater
-            updater.terminate()
-
-    # start workflow "one"
-    async with run(one):
-        # mark "1/a" as running and "1/b" as succeeded
-        one_a = one.pool.get_task(IntegerPoint('1'), 'a')
-        one_a.state_reset('running')
-        one.data_store_mgr.delta_task_state(one_a)
-        one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded')
-
-        # start workflow "two"
-        async with run(two):
-            # run the updater and the test
-            async with timeout(10):
-                await asyncio.gather(
-                    asyncio.create_task(the_test()),
-                    asyncio.create_task(updater.run(filters)),
-                )

From 6c03bc03fee17bd2d2a693bc7ae43ab146aed7e7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 11:54:48 +0000
Subject: [PATCH 047/110] Bump pypa/gh-action-pypi-publish from 1.8.10 to
 1.8.11 (#5851)

---
 .github/workflows/2_auto_publish_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 02ea52d678c..9223bd2106c 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.10
+      uses: pypa/gh-action-pypi-publish@v1.8.11
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

From 3f8edcd53b17a7f24a1a55201e4db15c547b1055 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 29 Nov 2023 14:57:25 +0000
Subject: [PATCH 048/110] tests/i: tui - attempt to stabilise tests

* Tests became unstable for unknown reasons.
* This change attempts to resolve some issues by:
  * Ensuring that the workflow has appeared in the scan results before
    attempting to "expand" it (i.e. subscribe to it).
  * Increasing the sleep time used to wait for nodes to appear to make
    it more robust to a slow running updater (e.g. if FS operations
    become slow).
  * Separate successfull and failed updates when waiting for nodes to
    appear to make it more robust to heavily loaded systems.
---
 cylc/flow/tui/app.py               | 34 ++++++++++++++++++++++++------
 tests/integration/tui/conftest.py  | 21 ++++++++++--------
 tests/integration/tui/test_app.py  | 14 ++++++++++--
 tests/integration/tui/test_logs.py | 12 +++++++++++
 4 files changed, 63 insertions(+), 18 deletions(-)

diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py
index 63e7fb36803..97abab94f81 100644
--- a/cylc/flow/tui/app.py
+++ b/cylc/flow/tui/app.py
@@ -337,7 +337,7 @@ def set_initial_filters(self, w_id, id_filter):
             # a custom workflow ID filter has been provided
             self.filters['workflows']['id'] = id_filter
 
-    def wait_until_loaded(self, *ids, retries=None):
+    def wait_until_loaded(self, *ids, retries=None, max_fails=50):
         """Wait for any requested nodes to be created.
 
         Warning:
@@ -352,21 +352,41 @@ def wait_until_loaded(self, *ids, retries=None):
             retries:
                 The maximum number of updates to perform whilst waiting
                 for the specified IDs to appear in the tree.
+            max_fails:
+                If there is no update received from the updater, then we call
+                it a failed attempt. This isn't necessarily an issue, the
+                updater might be running a little slow. But if there are a
+                large number of fails, it likely means the condition won't be
+                satisfied.
 
         Returns:
             A list of the IDs which NOT not appear in the store.
 
+        Raises:
+            Exception:
+                If the "max_fails" are exhausted.
+
         """
         from time import sleep
         ids = set(ids)
         self.expand_on_load.update(ids)
-        try_ = 0
+        successful_updates = 0
+        failed_updates = 0
         while ids & self.expand_on_load:
-            self.update()
-            sleep(0.1)  # blocking
-            try_ += 1
-            if retries is not None and try_ > retries:
-                return list(self.expand_on_load)
+            if self.update():
+                successful_updates += 1
+                if retries is not None and successful_updates > retries:
+                    return list(self.expand_on_load)
+            else:
+                failed_updates += 1
+                if failed_updates > max_fails:
+                    raise Exception(
+                        f'No update was received after {max_fails} attempts.'
+                        f'\nThere were {successful_updates} successful'
+                        ' updates.'
+                    )
+
+            sleep(0.15)  # blocking to Tui but not to the updater process
         return None
 
     def unhandled_input(self, key):
diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py
index 5b9ddcae61b..ed716bd3e8d 100644
--- a/tests/integration/tui/conftest.py
+++ b/tests/integration/tui/conftest.py
@@ -198,17 +198,20 @@ def wait_until_loaded(self, *ids, retries=20):
 
         Note, this is a blocking wait with no timeout!
         """
-        ids = self.app.wait_until_loaded(*ids, retries=retries)
+        exc = None
+        try:
+            ids = self.app.wait_until_loaded(*ids, retries=retries)
+        except Exception as _exc:
+            exc = _exc
         if ids:
-            self.compare_screenshot(
-                f'fail-{uuid1()}',
-                (
-                    'Requested nodes did not appear in Tui after'
-                    f' {retries} retries: '
-                    + ', '.join(ids)
-                ),
-                1,
+            msg = (
+                'Requested nodes did not appear in Tui after'
+                f' {retries} retries: '
+                + ', '.join(ids)
             )
+            if exc:
+                msg += f'\n{msg}'
+            self.compare_screenshot(f'fail-{uuid1()}', msg, 1)
 
 
 @pytest.fixture
diff --git a/tests/integration/tui/test_app.py b/tests/integration/tui/test_app.py
index d7dc26457b3..e6ed8c6e24d 100644
--- a/tests/integration/tui/test_app.py
+++ b/tests/integration/tui/test_app.py
@@ -258,6 +258,9 @@ async def test_navigation(flow, scheduler, start, raikura):
         await schd.update_data_structure()
 
         with raikura(size='80,30') as rk:
+            # wait for the workflow to appear (collapsed)
+            rk.wait_until_loaded('#spring')
+
             rk.compare_screenshot(
                 'on-load',
                 'the workflow should be collapsed when Tui is loaded',
@@ -314,6 +317,8 @@ async def test_auto_expansion(flow, scheduler, start, raikura):
     with raikura(size='80,20') as rk:
         async with start(schd):
             await schd.update_data_structure()
+            # wait for the workflow to appear (collapsed)
+            rk.wait_until_loaded('#spring')
 
             # open the workflow
             rk.force_update()
@@ -351,12 +356,18 @@ async def test_restart_reconnect(one_conf, flow, scheduler, start, raikura):
         # 1- start the workflow
         async with start(schd):
             await schd.update_data_structure()
+            # wait for the workflow to appear (collapsed)
+            rk.wait_until_loaded('#spring')
+
+            # expand the workflow (subscribes to updates from it)
             rk.force_update()
             rk.user_input('down', 'right')
+
+            # wait for workflow to appear (expanded)
             rk.wait_until_loaded(schd.tokens.id)
             rk.compare_screenshot(
                 '1-workflow-running',
-                'the workflow should appear in tui and be expanded on load',
+                'the workflow should appear in tui and be expanded',
             )
 
         # 2 - stop the worlflow
@@ -371,7 +382,6 @@ async def test_restart_reconnect(one_conf, flow, scheduler, start, raikura):
         async with start(schd):
             await schd.update_data_structure()
             rk.wait_until_loaded(schd.tokens.id)
-            # rk.user_input('down', 'right')
             rk.compare_screenshot(
                 '3-workflow-restarted',
                 'the restarted workflow should be expanded',
diff --git a/tests/integration/tui/test_logs.py b/tests/integration/tui/test_logs.py
index 7b734030bb7..66531a1e42d 100644
--- a/tests/integration/tui/test_logs.py
+++ b/tests/integration/tui/test_logs.py
@@ -201,6 +201,9 @@ async def test_scheduler_logs(
 ):
     """Test viewing the scheduler log files."""
     with mod_raikura(size='80,30') as rk:
+        # wait for the workflow to appear (collapsed)
+        rk.wait_until_loaded('#spring')
+
         # open the workflow in Tui
         rk.user_input('down', 'right')
         rk.wait_until_loaded(workflow.tokens.id)
@@ -244,6 +247,9 @@ async def test_task_logs(
     I.E. Test viewing job log files by opening the log view on a task.
     """
     with mod_raikura(size='80,30') as rk:
+        # wait for the workflow to appear (collapsed)
+        rk.wait_until_loaded('#spring')
+
         # open the workflow in Tui
         rk.user_input('down', 'right')
         rk.wait_until_loaded(workflow.tokens.id)
@@ -282,6 +288,9 @@ async def test_job_logs(
     I.E. Test viewing job log files by opening the log view on a job.
     """
     with mod_raikura(size='80,30') as rk:
+        # wait for the workflow to appear (collapsed)
+        rk.wait_until_loaded('#spring')
+
         # open the workflow in Tui
         rk.user_input('down', 'right')
         rk.wait_until_loaded(workflow.tokens.id)
@@ -331,6 +340,9 @@ def cli_cmd_fail(*args, **kwargs):
     )
 
     with mod_raikura(size='80,30') as rk:
+        # wait for the workflow to appear (collapsed)
+        rk.wait_until_loaded('#spring')
+
         # open the log view on scheduler
         rk.user_input('down', 'enter', 'down', 'down', 'enter')
 

From 732ef98682cdc8eeccdc4aec71e3b6672940ddc4 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 30 Nov 2023 10:41:07 +0000
Subject: [PATCH 049/110] tests/i: fix flaky tui/test_updater tests (#5849)

* These tests were trying to be a bit too clever, testing the updater
  whilst it was running.
* This wasn't really needed for what the tests were tring to assert,
  there are other tests which run Tui for real (live updater and all),
  those tests didn't exist when these ones were written.
* This commit freezes the updater so we can run it in lock-step with
  the tests.
---
 cylc/flow/tui/updater.py              |  49 ++++----
 tests/integration/tui/test_updater.py | 171 +++++++++-----------------
 2 files changed, 83 insertions(+), 137 deletions(-)

diff --git a/cylc/flow/tui/updater.py b/cylc/flow/tui/updater.py
index 5321a4d6c1f..f9bd0cf2c00 100644
--- a/cylc/flow/tui/updater.py
+++ b/cylc/flow/tui/updater.py
@@ -155,7 +155,11 @@ async def run(self, filters):
         """
         with suppress_logging():
             self._update_filters(filters)
-            await self._update()
+            while True:
+                ret = await self._update()
+                if ret == self.SIGNAL_TERMINATE:
+                    break
+                self.update_queue.put(ret)
 
     def _subscribe(self, w_id):
         if w_id not in self._clients:
@@ -183,27 +187,30 @@ def _update_filters(self, filters):
         self.filters = filters
 
     async def _update(self):
-        last_scan_time = 0
-        while True:
-            # process any pending commands
-            if not self._command_queue.empty():
-                (command, payload) = self._command_queue.get()
-                if command == self.SIGNAL_TERMINATE:
-                    break
-                getattr(self, command)(payload)
-                continue
+        """Run one iteration of the updater.
 
-            # do a workflow scan if it's due
-            update_start_time = time()
-            if update_start_time - last_scan_time > self.BASE_SCAN_INTERVAL:
-                data = await self._scan()
-
-            # get the next snapshot from workflows we are subscribed to
-            self.update_queue.put(await self._run_update(data))
-
-            # schedule the next update
-            update_time = time() - update_start_time
-            await sleep(self.BASE_UPDATE_INTERVAL - update_time)
+        Either returns the next update or "self.SIGNAL_TERMINATE".
+        """
+        last_scan_time = 0
+        # process any pending commands
+        while not self._command_queue.empty():
+            (command, payload) = self._command_queue.get()
+            if command == self.SIGNAL_TERMINATE:
+                return command
+            getattr(self, command)(payload)
+
+        # do a workflow scan if it's due
+        update_start_time = time()
+        if update_start_time - last_scan_time > self.BASE_SCAN_INTERVAL:
+            data = await self._scan()
+
+        # get the next snapshot from workflows we are subscribed to
+        update = await self._run_update(data)
+
+        # schedule the next update
+        update_time = time() - update_start_time
+        await sleep(self.BASE_UPDATE_INTERVAL - update_time)
+        return update
 
     async def _run_update(self, data):
         # copy the scanned data so it can be reused for future updates
diff --git a/tests/integration/tui/test_updater.py b/tests/integration/tui/test_updater.py
index 39a9a606eff..b3daac5a328 100644
--- a/tests/integration/tui/test_updater.py
+++ b/tests/integration/tui/test_updater.py
@@ -15,12 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import asyncio
 from copy import deepcopy
 from pathlib import Path
+from queue import Queue
 import re
 
 from async_timeout import timeout
+import pytest
 
 from cylc.flow.cycling.integer import IntegerPoint
 from cylc.flow.id import Tokens
@@ -31,57 +32,35 @@
 from cylc.flow.workflow_status import WorkflowStatus
 
 
-async def await_update(updater):
-    """Wait for the latest update from the Updater.
-
-    Note:
-        If multiple updates are waiting, this returns the most recent.Q
-
-    Returns:
-        The latest update from the Updater's "update_queue".
-
-    """
-    while updater.update_queue.empty():
-        await asyncio.sleep(0.1)
-    while not updater.update_queue.empty():
-        # flush out any older updates to avoid race conditions in tests
-        update = updater.update_queue.get()
-    return update
-
-
-async def wait_workflow_connected(updater, tokens, connected=True, time=3):
-    """Wait for the Updater to connect to a workflow.
-
-    This will return once the updater has connected to a workflow and returned
-    the first data from it.
+@pytest.fixture
+def updater(monkeypatch, test_dir):
+    """Return an updater ready for testing."""
+    # patch the update intervals so that everything runs for every update
+    monkeypatch.setattr(
+        'cylc.flow.tui.updater.Updater.BASE_UPDATE_INTERVAL',
+        0,
+    )
+    monkeypatch.setattr(
+        'cylc.flow.tui.updater.Updater.BASE_SCAN_INTERVAL',
+        0,
+    )
+
+    # create the updater
+    updater = Updater()
 
-    Arugments:
-        tokens:
-            The tokens of the workflow you're waiting for.
-        connected:
-            If True this waits for the updater to connect to the workflow, if
-            False it waits for the updater to disconnect from it.
-        time:
-            The maximum time to wait for this to happen.
+    # swap multiprocessing.Queue for queue.Queue
+    # (this means queued operations are instant making tests more stable)
+    updater.update_queue = Queue()
+    updater._command_queue = Queue()
 
-    Returns:
-        The first update from the Updater which contains the workflow data.
+    # set up the filters
+    # (these filter for the workflows created in this test only)
+    filters = get_default_filters()
+    id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser()))
+    filters['workflows']['id'] = f'^{re.escape(id_base)}/.*'
+    updater._update_filters(filters)
 
-    """
-    async with timeout(time):
-        while True:
-            root_node = await await_update(updater)
-            workflow_node = root_node['children'][0]
-            for workflow_node in root_node['children']:
-                if (
-                    workflow_node['id_'] == tokens.id
-                    and (
-                        workflow_node['children'][0]['id_'] != '#spring'
-                    ) == connected
-                ):
-                    # if the spring node is still there then we haven't
-                    # recieved the first update from the workflow yet
-                    return root_node
+    return updater
 
 
 def get_child_tokens(root_node, types, relative=False):
@@ -111,19 +90,16 @@ def get_child_tokens(root_node, types, relative=False):
     return ret
 
 
-async def test_subscribe(one_conf, flow, scheduler, run, test_dir):
+async def test_subscribe(one_conf, flow, scheduler, run, updater):
     """It should subscribe and unsubscribe from workflows."""
     id_ = flow(one_conf)
     schd = scheduler(id_)
 
-    updater = Updater()
-
-    async def the_test():
-        nonlocal updater
-
-        try:
+    async with run(schd):
+        # run the updater and the test
+        async with timeout(10):
             # wait for the first update
-            root_node = await await_update(updater)
+            root_node = await updater._update()
 
             # there should be a root root_node
             assert root_node['id_'] == 'root'
@@ -135,9 +111,7 @@ async def the_test():
 
             # subscribe to the workflow
             updater.subscribe(schd.tokens.id)
-
-            # wait for it to connect to the workflow
-            root_node = await wait_workflow_connected(updater, schd.tokens)
+            root_node = await updater._update()
 
             # check the workflow contains one cycle with one task in it
             workflow_node = root_node['children'][0]
@@ -150,31 +124,13 @@ async def the_test():
 
             # unsubscribe from the workflow
             updater.unsubscribe(schd.tokens.id)
+            root_node = await updater._update()
 
-            # wait for it to disconnect from the workflow
-            root_node = await wait_workflow_connected(
-                updater,
-                schd.tokens,
-                connected=False,
-            )
-
-        finally:
-            # shut down the updater
-            updater.terminate()
-
-    async with run(schd):
-        filters = get_default_filters()
-        filters['workflows']['id'] = f'{re.escape(str(test_dir.relative_to(Path("~/cylc-run").expanduser())))}/.*'
-
-        # run the updater and the test
-        async with timeout(10):
-            await asyncio.gather(
-                asyncio.create_task(updater.run(filters)),
-                asyncio.create_task(the_test()),
-            )
+            # the workflow should be replaced by a "spring" node again
+            assert root_node['children'][0]['children'][0]['id_'] == '#spring'
 
 
-async def test_filters(one_conf, flow, scheduler, run, test_dir):
+async def test_filters(one_conf, flow, scheduler, run, updater):
     """It should filter workflow and task states.
 
     Note:
@@ -196,16 +152,20 @@ async def test_filters(one_conf, flow, scheduler, run, test_dir):
     two = scheduler(flow(one_conf, name='two'))
     tre = scheduler(flow(one_conf, name='tre'))
 
-    filters = get_default_filters()
-    id_base = str(test_dir.relative_to(Path("~/cylc-run").expanduser()))
-    filters['workflows']['id'] = f'^{re.escape(id_base)}/.*'
+    # start workflow "one"
+    async with run(one):
+        # mark "1/a" as running and "1/b" as succeeded
+        one_a = one.pool.get_task(IntegerPoint('1'), 'a')
+        one_a.state_reset('running')
+        one.data_store_mgr.delta_task_state(one_a)
+        one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded')
 
-    updater = Updater()
+        # start workflow "two"
+        async with run(two):
+            # run the updater and the test
+            filters = deepcopy(updater.filters)
 
-    async def the_test():
-        nonlocal filters
-        try:
-            root_node = await await_update(updater)
+            root_node = await updater._update()
             assert {child['id_'] for child in root_node['children']} == {
                 one.tokens.id,
                 two.tokens.id,
@@ -219,7 +179,7 @@ async def the_test():
             updater.update_filters(filters)
 
             # "one" and "two" should now be filtered out
-            root_node = await await_update(updater)
+            root_node = await updater._update()
             assert {child['id_'] for child in root_node['children']} == {
                 tre.tokens.id,
             }
@@ -231,15 +191,15 @@ async def the_test():
             updater.update_filters(filters)
 
             # "tre" should now be filtered out
-            root_node = await await_update(updater)
+            root_node = await updater._update()
             assert {child['id_'] for child in root_node['children']} == {
                 one.tokens.id,
                 two.tokens.id,
             }
 
             # subscribe to "one"
-            updater.subscribe(one.tokens.id)
-            root_node = await wait_workflow_connected(updater, one.tokens)
+            updater._subscribe(one.tokens.id)
+            root_node = await updater._update()
             assert get_child_tokens(
                 root_node, types={'task'}, relative=True
             ) == {
@@ -254,7 +214,7 @@ async def the_test():
             # filters['tasks'][TASK_STATUS_RUNNING] = False
             # updater.update_filters(filters)
 
-            # root_node = await await_update(updater)
+            # root_node = await updater._update()
             # assert get_child_tokens(
             #   root_node,
             #   types={'task'},
@@ -263,24 +223,3 @@ async def the_test():
             #     '1/b',
             #     '1/c',
             # }
-
-        finally:
-            # shut down the updater
-            updater.terminate()
-
-    # start workflow "one"
-    async with run(one):
-        # mark "1/a" as running and "1/b" as succeeded
-        one_a = one.pool.get_task(IntegerPoint('1'), 'a')
-        one_a.state_reset('running')
-        one.data_store_mgr.delta_task_state(one_a)
-        one.pool.get_task(IntegerPoint('1'), 'b').state_reset('succeeded')
-
-        # start workflow "two"
-        async with run(two):
-            # run the updater and the test
-            async with timeout(10):
-                await asyncio.gather(
-                    asyncio.create_task(the_test()),
-                    asyncio.create_task(updater.run(filters)),
-                )

From b0115e6c8b154dbd144bf4979c5b2578195c2e46 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 30 Nov 2023 13:42:54 +0000
Subject: [PATCH 050/110] Update tests/integration/tui/conftest.py

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 tests/integration/tui/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py
index ed716bd3e8d..813d81ad4f4 100644
--- a/tests/integration/tui/conftest.py
+++ b/tests/integration/tui/conftest.py
@@ -210,7 +210,7 @@ def wait_until_loaded(self, *ids, retries=20):
                 + ', '.join(ids)
             )
             if exc:
-                msg += f'\n{msg}'
+                msg += f'\n{exc}'
             self.compare_screenshot(f'fail-{uuid1()}', msg, 1)
 
 

From da365bf8403ac9ac32441264cbc91068d7543d64 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 30 Nov 2023 18:27:05 +0000
Subject: [PATCH 051/110] Maintain order for CYLC_PYTHONPATH (#5853)

---
 cylc/flow/scripts/cylc.py       |  9 ++++++---
 tests/unit/scripts/test_cylc.py | 12 +++++-------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 46c8127585a..700a70f8d7b 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -31,9 +31,12 @@ def pythonpath_manip():
         https://github.com/cylc/cylc-flow/issues/5124
     """
     if 'CYLC_PYTHONPATH' in os.environ:
-        for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep):
-            abspath = os.path.abspath(item)
-            sys.path.insert(0, abspath)
+        paths = [
+            os.path.abspath(item)
+            for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep)
+        ]
+        paths.extend(sys.path)
+        sys.path = paths
     if 'PYTHONPATH' in os.environ:
         for item in os.environ['PYTHONPATH'].split(os.pathsep):
             abspath = os.path.abspath(item)
diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py
index 819583a296c..9928024bb66 100644
--- a/tests/unit/scripts/test_cylc.py
+++ b/tests/unit/scripts/test_cylc.py
@@ -133,18 +133,16 @@ def test_pythonpath_manip(monkeypatch):
 
     and adds items from CYLC_PYTHONPATH
     """
-    # If PYTHONPATH is set...
-    monkeypatch.setenv('PYTHONPATH', '/remove-from-sys.path')
-    monkeypatch.setattr('sys.path', ['/leave-alone', '/remove-from-sys.path'])
+    monkeypatch.setenv('PYTHONPATH', '/remove1:/remove2')
+    monkeypatch.setattr('sys.path', ['/leave-alone', '/remove1', '/remove2'])
     pythonpath_manip()
     # ... we don't change PYTHONPATH
-    assert os.environ['PYTHONPATH'] == '/remove-from-sys.path'
+    assert os.environ['PYTHONPATH'] == '/remove1:/remove2'
     # ... but we do remove PYTHONPATH items from sys.path, and don't remove
     # items there not in PYTHONPATH
     assert sys.path == ['/leave-alone']
-
     # If CYLC_PYTHONPATH is set we retrieve its contents and
     # add them to the sys.path:
-    monkeypatch.setenv('CYLC_PYTHONPATH', '/add-to-sys.path')
+    monkeypatch.setenv('CYLC_PYTHONPATH', '/add1:/add2')
     pythonpath_manip()
-    assert sys.path == ['/add-to-sys.path', '/leave-alone']
+    assert sys.path == ['/add1', '/add2', '/leave-alone']

From 64417e5ba77e119525e905f12e41eacb8770ae9f Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Mon, 4 Dec 2023 15:41:21 +0000
Subject: [PATCH 052/110] tui: avoid deprecated import (#5858)

---
 cylc/flow/tui/app.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py
index 97abab94f81..fff634ea894 100644
--- a/cylc/flow/tui/app.py
+++ b/cylc/flow/tui/app.py
@@ -22,7 +22,13 @@
 import re
 
 import urwid
-from urwid.wimp import SelectableIcon
+try:
+    from urwid.widget import SelectableIcon
+except ImportError:
+    # BACK COMPAT: urwid.wimp
+    # From: urwid 2.0
+    # To: urwid 2.2
+    from urwid.wimp import SelectableIcon
 
 from cylc.flow.id import Tokens
 from cylc.flow.task_state import (

From b6efda45b62efc72e4497eb1917b17c860c03ec0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:46:23 +0000
Subject: [PATCH 053/110] Bump actions/setup-python from 4 to 5 (#5870)

---
 .github/workflows/1_create_release_pr.yml    | 2 +-
 .github/workflows/2_auto_publish_release.yml | 2 +-
 .github/workflows/build.yml                  | 2 +-
 .github/workflows/test_fast.yml              | 4 ++--
 .github/workflows/test_functional.yml        | 2 +-
 .github/workflows/test_manylinux.yml         | 2 +-
 .github/workflows/test_tutorial_workflow.yml | 2 +-
 7 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml
index 6d792304196..10a700b7ab4 100644
--- a/.github/workflows/1_create_release_pr.yml
+++ b/.github/workflows/1_create_release_pr.yml
@@ -30,7 +30,7 @@ jobs:
         uses: cylc/release-actions/check-shortlog@v1
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 
diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 9223bd2106c..f76f8afde2d 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -27,7 +27,7 @@ jobs:
         ref: ${{ env.MERGE_SHA }}
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
 
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 51bf49500f9..d1d76b4e4d1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 2f303e36ee7..5fb95def4e8 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -31,7 +31,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -92,7 +92,7 @@ jobs:
       # note: exclude python 3.10+ from mypy checks as these produce false
       # positives in installed libraries for python 3.7
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index d3a6a38dad3..6fad4147345 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -99,7 +99,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/test_manylinux.yml b/.github/workflows/test_manylinux.yml
index 5b57b4973e7..84e1e041283 100644
--- a/.github/workflows/test_manylinux.yml
+++ b/.github/workflows/test_manylinux.yml
@@ -44,7 +44,7 @@ jobs:
         uses: cylc/release-actions/configure-git@v1
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/test_tutorial_workflow.yml b/.github/workflows/test_tutorial_workflow.yml
index c95439923d6..7859b8588e2 100644
--- a/.github/workflows/test_tutorial_workflow.yml
+++ b/.github/workflows/test_tutorial_workflow.yml
@@ -26,7 +26,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: configure python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

From d9669a86a74b5cda9a637bbe97bc6db0638c7a38 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Fri, 8 Dec 2023 23:29:43 +1300
Subject: [PATCH 054/110] Fix misspelling of 'Rakiura'. (#5869)

Fix misspelling of 'Rakiura'.
---
 tests/integration/tui/conftest.py             | 30 +++++++--------
 ...> test_tui_basics.test-rakiura-enter.html} |  0
 ...=> test_tui_basics.test-rakiura-help.html} |  0
 ...html => test_tui_basics.test-rakiura.html} |  0
 tests/integration/tui/test_app.py             | 38 +++++++++----------
 tests/integration/tui/test_logs.py            | 16 ++++----
 tests/integration/tui/test_mutations.py       |  8 ++--
 tests/integration/tui/test_show.py            |  4 +-
 8 files changed, 48 insertions(+), 48 deletions(-)
 rename tests/integration/tui/screenshots/{test_tui_basics.test-raikura-enter.html => test_tui_basics.test-rakiura-enter.html} (100%)
 rename tests/integration/tui/screenshots/{test_tui_basics.test-raikura-help.html => test_tui_basics.test-rakiura-help.html} (100%)
 rename tests/integration/tui/screenshots/{test_tui_basics.test-raikura.html => test_tui_basics.test-rakiura.html} (100%)

diff --git a/tests/integration/tui/conftest.py b/tests/integration/tui/conftest.py
index 813d81ad4f4..f99bac66581 100644
--- a/tests/integration/tui/conftest.py
+++ b/tests/integration/tui/conftest.py
@@ -68,8 +68,8 @@ def format_test_failure(expected, got, description):
     '''
 
 
-class RaikuraSession:
-    """Convenience class for accessing Raikura functionality."""
+class RakiuraSession:
+    """Convenience class for accessing Rakiura functionality."""
 
     def __init__(self, app, html_fragment, test_dir, test_name):
         self.app = app
@@ -177,7 +177,7 @@ def force_update(self):
         This is done automatically by compare_screenshot but you may want to
         call it in a test, e.g. before pressing navigation keys.
 
-        With Raikura, the Tui event loop is not running so the data is never
+        With Rakiura, the Tui event loop is not running so the data is never
         refreshed.
 
         You do NOT need to call this method for key presses, but you do need to
@@ -215,16 +215,16 @@ def wait_until_loaded(self, *ids, retries=20):
 
 
 @pytest.fixture
-def raikura(test_dir, request, monkeypatch):
+def rakiura(test_dir, request, monkeypatch):
     """Visual regression test framework for Urwid apps.
 
     Like Cypress but for Tui so named after a NZ island with lots of Tuis.
 
-    When called this yields a RaikuraSession object loaded with test
+    When called this yields a RakiuraSession object loaded with test
     utilities. All tests have default retries to avoid flaky tests.
 
     Similar to the "start" fixture, which starts a Scheduler without running
-    the main loop, raikura starts Tui without running the main loop.
+    the main loop, rakiura starts Tui without running the main loop.
 
     Arguments:
         workflow_id:
@@ -234,25 +234,25 @@ def raikura(test_dir, request, monkeypatch):
             separated string e.g. "80,50" for 80 cols wide by 50 rows tall.
 
     Returns:
-        A RaikuraSession context manager which provides useful utilities for
+        A RakiuraSession context manager which provides useful utilities for
         testing.
 
     """
-    return _raikura(test_dir, request, monkeypatch)
+    return _rakiura(test_dir, request, monkeypatch)
 
 
 @pytest.fixture
-def mod_raikura(test_dir, request, monkeypatch):
-    """Same as raikura but configured to view module-scoped workflows.
+def mod_rakiura(test_dir, request, monkeypatch):
+    """Same as rakiura but configured to view module-scoped workflows.
 
     Note: This is *not* a module-scoped fixture (no need, creating Tui sessions
     is not especially slow), it is configured to display module-scoped
     "scheduler" fixtures (which may be more expensive to create/destroy).
     """
-    return _raikura(test_dir.parent, request, monkeypatch)
+    return _rakiura(test_dir.parent, request, monkeypatch)
 
 
-def _raikura(test_dir, request, monkeypatch):
+def _rakiura(test_dir, request, monkeypatch):
     # make the workflow and scan update intervals match (more reliable)
     # and speed things up a little whilst we're at it
     monkeypatch.setattr(
@@ -289,7 +289,7 @@ def get_display_id(id_):
     workflow_filter = re.escape(id_base) + r'/.*'
 
     @contextmanager
-    def _raikura(workflow_id=None, size='80,50'):
+    def _rakiura(workflow_id=None, size='80,50'):
         screen, html_fragment = configure_screenshot(size)
         app = TuiApp(screen=screen)
         with app.main(
@@ -297,11 +297,11 @@ def _raikura(workflow_id=None, size='80,50'):
             id_filter=workflow_filter,
             interactive=False,
         ):
-            yield RaikuraSession(
+            yield RakiuraSession(
                 app,
                 html_fragment,
                 test_dir,
                 request.function.__name__,
             )
 
-    return _raikura
+    return _rakiura
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
similarity index 100%
rename from tests/integration/tui/screenshots/test_tui_basics.test-raikura-enter.html
rename to tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html
similarity index 100%
rename from tests/integration/tui/screenshots/test_tui_basics.test-raikura-help.html
rename to tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-raikura.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
similarity index 100%
rename from tests/integration/tui/screenshots/test_tui_basics.test-raikura.html
rename to tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
diff --git a/tests/integration/tui/test_app.py b/tests/integration/tui/test_app.py
index e6ed8c6e24d..908866fc948 100644
--- a/tests/integration/tui/test_app.py
+++ b/tests/integration/tui/test_app.py
@@ -46,37 +46,37 @@ def set_task_state(schd, task_states):
         )
 
 
-async def test_tui_basics(raikura):
+async def test_tui_basics(rakiura):
     """Test basic Tui interaction with no workflows."""
-    with raikura(size='80,40') as rk:
+    with rakiura(size='80,40') as rk:
         # the app should open
-        rk.compare_screenshot('test-raikura', 'the app should have loaded')
+        rk.compare_screenshot('test-rakiura', 'the app should have loaded')
 
         # "h" should bring up the onscreen help
         rk.user_input('h')
         rk.compare_screenshot(
-            'test-raikura-help',
+            'test-rakiura-help',
             'the help screen should be visible'
         )
 
         # "q" should close the popup
         rk.user_input('q')
         rk.compare_screenshot(
-            'test-raikura',
+            'test-rakiura',
             'the help screen should have closed',
         )
 
         # "enter" should bring up the context menu
         rk.user_input('enter')
         rk.compare_screenshot(
-            'test-raikura-enter',
+            'test-rakiura-enter',
             'the context menu should have opened',
         )
 
         # "enter" again should close it via the "cancel" button
         rk.user_input('enter')
         rk.compare_screenshot(
-            'test-raikura',
+            'test-rakiura',
             'the context menu should have closed',
         )
 
@@ -89,13 +89,13 @@ async def test_tui_basics(raikura):
             rk.user_input('q')
 
 
-async def test_subscribe_unsubscribe(one_conf, flow, scheduler, start, raikura):
+async def test_subscribe_unsubscribe(one_conf, flow, scheduler, start, rakiura):
     """Test a simple workflow with one task."""
     id_ = flow(one_conf, name='one')
     schd = scheduler(id_)
     async with start(schd):
         await schd.update_data_structure()
-        with raikura(size='80,15') as rk:
+        with rakiura(size='80,15') as rk:
             rk.compare_screenshot(
                 'unsubscribed',
                 'the workflow should be collapsed'
@@ -120,7 +120,7 @@ async def test_subscribe_unsubscribe(one_conf, flow, scheduler, start, raikura):
             )
 
 
-async def test_workflow_states(one_conf, flow, scheduler, start, raikura):
+async def test_workflow_states(one_conf, flow, scheduler, start, rakiura):
     """Test viewing multiple workflows in different states."""
     # one => stopping
     id_1 = flow(one_conf, name='one')
@@ -137,7 +137,7 @@ async def test_workflow_states(one_conf, flow, scheduler, start, raikura):
 
         async with start(schd_2):
             await schd_2.update_data_structure()
-            with raikura(size='80,15') as rk:
+            with rakiura(size='80,15') as rk:
                 rk.compare_screenshot(
                     'unfiltered',
                     'All workflows should be visible (one, two, tree)',
@@ -193,7 +193,7 @@ async def test_workflow_states(one_conf, flow, scheduler, start, raikura):
 # TODO: Task state filtering is currently broken
 # see: https://github.com/cylc/cylc-flow/issues/5716
 #
-# async def test_task_states(flow, scheduler, start, raikura):
+# async def test_task_states(flow, scheduler, start, rakiura):
 #     id_ = flow({
 #         'scheduler': {
 #             'allow implicit tasks': 'true',
@@ -224,7 +224,7 @@ async def test_workflow_states(one_conf, flow, scheduler, start, raikura):
 #         )
 #         await schd.update_data_structure()
 #
-#         with raikura(schd.tokens.id, size='80,20') as rk:
+#         with rakiura(schd.tokens.id, size='80,20') as rk:
 #             rk.compare_screenshot('unfiltered')
 #
 #             # filter out waiting tasks
@@ -232,7 +232,7 @@ async def test_workflow_states(one_conf, flow, scheduler, start, raikura):
 #             rk.compare_screenshot('filter-not-waiting')
 
 
-async def test_navigation(flow, scheduler, start, raikura):
+async def test_navigation(flow, scheduler, start, rakiura):
     """Test navigating with the arrow keys."""
     id_ = flow({
         'scheduling': {
@@ -257,7 +257,7 @@ async def test_navigation(flow, scheduler, start, raikura):
     async with start(schd):
         await schd.update_data_structure()
 
-        with raikura(size='80,30') as rk:
+        with rakiura(size='80,30') as rk:
             # wait for the workflow to appear (collapsed)
             rk.wait_until_loaded('#spring')
 
@@ -291,7 +291,7 @@ async def test_navigation(flow, scheduler, start, raikura):
             )
 
 
-async def test_auto_expansion(flow, scheduler, start, raikura):
+async def test_auto_expansion(flow, scheduler, start, rakiura):
     """It should automatically expand cycles and top-level families.
 
     When a workflow is expanded, Tui should auto expand cycles and top-level
@@ -314,7 +314,7 @@ async def test_auto_expansion(flow, scheduler, start, raikura):
         },
     }, name='one')
     schd = scheduler(id_)
-    with raikura(size='80,20') as rk:
+    with rakiura(size='80,20') as rk:
         async with start(schd):
             await schd.update_data_structure()
             # wait for the workflow to appear (collapsed)
@@ -342,7 +342,7 @@ async def test_auto_expansion(flow, scheduler, start, raikura):
             )
 
 
-async def test_restart_reconnect(one_conf, flow, scheduler, start, raikura):
+async def test_restart_reconnect(one_conf, flow, scheduler, start, rakiura):
     """It should handle workflow shutdown and restart.
 
     The Cylc client can raise exceptions e.g. WorkflowStopped. Any text written
@@ -350,7 +350,7 @@ async def test_restart_reconnect(one_conf, flow, scheduler, start, raikura):
     can handle shutdown / restart without any errors occuring and any spurious
     text appearing on the screen.
     """
-    with raikura(size='80,20') as rk:
+    with rakiura(size='80,20') as rk:
         schd = scheduler(flow(one_conf, name='one'))
 
         # 1- start the workflow
diff --git a/tests/integration/tui/test_logs.py b/tests/integration/tui/test_logs.py
index 66531a1e42d..7c63ad39fcc 100644
--- a/tests/integration/tui/test_logs.py
+++ b/tests/integration/tui/test_logs.py
@@ -196,11 +196,11 @@ async def workflow(mod_flow, mod_scheduler, mod_start, standarise_host_and_path)
 
 async def test_scheduler_logs(
     workflow,
-    mod_raikura,
+    mod_rakiura,
     wait_log_loaded,
 ):
     """Test viewing the scheduler log files."""
-    with mod_raikura(size='80,30') as rk:
+    with mod_rakiura(size='80,30') as rk:
         # wait for the workflow to appear (collapsed)
         rk.wait_until_loaded('#spring')
 
@@ -239,14 +239,14 @@ async def test_scheduler_logs(
 
 async def test_task_logs(
     workflow,
-    mod_raikura,
+    mod_rakiura,
     wait_log_loaded,
 ):
     """Test viewing task log files.
 
     I.E. Test viewing job log files by opening the log view on a task.
     """
-    with mod_raikura(size='80,30') as rk:
+    with mod_rakiura(size='80,30') as rk:
         # wait for the workflow to appear (collapsed)
         rk.wait_until_loaded('#spring')
 
@@ -280,14 +280,14 @@ async def test_task_logs(
 
 async def test_job_logs(
     workflow,
-    mod_raikura,
+    mod_rakiura,
     wait_log_loaded,
 ):
     """Test viewing the job log files.
 
     I.E. Test viewing job log files by opening the log view on a job.
     """
-    with mod_raikura(size='80,30') as rk:
+    with mod_rakiura(size='80,30') as rk:
         # wait for the workflow to appear (collapsed)
         rk.wait_until_loaded('#spring')
 
@@ -325,7 +325,7 @@ async def test_job_logs(
 
 async def test_errors(
     workflow,
-    mod_raikura,
+    mod_rakiura,
     wait_log_loaded,
     monkeypatch,
 ):
@@ -339,7 +339,7 @@ def cli_cmd_fail(*args, **kwargs):
         cli_cmd_fail,
     )
 
-    with mod_raikura(size='80,30') as rk:
+    with mod_rakiura(size='80,30') as rk:
         # wait for the workflow to appear (collapsed)
         rk.wait_until_loaded('#spring')
 
diff --git a/tests/integration/tui/test_mutations.py b/tests/integration/tui/test_mutations.py
index 659c74ac1d0..18da88d227d 100644
--- a/tests/integration/tui/test_mutations.py
+++ b/tests/integration/tui/test_mutations.py
@@ -35,13 +35,13 @@ async def test_online_mutation(
     flow,
     scheduler,
     start,
-    raikura,
+    rakiura,
     monkeypatch,
 ):
     """Test a simple workflow with one task."""
     id_ = flow(one_conf, name='one')
     schd = scheduler(id_)
-    with raikura(size='80,15') as rk:
+    with rakiura(size='80,15') as rk:
         async with start(schd):
             await schd.update_data_structure()
             assert schd.command_queue.empty()
@@ -157,14 +157,14 @@ def returncode(self):
 async def test_offline_mutation(
     one_conf,
     flow,
-    raikura,
+    rakiura,
     capture_commands,
     standardise_cli_cmds,
 ):
     id_ = flow(one_conf, name='one')
     commands, returncode = capture_commands
 
-    with raikura(size='80,15') as rk:
+    with rakiura(size='80,15') as rk:
         # run the stop-all mutation
         rk.wait_until_loaded('root')
         rk.user_input('enter', 'down')
diff --git a/tests/integration/tui/test_show.py b/tests/integration/tui/test_show.py
index ac6aa8532a6..c664cdd1393 100644
--- a/tests/integration/tui/test_show.py
+++ b/tests/integration/tui/test_show.py
@@ -19,7 +19,7 @@
 from cylc.flow.tui.data import _show
 
 
-async def test_show(flow, scheduler, start, raikura, monkeypatch):
+async def test_show(flow, scheduler, start, rakiura, monkeypatch):
     """Test "cylc show" support in Tui."""
     id_ = flow({
         'scheduling': {
@@ -40,7 +40,7 @@ async def test_show(flow, scheduler, start, raikura, monkeypatch):
     async with start(schd):
         await schd.update_data_structure()
 
-        with raikura(size='80,40') as rk:
+        with rakiura(size='80,40') as rk:
             rk.user_input('down', 'right')
             rk.wait_until_loaded(schd.tokens.id)
 

From 043b5e4e64e393863231b728708564a77f0ac9e1 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 8 Dec 2023 11:11:34 +0000
Subject: [PATCH 055/110] Increase default `cylc clean` remote timeout to 5m &
 allow ISO duration

---
 cylc/flow/clean.py         | 14 +++++++++++---
 cylc/flow/scripts/clean.py |  8 +++++---
 tests/unit/test_clean.py   | 39 ++++++++++++++++++++++++++++++++++----
 3 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index 81259e12a25..d17214ca552 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -44,6 +44,9 @@
     Union,
 )
 
+from metomi.isodatetime.exceptions import ISO8601SyntaxError
+from metomi.isodatetime.parsers import DurationParser
+
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.exceptions import (
@@ -187,7 +190,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
 
         if platform_names and platform_names != {'localhost'}:
             remote_clean(
-                id_, platform_names, opts.rm_dirs, opts.remote_timeout
+                id_, platform_names, opts.remote_timeout, opts.rm_dirs
             )
 
     if not opts.remote_only:
@@ -338,8 +341,8 @@ def _clean_using_glob(
 def remote_clean(
     id_: str,
     platform_names: Iterable[str],
+    timeout: str,
     rm_dirs: Optional[List[str]] = None,
-    timeout: str = '120'
 ) -> None:
     """Run subprocesses to clean a workflow on its remote install targets
     (skip localhost), given a set of platform names to look up.
@@ -348,8 +351,9 @@ def remote_clean(
         id_: Workflow name.
         platform_names: List of platform names to look up in the global
             config, in order to determine the install targets to clean on.
+        timeout: ISO 8601 duration or number of seconds to wait before
+            cancelling.
         rm_dirs: Sub dirs to remove instead of the whole run dir.
-        timeout: Number of seconds to wait before cancelling.
     """
     try:
         install_targets_map = (
@@ -358,6 +362,10 @@ def remote_clean(
         raise PlatformLookupError(
             f"Cannot clean {id_} on remote platforms as the workflow database "
             f"is out of date/inconsistent with the global config - {exc}")
+
+    with suppress(ISO8601SyntaxError):
+        timeout = str(DurationParser().parse(timeout).get_seconds())
+
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(
         _remote_clean_cmd, id_=id_, rm_dirs=rm_dirs, timeout=timeout
diff --git a/cylc/flow/scripts/clean.py b/cylc/flow/scripts/clean.py
index 94fe92b6dab..a002f1d90ab 100644
--- a/cylc/flow/scripts/clean.py
+++ b/cylc/flow/scripts/clean.py
@@ -120,9 +120,11 @@ def get_option_parser():
 
     parser.add_option(
         '--timeout',
-        help=("The number of seconds to wait for cleaning to take place on "
-              r"remote hosts before cancelling. Default: %default."),
-        action='store', default='120', dest='remote_timeout'
+        help=(
+            "The length of time to wait for cleaning to take place on "
+            r"remote hosts before cancelling. Default: %default."
+        ),
+        action='store', default='PT5M', dest='remote_timeout'
     )
 
     parser.add_option(
diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py
index b7769cfb54d..5b6ebbe0303 100644
--- a/tests/unit/test_clean.py
+++ b/tests/unit/test_clean.py
@@ -19,6 +19,7 @@
 import shutil
 from glob import iglob
 from pathlib import Path
+from subprocess import Popen
 from typing import (
     Any,
     Callable,
@@ -274,7 +275,8 @@ def test_init_clean__rm_dirs(
     init_clean(id_, opts=opts)
     mock_clean.assert_called_with(id_, run_dir, expected_clean)
     mock_remote_clean.assert_called_with(
-        id_, platforms, expected_remote_clean, opts.remote_timeout)
+        id_, platforms, opts.remote_timeout, expected_remote_clean
+    )
 
 
 @pytest.mark.parametrize(
@@ -920,7 +922,7 @@ def test_remote_clean(
     # Remove randomness:
     monkeymock('cylc.flow.clean.shuffle')
 
-    def mocked_remote_clean_cmd_side_effect(id_, platform, rm_dirs, timeout):
+    def mocked_remote_clean_cmd_side_effect(id_, platform, timeout, rm_dirs):
         proc_ret_code = 0
         if failed_platforms and platform['name'] in failed_platforms:
             proc_ret_code = failed_platforms[platform['name']]
@@ -942,11 +944,13 @@ def mocked_remote_clean_cmd_side_effect(id_, platform, rm_dirs, timeout):
     if exc_expected:
         with pytest.raises(CylcError) as exc:
             cylc_clean.remote_clean(
-                id_, platform_names, rm_dirs, timeout='irrelevant')
+                id_, platform_names, timeout='irrelevant', rm_dirs=rm_dirs
+            )
         assert "Remote clean failed" in str(exc.value)
     else:
         cylc_clean.remote_clean(
-            id_, platform_names, rm_dirs, timeout='irrelevant')
+            id_, platform_names, timeout='irrelevant', rm_dirs=rm_dirs
+        )
     for msg in expected_err_msgs:
         assert log_filter(caplog, level=logging.ERROR, contains=msg)
     if expected_platforms:
@@ -960,6 +964,33 @@ def mocked_remote_clean_cmd_side_effect(id_, platform, rm_dirs, timeout):
             assert f"{p_name} - {PlatformError.MSG_TIDY}" in caplog.text
 
 
+@pytest.mark.parametrize(
+    'timeout, expected',
+    [('100', '100'),
+     ('PT1M2S', '62.0')]
+)
+def test_remote_clean__timeout(
+    monkeymock: MonkeyMock, monkeypatch: pytest.MonkeyPatch,
+    timeout: str, expected: str,
+):
+    """Test that remote_clean() accepts a timeout in ISO 8601 format or
+    number of seconds."""
+    mock_remote_clean_cmd = monkeymock(
+        'cylc.flow.clean._remote_clean_cmd',
+        spec=_remote_clean_cmd,
+        return_value=mock.Mock(
+            spec=Popen, poll=lambda: 0, communicate=lambda: ('', '')
+        )
+    )
+    monkeypatch.setattr(
+        'cylc.flow.clean.get_install_target_to_platforms_map',
+        lambda *a, **k: {'picard': [PLATFORMS['stargazer']]}
+    )
+
+    cylc_clean.remote_clean('blah', 'blah', timeout)
+    assert mock_remote_clean_cmd.call_args.kwargs['timeout'] == expected
+
+
 @pytest.mark.parametrize(
     'rm_dirs, expected_args',
     [

From 4d17bb1a7d2004d08eb96bdbfa5bd451f5919bfd Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 8 Dec 2023 11:40:08 +0000
Subject: [PATCH 056/110] Improve error message for `cylc clean` remote timeout

---
 cylc/flow/clean.py                      | 13 +++++++++----
 tests/functional/cylc-clean/01-remote.t | 10 ++++++++--
 tests/unit/test_clean.py                | 24 +++++++++++++++++-------
 3 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index d17214ca552..f16996ee635 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -384,7 +384,7 @@ def remote_clean(
                 remote_clean_cmd(platform=platforms[0]), target, platforms
             )
         )
-    failed_targets: Dict[str, PlatformError] = {}
+    failed_targets: Dict[str, Union[PlatformError, str]] = {}
     # Handle subproc pool results almost concurrently:
     while queue:
         item = queue.popleft()
@@ -395,7 +395,12 @@ def remote_clean(
         out, err = item.proc.communicate()
         if out:
             LOG.info(f"[{item.install_target}]\n{out}")
-        if ret_code:
+        if ret_code == 124:
+            failed_targets[item.install_target] = (
+                f"cylc clean timed out after {timeout}s. You can increase "
+                "this timeout using the --timeout option."
+            )
+        elif ret_code:
             this_platform = item.platforms.pop(0)
             excp = PlatformError(
                 PlatformError.MSG_TIDY,
@@ -423,9 +428,9 @@ def remote_clean(
             LOG.debug(f"[{item.install_target}]\n{err}")
         sleep(0.2)
     if failed_targets:
-        for target, excp in failed_targets.items():
+        for target, info in failed_targets.items():
             LOG.error(
-                f"Could not clean {id_} on install target: {target}\n{excp}"
+                f"Could not clean {id_} on install target: {target}\n{info}"
             )
         raise CylcError(f"Remote clean failed for {id_}")
 
diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
index 6bc4a9a40ea..8b15caa4162 100644
--- a/tests/functional/cylc-clean/01-remote.t
+++ b/tests/functional/cylc-clean/01-remote.t
@@ -25,7 +25,7 @@ SSH_CMD="$(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") ${
 if ! $SSH_CMD command -v 'tree' > '/dev/null'; then
     skip_all "'tree' command not available on remote host ${CYLC_TEST_HOST}"
 fi
-set_test_number 10
+set_test_number 12
 
 # Generate random name for symlink dirs to avoid any clashes with other tests
 SYM_NAME="$(mktemp -u)"
@@ -108,7 +108,13 @@ __TREE__
 
 # -----------------------------------------------------------------------------
 
-TEST_NAME="cylc-clean"
+TEST_NAME="cylc-clean-timeout"
+run_fail "$TEST_NAME" cylc clean --timeout 'PT0,1S' "$WORKFLOW_NAME"
+dump_std "$TEST_NAME"
+grep_ok 'cylc clean timed out after 0.1s' "${TEST_NAME}.stderr"
+
+
+TEST_NAME="cylc-clean-ok"
 run_ok "$TEST_NAME" cylc clean "$WORKFLOW_NAME"
 dump_std "$TEST_NAME"
 
diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py
index 5b6ebbe0303..f5c46ee3f73 100644
--- a/tests/unit/test_clean.py
+++ b/tests/unit/test_clean.py
@@ -970,16 +970,23 @@ def mocked_remote_clean_cmd_side_effect(id_, platform, timeout, rm_dirs):
      ('PT1M2S', '62.0')]
 )
 def test_remote_clean__timeout(
-    monkeymock: MonkeyMock, monkeypatch: pytest.MonkeyPatch,
-    timeout: str, expected: str,
+    monkeymock: MonkeyMock,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    timeout: str,
+    expected: str,
 ):
-    """Test that remote_clean() accepts a timeout in ISO 8601 format or
-    number of seconds."""
+    """Test remote_clean() timeout.
+
+    - It should accept ISO 8601 format or number of seconds.
+    - It should give a sensible error message for return code 124.
+    """
+    caplog.set_level(logging.ERROR, CYLC_LOG)
     mock_remote_clean_cmd = monkeymock(
         'cylc.flow.clean._remote_clean_cmd',
         spec=_remote_clean_cmd,
         return_value=mock.Mock(
-            spec=Popen, poll=lambda: 0, communicate=lambda: ('', '')
+            spec=Popen, poll=lambda: 124, communicate=lambda: ('', '')
         )
     )
     monkeypatch.setattr(
@@ -987,8 +994,11 @@ def test_remote_clean__timeout(
         lambda *a, **k: {'picard': [PLATFORMS['stargazer']]}
     )
 
-    cylc_clean.remote_clean('blah', 'blah', timeout)
-    assert mock_remote_clean_cmd.call_args.kwargs['timeout'] == expected
+    with pytest.raises(CylcError):
+        cylc_clean.remote_clean('blah', 'blah', timeout)
+    _, kwargs = mock_remote_clean_cmd.call_args
+    assert kwargs['timeout'] == expected
+    assert f"cylc clean timed out after {expected}s" in caplog.text
 
 
 @pytest.mark.parametrize(

From 6f34d1568455a9b8299c8637a99fc62929f37dc3 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 8 Dec 2023 12:01:16 +0000
Subject: [PATCH 057/110] Changelog

---
 changes.d/5872.feat.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes.d/5872.feat.md

diff --git a/changes.d/5872.feat.md b/changes.d/5872.feat.md
new file mode 100644
index 00000000000..d88b0dd8116
--- /dev/null
+++ b/changes.d/5872.feat.md
@@ -0,0 +1 @@
+Improvements to `cylc clean` remote timeout handling.

From ec8eec84476980357f2502b41464d68194c030e2 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Tue, 12 Dec 2023 11:20:20 +0000
Subject: [PATCH 058/110] Do not allow plugins to act on global.cylc (#5839)

---
 cylc/flow/parsec/fileparse.py            | 66 +++++++++++++-----------
 tests/unit/parsec/test_fileparse.py      | 39 ++++++++++++--
 tests/unit/plugins/test_pre_configure.py |  8 +--
 3 files changed, 74 insertions(+), 39 deletions(-)

diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py
index 42e8a4aa40b..4ba55d18180 100644
--- a/cylc/flow/parsec/fileparse.py
+++ b/cylc/flow/parsec/fileparse.py
@@ -30,6 +30,7 @@
       value type is known).
 """
 
+from copy import deepcopy
 import os
 from pathlib import Path
 import re
@@ -51,6 +52,7 @@
 
 if t.TYPE_CHECKING:
     from optparse import Values
+    from typing import Union
 
 
 # heading/sections can contain commas (namespace name lists) and any
@@ -105,6 +107,13 @@
 _UNCLOSED_MULTILINE = re.compile(
     r'(?<![\w>])\[.*\]'
 )
+TEMPLATING_DETECTED = 'templating_detected'
+TEMPLATE_VARIABLES = 'template_variables'
+EXTRA_VARS_TEMPLATE: t.Dict[str, t.Any] = {
+    'env': {},
+    TEMPLATE_VARIABLES: {},
+    TEMPLATING_DETECTED: None
+}
 
 
 def get_cylc_env_vars() -> t.Dict[str, str]:
@@ -242,7 +251,7 @@ def multiline(flines, value, index, maxline):
     return quot + newvalue + line, index
 
 
-def process_plugins(fpath, opts):
+def process_plugins(fpath: 'Union[str, Path]', opts: 'Values'):
     """Run a Cylc pre-configuration plugin.
 
     Plugins should return a dictionary containing:
@@ -253,22 +262,25 @@ def process_plugins(fpath, opts):
             or ``empy``.
 
     args:
-        fpath: Directory where the plugin will look for a config.
+        fpath: Path to a config file. Plugin will look at the parent
+            directory of this file.
         opts: Command line options to be passed to the plugin.
 
     Returns: Dictionary in the form:
         extra_vars = {
             'env': {},
             'template_variables': {},
-            'templating_detected': None
+            TEMPLATING_DETECTED: None
         }
     """
+    fpath = Path(fpath)
+
     # Set out blank dictionary for return:
-    extra_vars = {
-        'env': {},
-        'template_variables': {},
-        'templating_detected': None
-    }
+    extra_vars = deepcopy(EXTRA_VARS_TEMPLATE)
+
+    # Don't run this on global configs:
+    if fpath.name == 'global.cylc':
+        return extra_vars
 
     # Run entry point pre_configure items, trying to merge values with each.:
     for entry_point in iter_entry_points(
@@ -278,7 +290,7 @@ def process_plugins(fpath, opts):
             # If you want it to work on sourcedirs you need to get the options
             # to here.
             plugin_result = entry_point.load()(
-                srcdir=fpath, opts=opts
+                srcdir=fpath.parent, opts=opts
             )
         except Exception as exc:
             # NOTE: except Exception (purposefully vague)
@@ -288,7 +300,7 @@ def process_plugins(fpath, opts):
                 entry_point.name,
                 exc
             ) from None
-        for section in ['env', 'template_variables']:
+        for section in ['env', TEMPLATE_VARIABLES]:
             if section in plugin_result and plugin_result[section] is not None:
                 # Raise error if multiple plugins try to update the same keys.
                 section_update = plugin_result.get(section, {})
@@ -302,26 +314,20 @@ def process_plugins(fpath, opts):
                     )
                 extra_vars[section].update(section_update)
 
+        templating_detected = plugin_result.get(TEMPLATING_DETECTED, None)
         if (
-            'templating_detected' in plugin_result and
-            plugin_result['templating_detected'] is not None and
-            extra_vars['templating_detected'] is not None and
-            extra_vars['templating_detected'] !=
-                plugin_result['templating_detected']
+            templating_detected is not None
+            and extra_vars[TEMPLATING_DETECTED] is not None
+            and extra_vars[TEMPLATING_DETECTED] != templating_detected
         ):
             # Don't allow subsequent plugins with different templating_detected
             raise ParsecError(
                 "Can't merge templating languages "
-                f"{extra_vars['templating_detected']} and "
-                f"{plugin_result['templating_detected']}"
+                f"{extra_vars[TEMPLATING_DETECTED]} and "
+                f"{templating_detected}"
             )
-        elif (
-            'templating_detected' in plugin_result and
-            plugin_result['templating_detected'] is not None
-        ):
-            extra_vars['templating_detected'] = plugin_result[
-                'templating_detected'
-            ]
+        else:
+            extra_vars[TEMPLATING_DETECTED] = templating_detected
 
     return extra_vars
 
@@ -352,8 +358,8 @@ def merge_template_vars(
         >>> merge_template_vars(a, b)
         {'FOO': 42, 'BAZ': 3.14159, 'BAR': 'Hello World'}
     """
-    if plugin_result['templating_detected'] is not None:
-        plugin_tvars = plugin_result['template_variables']
+    if plugin_result[TEMPLATING_DETECTED] is not None:
+        plugin_tvars = plugin_result[TEMPLATE_VARIABLES]
         will_be_overwritten = (
             native_tvars.keys() &
             plugin_tvars.keys()
@@ -456,7 +462,7 @@ def read_and_proc(
     do_jinja2 = True
     do_contin = True
 
-    extra_vars = process_plugins(Path(fpath).parent, opts)
+    extra_vars = process_plugins(fpath, opts)
 
     if not template_vars:
         template_vars = {}
@@ -483,12 +489,12 @@ def read_and_proc(
 
     # Fail if templating_detected ≠ hashbang
     process_with = hashbang_and_plugin_templating_clash(
-        extra_vars['templating_detected'], flines
+        extra_vars[TEMPLATING_DETECTED], flines
     )
     # process with EmPy
     if do_empy:
         if (
-            extra_vars['templating_detected'] == 'empy' and
+            extra_vars[TEMPLATING_DETECTED] == 'empy' and
             not process_with and
             process_with != 'empy'
         ):
@@ -508,7 +514,7 @@ def read_and_proc(
     # process with Jinja2
     if do_jinja2:
         if (
-            extra_vars['templating_detected'] == 'jinja2' and
+            extra_vars[TEMPLATING_DETECTED] == 'jinja2' and
             not process_with and
             process_with != 'jinja2'
         ):
diff --git a/tests/unit/parsec/test_fileparse.py b/tests/unit/parsec/test_fileparse.py
index c9732e9a315..569632e131d 100644
--- a/tests/unit/parsec/test_fileparse.py
+++ b/tests/unit/parsec/test_fileparse.py
@@ -32,6 +32,7 @@
 )
 from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 from cylc.flow.parsec.fileparse import (
+    EXTRA_VARS_TEMPLATE,
     _prepend_old_templatevars,
     _get_fpath_for_source,
     get_cylc_env_vars,
@@ -39,6 +40,7 @@
     addsect,
     multiline,
     parse,
+    process_plugins,
     read_and_proc,
     merge_template_vars
 )
@@ -741,7 +743,7 @@ def test_get_fpath_for_source(tmp_path):
 
 def test_user_has_no_cwd(tmp_path):
     """Test we can parse a config file even if cwd does not exist."""
-    cwd = tmp_path/"cwd"
+    cwd = tmp_path / "cwd"
     os.mkdir(cwd)
     os.chdir(cwd)
     os.rmdir(cwd)
@@ -765,15 +767,42 @@ def test_get_cylc_env_vars(monkeypatch):
         {
             "CYLC_VERSION": "betwixt",
             "CYLC_ENV_NAME": "between",
-            "CYLC_QUESTION": "que?", 
-            "CYLC_ANSWER": "42", 
+            "CYLC_QUESTION": "que?",
+            "CYLC_ANSWER": "42",
             "FOO": "foo"
         }
     )
     assert (
         get_cylc_env_vars() == {
-            "CYLC_QUESTION": "que?", 
-            "CYLC_ANSWER": "42", 
+            "CYLC_QUESTION": "que?",
+            "CYLC_ANSWER": "42",
         }
     )
 
+
+class EntryPointWrapper:
+    """Wraps a method to make it look like an entry point."""
+
+    def __init__(self, fcn):
+        self.name = fcn.__name__
+        self.fcn = fcn
+
+    def load(self):
+        return self.fcn
+
+
+@EntryPointWrapper
+def pre_configure_basic(*_, **__):
+    """Simple plugin that returns one env var and one template var."""
+    return {'env': {'foo': 44}, 'template_variables': {}}
+
+
+def test_plugins_not_called_on_global_config(monkeypatch):
+    monkeypatch.setattr(
+        'cylc.flow.parsec.fileparse.iter_entry_points',
+        lambda x: [pre_configure_basic]
+    )
+    result = process_plugins('/pennine/way/flow.cylc', {})
+    assert result != EXTRA_VARS_TEMPLATE
+    result = process_plugins('/appalachian/trail/global.cylc', {})
+    assert result == EXTRA_VARS_TEMPLATE
diff --git a/tests/unit/plugins/test_pre_configure.py b/tests/unit/plugins/test_pre_configure.py
index e9f91054e12..5f54571e741 100644
--- a/tests/unit/plugins/test_pre_configure.py
+++ b/tests/unit/plugins/test_pre_configure.py
@@ -68,7 +68,7 @@ def test_pre_configure(monkeypatch):
         'cylc.flow.parsec.fileparse.iter_entry_points',
         lambda x: [pre_configure_basic]
     )
-    extra_vars = process_plugins(None, None)
+    extra_vars = process_plugins('/', None)
     assert extra_vars == {
         'env': {
             'ANSWER': '42'
@@ -90,7 +90,7 @@ def test_pre_configure_duplicate(monkeypatch):
         ]
     )
     with pytest.raises(ParsecError):
-        process_plugins(None, None)
+        process_plugins('/', None)
 
 
 def test_pre_configure_templating_detected(monkeypatch):
@@ -103,7 +103,7 @@ def test_pre_configure_templating_detected(monkeypatch):
         ]
     )
     with pytest.raises(ParsecError):
-        process_plugins(None, None)
+        process_plugins('/', None)
 
 
 def test_pre_configure_exception(monkeypatch):
@@ -113,7 +113,7 @@ def test_pre_configure_exception(monkeypatch):
         lambda x: [pre_configure_error]
     )
     with pytest.raises(PluginError) as exc_ctx:
-        process_plugins(None, None)
+        process_plugins('/', None)
     # the context of the original error should be preserved in the raised
     # exception
     assert exc_ctx.value.entry_point == 'cylc.pre_configure'

From 5518b48dccb8d785c998b6c3173a045534ffa49e Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 12 Dec 2023 12:54:56 +0000
Subject: [PATCH 059/110] Validate `cylc clean --timeout` option more
 thoroughly

---
 cylc/flow/clean.py                      |  6 ----
 cylc/flow/scripts/clean.py              | 33 ++++++++++++++++++-
 tests/functional/cylc-clean/01-remote.t | 11 ++-----
 tests/unit/scripts/test_clean.py        | 44 +++++++++++++++++++++++--
 tests/unit/test_clean.py                | 25 +++++---------
 5 files changed, 86 insertions(+), 33 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index f16996ee635..c4434dc9244 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -44,9 +44,6 @@
     Union,
 )
 
-from metomi.isodatetime.exceptions import ISO8601SyntaxError
-from metomi.isodatetime.parsers import DurationParser
-
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.exceptions import (
@@ -363,9 +360,6 @@ def remote_clean(
             f"Cannot clean {id_} on remote platforms as the workflow database "
             f"is out of date/inconsistent with the global config - {exc}")
 
-    with suppress(ISO8601SyntaxError):
-        timeout = str(DurationParser().parse(timeout).get_seconds())
-
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(
         _remote_clean_cmd, id_=id_, rm_dirs=rm_dirs, timeout=timeout
diff --git a/cylc/flow/scripts/clean.py b/cylc/flow/scripts/clean.py
index a002f1d90ab..2166a98f055 100644
--- a/cylc/flow/scripts/clean.py
+++ b/cylc/flow/scripts/clean.py
@@ -64,6 +64,9 @@
 import sys
 from typing import TYPE_CHECKING, Iterable, List, Tuple
 
+from metomi.isodatetime.exceptions import ISO8601SyntaxError
+from metomi.isodatetime.parsers import DurationParser
+
 from cylc.flow import LOG
 from cylc.flow.clean import init_clean, get_contained_workflows
 from cylc.flow.exceptions import CylcError, InputError
@@ -140,6 +143,24 @@ def get_option_parser():
 CleanOptions = Options(get_option_parser())
 
 
+def parse_timeout(opts: 'Values') -> None:
+    """Parse timeout as ISO 8601 duration or number of seconds."""
+    if opts.remote_timeout:
+        try:
+            timeout = int(
+                DurationParser().parse(opts.remote_timeout).get_seconds()
+            )
+        except ISO8601SyntaxError:
+            try:
+                timeout = int(opts.remote_timeout)
+            except ValueError:
+                raise InputError(
+                    f"Invalid timeout: {opts.remote_timeout}. Must be "
+                    "an ISO 8601 duration or number of seconds."
+                )
+        opts.remote_timeout = str(timeout)
+
+
 def prompt(workflows: Iterable[str]) -> None:
     """Ask user if they want to clean the given set of workflows."""
     print("Would clean the following workflows:")
@@ -220,7 +241,15 @@ async def run(*ids: str, opts: 'Values') -> None:
 
 
 @cli_function(get_option_parser)
-def main(_, opts: 'Values', *ids: str):
+def main(_parser, opts: 'Values', *ids: str):
+    _main(opts, *ids)
+
+
+def _main(opts: 'Values', *ids: str):
+    """Run the clean command.
+
+    This is a separate function for ease of testing.
+    """
     if cylc.flow.flags.verbosity < 2:
         set_timestamps(LOG, False)
 
@@ -229,4 +258,6 @@ def main(_, opts: 'Values', *ids: str):
             "--local and --remote options are mutually exclusive"
         )
 
+    parse_timeout(opts)
+
     asyncio.run(run(*ids, opts=opts))
diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
index 8b15caa4162..67148478eb0 100644
--- a/tests/functional/cylc-clean/01-remote.t
+++ b/tests/functional/cylc-clean/01-remote.t
@@ -25,7 +25,7 @@ SSH_CMD="$(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") ${
 if ! $SSH_CMD command -v 'tree' > '/dev/null'; then
     skip_all "'tree' command not available on remote host ${CYLC_TEST_HOST}"
 fi
-set_test_number 12
+set_test_number 10
 
 # Generate random name for symlink dirs to avoid any clashes with other tests
 SYM_NAME="$(mktemp -u)"
@@ -108,14 +108,9 @@ __TREE__
 
 # -----------------------------------------------------------------------------
 
-TEST_NAME="cylc-clean-timeout"
-run_fail "$TEST_NAME" cylc clean --timeout 'PT0,1S' "$WORKFLOW_NAME"
-dump_std "$TEST_NAME"
-grep_ok 'cylc clean timed out after 0.1s' "${TEST_NAME}.stderr"
-
-
 TEST_NAME="cylc-clean-ok"
-run_ok "$TEST_NAME" cylc clean "$WORKFLOW_NAME"
+run_ok "$TEST_NAME" cylc clean "$WORKFLOW_NAME" --timeout PT2M
+# (timeout opt is covered by unit tests but no harm double-checking here)
 dump_std "$TEST_NAME"
 
 TEST_NAME="run-dir-not-exist-post-clean.local"
diff --git a/tests/unit/scripts/test_clean.py b/tests/unit/scripts/test_clean.py
index dc29d7ba1b6..cc1bf2eff2f 100644
--- a/tests/unit/scripts/test_clean.py
+++ b/tests/unit/scripts/test_clean.py
@@ -16,11 +16,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from typing import Callable, List
+from typing import Callable, List, Type, Union
 
 import pytest
 
-from cylc.flow.scripts.clean import CleanOptions, scan, run
+from cylc.flow.exceptions import InputError
+from cylc.flow.scripts.clean import (
+    CleanOptions, _main, parse_timeout, scan, run
+)
 
 
 async def test_scan(tmp_run_dir):
@@ -88,3 +91,40 @@ async def test_multi(tmp_run_dir: Callable, mute: List[str]):
     mute.clear()
     await run('*', opts=opts)
     assert mute == ['bar/pub/beer', 'baz/run1', 'foo']
+
+
+@pytest.mark.parametrize(
+    'timeout, expected',
+    [('100', '100'),
+     ('PT1M2S', '62'),
+     ('', ''),
+     ('oopsie', InputError),
+     (' ', InputError)]
+)
+def test_parse_timeout(
+    timeout: str,
+    expected: Union[str, Type[InputError]]
+):
+    """It should accept ISO 8601 format or number of seconds."""
+    opts = CleanOptions(remote_timeout=timeout)
+
+    if expected is InputError:
+        with pytest.raises(expected):
+            parse_timeout(opts)
+    else:
+        parse_timeout(opts)
+        assert opts.remote_timeout == expected
+
+
+@pytest.mark.parametrize(
+    'opts, expected_msg',
+    [
+        ({'local_only': True, 'remote_only': True}, "mutually exclusive"),
+        ({'remote_timeout': 'oops'}, "Invalid timeout"),
+    ]
+)
+def test_bad_user_input(opts: dict, expected_msg: str, mute):
+    """It should raise an InputError for bad user input."""
+    with pytest.raises(InputError) as exc_info:
+        _main(CleanOptions(**opts), 'blah')
+    assert expected_msg in str(exc_info.value)
diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py
index f5c46ee3f73..a0c0ccdda1e 100644
--- a/tests/unit/test_clean.py
+++ b/tests/unit/test_clean.py
@@ -964,25 +964,15 @@ def mocked_remote_clean_cmd_side_effect(id_, platform, timeout, rm_dirs):
             assert f"{p_name} - {PlatformError.MSG_TIDY}" in caplog.text
 
 
-@pytest.mark.parametrize(
-    'timeout, expected',
-    [('100', '100'),
-     ('PT1M2S', '62.0')]
-)
 def test_remote_clean__timeout(
     monkeymock: MonkeyMock,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    timeout: str,
-    expected: str,
 ):
-    """Test remote_clean() timeout.
-
-    - It should accept ISO 8601 format or number of seconds.
-    - It should give a sensible error message for return code 124.
+    """Test remote_clean() gives a sensible error message for return code 124.
     """
     caplog.set_level(logging.ERROR, CYLC_LOG)
-    mock_remote_clean_cmd = monkeymock(
+    monkeymock(
         'cylc.flow.clean._remote_clean_cmd',
         spec=_remote_clean_cmd,
         return_value=mock.Mock(
@@ -995,10 +985,13 @@ def test_remote_clean__timeout(
     )
 
     with pytest.raises(CylcError):
-        cylc_clean.remote_clean('blah', 'blah', timeout)
-    _, kwargs = mock_remote_clean_cmd.call_args
-    assert kwargs['timeout'] == expected
-    assert f"cylc clean timed out after {expected}s" in caplog.text
+        cylc_clean.remote_clean(
+            'blah', platform_names=['blah'], timeout='blah'
+        )
+    assert "cylc clean timed out" in caplog.text
+    # No need to log the remote clean cmd etc. for timeout
+    assert "ssh" not in caplog.text.lower()
+    assert "stderr" not in caplog.text.lower()
 
 
 @pytest.mark.parametrize(

From ae25de01971641543b41ec2df03564a9de59212f Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 17 Nov 2023 11:53:08 +0000
Subject: [PATCH 060/110] tests/i: add reflog utility

* Add an itegration test pattern for implementing reference tests.
---
 cylc/flow/task_pool.py                |   5 +
 tests/integration/conftest.py         | 140 ++++++++++++++++++++++++++
 tests/integration/test_examples.py    |  42 ++++++++
 tests/integration/tui/test_updater.py |  11 +-
 tests/integration/utils/flow_tools.py |  17 +++-
 5 files changed, 211 insertions(+), 4 deletions(-)

diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index ac0e9f44e31..851ee1e39b7 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1400,9 +1400,11 @@ def remove_if_complete(self, itask):
                 - retain and recompute runahead
                   (C7 failed tasks don't count toward runahead limit)
         """
+        ret = False
         if cylc.flow.flags.cylc7_back_compat:
             if not itask.state(TASK_STATUS_FAILED, TASK_OUTPUT_SUBMIT_FAILED):
                 self.remove(itask, 'finished')
+                ret = True
             if self.compute_runahead():
                 self.release_runahead_tasks()
         else:
@@ -1416,11 +1418,14 @@ def remove_if_complete(self, itask):
             else:
                 # Remove as completed.
                 self.remove(itask, 'finished')
+                ret = True
                 if itask.identity == self.stop_task_id:
                     self.stop_task_finished = True
                 if self.compute_runahead():
                     self.release_runahead_tasks()
 
+        return ret
+
     def spawn_on_all_outputs(
         self, itask: TaskProxy, completed_only: bool = False
     ) -> None:
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index f2f24b09ab5..120d64d24ce 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -20,6 +20,7 @@
 from pathlib import Path
 import pytest
 from shutil import rmtree
+from time import time
 from typing import List, TYPE_CHECKING, Set, Tuple, Union
 
 from cylc.flow.config import WorkflowConfig
@@ -32,8 +33,10 @@
     install as cylc_install,
     get_option_parser as install_gop
 )
+from cylc.flow.util import serialise
 from cylc.flow.wallclock import get_current_time_string
 from cylc.flow.workflow_files import infer_latest_run_from_id
+from cylc.flow.workflow_status import StopMode
 
 from .utils import _rm_if_empty
 from .utils.flow_tools import (
@@ -473,3 +476,140 @@ def _inner(source, **kwargs):
         workflow_id = infer_latest_run_from_id(workflow_id)
         return workflow_id
     yield _inner
+
+
+@pytest.fixture
+def reflog():
+    """Integration test version of the --reflog CLI option.
+
+    This returns a set which captures task triggers.
+
+    Note, you'll need to call this on the scheduler *after* you have started
+    it.
+
+    Args:
+        schd:
+            The scheduler to capture triggering information for.
+        flow_nums:
+            If True, the flow numbers of the task being triggered will be added
+            to the end of each entry.
+
+    Returns:
+        tuple
+
+        (task, triggers):
+            If flow_nums == False
+        (task, flow_nums, triggers):
+            If flow_nums == True
+
+        task:
+            The [relative] task ID e.g. "1/a".
+        flow_nums:
+            The serialised flow nums e.g. ["1"].
+        triggers:
+            Sorted tuple of the trigger IDs, e.g. ("1/a", "2/b").
+
+    """
+
+    def _reflog(schd, flow_nums=False):
+        submit_task_jobs = schd.task_job_mgr.submit_task_jobs
+        triggers = set()
+
+        def _submit_task_jobs(*args, **kwargs):
+            nonlocal submit_task_jobs, triggers, flow_nums
+            itasks = submit_task_jobs(*args, **kwargs)
+            for itask in itasks:
+                deps = tuple(sorted(itask.state.get_resolved_dependencies()))
+                if flow_nums:
+                    triggers.add(
+                        (itask.identity, serialise(itask.flow_nums), deps or None)
+                    )
+                else:
+                    triggers.add((itask.identity, deps or None))
+            return itasks
+
+        schd.task_job_mgr.submit_task_jobs = _submit_task_jobs
+
+        return triggers
+
+    return _reflog
+
+
+@pytest.fixture
+def complete():
+    """Wait for the workflow, or tasks within it to complete.
+
+    Args:
+        schd:
+            The scheduler to await.
+        tokens_list:
+            If specified, this will wait for the tasks represented by these
+            tokens to be marked as completed by the task pool.
+        stop_mode:
+            If tokens_list is not provided, this will wait for the scheduler
+            to be shutdown with the specified mode (default = AUTO, i.e.
+            workflow completed normally).
+        timeout:
+            Max time to wait for the condition to be met.
+
+            Note, if you need to increase this, you might want to rethink your
+            test.
+
+            Note, use this timeout rather than wrapping the complete call with
+            async_timeout (handles shutdown logic more cleanly).
+
+    """
+    async def _complete(
+        schd,
+        *tokens_list,
+        stop_mode=StopMode.AUTO,
+        timeout=60,
+    ):
+        start_time = time()
+        tokens_list = [tokens.task for tokens in tokens_list]
+
+        # capture task completion
+        remove_if_complete = schd.pool.remove_if_complete
+
+        def _remove_if_complete(itask):
+            ret = remove_if_complete(itask)
+            if ret and itask.tokens.task in tokens_list:
+                tokens_list.remove(itask.tokens.task)
+            return ret
+
+        schd.pool.remove_if_complete = _remove_if_complete
+
+        # capture workflow shutdown
+        set_stop = schd._set_stop
+        has_shutdown = False
+
+        def _set_stop(mode=None):
+            nonlocal has_shutdown, stop_mode
+            if mode == stop_mode:
+                has_shutdown = True
+                return set_stop(mode)
+            else:
+                set_stop(mode)
+                raise Exception(f'Workflow bailed with stop mode = {mode}')
+
+        schd._set_stop = _set_stop
+
+        # determine the completion condition
+        if tokens_list:
+            condition = lambda: bool(tokens_list)
+        else:
+            condition = lambda: bool(not has_shutdown)
+
+        # wait for the condition to be met
+        while condition():
+            # allow the main loop to advance
+            await asyncio.sleep(0)
+            if time() - start_time > timeout:
+                raise Exception(
+                    f'Timeout waiting for {", ".join(map(str, tokens_list))}'
+                )
+
+        # restore regular shutdown logic
+        schd._set_stop = set_stop
+
+    return _complete
diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py
index 02a1bb0a497..da8b156503d 100644
--- a/tests/integration/test_examples.py
+++ b/tests/integration/test_examples.py
@@ -229,3 +229,45 @@ async def test_db_select(one, start, db_select):
     results = db_select(
         schd, False, 'task_states', name='one', status='waiting')
     assert len(results) == 1
+
+
+async def test_reflog(flow, scheduler, run, reflog, complete):
+    """Test the triggering of tasks.
+
+    This is the integration test version of "reftest" in the funtional tests.
+
+    It works by capturing the triggers which caused each submission so that
+    they can be compared with the expected outcome.
+    """
+    id_ = flow({
+        'scheduler': {
+            'allow implicit tasks': 'True',
+        },
+        'scheduling': {
+            'initial cycle point': '1',
+            'final cycle point': '1',
+            'cycling mode': 'integer',
+            'graph': {
+                'P1': '''
+                    a => b => c
+                    x => b => z
+                    b[-P1] => b
+                '''
+            }
+        }
+    })
+    schd = scheduler(id_, paused_start=False)
+
+    async with run(schd):
+        triggers = reflog(schd)  # Note: add flow_nums=True to capture flows
+        await complete(schd)
+
+    assert triggers == {
+        # 1/a was triggered by nothing (i.e. it's parentless)
+        ('1/a', None),
+        # 1/b was triggered by three tasks (note the pre-initial dependency)
+        ('1/b', ('0/b', '1/a', '1/x')),
+        ('1/c', ('1/b',)),
+        ('1/x', None),
+        ('1/z', ('1/b',)),
+    }
diff --git a/tests/integration/tui/test_updater.py b/tests/integration/tui/test_updater.py
index b3daac5a328..2d9927ca5eb 100644
--- a/tests/integration/tui/test_updater.py
+++ b/tests/integration/tui/test_updater.py
@@ -147,7 +147,16 @@ async def test_filters(one_conf, flow, scheduler, run, updater):
             'graph': {
                 'R1': 'a & b & c',
             }
-        }
+        },
+        'runtime': {
+            # TODO: remove this runtime section in
+            # https://github.com/cylc/cylc-flow/pull/5721
+            'root': {
+                'simulation': {
+                    'default run length': 'PT1M',
+                },
+            },
+        },
     }, name='one'), paused_start=True)
     two = scheduler(flow(one_conf, name='two'))
     tre = scheduler(flow(one_conf, name='tre'))
diff --git a/tests/integration/utils/flow_tools.py b/tests/integration/utils/flow_tools.py
index 69cf3e065f1..9270f12ee8d 100644
--- a/tests/integration/utils/flow_tools.py
+++ b/tests/integration/utils/flow_tools.py
@@ -53,7 +53,7 @@ def _make_src_flow(src_path, conf):
 def _make_flow(
     cylc_run_dir: Union[Path, str],
     test_dir: Path,
-    conf: Union[dict, str],
+    conf: dict,
     name: Optional[str] = None,
     id_: Optional[str] = None,
 ) -> str:
@@ -66,8 +66,19 @@ def _make_flow(
         flow_run_dir = (test_dir / name)
     flow_run_dir.mkdir(parents=True, exist_ok=True)
     id_ = str(flow_run_dir.relative_to(cylc_run_dir))
-    if isinstance(conf, dict):
-        conf = flow_config_str(conf)
+    conf = flow_config_str({
+        # override the default simulation runtime logic to make
+        # tasks execute instantly
+        # NOTE: this is prepended so it can be overwritten
+        'runtime': {
+            'root': {
+                'simulation': {
+                    'default run length': 'PT0S',
+                },
+            },
+        },
+        **conf,
+    })
     with open((flow_run_dir / WorkflowFiles.FLOW_FILE), 'w+') as flow_file:
         flow_file.write(conf)
     return id_

From be46a11115294ab41f00b3a6a76ffacb29b28d32 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 17 Nov 2023 11:58:59 +0000
Subject: [PATCH 061/110] tests/u, tests/i: set TZ variable before running
 tests

* We were using the pytest-env plugin to run the tests in a non-UTC time
  zone.
* The pytest-env plugin doesn't work with pytest-xdist so this was being
  ignored.
* Also due to the way TZ support works in Python, changing the env var
  whilst Python is running may or may not result in changes.
---
 .github/workflows/test_fast.yml | 4 ++++
 pytest.ini                      | 5 +----
 setup.cfg                       | 1 -
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 5fb95def4e8..ad342b3dfa0 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -24,8 +24,12 @@ jobs:
         include:
           - os: 'macos-latest'
             python-version: '3.7'
+
     env:
+      # Use non-UTC time zone
+      TZ: XXX-09:35
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
diff --git a/pytest.ini b/pytest.ini
index 81df3785cec..9be86cb507c 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -32,13 +32,10 @@ testpaths =
     cylc/flow/
     tests/unit/
     tests/integration/
-env =
-    # a weird timezone to check that tests aren't assuming the local timezone
-    TZ=XXX-09:35
 doctest_optionflags =
     NORMALIZE_WHITESPACE
     IGNORE_EXCEPTION_DETAIL
     ELLIPSIS
 asyncio_mode = auto
 markers=
-    linkcheck: Test links
\ No newline at end of file
+    linkcheck: Test links
diff --git a/setup.cfg b/setup.cfg
index 53b2024a9a3..9a3838de1dc 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -121,7 +121,6 @@ tests =
     pytest-asyncio>=0.17,!=0.23.*
     pytest-cov>=2.8.0
     pytest-xdist>=2
-    pytest-env>=0.6.2
     pytest>=6
     testfixtures>=6.11.0
     towncrier>=23

From 49879a29f1f50bed3db25df816cc8aad24256742 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Mon, 27 Nov 2023 15:41:57 +0000
Subject: [PATCH 062/110] actions: add non-utc job to fast tests

---
 .github/workflows/test_fast.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index ad342b3dfa0..34d151b7737 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -20,14 +20,20 @@ jobs:
       fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3']
+        python-version: ['3.7', '3.8', '3.10', '3.11', '3']
         include:
+          # mac os test
           - os: 'macos-latest'
-            python-version: '3.7'
+            python-version: '3.7'  # oldest supported version
+
+          # non-utc timezone test
+          - os: 'ubuntu-latest'
+            python-version: '3.9'  # not the oldest, not the most recent version
+            time-zone: ['XXX-09:35']
 
     env:
       # Use non-UTC time zone
-      TZ: XXX-09:35
+      TZ: ${{ matrix.time-zone }}
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
 
     steps:

From de7cb176523cbe4f915564d60bf308207cfe8380 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 28 Nov 2023 11:42:05 +0000
Subject: [PATCH 063/110] Update .github/workflows/test_fast.yml

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 .github/workflows/test_fast.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 34d151b7737..6530bb85cd6 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -29,7 +29,7 @@ jobs:
           # non-utc timezone test
           - os: 'ubuntu-latest'
             python-version: '3.9'  # not the oldest, not the most recent version
-            time-zone: ['XXX-09:35']
+            time-zone: 'XXX-09:35'
 
     env:
       # Use non-UTC time zone

From fa340fa24d69fde3ce00a8af44253ece70cbaa58 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 13 Dec 2023 11:12:26 +0000
Subject: [PATCH 064/110] tests/i: update tui screenshot

---
 .../test_scheduler_logs.workflow-configuration-file.html      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
index e3fcdfbab22..8cc2f9af73b 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -3,11 +3,11 @@
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[runtime]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[a]]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[scheduling]                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[graph]]                                                             </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        R1 = a                                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[runtime]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[a]]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>

From 2f599a4dd94416ff8355919b51a4b1838ce289da Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Dec 2023 09:04:50 +0000
Subject: [PATCH 065/110] Bump actions/download-artifact from 3 to 4

Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test_fast.yml       | 2 +-
 .github/workflows/test_functional.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 6530bb85cd6..e954d6ea598 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -138,7 +138,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Download coverage artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       - name: Codecov upload
         uses: codecov/codecov-action@v3
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index 6fad4147345..e200aae156c 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -312,7 +312,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Download coverage artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       - name: Codecov upload
         uses: codecov/codecov-action@v3

From 58b76079d02f5b897f41610a7fc966bcbd2a00f2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Dec 2023 09:04:52 +0000
Subject: [PATCH 066/110] Bump actions/upload-artifact from 3 to 4

Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/bash.yml            | 2 +-
 .github/workflows/test_fast.yml       | 4 ++--
 .github/workflows/test_functional.yml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml
index 4d740df23ac..e7daf22be59 100644
--- a/.github/workflows/bash.yml
+++ b/.github/workflows/bash.yml
@@ -85,7 +85,7 @@ jobs:
 
       - name: Upload artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Upload cylc-run artifact
           path: cylc-run
diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 6530bb85cd6..5c0c9ead20d 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -70,7 +70,7 @@ jobs:
 
       - name: Upload failed tests artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cylc-run (${{ matrix.os }} py-${{ matrix.python-version }})
           path: ~/cylc-run/
@@ -81,7 +81,7 @@ jobs:
           coverage report
 
       - name: Upload coverage artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }}
           path: coverage.xml
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index 6fad4147345..e284a9906b1 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -259,7 +259,7 @@ jobs:
 
       - name: Upload failed tests artifact
         if: failure() && steps.test.outcome == 'failure'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cylc-run (${{ steps.uploadname.outputs.uploadname }})
           path: ~/cylc-run/
@@ -297,7 +297,7 @@ jobs:
           coverage report
 
       - name: Upload coverage artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_${{ steps.uploadname.outputs.uploadname }}
           path: coverage.xml

From b878692950bb9af93b87359d2898f91fddb924a3 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 15 Dec 2023 16:10:00 +0000
Subject: [PATCH 067/110] GH Actions artifact name fixes

---
 .github/workflows/bash.yml            | 2 +-
 .github/workflows/test_functional.yml | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml
index e7daf22be59..eb34e3ad76d 100644
--- a/.github/workflows/bash.yml
+++ b/.github/workflows/bash.yml
@@ -87,5 +87,5 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: Upload cylc-run artifact
+          name: 'cylc-run (bash-${{ matrix.bash-version }})'
           path: cylc-run
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index e284a9906b1..35c01ba7f2f 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -251,6 +251,7 @@ jobs:
               -exec echo '====== {} ======' \; -exec cat '{}' \;
 
       - name: Set artifact upload name
+        if: always()
         id: uploadname
         run: |
           # artifact name cannot contain '/' characters

From 8c4cd7e5b79e6258a79332bacd6b181a368d5be6 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Mon, 18 Dec 2023 09:39:08 +0000
Subject: [PATCH 068/110] Test.example replace a reftest (#5860)

Update tests/integration/triggering.py
Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 tests/functional/triggering/09-fail.t         | 22 ---------
 tests/functional/triggering/09-fail/flow.cylc | 12 -----
 .../triggering/09-fail/reference.log          |  4 --
 tests/integration/test_scan_api.py            |  6 +--
 tests/integration/test_triggering.py          | 46 +++++++++++++++++++
 ...uler_logs.workflow-configuration-file.html | 10 ++--
 tests/integration/utils/flow_tools.py         | 21 ++++-----
 7 files changed, 62 insertions(+), 59 deletions(-)
 delete mode 100644 tests/functional/triggering/09-fail.t
 delete mode 100644 tests/functional/triggering/09-fail/flow.cylc
 delete mode 100644 tests/functional/triggering/09-fail/reference.log
 create mode 100644 tests/integration/test_triggering.py

diff --git a/tests/functional/triggering/09-fail.t b/tests/functional/triggering/09-fail.t
deleted file mode 100644
index 0f4698ec2ac..00000000000
--- a/tests/functional/triggering/09-fail.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test fail triggering
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/triggering/09-fail/flow.cylc b/tests/functional/triggering/09-fail/flow.cylc
deleted file mode 100644
index b4048e27ae5..00000000000
--- a/tests/functional/triggering/09-fail/flow.cylc
+++ /dev/null
@@ -1,12 +0,0 @@
-[scheduler]
-    [[events]]
-        expected task failures = 1/foo
-
-[scheduling]
-    [[graph]]
-        R1 = "foo:fail => bar"
-[runtime]
-    [[foo]]
-        script = "false"
-    [[bar]]
-        script = "true"
diff --git a/tests/functional/triggering/09-fail/reference.log b/tests/functional/triggering/09-fail/reference.log
deleted file mode 100644
index 1a9f846c98d..00000000000
--- a/tests/functional/triggering/09-fail/reference.log
+++ /dev/null
@@ -1,4 +0,0 @@
-Initial point: 1
-Final point: 1
-1/foo -triggered off []
-1/bar -triggered off ['1/foo']
diff --git a/tests/integration/test_scan_api.py b/tests/integration/test_scan_api.py
index ea25035e804..71d83124f77 100644
--- a/tests/integration/test_scan_api.py
+++ b/tests/integration/test_scan_api.py
@@ -80,14 +80,14 @@ async def flows(mod_flow, mod_scheduler, mod_run, mod_one_conf):
                 }
             },
             'runtime': {
-                'one': {
-                    'execution time limit': 'PT10S'
+                'foo': {
+                    'simulation': {'default run length': 'PT10S'}
                 }
             }
         },
         name='-running-'
     )
-    schd2 = mod_scheduler(reg2, run_mode='simulation', paused_start=False)
+    schd2 = mod_scheduler(reg2, paused_start=False)
 
     # run cylc run
     async with mod_run(schd1):
diff --git a/tests/integration/test_triggering.py b/tests/integration/test_triggering.py
new file mode 100644
index 00000000000..f086d46aa4c
--- /dev/null
+++ b/tests/integration/test_triggering.py
@@ -0,0 +1,46 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+async def test_fail(flow, scheduler, run, reflog, complete, validate):
+    """Test triggering on :fail"""
+    id_ = flow({
+        'scheduler': {
+            'allow implicit tasks': 'true'
+        },
+        'scheduling': {
+            'graph': {
+                'R1': 'foo:failed => bar'
+            }
+        },
+        'runtime': {
+            'root': {
+                'simulation': {'default run length': 'PT0S'}
+            },
+            'foo': {
+                'simulation': {'fail cycle points': 'all'}
+            }
+        }
+    })
+    schd = scheduler(id_, paused_start=False)
+
+    async with run(schd):
+        triggers = reflog(schd)
+        await complete(schd)
+
+    assert triggers == {
+        ('1/foo', None),
+        ('1/bar', ('1/foo',)),
+    }
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
index 8cc2f9af73b..04ebb27ff79 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -3,14 +3,14 @@
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[runtime]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[a]]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[scheduling]                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[graph]]                                                             </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        R1 = a                                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[runtime]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[a]]                                                                 </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[root]]                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        [[[simulation]]]                                                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">            default run length = PT0S                                     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/utils/flow_tools.py b/tests/integration/utils/flow_tools.py
index 9270f12ee8d..c26f79c04ac 100644
--- a/tests/integration/utils/flow_tools.py
+++ b/tests/integration/utils/flow_tools.py
@@ -66,19 +66,14 @@ def _make_flow(
         flow_run_dir = (test_dir / name)
     flow_run_dir.mkdir(parents=True, exist_ok=True)
     id_ = str(flow_run_dir.relative_to(cylc_run_dir))
-    conf = flow_config_str({
-        # override the default simulation runtime logic to make
-        # tasks execute instantly
-        # NOTE: this is prepended so it can be overwritten
-        'runtime': {
-            'root': {
-                'simulation': {
-                    'default run length': 'PT0S',
-                },
-            },
-        },
-        **conf,
-    })
+    # set the default simulation runtime to zero (can be overridden)
+    (
+        conf.setdefault('runtime', {})
+        .setdefault('root', {})
+        .setdefault('simulation', {})
+        .setdefault('default run length', 'PT0S')
+    )
+    conf = flow_config_str(conf)
     with open((flow_run_dir / WorkflowFiles.FLOW_FILE), 'w+') as flow_file:
         flow_file.write(conf)
     return id_

From 3320c050a350a3f0c2622405a2739d28ff827eb9 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 19 Dec 2023 12:28:47 +0000
Subject: [PATCH 069/110]  dump: restrict window to n=0 (#5600)

This restores the old `cylc dump` behaviour of displaying only the pool contents.
---
 changes.d/5600.break.md                       |  3 ++
 cylc/flow/scripts/dump.py                     | 40 +++++++++-----
 tests/functional/runahead/06-release-update.t |  1 -
 tests/integration/scripts/test_dump.py        | 52 +++++++++++++++++++
 4 files changed, 81 insertions(+), 15 deletions(-)
 create mode 100644 changes.d/5600.break.md
 create mode 100644 tests/integration/scripts/test_dump.py

diff --git a/changes.d/5600.break.md b/changes.d/5600.break.md
new file mode 100644
index 00000000000..fd11f650ff5
--- /dev/null
+++ b/changes.d/5600.break.md
@@ -0,0 +1,3 @@
+The `cylc dump` command now only shows active tasks (e.g. running & queued
+tasks). This restores its behaviour of only showing the tasks which currently
+exist in the pool as it did in Cylc 7 and earlier versions of Cylc 8.
diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py
index c62bf6719aa..d3ffda59fce 100755
--- a/cylc/flow/scripts/dump.py
+++ b/cylc/flow/scripts/dump.py
@@ -20,6 +20,9 @@
 
 Print information about a running workflow.
 
+This command can provide information about active tasks, e.g. running or queued
+tasks. For more detailed view of the workflow see `cylc tui` or `cylc gui`.
+
 For command line monitoring:
 * `cylc tui`
 * `watch cylc dump WORKFLOW_ID` works for small simple workflows
@@ -28,20 +31,21 @@
 its prerequisites and outputs, see 'cylc show'.
 
 Examples:
-  # Display the state of all running tasks, sorted by cycle point:
+  # Display the state of all active tasks, sorted by cycle point:
   $ cylc dump --tasks --sort WORKFLOW_ID | grep running
 
-  # Display the state of all tasks in a particular cycle point:
+  # Display the state of all active in a particular cycle point:
   $ cylc dump -t WORKFLOW_ID | grep 2010082406
 """
 
-from graphene.utils.str_converters import to_snake_case
+import asyncio
 import json
-import sys
 from typing import TYPE_CHECKING
 
+from graphene.utils.str_converters import to_snake_case
+
 from cylc.flow.exceptions import CylcError
-from cylc.flow.id_cli import parse_id
+from cylc.flow.id_cli import parse_id_async
 from cylc.flow.option_parsers import (
     WORKFLOW_ID_ARG_DOC,
     CylcOptionParser as COP,
@@ -59,6 +63,7 @@
   name
   cyclePoint
   state
+  graphDepth
   isHeld
   isQueued
   isRunahead
@@ -179,7 +184,11 @@ def get_option_parser():
 
 @cli_function(get_option_parser)
 def main(_, options: 'Values', workflow_id: str) -> None:
-    workflow_id, *_ = parse_id(
+    asyncio.run(dump(workflow_id, options))
+
+
+async def dump(workflow_id, options, write=print):
+    workflow_id, *_ = await parse_id_async(
         workflow_id,
         constraint='workflows',
     )
@@ -195,6 +204,9 @@ def main(_, options: 'Values', workflow_id: str) -> None:
     else:
         sort_args = {'keys': ['name', 'cyclePoint']}
 
+    # retrict to the n=0 window
+    graph_depth = 0
+
     if options.disp_form == "raw":
         query = f'''
             {TASK_SUMMARY_FRAGMENT}
@@ -203,10 +215,10 @@ def main(_, options: 'Values', workflow_id: str) -> None:
             query ($wFlows: [ID]!, $sortBy: SortArgs) {{
               workflows (ids: $wFlows, stripNull: false) {{
                 ...wFlow
-                taskProxies (sort: $sortBy) {{
+                taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{
                   ...tProxy
                 }}
-                familyProxies (sort: $sortBy) {{
+                familyProxies (sort: $sortBy, graphDepth: {graph_depth}) {{
                   ...fProxy
                 }}
               }}
@@ -224,7 +236,7 @@ def main(_, options: 'Values', workflow_id: str) -> None:
             {TASK_SUMMARY_FRAGMENT}
             query ($wFlows: [ID]!, $sortBy: SortArgs) {{
               workflows (ids: $wFlows, stripNull: false) {{
-                taskProxies (sort: $sortBy) {{
+                taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{
                   ...tProxy
                 }}
               }}
@@ -235,15 +247,15 @@ def main(_, options: 'Values', workflow_id: str) -> None:
         'variables': {'wFlows': [workflow_id], 'sortBy': sort_args}
     }
 
-    workflows = pclient('graphql', query_kwargs)
+    workflows = await pclient.async_request('graphql', query_kwargs)
 
     try:
         for summary in workflows['workflows']:
             if options.disp_form == "raw":
                 if options.pretty:
-                    sys.stdout.write(json.dumps(summary, indent=4) + '\n')
+                    write(json.dumps(summary, indent=4))
                 else:
-                    print(summary)
+                    write(summary)
             else:
                 if options.disp_form != "tasks":
                     node_urls = {
@@ -261,7 +273,7 @@ def main(_, options: 'Values', workflow_id: str) -> None:
                     del summary['families']
                     del summary['meta']
                     for key, value in sorted(summary.items()):
-                        print(
+                        write(
                             f'{to_snake_case(key).replace("_", " ")}={value}')
                 else:
                     for item in summary['taskProxies']:
@@ -282,7 +294,7 @@ def main(_, options: 'Values', workflow_id: str) -> None:
                                       else 'not-runahead')
                         if options.show_flows:
                             values.append(item['flowNums'])
-                        print(', '.join(values))
+                        write(', '.join(values))
     except Exception as exc:
         raise CylcError(
             json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n')
diff --git a/tests/functional/runahead/06-release-update.t b/tests/functional/runahead/06-release-update.t
index d52b12272a6..c4ab28530e3 100644
--- a/tests/functional/runahead/06-release-update.t
+++ b/tests/functional/runahead/06-release-update.t
@@ -35,7 +35,6 @@ sleep 10
 # (gratuitous use of --flows for test coverage)
 cylc dump --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log'
 cmp_ok 'log' - <<__END__
-bar,$NEXT1,waiting,[1]
 foo,$NEXT1,waiting,[1]
 __END__
 
diff --git a/tests/integration/scripts/test_dump.py b/tests/integration/scripts/test_dump.py
new file mode 100644
index 00000000000..681167a564f
--- /dev/null
+++ b/tests/integration/scripts/test_dump.py
@@ -0,0 +1,52 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the "cylc dump" command."""
+
+from cylc.flow.option_parsers import (
+    Options,
+)
+from cylc.flow.scripts.dump import (
+    dump,
+    get_option_parser,
+)
+
+
+DumpOptions = Options(get_option_parser())
+
+
+async def test_dump_tasks(flow, scheduler, start):
+    """It should show n=0 tasks.
+
+    See: https://github.com/cylc/cylc-flow/pull/5600
+    """
+    id_ = flow({
+        'scheduler': {
+            'allow implicit tasks': 'true',
+        },
+        'scheduling': {
+            'graph': {
+                'R1': 'a => b => c',
+            },
+        },
+    })
+    schd = scheduler(id_)
+    async with start(schd):
+        # schd.release_queued_tasks()
+        await schd.update_data_structure()
+        ret = []
+        await dump(id_, DumpOptions(disp_form='tasks'), write=ret.append)
+        assert ret == ['a, 1, waiting, not-held, queued, not-runahead']

From 031fc0560e930fcd63d5e6a543396d668468e7f1 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 20 Dec 2023 10:03:13 +0000
Subject: [PATCH 070/110] Feat.lint obsolete vars (#5879)

* lint.warn about deprecated templating

* Update changes.d/5879.feat.md

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>

* Update cylc/flow/scripts/lint.py

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>

* Clarify templated vars which have to be set in metadata sections.
Fix an rst bug.

* fix broken code

---------

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 changes.d/5879.feat.md          |  1 +
 cylc/flow/scripts/lint.py       | 85 ++++++++++++++++++++++++++++++---
 tests/unit/scripts/test_lint.py |  2 +-
 3 files changed, 81 insertions(+), 7 deletions(-)
 create mode 100644 changes.d/5879.feat.md

diff --git a/changes.d/5879.feat.md b/changes.d/5879.feat.md
new file mode 100644
index 00000000000..be4c7e14e94
--- /dev/null
+++ b/changes.d/5879.feat.md
@@ -0,0 +1 @@
+`cylc lint` now warns of use of old templated items such as `%(suite)s`
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index cc06512988c..ef7635f1f0d 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -67,7 +67,7 @@
         TOMLDecodeError,
     )
 from typing import (
-    TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Union)
+    TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union)
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
@@ -106,6 +106,31 @@
 }
 
 
+DEPRECATED_STRING_TEMPLATES = {
+    'suite': 'workflow',
+    'suite_uuid': 'uuid',
+    'batch_sys_name': 'job_runner_name',
+    'batch_sys_job_id': 'job_id',
+    'user@host': 'platform_name',
+    'task_url': '``URL`` (if set in :cylc:conf:`[meta]URL`)',
+    'workflow_url': (
+        '``workflow_URL`` (if set in '
+        ':cylc:conf:`[runtime][<namespace>][meta]URL`)'),
+}
+
+
+LIST_ITEM = '    * '
+
+
+deprecated_string_templates = {
+    key: (
+        re.compile(r'%\(' + key + r'\)s'),
+        value
+    )
+    for key, value in DEPRECATED_STRING_TEMPLATES.items()
+}
+
+
 def check_jinja2_no_shebang(
     line: str,
     file: Path,
@@ -233,6 +258,36 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]:
     return [i for i in OBSOLETE_ENV_VARS if i in line]
 
 
+def check_for_deprecated_task_event_template_vars(
+    line: str
+) -> Optional[Dict[str, str]]:
+    """Look for string variables which are no longer supported
+
+    Examples:
+        >>> this = check_for_deprecated_task_event_template_vars
+
+        >>> this('hello = "My name is %(suite)s"')
+        {'list': '    * %(suite)s ⇒ %(workflow)s'}
+
+        >>> expect = {'list': (
+        ...     '    * %(suite)s ⇒ %(workflow)s    * %(task_url)s'
+        ... ' - get ``URL`` (if set in :cylc:conf:`[meta]URL`)')}
+        >>> this('hello = "My name is %(suite)s, %(task_url)s"') == expect
+        True
+    """
+    result = []
+    for key, (regex, replacement) in deprecated_string_templates.items():
+        search_outcome = regex.findall(line)
+        if search_outcome and ' ' in replacement:
+            result.append(f'%({key})s - get {replacement}')
+        elif search_outcome:
+            result.append(f'%({key})s ⇒ %({replacement})s')
+
+    if result:
+        return {'list': LIST_ITEM + LIST_ITEM.join(result)}
+    return None
+
+
 INDENTATION = re.compile(r'^(\s*)(.*)')
 
 
@@ -584,6 +639,23 @@ def check_indentation(line: str) -> bool:
         ),
         FUNCTION: re.compile(r'rose +date').findall,
     },
+    'U015': {
+        'short': (
+            'Deprecated template variables.'),
+        'rst': (
+            'The following template variables, mostly used in event handlers,'
+            'are deprecated, and should be replaced:'
+            + ''.join([
+                f'\n * ``{old}`` ⇒ {new}'
+                for old, new in DEPRECATED_STRING_TEMPLATES.items()
+            ])
+        ),
+        'url': (
+            'https://cylc.github.io/cylc-doc/stable/html/user-guide/'
+            'writing-workflows/runtime.html#task-event-template-variables'
+        ),
+        FUNCTION: check_for_deprecated_task_event_template_vars,
+    }
 }
 RULESETS = ['728', 'style', 'all']
 EXTRA_TOML_VALIDATION = {
@@ -1077,7 +1149,7 @@ def get_cylc_files(
     'section heading': '\n{title}\n{underline}\n',
     'issue heading': {
         'text': '\n{check}:\n    {summary}\n    {url}\n\n',
-        'rst': '\n`{check} <{url}>`_\n{underline}\n{summary}\n\n',
+        'rst': '\n{url}_\n{underline}\n{summary}\n\n',
     },
     'auto gen message': (
         'U998 and U999 represent automatically generated'
@@ -1122,16 +1194,17 @@ def get_reference(linter, output_type):
                 output += '\n'
             output += '\n* ' + summary
         else:
+            check = get_index_str(meta, index)
             template = issue_heading_template
             url = get_url(meta)
+            if output_type == 'rst':
+                url = f'`{check} <{url}>`' if url else f'{check}'
             msg = template.format(
                 title=index,
-                check=get_index_str(meta, index),
+                check=check,
                 summary=summary,
                 url=url,
-                underline=(
-                    len(get_index_str(meta, index)) + len(url) + 6
-                ) * '^'
+                underline=(len(url) + 1) * '^'
             )
             output += msg
     output += '\n'
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 6ee1018cf96..349854be312 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -119,7 +119,7 @@
             submission failed handler = giaSEHFUIHJ
             failed handler = woo
             execution timeout handler = sdfghjkl
-            expired handler = dafuhj
+            expired handler = %(suite_uuid)s %(user@host)s
             late handler = dafuhj
             submitted handler = dafuhj
             started handler = dafuhj

From 2ccc8ecf2bdbc8e2d1701eb48fd71247528d09dd Mon Sep 17 00:00:00 2001
From: WXTIM <26465611+wxtim@users.noreply.github.com>
Date: Wed, 20 Dec 2023 16:28:52 +0000
Subject: [PATCH 071/110] auto update syntax files

---
 cylc/flow/etc/syntax/cylc.lang | 46 ++++++++++++++++---------------
 cylc/flow/etc/syntax/cylc.xml  | 49 ++++++++++++++++++----------------
 2 files changed, 50 insertions(+), 45 deletions(-)

diff --git a/cylc/flow/etc/syntax/cylc.lang b/cylc/flow/etc/syntax/cylc.lang
index 2de9106f360..0270c7f3cfc 100644
--- a/cylc/flow/etc/syntax/cylc.lang
+++ b/cylc/flow/etc/syntax/cylc.lang
@@ -87,35 +87,37 @@
         -->
         <!--TAG_FOR_AUTO_UPDATE-->
         <keyword>xtriggers</keyword>
+        <keyword>workflow timeout handlers</keyword>
+        <keyword>workflow timeout</keyword>
         <keyword>workflow state polling</keyword>
         <keyword>work sub-directory</keyword>
-        <keyword>warning handler</keyword>
+        <keyword>warning handlers</keyword>
         <keyword>verbose mode</keyword>
         <keyword>user</keyword>
         <keyword>to</keyword>
         <keyword>title</keyword>
-        <keyword>workflow timeout handler</keyword>
-        <keyword>workflow timeout</keyword>
         <keyword>time limit buffer</keyword>
         <keyword>templates</keyword>
         <keyword>task parameters</keyword>
         <keyword>task event batch interval</keyword>
-        <keyword>succeeded handler</keyword>
-        <keyword>submitted handler</keyword>
-        <keyword>submission timeout handler</keyword>
+        <keyword>succeeded handlers</keyword>
+        <keyword>submitted handlers</keyword>
+        <keyword>submission timeout handlers</keyword>
         <keyword>submission timeout</keyword>
-        <keyword>submission retry handler</keyword>
+        <keyword>submission retry handlers</keyword>
         <keyword>submission retry delays</keyword>
         <keyword>submission polling intervals</keyword>
-        <keyword>submission failed handler</keyword>
+        <keyword>submission failed handlers</keyword>
         <keyword>stop after cycle point</keyword>
-        <keyword>startup handler</keyword>
-        <keyword>started handler</keyword>
-        <keyword>stall handler</keyword>
+        <keyword>startup handlers</keyword>
+        <keyword>started handlers</keyword>
+        <keyword>stall timeout handlers</keyword>
+        <keyword>stall timeout</keyword>
+        <keyword>stall handlers</keyword>
         <keyword>speedup factor</keyword>
         <keyword>special tasks</keyword>
         <keyword>simulation</keyword>
-        <keyword>shutdown handler</keyword>
+        <keyword>shutdown handlers</keyword>
         <keyword>sequential</keyword>
         <keyword>script</keyword>
         <keyword>scheduling</keyword>
@@ -123,10 +125,11 @@
         <keyword>runtime</keyword>
         <keyword>runahead limit</keyword>
         <keyword>run-dir</keyword>
-        <keyword>retry handler</keyword>
+        <keyword>retry handlers</keyword>
         <keyword>retrieve job logs retry delays</keyword>
         <keyword>retrieve job logs max size</keyword>
         <keyword>retrieve job logs</keyword>
+        <keyword>restart timeout</keyword>
         <keyword>remote</keyword>
         <keyword>queues</keyword>
         <keyword>pre-script</keyword>
@@ -144,7 +147,7 @@
         <keyword>mail</keyword>
         <keyword>limit</keyword>
         <keyword>late offset</keyword>
-        <keyword>late handler</keyword>
+        <keyword>late handlers</keyword>
         <keyword>job</keyword>
         <keyword>interval</keyword>
         <keyword>install</keyword>
@@ -153,7 +156,7 @@
         <keyword>init-script</keyword>
         <keyword>inherit</keyword>
         <keyword>include</keyword>
-        <keyword>inactivity timeout handler</keyword>
+        <keyword>inactivity timeout handlers</keyword>
         <keyword>inactivity timeout</keyword>
         <keyword>host</keyword>
         <keyword>hold after cycle point</keyword>
@@ -166,14 +169,14 @@
         <keyword>flow.cylc</keyword>
         <keyword>final cycle point constraints</keyword>
         <keyword>final cycle point</keyword>
-        <keyword>failed handler</keyword>
+        <keyword>failed handlers</keyword>
         <keyword>fail try 1 only</keyword>
         <keyword>fail cycle points</keyword>
         <keyword>external-trigger</keyword>
-        <keyword>expired handler</keyword>
+        <keyword>expired handlers</keyword>
         <keyword>expected task failures</keyword>
         <keyword>exit-script</keyword>
-        <keyword>execution timeout handler</keyword>
+        <keyword>execution timeout handlers</keyword>
         <keyword>execution timeout</keyword>
         <keyword>execution time limit</keyword>
         <keyword>execution retry delays</keyword>
@@ -184,7 +187,6 @@
         <keyword>environment filter</keyword>
         <keyword>environment</keyword>
         <keyword>env-script</keyword>
-        <keyword>edge penwidth</keyword>
         <keyword>disable task event handlers</keyword>
         <keyword>directives</keyword>
         <keyword>description</keyword>
@@ -194,17 +196,17 @@
         <keyword>cycle point time zone</keyword>
         <keyword>cycle point num expanded year digits</keyword>
         <keyword>cycle point format</keyword>
-        <keyword>custom handler</keyword>
-        <keyword>critical handler</keyword>
+        <keyword>custom handlers</keyword>
+        <keyword>critical handlers</keyword>
         <keyword>clock-trigger</keyword>
         <keyword>clock-expire</keyword>
         <keyword>batch system</keyword>
         <keyword>batch submit command template</keyword>
         <keyword>allow implicit tasks</keyword>
-        <keyword>aborted handler</keyword>
         <keyword>abort on workflow timeout</keyword>
         <keyword>abort on stall timeout</keyword>
         <keyword>abort on inactivity timeout</keyword>
+        <keyword>abort handlers</keyword>
         <keyword>UTC mode</keyword>
         <keyword>URL</keyword>
         <!--END_TAG_FOR_AUTO_UPDATE-->
diff --git a/cylc/flow/etc/syntax/cylc.xml b/cylc/flow/etc/syntax/cylc.xml
index 9f3a90287e8..da11c1215e1 100644
--- a/cylc/flow/etc/syntax/cylc.xml
+++ b/cylc/flow/etc/syntax/cylc.xml
@@ -14,35 +14,37 @@
         The sort is in reverse order to allow longer keywords to take precedence over sub-sets. -->
         <!--TAG_FOR_AUTO_UPDATE-->
         <RegExpr attribute='Keyword' String=' xtriggers '/>
+        <RegExpr attribute='Keyword' String=' workflow timeout handlers '/>
+        <RegExpr attribute='Keyword' String=' workflow timeout '/>
         <RegExpr attribute='Keyword' String=' workflow state polling '/>
         <RegExpr attribute='Keyword' String=' work sub-directory '/>
-        <RegExpr attribute='Keyword' String=' warning handler '/>
+        <RegExpr attribute='Keyword' String=' warning handlers '/>
         <RegExpr attribute='Keyword' String=' verbose mode '/>
         <RegExpr attribute='Keyword' String=' user '/>
         <RegExpr attribute='Keyword' String=' to '/>
         <RegExpr attribute='Keyword' String=' title '/>
-        <RegExpr attribute='Keyword' String=' workflow timeout handler '/>
-        <RegExpr attribute='Keyword' String=' workflow timeout '/>
         <RegExpr attribute='Keyword' String=' time limit buffer '/>
         <RegExpr attribute='Keyword' String=' templates '/>
         <RegExpr attribute='Keyword' String=' task parameters '/>
         <RegExpr attribute='Keyword' String=' task event batch interval '/>
-        <RegExpr attribute='Keyword' String=' succeeded handler '/>
-        <RegExpr attribute='Keyword' String=' submitted handler '/>
-        <RegExpr attribute='Keyword' String=' submission timeout handler '/>
+        <RegExpr attribute='Keyword' String=' succeeded handlers '/>
+        <RegExpr attribute='Keyword' String=' submitted handlers '/>
+        <RegExpr attribute='Keyword' String=' submission timeout handlers '/>
         <RegExpr attribute='Keyword' String=' submission timeout '/>
-        <RegExpr attribute='Keyword' String=' submission retry handler '/>
+        <RegExpr attribute='Keyword' String=' submission retry handlers '/>
         <RegExpr attribute='Keyword' String=' submission retry delays '/>
         <RegExpr attribute='Keyword' String=' submission polling intervals '/>
-        <RegExpr attribute='Keyword' String=' submission failed handler '/>
+        <RegExpr attribute='Keyword' String=' submission failed handlers '/>
         <RegExpr attribute='Keyword' String=' stop after cycle point '/>
-        <RegExpr attribute='Keyword' String=' startup handler '/>
-        <RegExpr attribute='Keyword' String=' started handler '/>
-        <RegExpr attribute='Keyword' String=' stall handler '/>
+        <RegExpr attribute='Keyword' String=' startup handlers '/>
+        <RegExpr attribute='Keyword' String=' started handlers '/>
+        <RegExpr attribute='Keyword' String=' stall timeout handlers '/>
+        <RegExpr attribute='Keyword' String=' stall timeout '/>
+        <RegExpr attribute='Keyword' String=' stall handlers '/>
         <RegExpr attribute='Keyword' String=' speedup factor '/>
         <RegExpr attribute='Keyword' String=' special tasks '/>
         <RegExpr attribute='Keyword' String=' simulation '/>
-        <RegExpr attribute='Keyword' String=' shutdown handler '/>
+        <RegExpr attribute='Keyword' String=' shutdown handlers '/>
         <RegExpr attribute='Keyword' String=' sequential '/>
         <RegExpr attribute='Keyword' String=' script '/>
         <RegExpr attribute='Keyword' String=' scheduling '/>
@@ -50,10 +52,11 @@
         <RegExpr attribute='Keyword' String=' runtime '/>
         <RegExpr attribute='Keyword' String=' runahead limit '/>
         <RegExpr attribute='Keyword' String=' run-dir '/>
-        <RegExpr attribute='Keyword' String=' retry handler '/>
+        <RegExpr attribute='Keyword' String=' retry handlers '/>
         <RegExpr attribute='Keyword' String=' retrieve job logs retry delays '/>
         <RegExpr attribute='Keyword' String=' retrieve job logs max size '/>
         <RegExpr attribute='Keyword' String=' retrieve job logs '/>
+        <RegExpr attribute='Keyword' String=' restart timeout '/>
         <RegExpr attribute='Keyword' String=' remote '/>
         <RegExpr attribute='Keyword' String=' queues '/>
         <RegExpr attribute='Keyword' String=' pre-script '/>
@@ -71,7 +74,7 @@
         <RegExpr attribute='Keyword' String=' mail '/>
         <RegExpr attribute='Keyword' String=' limit '/>
         <RegExpr attribute='Keyword' String=' late offset '/>
-        <RegExpr attribute='Keyword' String=' late handler '/>
+        <RegExpr attribute='Keyword' String=' late handlers '/>
         <RegExpr attribute='Keyword' String=' job '/>
         <RegExpr attribute='Keyword' String=' interval '/>
         <RegExpr attribute='Keyword' String=' install '/>
@@ -80,8 +83,8 @@
         <RegExpr attribute='Keyword' String=' init-script '/>
         <RegExpr attribute='Keyword' String=' inherit '/>
         <RegExpr attribute='Keyword' String=' include '/>
-        <RegExpr attribute='Keyword' String=' inactivity timeout handler '/>
-        <RegExpr attribute='Keyword' String=' inactivity timeout'/>
+        <RegExpr attribute='Keyword' String=' inactivity timeout handlers '/>
+        <RegExpr attribute='Keyword' String=' inactivity timeout '/>
         <RegExpr attribute='Keyword' String=' host '/>
         <RegExpr attribute='Keyword' String=' hold after cycle point '/>
         <RegExpr attribute='Keyword' String=' handlers '/>
@@ -93,14 +96,14 @@
         <RegExpr attribute='Keyword' String=' flow.cylc '/>
         <RegExpr attribute='Keyword' String=' final cycle point constraints '/>
         <RegExpr attribute='Keyword' String=' final cycle point '/>
-        <RegExpr attribute='Keyword' String=' failed handler '/>
+        <RegExpr attribute='Keyword' String=' failed handlers '/>
         <RegExpr attribute='Keyword' String=' fail try 1 only '/>
         <RegExpr attribute='Keyword' String=' fail cycle points '/>
         <RegExpr attribute='Keyword' String=' external-trigger '/>
-        <RegExpr attribute='Keyword' String=' expired handler '/>
+        <RegExpr attribute='Keyword' String=' expired handlers '/>
         <RegExpr attribute='Keyword' String=' expected task failures '/>
         <RegExpr attribute='Keyword' String=' exit-script '/>
-        <RegExpr attribute='Keyword' String=' execution timeout handler '/>
+        <RegExpr attribute='Keyword' String=' execution timeout handlers '/>
         <RegExpr attribute='Keyword' String=' execution timeout '/>
         <RegExpr attribute='Keyword' String=' execution time limit '/>
         <RegExpr attribute='Keyword' String=' execution retry delays '/>
@@ -120,17 +123,17 @@
         <RegExpr attribute='Keyword' String=' cycle point time zone '/>
         <RegExpr attribute='Keyword' String=' cycle point num expanded year digits '/>
         <RegExpr attribute='Keyword' String=' cycle point format '/>
-        <RegExpr attribute='Keyword' String=' custom handler '/>
-        <RegExpr attribute='Keyword' String=' critical handler '/>
+        <RegExpr attribute='Keyword' String=' custom handlers '/>
+        <RegExpr attribute='Keyword' String=' critical handlers '/>
         <RegExpr attribute='Keyword' String=' clock-trigger '/>
         <RegExpr attribute='Keyword' String=' clock-expire '/>
         <RegExpr attribute='Keyword' String=' batch system '/>
         <RegExpr attribute='Keyword' String=' batch submit command template '/>
         <RegExpr attribute='Keyword' String=' allow implicit tasks '/>
-        <RegExpr attribute='Keyword' String=' aborted handler '/>
         <RegExpr attribute='Keyword' String=' abort on workflow timeout '/>
-        <RegExpr attribute='Keyword' String=' abort on stall timeout'/>
+        <RegExpr attribute='Keyword' String=' abort on stall timeout '/>
         <RegExpr attribute='Keyword' String=' abort on inactivity timeout '/>
+        <RegExpr attribute='Keyword' String=' abort handlers '/>
         <RegExpr attribute='Keyword' String=' UTC mode '/>
         <RegExpr attribute='Keyword' String=' URL '/>
         <!--END_TAG_FOR_AUTO_UPDATE-->

From 7a6f25660184926f79bb293026f46ef4ab6d967e Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 15 Dec 2023 16:58:34 +0000
Subject: [PATCH 072/110] Integration tests: add a simpler reftest fixture

---
 tests/integration/conftest.py                 | 25 ++++++++++++++++---
 tests/integration/test_examples.py            | 25 ++++++++++++++++---
 ...uler_logs.workflow-configuration-file.html |  4 +--
 tests/integration/utils/flow_tools.py         |  5 ++--
 4 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 120d64d24ce..ca7ee981bd9 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -25,7 +25,6 @@
 
 from cylc.flow.config import WorkflowConfig
 from cylc.flow.option_parsers import Options
-from cylc.flow.network.client import WorkflowRuntimeClient
 from cylc.flow.pathutil import get_cylc_run_dir
 from cylc.flow.rundb import CylcWorkflowDAO
 from cylc.flow.scripts.validate import ValidateOptions
@@ -48,6 +47,7 @@
 )
 
 if TYPE_CHECKING:
+    from cylc.flow.network.client import WorkflowRuntimeClient
     from cylc.flow.scheduler import Scheduler
     from cylc.flow.task_proxy import TaskProxy
 
@@ -326,7 +326,7 @@ def _inner(
 def gql_query():
     """Execute a GraphQL query given a workflow runtime client."""
     async def _gql_query(
-        client: WorkflowRuntimeClient, query_str: str
+        client: 'WorkflowRuntimeClient', query_str: str
     ) -> object:
         ret = await client.async_request(
             'graphql', {
@@ -511,7 +511,7 @@ def reflog():
 
     """
 
-    def _reflog(schd, flow_nums=False):
+    def _reflog(schd: 'Scheduler', flow_nums: bool = False) -> Set[tuple]:
         submit_task_jobs = schd.task_job_mgr.submit_task_jobs
         triggers = set()
 
@@ -613,3 +613,22 @@ def _set_stop(mode=None):
         schd._set_stop = set_stop
 
     return _complete
+
+
+@pytest.fixture
+def reftest(run, reflog, complete):
+    """Fixture that runs a simple reftest.
+
+    Combines the `reflog` and `complete` fixtures.
+    """
+    async def _reftest(
+        schd: 'Scheduler',
+        flow_nums: bool = False,
+    ) -> Set[tuple]:
+        async with run(schd):
+            triggers = reflog(schd, flow_nums)
+            await complete(schd)
+
+        return triggers
+
+    return _reftest
diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py
index da8b156503d..882d4c10163 100644
--- a/tests/integration/test_examples.py
+++ b/tests/integration/test_examples.py
@@ -240,9 +240,6 @@ async def test_reflog(flow, scheduler, run, reflog, complete):
     they can be compared with the expected outcome.
     """
     id_ = flow({
-        'scheduler': {
-            'allow implicit tasks': 'True',
-        },
         'scheduling': {
             'initial cycle point': '1',
             'final cycle point': '1',
@@ -271,3 +268,25 @@ async def test_reflog(flow, scheduler, run, reflog, complete):
         ('1/x', None),
         ('1/z', ('1/b',)),
     }
+
+
+async def test_reftest(flow, scheduler, reftest):
+    """Test the triggering of tasks.
+
+    This uses the reftest fixture which combines the reflog and
+    complete fixtures. Suitable for use when you just want to do a simple
+    reftest.
+    """
+    id_ = flow({
+        'scheduling': {
+            'graph': {
+                'R1': 'a => b'
+            }
+        }
+    })
+    schd = scheduler(id_, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('1/a', None),
+        ('1/b', ('1/a',)),
+    }
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
index 04ebb27ff79..011734bc410 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -11,8 +11,8 @@
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[root]]                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        [[[simulation]]]                                                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">            default run length = PT0S                                     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[scheduler]                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    allow implicit tasks = True                                           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/utils/flow_tools.py b/tests/integration/utils/flow_tools.py
index c26f79c04ac..3b196c0113d 100644
--- a/tests/integration/utils/flow_tools.py
+++ b/tests/integration/utils/flow_tools.py
@@ -73,9 +73,10 @@ def _make_flow(
         .setdefault('simulation', {})
         .setdefault('default run length', 'PT0S')
     )
-    conf = flow_config_str(conf)
+    # allow implicit tasks by default:
+    conf.setdefault('scheduler', {}).setdefault('allow implicit tasks', 'True')
     with open((flow_run_dir / WorkflowFiles.FLOW_FILE), 'w+') as flow_file:
-        flow_file.write(conf)
+        flow_file.write(flow_config_str(conf))
     return id_
 
 

From aff3dd18cf86673c0a763e8896cb0594e5a971f7 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 15 Dec 2023 16:59:29 +0000
Subject: [PATCH 073/110] Replace pre-initial functional reftests with
 integration reftests

---
 tests/functional/pre-initial/01-basic.t       |  22 ----
 .../functional/pre-initial/01-basic/flow.cylc |  10 --
 .../pre-initial/01-basic/reference.log        |   6 -
 tests/functional/pre-initial/02-advanced.t    |  22 ----
 .../pre-initial/02-advanced/flow.cylc         |  10 --
 .../pre-initial/02-advanced/reference.log     |  17 ---
 tests/functional/pre-initial/03-drop.t        |  22 ----
 .../functional/pre-initial/03-drop/flow.cylc  |  10 --
 .../pre-initial/03-drop/reference.log         |  12 --
 .../pre-initial/06-over-bracketed.t           |  22 ----
 .../pre-initial/06-over-bracketed/flow.cylc   |  14 --
 .../06-over-bracketed/reference.log           |   6 -
 .../integration/reftests/test_pre_initial.py  | 122 ++++++++++++++++++
 13 files changed, 122 insertions(+), 173 deletions(-)
 delete mode 100644 tests/functional/pre-initial/01-basic.t
 delete mode 100644 tests/functional/pre-initial/01-basic/flow.cylc
 delete mode 100644 tests/functional/pre-initial/01-basic/reference.log
 delete mode 100644 tests/functional/pre-initial/02-advanced.t
 delete mode 100644 tests/functional/pre-initial/02-advanced/flow.cylc
 delete mode 100644 tests/functional/pre-initial/02-advanced/reference.log
 delete mode 100644 tests/functional/pre-initial/03-drop.t
 delete mode 100644 tests/functional/pre-initial/03-drop/flow.cylc
 delete mode 100644 tests/functional/pre-initial/03-drop/reference.log
 delete mode 100644 tests/functional/pre-initial/06-over-bracketed.t
 delete mode 100644 tests/functional/pre-initial/06-over-bracketed/flow.cylc
 delete mode 100644 tests/functional/pre-initial/06-over-bracketed/reference.log
 create mode 100644 tests/integration/reftests/test_pre_initial.py

diff --git a/tests/functional/pre-initial/01-basic.t b/tests/functional/pre-initial/01-basic.t
deleted file mode 100644
index b1f413ddace..00000000000
--- a/tests/functional/pre-initial/01-basic.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test simplification of basic conditionals in pre-initial cycling
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/pre-initial/01-basic/flow.cylc b/tests/functional/pre-initial/01-basic/flow.cylc
deleted file mode 100644
index bca29e50d8e..00000000000
--- a/tests/functional/pre-initial/01-basic/flow.cylc
+++ /dev/null
@@ -1,10 +0,0 @@
-[scheduler]
-   UTC mode = True
-[scheduling]
-    initial cycle point = 20100101T00
-    final cycle point = 20100102T00
-    [[graph]]
-        T00 = "a[-P1D] & b => a"
-[runtime]
-    [[a, b]]
-        script = true
diff --git a/tests/functional/pre-initial/01-basic/reference.log b/tests/functional/pre-initial/01-basic/reference.log
deleted file mode 100644
index f8e9ebfb9b9..00000000000
--- a/tests/functional/pre-initial/01-basic/reference.log
+++ /dev/null
@@ -1,6 +0,0 @@
-Initial point: 20100101T0000Z
-Final point: 20100102T0000Z
-20100101T0000Z/b -triggered off []
-20100102T0000Z/b -triggered off []
-20100101T0000Z/a -triggered off ['20091231T0000Z/a', '20100101T0000Z/b']
-20100102T0000Z/a -triggered off ['20100101T0000Z/a', '20100102T0000Z/b']
diff --git a/tests/functional/pre-initial/02-advanced.t b/tests/functional/pre-initial/02-advanced.t
deleted file mode 100644
index e47da93e022..00000000000
--- a/tests/functional/pre-initial/02-advanced.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test nested conditional simplification for pre-initial cycling.
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/pre-initial/02-advanced/flow.cylc b/tests/functional/pre-initial/02-advanced/flow.cylc
deleted file mode 100644
index 3fac2378990..00000000000
--- a/tests/functional/pre-initial/02-advanced/flow.cylc
+++ /dev/null
@@ -1,10 +0,0 @@
-[scheduler]
-   UTC mode = True
-[scheduling]
-    initial cycle point = 20100101T00
-    final cycle point = 20100102T00
-    [[graph]]
-        PT6H = "(a[-PT6H] & b) & c[-PT6H] => a & c"
-[runtime]
-    [[a, b, c]]
-        script = true
diff --git a/tests/functional/pre-initial/02-advanced/reference.log b/tests/functional/pre-initial/02-advanced/reference.log
deleted file mode 100644
index 9592b07f619..00000000000
--- a/tests/functional/pre-initial/02-advanced/reference.log
+++ /dev/null
@@ -1,17 +0,0 @@
-Initial point: 20100101T0000Z
-Final point: 20100102T0000Z
-20100101T0000Z/b -triggered off []
-20100101T0600Z/b -triggered off []
-20100101T0000Z/a -triggered off ['20091231T1800Z/a', '20091231T1800Z/c', '20100101T0000Z/b']
-20100101T0000Z/c -triggered off ['20091231T1800Z/a', '20091231T1800Z/c', '20100101T0000Z/b']
-20100101T1200Z/b -triggered off []
-20100101T0600Z/a -triggered off ['20100101T0000Z/a', '20100101T0000Z/c', '20100101T0600Z/b']
-20100101T0600Z/c -triggered off ['20100101T0000Z/a', '20100101T0000Z/c', '20100101T0600Z/b']
-20100101T1800Z/b -triggered off []
-20100101T1200Z/a -triggered off ['20100101T0600Z/a', '20100101T0600Z/c', '20100101T1200Z/b']
-20100101T1200Z/c -triggered off ['20100101T0600Z/a', '20100101T0600Z/c', '20100101T1200Z/b']
-20100102T0000Z/b -triggered off []
-20100101T1800Z/c -triggered off ['20100101T1200Z/a', '20100101T1200Z/c', '20100101T1800Z/b']
-20100101T1800Z/a -triggered off ['20100101T1200Z/a', '20100101T1200Z/c', '20100101T1800Z/b']
-20100102T0000Z/a -triggered off ['20100101T1800Z/a', '20100101T1800Z/c', '20100102T0000Z/b']
-20100102T0000Z/c -triggered off ['20100101T1800Z/a', '20100101T1800Z/c', '20100102T0000Z/b']
diff --git a/tests/functional/pre-initial/03-drop.t b/tests/functional/pre-initial/03-drop.t
deleted file mode 100644
index 6afb98480a9..00000000000
--- a/tests/functional/pre-initial/03-drop.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test the case of dropping a conditional based on pre-initial cycling
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/pre-initial/03-drop/flow.cylc b/tests/functional/pre-initial/03-drop/flow.cylc
deleted file mode 100644
index f54faf43fdf..00000000000
--- a/tests/functional/pre-initial/03-drop/flow.cylc
+++ /dev/null
@@ -1,10 +0,0 @@
-[scheduler]
-   UTC mode = True
-[scheduling]
-    initial cycle point = 20100101T00
-    final cycle point = 20100102T00
-    [[graph]]
-        PT6H = "a[-PT6H] & b[-PT6H] => a => b"
-[runtime]
-    [[a, b]]
-        script = true
diff --git a/tests/functional/pre-initial/03-drop/reference.log b/tests/functional/pre-initial/03-drop/reference.log
deleted file mode 100644
index cecb29f06a0..00000000000
--- a/tests/functional/pre-initial/03-drop/reference.log
+++ /dev/null
@@ -1,12 +0,0 @@
-Initial point: 20100101T0000Z
-Final point: 20100102T0000Z
-20100101T0000Z/a -triggered off ['20091231T1800Z/a', '20091231T1800Z/b']
-20100101T0000Z/b -triggered off ['20100101T0000Z/a']
-20100101T0600Z/a -triggered off ['20100101T0000Z/a', '20100101T0000Z/b']
-20100101T0600Z/b -triggered off ['20100101T0600Z/a']
-20100101T1200Z/a -triggered off ['20100101T0600Z/a', '20100101T0600Z/b']
-20100101T1200Z/b -triggered off ['20100101T1200Z/a']
-20100101T1800Z/a -triggered off ['20100101T1200Z/a', '20100101T1200Z/b']
-20100101T1800Z/b -triggered off ['20100101T1800Z/a']
-20100102T0000Z/a -triggered off ['20100101T1800Z/a', '20100101T1800Z/b']
-20100102T0000Z/b -triggered off ['20100102T0000Z/a']
diff --git a/tests/functional/pre-initial/06-over-bracketed.t b/tests/functional/pre-initial/06-over-bracketed.t
deleted file mode 100644
index e47da93e022..00000000000
--- a/tests/functional/pre-initial/06-over-bracketed.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test nested conditional simplification for pre-initial cycling.
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/pre-initial/06-over-bracketed/flow.cylc b/tests/functional/pre-initial/06-over-bracketed/flow.cylc
deleted file mode 100644
index 64203d91e96..00000000000
--- a/tests/functional/pre-initial/06-over-bracketed/flow.cylc
+++ /dev/null
@@ -1,14 +0,0 @@
-[scheduler]
-    UTC mode = True
-    allow implicit tasks = True
-[scheduling]
-    initial cycle point = 20131225T1200Z
-    final cycle point = 20131225T1200Z
-    [[graph]]
-        T12 = """
-            (a[-P1D]:fail | b[-P1D]:fail | c[-P1D]:fail) => d
-            a & b & c  # Implied by implicit cycling now...
-        """
-[runtime]
-    [[root]]
-        script = true
diff --git a/tests/functional/pre-initial/06-over-bracketed/reference.log b/tests/functional/pre-initial/06-over-bracketed/reference.log
deleted file mode 100644
index 5246592f6c6..00000000000
--- a/tests/functional/pre-initial/06-over-bracketed/reference.log
+++ /dev/null
@@ -1,6 +0,0 @@
-Initial point: 20131225T1200Z
-Final point: 20131225T1200Z
-20131225T1200Z/c -triggered off []
-20131225T1200Z/d -triggered off ['20131224T1200Z/a', '20131224T1200Z/b', '20131224T1200Z/c']
-20131225T1200Z/a -triggered off []
-20131225T1200Z/b -triggered off []
diff --git a/tests/integration/reftests/test_pre_initial.py b/tests/integration/reftests/test_pre_initial.py
new file mode 100644
index 00000000000..1cf48cc4d7f
--- /dev/null
+++ b/tests/integration/reftests/test_pre_initial.py
@@ -0,0 +1,122 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+async def test_basic(flow, scheduler, reftest):
+    """Test simplification of basic conditionals in pre-initial cycling"""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2010-01-01',
+            'final cycle point': '2010-01-02',
+            'graph': {
+                'T00': 'a[-P1D] & b => a',
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    # 20100101T0000Z/b -triggered off []
+    # 20100102T0000Z/b -triggered off []
+    # 20100101T0000Z/a -triggered off ['20091231T0000Z/a', '20100101T0000Z/b']
+    # 20100102T0000Z/a -triggered off ['20100101T0000Z/a', '20100102T0000Z/b']
+    assert await reftest(schd) == {
+        ('20100101T0000Z/b', None),
+        ('20100102T0000Z/b', None),
+        ('20100101T0000Z/a', ('20091231T0000Z/a', '20100101T0000Z/b')),
+        ('20100102T0000Z/a', ('20100101T0000Z/a', '20100102T0000Z/b')),
+    }
+
+
+async def test_advanced(flow, scheduler, reftest):
+    """Test nested conditional simplification for pre-initial cycling."""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2010-01-01',
+            'final cycle point': '2010-01-02',
+            'graph': {
+                'PT6H': '(a[-PT6H] & b) & c[-PT6H] => a & c',
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20100101T0000Z/b', None),
+        ('20100101T0600Z/b', None),
+        ('20100101T0000Z/a', ('20091231T1800Z/a', '20091231T1800Z/c', '20100101T0000Z/b')),
+        ('20100101T0000Z/c', ('20091231T1800Z/a', '20091231T1800Z/c', '20100101T0000Z/b')),
+        ('20100101T1200Z/b', None),
+        ('20100101T0600Z/a', ('20100101T0000Z/a', '20100101T0000Z/c', '20100101T0600Z/b')),
+        ('20100101T0600Z/c', ('20100101T0000Z/a', '20100101T0000Z/c', '20100101T0600Z/b')),
+        ('20100101T1800Z/b', None),
+        ('20100101T1200Z/a', ('20100101T0600Z/a', '20100101T0600Z/c', '20100101T1200Z/b')),
+        ('20100101T1200Z/c', ('20100101T0600Z/a', '20100101T0600Z/c', '20100101T1200Z/b')),
+        ('20100102T0000Z/b', None),
+        ('20100101T1800Z/c', ('20100101T1200Z/a', '20100101T1200Z/c', '20100101T1800Z/b')),
+        ('20100101T1800Z/a', ('20100101T1200Z/a', '20100101T1200Z/c', '20100101T1800Z/b')),
+        ('20100102T0000Z/a', ('20100101T1800Z/a', '20100101T1800Z/c', '20100102T0000Z/b')),
+        ('20100102T0000Z/c', ('20100101T1800Z/a', '20100101T1800Z/c', '20100102T0000Z/b')),
+    }
+
+
+async def test_drop(flow, scheduler, reftest):
+    """Test the case of dropping a conditional based on pre-initial cycling"""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2010-01-01',
+            'final cycle point': '2010-01-02',
+            'graph': {
+                'PT6H': 'a[-PT6H] & b[-PT6H] => a => b',
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20100101T0000Z/a', ('20091231T1800Z/a', '20091231T1800Z/b')),
+        ('20100101T0000Z/b', ('20100101T0000Z/a',)),
+        ('20100101T0600Z/a', ('20100101T0000Z/a', '20100101T0000Z/b')),
+        ('20100101T0600Z/b', ('20100101T0600Z/a',)),
+        ('20100101T1200Z/a', ('20100101T0600Z/a', '20100101T0600Z/b')),
+        ('20100101T1200Z/b', ('20100101T1200Z/a',)),
+        ('20100101T1800Z/a', ('20100101T1200Z/a', '20100101T1200Z/b')),
+        ('20100101T1800Z/b', ('20100101T1800Z/a',)),
+        ('20100102T0000Z/a', ('20100101T1800Z/a', '20100101T1800Z/b')),
+        ('20100102T0000Z/b', ('20100102T0000Z/a',)),
+    }
+
+
+async def test_over_bracketed(flow, scheduler, reftest):
+    """Test nested conditional simplification for pre-initial cycling."""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2013-12-25T12:00Z',
+            'final cycle point': '2013-12-25T12:00Z',
+            'graph': {
+                'T12': '''
+                    (a[-P1D]:fail | b[-P1D]:fail | c[-P1D]:fail) => d
+                    a & b & c  # Implied by implicit cycling now...
+                ''',
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20131225T1200Z/c', None),
+        ('20131225T1200Z/d', ('20131224T1200Z/a', '20131224T1200Z/b', '20131224T1200Z/c')),
+        ('20131225T1200Z/a', None),
+        ('20131225T1200Z/b', None),
+    }

From c397262c323abda4e43f686b425de20283086891 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Mon, 18 Dec 2023 12:25:38 +0000
Subject: [PATCH 074/110] Simplify integration reftest

---
 .../integration/{ => reftests}/test_triggering.py  | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)
 rename tests/integration/{ => reftests}/test_triggering.py (76%)

diff --git a/tests/integration/test_triggering.py b/tests/integration/reftests/test_triggering.py
similarity index 76%
rename from tests/integration/test_triggering.py
rename to tests/integration/reftests/test_triggering.py
index f086d46aa4c..bc1c0df402e 100644
--- a/tests/integration/test_triggering.py
+++ b/tests/integration/reftests/test_triggering.py
@@ -14,21 +14,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-async def test_fail(flow, scheduler, run, reflog, complete, validate):
+async def test_fail(flow, scheduler, reftest):
     """Test triggering on :fail"""
     id_ = flow({
-        'scheduler': {
-            'allow implicit tasks': 'true'
-        },
         'scheduling': {
             'graph': {
                 'R1': 'foo:failed => bar'
             }
         },
         'runtime': {
-            'root': {
-                'simulation': {'default run length': 'PT0S'}
-            },
             'foo': {
                 'simulation': {'fail cycle points': 'all'}
             }
@@ -36,11 +30,7 @@ async def test_fail(flow, scheduler, run, reflog, complete, validate):
     })
     schd = scheduler(id_, paused_start=False)
 
-    async with run(schd):
-        triggers = reflog(schd)
-        await complete(schd)
-
-    assert triggers == {
+    assert await reftest(schd) == {
         ('1/foo', None),
         ('1/bar', ('1/foo',)),
     }

From 3404f6ca910a74b6af9387c1db46b087d03f890d Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 20 Dec 2023 16:25:51 +0000
Subject: [PATCH 075/110] Replace cyclers functional reftests with integration
 reftests

---
 tests/functional/cyclers/21-360_calendar.t    |  22 ---
 .../cyclers/21-360_calendar/flow.cylc         |  11 --
 .../cyclers/21-360_calendar/graph.plain.ref   |   9 --
 .../cyclers/21-360_calendar/reference.log     |   6 -
 .../functional/cyclers/36-icp_fcp_notation.t  |  22 ---
 .../cyclers/36-icp_fcp_notation/flow.cylc     |  15 ---
 .../cyclers/36-icp_fcp_notation/reference.log |   7 -
 .../functional/cyclers/47-icp_fcp_notation.t  |  36 -----
 .../cyclers/47-icp_fcp_notation/flow.cylc     |  15 ---
 tests/functional/cyclers/49-365_calendar.t    |  22 ---
 .../cyclers/49-365_calendar/flow.cylc         |  11 --
 .../cyclers/49-365_calendar/graph.plain.ref   |   5 -
 .../cyclers/49-365_calendar/reference.log     |   4 -
 tests/functional/cyclers/50-366_calendar.t    |  22 ---
 .../cyclers/50-366_calendar/flow.cylc         |  11 --
 .../cyclers/50-366_calendar/graph.plain.ref   |   7 -
 .../cyclers/50-366_calendar/reference.log     |   5 -
 .../functional/cyclers/51-recurrence_fmt_1.t  |  22 ---
 .../cyclers/51-recurrence_fmt_1/flow.cylc     |  11 --
 .../cyclers/51-recurrence_fmt_1/reference.log |   6 -
 tests/integration/reftests/test_cyclers.py    | 127 ++++++++++++++++++
 21 files changed, 127 insertions(+), 269 deletions(-)
 delete mode 100755 tests/functional/cyclers/21-360_calendar.t
 delete mode 100644 tests/functional/cyclers/21-360_calendar/flow.cylc
 delete mode 100644 tests/functional/cyclers/21-360_calendar/graph.plain.ref
 delete mode 100644 tests/functional/cyclers/21-360_calendar/reference.log
 delete mode 100755 tests/functional/cyclers/36-icp_fcp_notation.t
 delete mode 100644 tests/functional/cyclers/36-icp_fcp_notation/flow.cylc
 delete mode 100644 tests/functional/cyclers/36-icp_fcp_notation/reference.log
 delete mode 100755 tests/functional/cyclers/47-icp_fcp_notation.t
 delete mode 100644 tests/functional/cyclers/47-icp_fcp_notation/flow.cylc
 delete mode 100644 tests/functional/cyclers/49-365_calendar.t
 delete mode 100644 tests/functional/cyclers/49-365_calendar/flow.cylc
 delete mode 100644 tests/functional/cyclers/49-365_calendar/graph.plain.ref
 delete mode 100644 tests/functional/cyclers/49-365_calendar/reference.log
 delete mode 100644 tests/functional/cyclers/50-366_calendar.t
 delete mode 100644 tests/functional/cyclers/50-366_calendar/flow.cylc
 delete mode 100644 tests/functional/cyclers/50-366_calendar/graph.plain.ref
 delete mode 100644 tests/functional/cyclers/50-366_calendar/reference.log
 delete mode 100644 tests/functional/cyclers/51-recurrence_fmt_1.t
 delete mode 100644 tests/functional/cyclers/51-recurrence_fmt_1/flow.cylc
 delete mode 100644 tests/functional/cyclers/51-recurrence_fmt_1/reference.log
 create mode 100644 tests/integration/reftests/test_cyclers.py

diff --git a/tests/functional/cyclers/21-360_calendar.t b/tests/functional/cyclers/21-360_calendar.t
deleted file mode 100755
index 5f58beef713..00000000000
--- a/tests/functional/cyclers/21-360_calendar.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test intercycle dependencies.
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/cyclers/21-360_calendar/flow.cylc b/tests/functional/cyclers/21-360_calendar/flow.cylc
deleted file mode 100644
index dd68f0ec60a..00000000000
--- a/tests/functional/cyclers/21-360_calendar/flow.cylc
+++ /dev/null
@@ -1,11 +0,0 @@
-[scheduler]
-    UTC mode = True
-[scheduling]
-    initial cycle point = 20130228T00
-    final cycle point = 20130301T00
-    cycling mode = 360day
-    [[graph]]
-        P1D = foo[-P1D] => foo
-[runtime]
-    [[foo]]
-        script = true
diff --git a/tests/functional/cyclers/21-360_calendar/graph.plain.ref b/tests/functional/cyclers/21-360_calendar/graph.plain.ref
deleted file mode 100644
index 56dba855ee2..00000000000
--- a/tests/functional/cyclers/21-360_calendar/graph.plain.ref
+++ /dev/null
@@ -1,9 +0,0 @@
-edge "20130228T00/foo" "20130229T00/foo"
-edge "20130229T00/foo" "20130230T00/foo"
-edge "20130230T00/foo" "20130301T00/foo"
-graph
-node "20130228T00/foo" "foo\n20130228T00"
-node "20130229T00/foo" "foo\n20130229T00"
-node "20130230T00/foo" "foo\n20130230T00"
-node "20130301T00/foo" "foo\n20130301T00"
-stop
diff --git a/tests/functional/cyclers/21-360_calendar/reference.log b/tests/functional/cyclers/21-360_calendar/reference.log
deleted file mode 100644
index 04bc0b86031..00000000000
--- a/tests/functional/cyclers/21-360_calendar/reference.log
+++ /dev/null
@@ -1,6 +0,0 @@
-Initial point: 20130228T00
-Final point: 20130301T00
-20130228T0000Z/foo -triggered off ['20130227T0000Z/foo']
-20130229T0000Z/foo -triggered off ['20130228T0000Z/foo']
-20130230T0000Z/foo -triggered off ['20130229T0000Z/foo']
-20130301T0000Z/foo -triggered off ['20130230T0000Z/foo']
diff --git a/tests/functional/cyclers/36-icp_fcp_notation.t b/tests/functional/cyclers/36-icp_fcp_notation.t
deleted file mode 100755
index 96d5a35e895..00000000000
--- a/tests/functional/cyclers/36-icp_fcp_notation.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test initial and final cycle point special notation (^, $)
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/cyclers/36-icp_fcp_notation/flow.cylc b/tests/functional/cyclers/36-icp_fcp_notation/flow.cylc
deleted file mode 100644
index d4a18a640a6..00000000000
--- a/tests/functional/cyclers/36-icp_fcp_notation/flow.cylc
+++ /dev/null
@@ -1,15 +0,0 @@
-[scheduler]
-    UTC mode = true
-    allow implicit tasks = True
-[scheduling]
-    initial cycle point = 20160101T00Z
-    final cycle point = 20160102T00Z
-    [[graph]]
-        R1 = foo
-        R1/^ = bar
-        R1/^+PT1H = baz
-        R1/$-PT1H = boo
-        R1/$ = foo[^] & bar[^] & baz[^+PT1H] & boo[^+PT23H] => bot
-[runtime]
-    [[ root ]]
-        script = echo "success"
diff --git a/tests/functional/cyclers/36-icp_fcp_notation/reference.log b/tests/functional/cyclers/36-icp_fcp_notation/reference.log
deleted file mode 100644
index 2d1494da8c1..00000000000
--- a/tests/functional/cyclers/36-icp_fcp_notation/reference.log
+++ /dev/null
@@ -1,7 +0,0 @@
-Initial point: 20160101T0000Z
-Final point: 20160102T0000Z
-20160101T0000Z/foo -triggered off []
-20160101T0000Z/bar -triggered off []
-20160101T0100Z/baz -triggered off []
-20160101T2300Z/boo -triggered off []
-20160102T0000Z/bot -triggered off ['20160101T0000Z/bar', '20160101T0000Z/foo', '20160101T0100Z/baz', '20160101T2300Z/boo']
diff --git a/tests/functional/cyclers/47-icp_fcp_notation.t b/tests/functional/cyclers/47-icp_fcp_notation.t
deleted file mode 100755
index 7f455983e9b..00000000000
--- a/tests/functional/cyclers/47-icp_fcp_notation.t
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test intercycle dependencies.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 6
-#-------------------------------------------------------------------------------
-# test initial and final cycle point special notation (^, $)
-TEST_NAME=${TEST_NAME_BASE}
-install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-TEST_NAME="${TEST_NAME_BASE}-run"
-run_ok "${TEST_NAME}" cylc play "${WORKFLOW_NAME}" --debug --no-detach
-
-TEST_NAME=${TEST_NAME_BASE}-out
-grep_ok "20160101T0000Z/foo" "$HOME/cylc-run/${WORKFLOW_NAME}/log/scheduler/log"
-grep_ok "20160101T0000Z/bar" "$HOME/cylc-run/${WORKFLOW_NAME}/log/scheduler/log"
-grep_ok "20160101T0100Z/baz" "$HOME/cylc-run/${WORKFLOW_NAME}/log/scheduler/log"
-grep_ok "20160101T2300Z/boo" "$HOME/cylc-run/${WORKFLOW_NAME}/log/scheduler/log"
-grep_ok "20160102T0000Z/bot" "$HOME/cylc-run/${WORKFLOW_NAME}/log/scheduler/log"
-#-------------------------------------------------------------------------------
-purge
diff --git a/tests/functional/cyclers/47-icp_fcp_notation/flow.cylc b/tests/functional/cyclers/47-icp_fcp_notation/flow.cylc
deleted file mode 100644
index d4a18a640a6..00000000000
--- a/tests/functional/cyclers/47-icp_fcp_notation/flow.cylc
+++ /dev/null
@@ -1,15 +0,0 @@
-[scheduler]
-    UTC mode = true
-    allow implicit tasks = True
-[scheduling]
-    initial cycle point = 20160101T00Z
-    final cycle point = 20160102T00Z
-    [[graph]]
-        R1 = foo
-        R1/^ = bar
-        R1/^+PT1H = baz
-        R1/$-PT1H = boo
-        R1/$ = foo[^] & bar[^] & baz[^+PT1H] & boo[^+PT23H] => bot
-[runtime]
-    [[ root ]]
-        script = echo "success"
diff --git a/tests/functional/cyclers/49-365_calendar.t b/tests/functional/cyclers/49-365_calendar.t
deleted file mode 100644
index 5f58beef713..00000000000
--- a/tests/functional/cyclers/49-365_calendar.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test intercycle dependencies.
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/cyclers/49-365_calendar/flow.cylc b/tests/functional/cyclers/49-365_calendar/flow.cylc
deleted file mode 100644
index 86073fe8c56..00000000000
--- a/tests/functional/cyclers/49-365_calendar/flow.cylc
+++ /dev/null
@@ -1,11 +0,0 @@
-[scheduler]
-    UTC mode = True
-[scheduling]
-    initial cycle point = 20120228T00
-    final cycle point = 20120301T00
-    cycling mode = 365day
-    [[graph]]
-        P1D = foo[-P1D] => foo
-[runtime]
-    [[foo]]
-        script = true
diff --git a/tests/functional/cyclers/49-365_calendar/graph.plain.ref b/tests/functional/cyclers/49-365_calendar/graph.plain.ref
deleted file mode 100644
index 7f0a0d0706e..00000000000
--- a/tests/functional/cyclers/49-365_calendar/graph.plain.ref
+++ /dev/null
@@ -1,5 +0,0 @@
-edge "20120228T00/foo" "20120330T00/foo"
-graph
-node "20120228T00/foo" "foo\n20120228T00"
-node "20120301T00/foo" "foo\n20120301T00"
-stop
diff --git a/tests/functional/cyclers/49-365_calendar/reference.log b/tests/functional/cyclers/49-365_calendar/reference.log
deleted file mode 100644
index 920807e835e..00000000000
--- a/tests/functional/cyclers/49-365_calendar/reference.log
+++ /dev/null
@@ -1,4 +0,0 @@
-Initial point: 20120228T00
-Final point: 20120301T00
-20120228T0000Z/foo -triggered off ['20120227T0000Z/foo']
-20120301T0000Z/foo -triggered off ['20120228T0000Z/foo']
diff --git a/tests/functional/cyclers/50-366_calendar.t b/tests/functional/cyclers/50-366_calendar.t
deleted file mode 100644
index 5f58beef713..00000000000
--- a/tests/functional/cyclers/50-366_calendar.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test intercycle dependencies.
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/cyclers/50-366_calendar/flow.cylc b/tests/functional/cyclers/50-366_calendar/flow.cylc
deleted file mode 100644
index 1c38cb46777..00000000000
--- a/tests/functional/cyclers/50-366_calendar/flow.cylc
+++ /dev/null
@@ -1,11 +0,0 @@
-[scheduler]
-    UTC mode = True
-[scheduling]
-    initial cycle point = 20130228T00
-    final cycle point = 20130301T00
-    cycling mode = 366day
-    [[graph]]
-        P1D = foo[-P1D] => foo
-[runtime]
-    [[foo]]
-        script = true
diff --git a/tests/functional/cyclers/50-366_calendar/graph.plain.ref b/tests/functional/cyclers/50-366_calendar/graph.plain.ref
deleted file mode 100644
index e49811ab8bf..00000000000
--- a/tests/functional/cyclers/50-366_calendar/graph.plain.ref
+++ /dev/null
@@ -1,7 +0,0 @@
-edge "20130228T00/foo" "20130229T00/foo"
-edge "20130229T00/foo" "20130301T00/foo"
-graph
-node "20130228T00/foo" "foo\n20130228T00"
-node "20130229T00/foo" "foo\n20130229T00"
-node "20130301T00/foo" "foo\n20130301T00"
-stop
diff --git a/tests/functional/cyclers/50-366_calendar/reference.log b/tests/functional/cyclers/50-366_calendar/reference.log
deleted file mode 100644
index a74c1954ed5..00000000000
--- a/tests/functional/cyclers/50-366_calendar/reference.log
+++ /dev/null
@@ -1,5 +0,0 @@
-Initial point: 20130228T00
-Final point: 20130301T00
-20130228T0000Z/foo -triggered off ['20130227T0000Z/foo']
-20130229T0000Z/foo -triggered off ['20130228T0000Z/foo']
-20130301T0000Z/foo -triggered off ['20130229T0000Z/foo']
diff --git a/tests/functional/cyclers/51-recurrence_fmt_1.t b/tests/functional/cyclers/51-recurrence_fmt_1.t
deleted file mode 100644
index 6f90a79e4b5..00000000000
--- a/tests/functional/cyclers/51-recurrence_fmt_1.t
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test ISO 8601 recurrence format no. 1 with unbounded repetitions
-. "$(dirname "$0")/test_header"
-set_test_number 2
-reftest
-exit
diff --git a/tests/functional/cyclers/51-recurrence_fmt_1/flow.cylc b/tests/functional/cyclers/51-recurrence_fmt_1/flow.cylc
deleted file mode 100644
index 8d83a47f1fa..00000000000
--- a/tests/functional/cyclers/51-recurrence_fmt_1/flow.cylc
+++ /dev/null
@@ -1,11 +0,0 @@
-[meta]
-    description = Test ISO 8601 recurrence format 1
-[scheduler]
-    UTC mode = True
-    allow implicit tasks = True
-    cycle point format = CCYY-MM-DD
-[scheduling]
-    initial cycle point = 2010-01-01
-    final cycle point = 2010-01-10
-    [[graph]]
-        R/2010-01-01/2010-01-04 = worf  # 3-day interval
diff --git a/tests/functional/cyclers/51-recurrence_fmt_1/reference.log b/tests/functional/cyclers/51-recurrence_fmt_1/reference.log
deleted file mode 100644
index dda1e323ebd..00000000000
--- a/tests/functional/cyclers/51-recurrence_fmt_1/reference.log
+++ /dev/null
@@ -1,6 +0,0 @@
-Initial point: 2010-01-01
-Final point: 2010-01-12
-2010-01-01/worf -triggered off []
-2010-01-04/worf -triggered off []
-2010-01-07/worf -triggered off []
-2010-01-10/worf -triggered off []
diff --git a/tests/integration/reftests/test_cyclers.py b/tests/integration/reftests/test_cyclers.py
new file mode 100644
index 00000000000..ccc6f718904
--- /dev/null
+++ b/tests/integration/reftests/test_cyclers.py
@@ -0,0 +1,127 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+async def test_360_calendar(flow, scheduler, reftest):
+    """Test 360 day calendar."""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2013-02-28',
+            'final cycle point': '2013-03-01',
+            'cycling mode': '360day',
+            'graph': {
+                'P1D': 'foo[-P1D] => foo'
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20130228T0000Z/foo', ('20130227T0000Z/foo',)),
+        ('20130229T0000Z/foo', ('20130228T0000Z/foo',)),
+        ('20130230T0000Z/foo', ('20130229T0000Z/foo',)),
+        ('20130301T0000Z/foo', ('20130230T0000Z/foo',)),
+    }
+
+
+async def test_365_calendar(flow, scheduler, reftest):
+    """Test 365 day calendar."""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2012-02-28',
+            'final cycle point': '2012-03-01',
+            'cycling mode': '365day',
+            'graph': {
+                'P1D': 'foo[-P1D] => foo'
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20120228T0000Z/foo', ('20120227T0000Z/foo',)),
+        ('20120301T0000Z/foo', ('20120228T0000Z/foo',)),
+    }
+
+
+async def test_366_calendar(flow, scheduler, reftest):
+    """Test 366 day calendar."""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2013-02-28',
+            'final cycle point': '2013-03-01',
+            'cycling mode': '366day',
+            'graph': {
+                'P1D': 'foo[-P1D] => foo'
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20130228T0000Z/foo', ('20130227T0000Z/foo',)),
+        ('20130229T0000Z/foo', ('20130228T0000Z/foo',)),
+        ('20130301T0000Z/foo', ('20130229T0000Z/foo',)),
+    }
+
+
+async def test_icp_fcp_notation(flow, scheduler, reftest):
+    """Test initial and final cycle point special notation (^, $)"""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2016-01-01',
+            'final cycle point': '2016-01-02',
+            'graph': {
+                'R1': 'foo',
+                'R1/^': 'bar',
+                'R1/^+PT1H': 'baz',
+                'R1/$-PT1H': 'boo',
+                'R1/$': 'foo[^] & bar[^] & baz[^+PT1H] & boo[^+PT23H] => bot'
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('20160101T0000Z/foo', None),
+        ('20160101T0000Z/bar', None),
+        ('20160101T0100Z/baz', None),
+        ('20160101T2300Z/boo', None),
+        ('20160102T0000Z/bot', ('20160101T0000Z/bar', '20160101T0000Z/foo', '20160101T0100Z/baz', '20160101T2300Z/boo')),
+    }
+
+
+async def test_recurrence_format_1(flow, scheduler, reftest):
+    """Test ISO 8601 recurrence format no. 1 with unbounded repetitions."""
+    wid = flow({
+        'scheduler': {
+            'cycle point format': 'CCYY-MM-DD',
+        },
+        'scheduling': {
+            'initial cycle point': '2010-01-01',
+            'final cycle point': '2010-01-10',
+            'graph': {
+                'R/2010-01-01/2010-01-04': 'worf',  # 3-day interval
+            },
+        },
+    })
+    schd = scheduler(wid, paused_start=False)
+
+    assert await reftest(schd) == {
+        ('2010-01-01/worf', None),
+        ('2010-01-04/worf', None),
+        ('2010-01-07/worf', None),
+        ('2010-01-10/worf', None),
+    }

From 9df0272225cd792c80b53a694831bd16633c95c6 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 20 Dec 2023 16:31:15 +0000
Subject: [PATCH 076/110] Tests: fix cached datetime arithmetic contamination
 between tests using different calendar modes

---
 cylc/flow/cycling/iso8601.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py
index 3c8bd3955fa..553b63e4b2f 100644
--- a/cylc/flow/cycling/iso8601.py
+++ b/cylc/flow/cycling/iso8601.py
@@ -88,7 +88,9 @@ def from_nonstandard_string(cls, point_string):
 
     def add(self, other):
         """Add an Interval to self."""
-        return ISO8601Point(self._iso_point_add(self.value, other.value))
+        return ISO8601Point(self._iso_point_add(
+            self.value, other.value, CALENDAR.mode
+        ))
 
     def standardise(self):
         """Reformat self.value into a standard representation."""
@@ -106,25 +108,27 @@ def standardise(self):
     def sub(self, other):
         """Subtract a Point or Interval from self."""
         if isinstance(other, ISO8601Point):
-            return ISO8601Interval(
-                self._iso_point_sub_point(self.value, other.value))
-        return ISO8601Point(
-            self._iso_point_sub_interval(self.value, other.value))
+            return ISO8601Interval(self._iso_point_sub_point(
+                self.value, other.value, CALENDAR.mode
+            ))
+        return ISO8601Point(self._iso_point_sub_interval(
+            self.value, other.value, CALENDAR.mode
+        ))
 
     @staticmethod
     @lru_cache(10000)
-    def _iso_point_add(point_string, interval_string):
+    def _iso_point_add(point_string, interval_string, _calendar_mode):
         """Add the parsed point_string to the parsed interval_string."""
         point = point_parse(point_string)
         interval = interval_parse(interval_string)
         return str(point + interval)
 
     def _cmp(self, other: 'ISO8601Point') -> int:
-        return self._iso_point_cmp(self.value, other.value)
+        return self._iso_point_cmp(self.value, other.value, CALENDAR.mode)
 
     @staticmethod
     @lru_cache(10000)
-    def _iso_point_cmp(point_string, other_point_string):
+    def _iso_point_cmp(point_string, other_point_string, _calendar_mode):
         """Compare the parsed point_string to the other one."""
         point = point_parse(point_string)
         other_point = point_parse(other_point_string)
@@ -132,7 +136,7 @@ def _iso_point_cmp(point_string, other_point_string):
 
     @staticmethod
     @lru_cache(10000)
-    def _iso_point_sub_interval(point_string, interval_string):
+    def _iso_point_sub_interval(point_string, interval_string, _calendar_mode):
         """Return the parsed point_string minus the parsed interval_string."""
         point = point_parse(point_string)
         interval = interval_parse(interval_string)
@@ -140,7 +144,7 @@ def _iso_point_sub_interval(point_string, interval_string):
 
     @staticmethod
     @lru_cache(10000)
-    def _iso_point_sub_point(point_string, other_point_string):
+    def _iso_point_sub_point(point_string, other_point_string, _calendar_mode):
         """Return the difference between the two parsed point strings."""
         point = point_parse(point_string)
         other_point = point_parse(other_point_string)

From b8a55d8f6ccfc57889f5e7b80e3baee5b8b613a4 Mon Sep 17 00:00:00 2001
From: ColemanTom <15375218+ColemanTom@users.noreply.github.com>
Date: Wed, 3 Jan 2024 01:54:24 +1100
Subject: [PATCH 077/110] Add a new entry_point for xtriggers (#5831)

* Add a new entry_point for xtriggers

* This creates a clean way to add xtriggers from python packages.
  xtriggers no longer need to be directly in the PYTHONPATH,
  but instead can be added via a cylc.xtriggers entry point.
* Built-in cylc-flow xtriggers are defined via new entry_points.

---------

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 changes.d/5831.feat.md                        |  1 +
 cylc/flow/subprocpool.py                      | 30 +++++++++-----
 setup.cfg                                     |  6 +++
 .../xtriggers/04-respect-cylc-pythonpath.t    | 39 +++++++++++++++++++
 .../04-respect-cylc-pythonpath/dir/echo.py    | 21 ++++++++++
 .../04-respect-cylc-pythonpath/flow.cylc      |  8 ++++
 6 files changed, 96 insertions(+), 9 deletions(-)
 create mode 100644 changes.d/5831.feat.md
 create mode 100644 tests/functional/xtriggers/04-respect-cylc-pythonpath.t
 create mode 100644 tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/echo.py
 create mode 100644 tests/functional/xtriggers/04-respect-cylc-pythonpath/flow.cylc

diff --git a/changes.d/5831.feat.md b/changes.d/5831.feat.md
new file mode 100644
index 00000000000..daecc5e7a87
--- /dev/null
+++ b/changes.d/5831.feat.md
@@ -0,0 +1 @@
+Add capability to install xtriggers via a new cylc.xtriggers entry point
diff --git a/cylc/flow/subprocpool.py b/cylc/flow/subprocpool.py
index c380bdaa3e8..9ac3845bb54 100644
--- a/cylc/flow/subprocpool.py
+++ b/cylc/flow/subprocpool.py
@@ -28,7 +28,7 @@
 from subprocess import DEVNULL, run  # nosec
 from typing import Any, Callable, List, Optional
 
-from cylc.flow import LOG
+from cylc.flow import LOG, iter_entry_points
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.cylc_subproc import procopen
 from cylc.flow.exceptions import PlatformLookupError
@@ -69,29 +69,41 @@ def _killpg(proc, signal):
 def get_func(func_name, src_dir):
     """Find and return an xtrigger function from a module of the same name.
 
-    Can be in <src_dir>/lib/python, CYLC_MOD_LOC, or in Python path.
+    These locations are checked in this order:
+    - <src_dir>/lib/python/
+    - `$CYLC_PYTHONPATH`
+    - defined via a `cylc.xtriggers` entry point for an
+      installed Python package.
+
     Workflow source directory passed in as this is executed in an independent
     process in the command pool and therefore doesn't know about the workflow.
 
     """
     if func_name in _XTRIG_FUNCS:
         return _XTRIG_FUNCS[func_name]
+
     # First look in <src-dir>/lib/python.
     sys.path.insert(0, os.path.join(src_dir, 'lib', 'python'))
     mod_name = func_name
     try:
         mod_by_name = __import__(mod_name, fromlist=[mod_name])
     except ImportError:
-        # Then look in built-in xtriggers.
-        mod_name = "%s.%s" % ("cylc.flow.xtriggers", func_name)
-        try:
-            mod_by_name = __import__(mod_name, fromlist=[mod_name])
-        except ImportError:
-            raise
+        # Look for xtriggers via entry_points for external sources.
+        # Do this after the lib/python and PYTHONPATH approaches to allow
+        # users to override entry_point definitions with local/custom
+        # implementations.
+        for entry_point in iter_entry_points('cylc.xtriggers'):
+            if func_name == entry_point.name:
+                _XTRIG_FUNCS[func_name] = entry_point.load()
+                return _XTRIG_FUNCS[func_name]
+
+        # Still unable to find anything so abort
+        raise
+
     try:
         _XTRIG_FUNCS[func_name] = getattr(mod_by_name, func_name)
     except AttributeError:
-        # Module func_name has no function func_name.
+        # Module func_name has no function func_name, nor an entry_point entry.
         raise
     return _XTRIG_FUNCS[func_name]
 
diff --git a/setup.cfg b/setup.cfg
index 9a3838de1dc..d723d2ed361 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -214,6 +214,12 @@ cylc.main_loop =
 cylc.pre_configure =
 cylc.post_install =
     log_vc_info = cylc.flow.install_plugins.log_vc_info:main
+# NOTE: Built-in xtriggers
+cylc.xtriggers =
+    echo = cylc.flow.xtriggers.echo:echo
+    wall_clock = cylc.flow.xtriggers.wall_clock:wall_clock
+    workflow_state = cylc.flow.xtriggers.workflow_state:workflow_state
+    xrandom = cylc.flow.xtriggers.xrandom:xrandom
 
 [bdist_rpm]
 requires =
diff --git a/tests/functional/xtriggers/04-respect-cylc-pythonpath.t b/tests/functional/xtriggers/04-respect-cylc-pythonpath.t
new file mode 100644
index 00000000000..5cc05cfcb89
--- /dev/null
+++ b/tests/functional/xtriggers/04-respect-cylc-pythonpath.t
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#------------------------------------------------------------------------------
+
+# An xtrigger added to $CYLC_PYTHONPATH should take precedence
+# over a `cylc.xtriggers` entry point of the same name
+
+. "$(dirname "$0")/test_header"
+set_test_number 3
+
+install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+# Install the succeeding xtrigger function.
+export CYLC_PYTHONPATH=${WORKFLOW_RUN_DIR}/dir:${CYLC_PYTHONPATH:-}
+
+# Validate the test workflow.
+run_ok "${TEST_NAME_BASE}-val" cylc validate --debug "${WORKFLOW_NAME}"
+
+TEST_NAME="${TEST_NAME_BASE}-run"
+workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}"
+
+# Check the result of xtrigger.
+grep_workflow_log_ok "${TEST_NAME_BASE}-grep" "echo overridden, args=('the_args',)"
+
+purge
diff --git a/tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/echo.py b/tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/echo.py
new file mode 100644
index 00000000000..1c38f3ff920
--- /dev/null
+++ b/tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/echo.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def echo(*args, **kwargs):
+    print(f"echo overridden, args={args}")
+    return (True, {})
diff --git a/tests/functional/xtriggers/04-respect-cylc-pythonpath/flow.cylc b/tests/functional/xtriggers/04-respect-cylc-pythonpath/flow.cylc
new file mode 100644
index 00000000000..da05eb9e5ce
--- /dev/null
+++ b/tests/functional/xtriggers/04-respect-cylc-pythonpath/flow.cylc
@@ -0,0 +1,8 @@
+[scheduling]
+    [[xtriggers]]
+        x1 = echo("the_args")
+    [[graph]]
+        R1 = @x1 => foo
+[runtime]
+    [[foo]]
+        script = true

From 5d274133d46a4a5e7f948f697bc7a4ed8e1dd89c Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 3 Jan 2024 12:01:16 +0000
Subject: [PATCH 078/110] lint: Warn users that `CYLC_VERSION={{CYLC_VERSION}}`
 is deprecated (#5890)

* Cylc Lint to warn users that CYLC_VERSION={{CYLC_VERSION}} is deprecated.
Additional warnings for ROSE_VERSION and FCM_VERSION in the same manner.

* Fix check which was returning false positives
Made the test for presence of lint messages optionally count those messages.

* fix-flake8

* fix

* Fixed merge

* correction

* Update cylc/flow/scripts/lint.py

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>

---------

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 changes.d/5890.feat.md          |  2 ++
 cylc/flow/scripts/lint.py       | 36 ++++++++++++++++++++++++++++++++-
 tests/unit/scripts/test_lint.py | 27 +++++++++++++++++++------
 3 files changed, 58 insertions(+), 7 deletions(-)
 create mode 100644 changes.d/5890.feat.md

diff --git a/changes.d/5890.feat.md b/changes.d/5890.feat.md
new file mode 100644
index 00000000000..5e6fce66c05
--- /dev/null
+++ b/changes.d/5890.feat.md
@@ -0,0 +1,2 @@
+Lint: Warn users that setting ``CYLC_VERSION``, ``ROSE_VERSION`` or
+``FCM_VERSION`` in the workflow config is deprecated.
\ No newline at end of file
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index ef7635f1f0d..636d611194f 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -320,6 +320,27 @@ def check_indentation(line: str) -> bool:
     return bool(len(match[0]) % 4 != 0)
 
 
+CHECK_FOR_OLD_VARS = re.compile(
+    r'CYLC_VERSION\s*=\s*\{\{\s*CYLC_VERSION\s*\}\}'
+    r'|ROSE_VERSION\s*=\s*\{\{\s*ROSE_VERSION\s*\}\}'
+    r'|FCM_VERSION\s*=\s*\{\{\s*FCM_VERSION\s*\}\}'
+)
+
+
+def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
+    """Take a line and a check function and return a Dict if there is a
+    result, None otherwise.
+
+    Returns
+
+        Dict, in for {'vars': Error string}
+    """
+    result = check(line)
+    if result:
+        return {'vars': '\n    * '.join(result)}
+    return None
+
+
 FUNCTION = 'function'
 
 STYLE_GUIDE = (
@@ -655,7 +676,20 @@ def check_indentation(line: str) -> bool:
             'writing-workflows/runtime.html#task-event-template-variables'
         ),
         FUNCTION: check_for_deprecated_task_event_template_vars,
-    }
+    },
+    'U016': {
+        'short': 'Deprecated template vars: {vars}',
+        'rst': (
+            'It is no longer necessary to configure the environment variables '
+            '``CYLC_VERSION``, ``ROSE_VERSION`` or ``FCM_VERSION``.'
+        ),
+        'url': (
+            'https://cylc.github.io/cylc-doc/stable/html/plugins/'
+            'cylc-rose.html#special-variables'
+        ),
+        FUNCTION: functools.partial(
+            list_wrapper, check=CHECK_FOR_OLD_VARS.findall),
+    },
 }
 RULESETS = ['728', 'style', 'all']
 EXTRA_TOML_VALIDATION = {
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 349854be312..db3ae052ca3 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -96,6 +96,12 @@
             graph = MyFaM:finish-all => remote => !mash_theme
 
 [runtime]
+    [[root]]
+        [[[environment]]]
+            CYLC_VERSION={{CYLC_VERSION}}
+            ROSE_VERSION  = {{ROSE_VERSION     }}
+            FCM_VERSION = {{   FCM_VERSION   }}
+
     [[MyFaM]]
         extra log files = True
         {% from 'cylc.flow' import LOG %}
@@ -151,7 +157,6 @@
  [scheduler]
 
 [[dependencies]]
-
 {% set   N = 009 %}
 {% foo %}
 {{foo}}
@@ -206,13 +211,22 @@ def filter_strings(items, contains):
     ]
 
 
-def assert_contains(items, contains):
+def assert_contains(items, contains, instances=None):
     """Pass if at least one item contains a given string."""
-    if not filter_strings(items, contains):
+    filtered = filter_strings(items, contains)
+    if not filtered:
         raise Exception(
             f'Could not find: "{contains}" in:\n'
-            + pformat(items)
-        )
+            + pformat(items))
+    elif instances and len(filtered) != instances:
+        raise Exception(
+            f'Expected "{contains}" to appear {instances} times'
+            f', got it {len(filtered)} times.')
+
+
+EXPECT_INSTANCES_OF_ERR = {
+    16: 3,
+}
 
 
 @pytest.mark.parametrize(
@@ -222,7 +236,8 @@ def assert_contains(items, contains):
 def test_check_cylc_file_7to8(number):
     """TEST File has one of each manual deprecation;"""
     lint = lint_text(TEST_FILE, ['728'])
-    assert_contains(lint.messages, f'[U{number:03d}]')
+    instances = EXPECT_INSTANCES_OF_ERR.get(number, None)
+    assert_contains(lint.messages, f'[U{number:03d}]', instances)
 
 
 def test_check_cylc_file_7to8_has_shebang():

From d4f90c35eae3829063c270005aec84f72978f338 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 11 Aug 2023 09:57:05 +0100
Subject: [PATCH 079/110] id: make Tokens objects immutable

* Having Tokens objects mutable seemed like a good idea at the time.
* But in practice, Tokens are rarely modified.
* And their mutability is a barrier to use as dictionary keys and
  caching.
* Implement the `__hash__` interface and block the `__setitem__`
  and `update` interfaces.
---
 cylc/flow/id.py                        | 83 +++++++++-----------------
 cylc/flow/id_cli.py                    |  8 ++-
 cylc/flow/scripts/completion_server.py |  1 -
 cylc/flow/task_id.py                   |  5 +-
 cylc/flow/task_pool.py                 |  4 +-
 tests/integration/test_task_pool.py    | 13 +++-
 tests/unit/test_id.py                  | 24 ++++++--
 tests/unit/test_id_cli.py              |  2 +-
 8 files changed, 69 insertions(+), 71 deletions(-)

diff --git a/cylc/flow/id.py b/cylc/flow/id.py
index 222d16f82b6..5e2a7da7d6d 100644
--- a/cylc/flow/id.py
+++ b/cylc/flow/id.py
@@ -74,7 +74,7 @@ class Tokens(dict):
         <id: w//c>
         >>> Tokens(workflow='w', cycle='c')['job']
 
-        # Make a copy (note Tokens are mutable):
+        # Make a copy (note Tokens are immutable):
         >>> tokens.duplicate()
         <id: ~u/w//c/t/01>
         >>> tokens.duplicate(job='02')  # make changes at the same time
@@ -118,9 +118,10 @@ def __init__(
         dict.__init__(self, **kwargs)
 
     def __setitem__(self, key, value):
-        if key not in self._KEYS:
-            raise ValueError(f'Invalid token: {key}')
-        dict.__setitem__(self, key, value)
+        raise Exception('Tokens objects are not mutable')
+
+    def update(self, other):
+        raise Exception('Tokens objects are not mutable')
 
     def __getitem__(self, key):
         try:
@@ -151,6 +152,9 @@ def __repr__(self):
             id_ = self.id
         return f'<id: {id_}>'
 
+    def __hash__(self):
+        return hash(tuple(self.values()))
+
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
             return False
@@ -336,11 +340,9 @@ def is_null(self) -> bool:
             >>> tokens = Tokens()
             >>> tokens.is_null
             True
-            >>> tokens['job_sel'] = 'x'
-            >>> tokens.is_null
+            >>> tokens.duplicate(job_sel='x').is_null
             True
-            >>> tokens['job'] = '01'
-            >>> tokens.is_null
+            >>> tokens.duplicate(job='01').is_null
             False
 
         """
@@ -348,50 +350,10 @@ def is_null(self) -> bool:
             self[key] for key in self._REGULAR_KEYS
         )
 
-    def update_tokens(
-        self,
-        tokens: 'Optional[Tokens]' = None,
-        **kwargs
-    ) -> None:
-        """Update the tokens dictionary.
-
-        Similar to dict.update but with an optional Tokens argument.
-
-        Examples:
-            >>> tokens = Tokens('x')
-            >>> tokens.update_tokens(workflow='y')
-            >>> tokens
-            <id: y>
-            >>> tokens.update_tokens(Tokens('z'))
-            >>> tokens
-            <id: z>
-            >>> tokens.update_tokens(Tokens('a'), cycle='b')
-            >>> tokens
-            <id: a//b>
-
-        """
-        if tokens:
-            for key, value in tokens.items():
-                self[key] = value
-        for key, value in kwargs.items():
-            self[key] = value
-
-    def update(self, other):
-        """dict.update.
-
-        Example:
-            >>> tokens = Tokens(workflow='w')
-            >>> tokens.update({'cycle': 'c'})
-            >>> tokens.id
-            'w//c'
-
-        """
-        return self.update_tokens(**other)
-
     def duplicate(
         self,
-        tokens: 'Optional[Tokens]' = None,
-        **kwargs
+        *tokens_list,
+        **kwargs,
     ) -> 'Tokens':
         """Duplicate a tokens object.
 
@@ -408,17 +370,28 @@ def duplicate(
             >>> id(tokens1) == id(tokens2)
             False
 
-            Make a copy and modify it:
+            Make a copy with a modification:
             >>> tokens1.duplicate(cycle='1').id
             '~u/w//1'
 
-            Original not changed
+            The Original is not changed:
             >>> tokens1.id
             '~u/w'
+
+            Arguments override in definition order:
+            >>> Tokens.duplicate(
+            ...     tokens1,
+            ...     Tokens(cycle='c', task='a', job='01'),
+            ...     task='b'
+            ... ).id
+            '~u/w//c/b/01'
+
         """
-        ret = Tokens(self)
-        ret.update_tokens(tokens, **kwargs)
-        return ret
+        _kwargs = {}
+        for tokens in (self, *tokens_list):
+            _kwargs.update(tokens)
+        _kwargs.update(kwargs)
+        return Tokens(**_kwargs)
 
 
 # //cycle[:sel][/task[:sel][/job[:sel]]]
diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py
index b47ba739bea..b6d2a9c4150 100644
--- a/cylc/flow/id_cli.py
+++ b/cylc/flow/id_cli.py
@@ -294,7 +294,7 @@ async def parse_ids_async(
 
     # infer the run number if not specified the ID (and if possible)
     if infer_latest_runs:
-        _infer_latest_runs(*tokens_list, src_path=src_path)
+        _infer_latest_runs(tokens_list, src_path=src_path)
 
     _validate_number(
         *tokens_list,
@@ -409,12 +409,14 @@ def _validate_workflow_ids(*tokens_list, src_path):
         detect_both_flow_and_suite(src_path)
 
 
-def _infer_latest_runs(*tokens_list, src_path):
+def _infer_latest_runs(tokens_list, src_path):
     for ind, tokens in enumerate(tokens_list):
         if ind == 0 and src_path:
             # source workflow passed in as a path
             continue
-        tokens['workflow'] = infer_latest_run_from_id(tokens['workflow'])
+        tokens_list[ind] = tokens.duplicate(
+            workflow=infer_latest_run_from_id(tokens['workflow'])
+        )
         pass
 
 
diff --git a/cylc/flow/scripts/completion_server.py b/cylc/flow/scripts/completion_server.py
index ead311955da..1677adb0d1e 100644
--- a/cylc/flow/scripts/completion_server.py
+++ b/cylc/flow/scripts/completion_server.py
@@ -390,7 +390,6 @@ async def list_in_workflow(tokens: Tokens, infer_run=True) -> t.List[str]:
             # list possible IDs
             cli_detokenise(
                 tokens.duplicate(
-                    tokens=None,
                     # use the workflow ID provided on the CLI to allow
                     # run name inference
                     workflow=input_workflow,
diff --git a/cylc/flow/task_id.py b/cylc/flow/task_id.py
index 4a17a59548d..dbe003222b6 100644
--- a/cylc/flow/task_id.py
+++ b/cylc/flow/task_id.py
@@ -107,5 +107,6 @@ def get_standardised_point(
     def get_standardised_taskid(cls, task_id):
         """Return task ID with standardised cycle point."""
         tokens = Tokens(task_id, relative=True)
-        tokens['cycle'] = cls.get_standardised_point_string(tokens['cycle'])
-        return tokens.relative_id
+        return tokens.duplicate(
+            cycle=cls.get_standardised_point_string(tokens['cycle'])
+        ).relative_id
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 851ee1e39b7..0fff50fc93e 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1917,7 +1917,7 @@ def match_future_tasks(
                 # Glob or task state was not matched by active tasks
                 if not tokens['task']:
                     # make task globs explicit to make warnings clearer
-                    tokens['task'] = '*'
+                    tokens = tokens.duplicate(task='*')
                 LOG.warning(
                     'No active tasks matching:'
                     # preserve :selectors when logging the id
@@ -1985,7 +1985,7 @@ def match_taskdefs(
             point_str = tokens['cycle']
             if not tokens['task']:
                 # make task globs explicit to make warnings clearer
-                tokens['task'] = '*'
+                tokens = tokens.duplicate(task='*')
             name_str = tokens['task']
             try:
                 point_str = standardise_point_string(point_str)
diff --git a/tests/integration/test_task_pool.py b/tests/integration/test_task_pool.py
index 97526088660..4688ca54586 100644
--- a/tests/integration/test_task_pool.py
+++ b/tests/integration/test_task_pool.py
@@ -216,7 +216,8 @@ async def test_filter_task_proxies(
     expected_bad_items: List[str],
     expected_warnings: List[str],
     mod_example_flow: Scheduler,
-    caplog: pytest.LogCaptureFixture
+    caplog: pytest.LogCaptureFixture,
+    monkeypatch,
 ) -> None:
     """Test TaskPool.filter_task_proxies().
 
@@ -256,7 +257,8 @@ async def test_filter_task_proxies_hidden(
     expected_bad_items: List[str],
     expected_warnings: List[str],
     mod_example_flow: Scheduler,
-    caplog: pytest.LogCaptureFixture
+    caplog: pytest.LogCaptureFixture,
+    monkeypatch,
 ) -> None:
     """Test TaskPool.filter_task_proxies().
 
@@ -277,6 +279,13 @@ async def test_filter_task_proxies_hidden(
         expected_bad_items: Expected to be returned.
         expected_warnings: Expected to be logged.
     """
+    monkeypatch.setattr(
+        # make Tokens objects mutable to allow deepcopy to work on TaskProxy
+        # objects
+        'cylc.flow.id.Tokens.__setitem__',
+        lambda self, key, value: dict.__setitem__(self, key, value),
+    )
+
     caplog.set_level(logging.WARNING, CYLC_LOG)
     task_pool = mod_example_flow.pool
 
diff --git a/tests/unit/test_id.py b/tests/unit/test_id.py
index ba47083b02f..4d46bebf725 100644
--- a/tests/unit/test_id.py
+++ b/tests/unit/test_id.py
@@ -316,9 +316,9 @@ def test_tokens():
     with pytest.raises(ValueError):
         Tokens(foo='a')
 
-    Tokens()['cycle'] = 'a'
+    Tokens().duplicate(cycle='a')
     with pytest.raises(ValueError):
-        Tokens()['foo'] = 'a'
+        Tokens(foo='a')
 
     # test equality
     assert Tokens('a') == Tokens('a')
@@ -335,10 +335,24 @@ def test_tokens():
     assert not Tokens('a') == 1
 
     tokens = Tokens('a//b')
-    tokens.update({'cycle': 'c', 'task': 'd'})
-    assert tokens == Tokens('a//c/d')
-    with pytest.raises(ValueError):
+    new_tokens = tokens.duplicate(cycle='c', task='d')
+    assert new_tokens == Tokens('a//c/d')
+    with pytest.raises(Exception):
         tokens.update({'foo': 'c'})
+    with pytest.raises(Exception):
+        tokens['cycle'] = 'a'
+
+    # test gt/lt
+    assert sorted(
+        tokens.id
+        for tokens in [
+            Tokens('~u/c'),
+            Tokens('~u/b//1'),
+            Tokens('~u/a'),
+            Tokens('~u/b'),
+            Tokens('~u/b//2'),
+        ]
+    ) == ['~u/a', '~u/b', '~u/b//1', '~u/b//2', '~u/c']
 
 
 def test_no_look_behind():
diff --git a/tests/unit/test_id_cli.py b/tests/unit/test_id_cli.py
index 9642a3c9874..699f69c4370 100644
--- a/tests/unit/test_id_cli.py
+++ b/tests/unit/test_id_cli.py
@@ -576,6 +576,6 @@ async def test_expand_workflow_tokens_impl_selector(no_scan):
     """It should reject filters it can't handle."""
     tokens = tokenise('~user/*')
     await _expand_workflow_tokens([tokens])
-    tokens['workflow_sel'] = 'stopped'
+    tokens = tokens.duplicate(workflow_sel='stopped')
     with pytest.raises(InputError):
         await _expand_workflow_tokens([tokens])

From 2fbcf4950ac89331657adff98d41259b3d385f61 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 3 Aug 2023 09:44:47 +0100
Subject: [PATCH 080/110] task_events_mgr: refactor event keys

* Use a named tuple to store task event keys.
* Rationalise the fields of task event keys.
* Centralise event ID code.
* Fixes an unrelated bug where the workflow port was not being included
  in emails.
---
 cylc/flow/task_events_mgr.py                 | 321 ++++++++++++++-----
 cylc/flow/task_pool.py                       |  40 +--
 cylc/flow/task_proxy.py                      |  12 +-
 cylc/flow/workflow_db_mgr.py                 |  14 +-
 tests/integration/events/test_task_events.py |   5 +-
 5 files changed, 279 insertions(+), 113 deletions(-)

diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py
index ec5e467b8aa..31514f88ca5 100644
--- a/cylc/flow/task_events_mgr.py
+++ b/cylc/flow/task_events_mgr.py
@@ -33,7 +33,16 @@
 from shlex import quote
 import shlex
 from time import time
-from typing import TYPE_CHECKING, List, Optional, Union, cast
+from typing import (
+    Any,
+    Dict,
+    List,
+    NamedTuple,
+    Optional,
+    TYPE_CHECKING,
+    Union,
+    cast,
+)
 
 from cylc.flow import LOG, LOG_LEVELS
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
@@ -85,6 +94,7 @@
 
 
 if TYPE_CHECKING:
+    from cylc.flow.id import Tokens
     from cylc.flow.task_proxy import TaskProxy
     from cylc.flow.scheduler import Scheduler
 
@@ -104,6 +114,56 @@
     ["key", "ctx_type", "platform_name", "max_size"])
 
 
+class EventKey(NamedTuple):
+    """Unique identifier for a task event.
+
+    This contains event context information for event handlers.
+    """
+
+    """The event handler name."""
+    handler: str
+
+    """The task event."""
+    event: str
+
+    """The job tokens."""
+    tokens: 'Tokens'
+
+
+def get_event_id(event: str, itask: 'TaskProxy') -> str:
+    """Return a unique event identifier.
+
+    Some events are not unique e.g. task "started" is unique in that it can
+    only happen once per-job, "warning", however, is not unique as this is a
+    message severity level which could be associated with any number of
+    custom task messages.
+
+    To handle this tasks track non-unique-events and number them to ensure
+    their EventKey's remain unique for ease of event tracking.
+
+    Examples:
+        >>> from types import SimpleNamespace
+
+        # regular events are passed straight through:
+        >>> get_event_id('whatever', SimpleNamespace())
+        'whatever'
+
+        # non-unique events get an integer added to the end:
+        >>> get_event_id('warning', SimpleNamespace(non_unique_events={
+        ...     'warning': None,
+        ... }))
+        'warning-1'
+        >>> get_event_id('warning', SimpleNamespace(non_unique_events={
+        ...     'warning': 2,
+        ... }))
+        'warning-2'
+
+    """
+    if event in TaskEventsManager.NON_UNIQUE_EVENTS:
+        event = f'{event}-{itask.non_unique_events[event] or 1:d}'
+    return event
+
+
 def log_task_job_activity(ctx, workflow, point, name, submit_num=None):
     """Log an activity for a task job."""
     ctx_str = str(ctx)
@@ -353,6 +413,11 @@ class TaskEventsManager():
     NON_UNIQUE_EVENTS = ('warning', 'critical', 'custom')
     JOB_SUBMIT_SUCCESS_FLAG = 0
     JOB_SUBMIT_FAIL_FLAG = 1
+    JOB_LOGS_RETRIEVAL_EVENTS = {
+        EVENT_FAILED,
+        EVENT_RETRY,
+        EVENT_SUCCEEDED
+    }
 
     def __init__(
         self, workflow, proc_pool, workflow_db_mgr, broadcast_mgr,
@@ -375,7 +440,7 @@ def __init__(
         self.reset_inactivity_timer_func = reset_inactivity_timer_func
         # NOTE: do not mutate directly
         # use the {add,remove,unset_waiting}_event_timers methods
-        self._event_timers = {}
+        self._event_timers: Dict[EventKey, Any] = {}
         # NOTE: flag for DB use
         self.event_timers_updated = True
         # To be set by the task pool:
@@ -458,15 +523,15 @@ def process_events(self, schd: 'Scheduler') -> None:
         ctx_groups: dict = {}
         now = time()
         for id_key, timer in self._event_timers.copy().items():
-            key1, point, name, submit_num = id_key
             if timer.is_waiting:
                 continue
             # Set timer if timeout is None.
             if not timer.is_timeout_set():
                 if timer.next() is None:
                     LOG.warning(
-                        f"{point}/{name}/{submit_num:02d}"
-                        f" handler:{key1[0]} for task event:{key1[1]} failed"
+                        f"{id_key.tokens.relative_id}"
+                        f" handler:{id_key.handler}"
+                        f" for task event:{id_key.event} failed"
                     )
                     self.remove_event_timer(id_key)
                     continue
@@ -474,16 +539,18 @@ def process_events(self, schd: 'Scheduler') -> None:
                 msg = None
                 if timer.num > 1:
                     msg = (
-                        f"handler:{key1[0]} for task event:{key1[1]} failed,"
+                        f"handler:{id_key.handler}"
+                        f" for task event:{id_key.event} failed,"
                         f" retrying in {timer.delay_timeout_as_str()}"
                     )
                 elif timer.delay:
                     msg = (
-                        f"handler:{key1[0]} for task event:{key1[1]} will"
+                        f"handler:{id_key.handler}"
+                        f" for task event:{id_key.event} will"
                         f" run after {timer.delay_timeout_as_str()}"
                     )
                 if msg:
-                    LOG.debug(f"{point}/{name}/{submit_num:02d} {msg}")
+                    LOG.debug("%s %s", id_key.tokens.relative_id, msg)
             # Ready to run?
             if not timer.is_delay_done() or (
                 # Avoid flooding user's mail box with mail notification.
@@ -501,7 +568,7 @@ def process_events(self, schd: 'Scheduler') -> None:
                 # Run custom event handlers on their own
                 self.proc_pool.put_command(
                     SubProcContext(
-                        (key1, submit_num),
+                        ((id_key.handler, id_key.event), id_key.tokens['job']),
                         timer.ctx.cmd,
                         env=os.environ,
                         shell=True,  # nosec
@@ -833,10 +900,21 @@ def setup_event_handlers(self, itask, event, message):
         self._setup_event_mail(itask, event)
         self._setup_custom_event_handlers(itask, event, message)
 
-    def _custom_handler_callback(self, ctx, schd, id_key):
+    def _custom_handler_callback(
+        self,
+        ctx,
+        schd: 'Scheduler',
+        id_key: EventKey,
+    ) -> None:
         """Callback when a custom event handler is done."""
-        _, point, name, submit_num = id_key
-        log_task_job_activity(ctx, schd.workflow, point, name, submit_num)
+        tokens = id_key.tokens
+        log_task_job_activity(
+            ctx,
+            schd.workflow,
+            tokens['cycle'],
+            tokens['task'],
+            tokens['job'],
+        )
         if ctx.ret_code == 0:
             self.remove_event_timer(id_key)
         else:
@@ -849,15 +927,21 @@ def _db_events_insert(self, itask, event="", message=""):
             "event": event,
             "message": message})
 
-    def _process_event_email(self, schd: 'Scheduler', ctx, id_keys) -> None:
+    def _process_event_email(
+        self,
+        schd: 'Scheduler',
+        ctx,
+        id_keys: List[EventKey],
+    ) -> None:
         """Process event notification, by email."""
         if len(id_keys) == 1:
-            # 1 event from 1 task
-            (_, event), point, name, submit_num = id_keys[0]
-            subject = "[%s/%s/%02d %s] %s" % (
-                point, name, submit_num, event, schd.workflow)
+            id_key = id_keys[0]
+            subject = (
+                f'[{id_key.tokens.relative_id} {id_key.event}]'
+                f' {schd.workflow}'
+            )
         else:
-            event_set = {id_key[0][1] for id_key in id_keys}
+            event_set = {id_key.event for id_key in id_keys}
             if len(event_set) == 1:
                 # 1 event from n tasks
                 subject = "[%d tasks %s] %s" % (
@@ -874,28 +958,33 @@ def _process_event_email(self, schd: 'Scheduler', ctx, id_keys) -> None:
         # STDIN for mail, tasks
         stdin_str = ""
         for id_key in sorted(id_keys):
-            (_, event), point, name, submit_num = id_key
-            stdin_str += "%s: %s/%s/%02d\n" % (event, point, name, submit_num)
+            stdin_str += f'{id_key.event}: {id_key.tokens.relative_id}\n'
+
         # STDIN for mail, event info + workflow detail
         stdin_str += "\n"
-        for key in (
-            WorkflowEventData.Workflow.value,
-            WorkflowEventData.Host.value,
-            WorkflowEventData.Port.value,
-            WorkflowEventData.Owner.value,
+        for key, value in (
+            (WorkflowEventData.Workflow.value, schd.workflow),
+            (WorkflowEventData.Host.value, schd.host),
+            (WorkflowEventData.Port.value, schd.server.port),
+            (WorkflowEventData.Owner.value, schd.owner),
         ):
-            value = getattr(schd, key, None)
-            if value:
-                stdin_str += '%s: %s\n' % (key, value)
+            stdin_str += '%s: %s\n' % (key, value)
 
         if self.mail_footer:
             stdin_str += process_mail_footer(
                 self.mail_footer,
-                get_workflow_template_variables(schd, event, ''),
+                get_workflow_template_variables(schd, id_keys[-1].event, ''),
             )
         self._send_mail(ctx, cmd, stdin_str, id_keys, schd)
 
-    def _send_mail(self, ctx, cmd, stdin_str, id_keys, schd):
+    def _send_mail(
+        self,
+        ctx,
+        cmd,
+        stdin_str,
+        id_keys: List[EventKey],
+        schd: 'Scheduler',
+    ) -> None:
         # SMTP server
         env = dict(os.environ)
         if self.mail_smtp:
@@ -906,17 +995,28 @@ def _send_mail(self, ctx, cmd, stdin_str, id_keys, schd):
             ),
             callback=self._event_email_callback, callback_args=[schd])
 
-    def _event_email_callback(self, proc_ctx, schd):
+    def _event_email_callback(self, proc_ctx, schd) -> None:
         """Call back when email notification command exits."""
+        id_key: EventKey
         for id_key in proc_ctx.cmd_kwargs["id_keys"]:
-            key1, point, name, submit_num = id_key
             try:
                 if proc_ctx.ret_code == 0:
                     self.remove_event_timer(id_key)
-                    log_ctx = SubProcContext((key1, submit_num), None)
+                    log_ctx = SubProcContext(
+                        (
+                            (id_key.handler, id_key.event),
+                            id_key.tokens['job']
+                        ),
+                        None,
+                    )
                     log_ctx.ret_code = 0
                     log_task_job_activity(
-                        log_ctx, schd.workflow, point, name, submit_num)
+                        log_ctx,
+                        schd.workflow,
+                        id_key.tokens['cycle'],
+                        id_key.tokens['task'],
+                        id_key.tokens['job'],
+                    )
                 else:
                     self.unset_waiting_event_timer(id_key)
             except KeyError as exc:
@@ -940,7 +1040,12 @@ def _get_events_conf(self, itask, key, default=None):
                     return value
         return default
 
-    def _process_job_logs_retrieval(self, schd, ctx, id_keys):
+    def _process_job_logs_retrieval(
+        self,
+        schd: 'Scheduler',
+        ctx,
+        id_keys: List[EventKey],
+    ) -> None:
         """Process retrieval of task job logs from remote user@host."""
         # get a host to run retrieval on
         try:
@@ -981,12 +1086,29 @@ def _process_job_logs_retrieval(self, schd, ctx, id_keys):
             cmd.append("--max-size=%s" % (ctx.max_size,))
         # Includes and excludes
         includes = set()
-        for _, point, name, submit_num in id_keys:
+        for id_key in id_keys:
             # Include relevant directories, all levels needed
-            includes.add("/%s" % (point))
-            includes.add("/%s/%s" % (point, name))
-            includes.add("/%s/%s/%02d" % (point, name, submit_num))
-            includes.add("/%s/%s/%02d/**" % (point, name, submit_num))
+            includes.add("/%s" % (id_key.tokens['cycle']))
+            includes.add(
+                "/%s/%s" % (
+                    id_key.tokens['cycle'],
+                    id_key.tokens['task']
+                )
+            )
+            includes.add(
+                "/%s/%s/%02d" % (
+                    id_key.tokens['cycle'],
+                    id_key.tokens['task'],
+                    id_key.tokens['job'],
+                )
+            )
+            includes.add(
+                "/%s/%s/%02d/**" % (
+                    id_key.tokens['cycle'],
+                    id_key.tokens['task'],
+                    id_key.tokens['job'],
+                )
+            )
         cmd += ["--include=%s" % (include) for include in sorted(includes)]
         cmd.append("--exclude=/**")  # exclude everything else
         # Remote source
@@ -1009,16 +1131,15 @@ def _process_job_logs_retrieval(self, schd, ctx, id_keys):
             callback_255=self._job_logs_retrieval_callback_255
         )
 
-    def _job_logs_retrieval_callback_255(self, proc_ctx, schd):
+    def _job_logs_retrieval_callback_255(self, proc_ctx, schd) -> None:
         """Call back when log job retrieval fails with a 255 error."""
         self.bad_hosts.add(proc_ctx.host)
-        for id_key in proc_ctx.cmd_kwargs["id_keys"]:
-            key1, point, name, submit_num = id_key
+        for _ in proc_ctx.cmd_kwargs["id_keys"]:
             for key in proc_ctx.cmd_kwargs['id_keys']:
                 timer = self._event_timers[key]
                 timer.reset()
 
-    def _job_logs_retrieval_callback(self, proc_ctx, schd):
+    def _job_logs_retrieval_callback(self, proc_ctx, schd) -> None:
         """Call back when log job retrieval completes."""
         if (
             (proc_ctx.ret_code and LOG.isEnabledFor(DEBUG))
@@ -1027,20 +1148,31 @@ def _job_logs_retrieval_callback(self, proc_ctx, schd):
             LOG.error(proc_ctx)
         else:
             LOG.debug(proc_ctx)
+        id_key: EventKey
         for id_key in proc_ctx.cmd_kwargs["id_keys"]:
-            key1, point, name, submit_num = id_key
             try:
                 # All completed jobs are expected to have a "job.out".
                 fnames = [JOB_LOG_OUT]
                 with suppress(TypeError):
-                    if key1[1] not in 'succeeded':
+                    if id_key.event not in 'succeeded':
                         fnames.append(JOB_LOG_ERR)
                 fname_oks = {}
                 for fname in fnames:
                     fname_oks[fname] = os.path.exists(get_task_job_log(
-                        schd.workflow, point, name, submit_num, fname))
+                        schd.workflow,
+                        id_key.tokens['cycle'],
+                        id_key.tokens['task'],
+                        id_key.tokens['job'],
+                        fname,
+                    ))
                 # All expected paths must exist to record a good attempt
-                log_ctx = SubProcContext((key1, submit_num), None)
+                log_ctx = SubProcContext(
+                    (
+                        (id_key.handler, id_key.event),
+                        id_key.tokens['job']
+                    ),
+                    None,
+                )
                 if all(fname_oks.values()):
                     log_ctx.ret_code = 0
                     self.remove_event_timer(id_key)
@@ -1052,7 +1184,12 @@ def _job_logs_retrieval_callback(self, proc_ctx, schd):
                             log_ctx.err += " %s" % fname
                     self.unset_waiting_event_timer(id_key)
                 log_task_job_activity(
-                    log_ctx, schd.workflow, point, name, submit_num)
+                    log_ctx,
+                    schd.workflow,
+                    id_key.tokens['cycle'],
+                    id_key.tokens['task'],
+                    id_key.tokens['job'],
+                )
             except KeyError as exc:
                 LOG.exception(exc)
 
@@ -1337,23 +1474,29 @@ def _insert_task_job(
             }
         )
 
-    def _setup_job_logs_retrieval(self, itask, event):
+    def _setup_job_logs_retrieval(self, itask, event) -> None:
         """Set up remote job logs retrieval.
 
         For a task with a job completion event, i.e. succeeded, failed,
         (execution) retry.
         """
-        id_key = (
-            (self.HANDLER_JOB_LOGS_RETRIEVE, event),
-            str(itask.point), itask.tdef.name, itask.submit_num)
-        events = (self.EVENT_FAILED, self.EVENT_RETRY, self.EVENT_SUCCEEDED)
         if (
-            event not in events or
-            not is_remote_platform(itask.platform) or
-            not self._get_remote_conf(itask, "retrieve job logs") or
-            id_key in self._event_timers
+            event not in self.JOB_LOGS_RETRIEVAL_EVENTS
+            or not is_remote_platform(itask.platform)
+            or not self._get_remote_conf(itask, "retrieve job logs")
         ):
+            # event does not need to be processed
             return
+
+        id_key = EventKey(
+            self.HANDLER_JOB_LOGS_RETRIEVE,
+            event,
+            itask.tokens.duplicate(job=itask.submit_num),
+        )
+        if id_key in self._event_timers:
+            # event already being processed
+            return
+
         retry_delays = self._get_remote_conf(
             itask, "retrieve job logs retry delays")
         if not retry_delays:
@@ -1371,18 +1514,19 @@ def _setup_job_logs_retrieval(self, itask, event):
             )
         )
 
-    def _setup_event_mail(self, itask, event):
+    def _setup_event_mail(self, itask: 'TaskProxy', event: str) -> None:
         """Set up task event notification, by email."""
-        if event in self.NON_UNIQUE_EVENTS:
-            key1 = (
-                self.HANDLER_MAIL,
-                '%s-%d' % (event, itask.non_unique_events[event] or 1)
-            )
-        else:
-            key1 = (self.HANDLER_MAIL, event)
-        id_key = (key1, str(itask.point), itask.tdef.name, itask.submit_num)
-        if (id_key in self._event_timers or
-                event not in self._get_events_conf(itask, "mail events", [])):
+        if event not in self._get_events_conf(itask, "mail events", []):
+            # event does not need to be processed
+            return
+
+        id_key = EventKey(
+            self.HANDLER_MAIL,
+            get_event_id(event, itask),
+            itask.tokens.duplicate(job=itask.submit_num),
+        )
+        if id_key in self._event_timers:
+            # event already being processed
             return
 
         self.add_event_timer(
@@ -1401,11 +1545,18 @@ def _setup_event_mail(self, itask, event):
             )
         )
 
-    def _setup_custom_event_handlers(self, itask, event, message):
+    def _setup_custom_event_handlers(
+        self,
+        itask: 'TaskProxy',
+        event: str,
+        message: str,
+    ) -> None:
         """Set up custom task event handlers."""
         handlers = self._get_events_conf(itask, f'{event} handlers')
-        if (handlers is None and
-                event in self._get_events_conf(itask, 'handler events', [])):
+        if (
+            handlers is None
+            and event in self._get_events_conf(itask, 'handler events', [])
+        ):
             handlers = self._get_events_conf(itask, 'handlers')
         if handlers is None:
             return
@@ -1417,15 +1568,12 @@ def _setup_custom_event_handlers(self, itask, event, message):
             retry_delays = [0]
         # There can be multiple custom event handlers
         for i, handler in enumerate(handlers):
-            if event in self.NON_UNIQUE_EVENTS:
-                key1 = (
-                    f'{self.HANDLER_CUSTOM}-{i:02d}',
-                    f'{event}-{itask.non_unique_events[event] or 1:d}'
-                )
-            else:
-                key1 = (f'{self.HANDLER_CUSTOM}-{i:02d}', event)
-            id_key = (
-                key1, str(itask.point), itask.tdef.name, itask.submit_num)
+            id_key = EventKey(
+                f'{self.HANDLER_CUSTOM}-{i:02d}',
+                get_event_id(event, itask),
+                itask.tokens.duplicate(job=itask.submit_num),
+            )
+
             if id_key in self._event_timers:
                 continue
             # Note: user@host may not always be set for a submit number, e.g.
@@ -1444,12 +1592,13 @@ def _setup_custom_event_handlers(self, itask, event, message):
                 message,
                 platform_name,
             )
+            key1 = (id_key.handler, id_key.event)
             try:
                 cmd = handler % template_variables
             except KeyError as exc:
                 LOG.error(
-                    f"{itask.point}/{itask.tdef.name}/{itask.submit_num:02d} "
-                    f"{key1} bad template: {exc}")
+                    f'{id_key.tokens.relative_id}'
+                    f" {key1} bad template: {exc}")
                 continue
 
             if cmd == handler:
@@ -1671,7 +1820,7 @@ def process_execution_polling_intervals(
         delays += time_limit_polling_intervals
         return delays
 
-    def add_event_timer(self, id_key, event_timer):
+    def add_event_timer(self, id_key: EventKey, event_timer) -> None:
         """Add a new event timer.
 
         Args:
@@ -1682,7 +1831,7 @@ def add_event_timer(self, id_key, event_timer):
         self._event_timers[id_key] = event_timer
         self.event_timers_updated = True
 
-    def remove_event_timer(self, id_key):
+    def remove_event_timer(self, id_key: EventKey) -> None:
         """Remove an event timer.
 
         Args:
@@ -1692,7 +1841,7 @@ def remove_event_timer(self, id_key):
         del self._event_timers[id_key]
         self.event_timers_updated = True
 
-    def unset_waiting_event_timer(self, id_key):
+    def unset_waiting_event_timer(self, id_key: EventKey) -> None:
         """Invoke unset_waiting on an event timer.
 
         Args:
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 0fff50fc93e..a7a2faa0a5a 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -43,8 +43,11 @@
 from cylc.flow.workflow_status import StopMode
 from cylc.flow.task_action_timer import TaskActionTimer, TimerFlags
 from cylc.flow.task_events_mgr import (
-    CustomTaskEventHandlerContext, TaskEventMailContext,
-    TaskJobLogsRetrieveContext)
+    CustomTaskEventHandlerContext,
+    EventKey,
+    TaskEventMailContext,
+    TaskJobLogsRetrieveContext,
+)
 from cylc.flow.task_id import TaskID
 from cylc.flow.task_proxy import TaskProxy
 from cylc.flow.task_state import (
@@ -617,16 +620,17 @@ def load_db_task_pool_for_restart(self, row_idx, row):
             self.compute_runahead()
             self.release_runahead_tasks()
 
-    def load_db_task_action_timers(self, row_idx, row):
+    def load_db_task_action_timers(self, row_idx, row) -> None:
         """Load a task action timer, e.g. event handlers, retry states."""
         if row_idx == 0:
             LOG.info("LOADING task action timers")
         (cycle, name, ctx_key_raw, ctx_raw, delays_raw, num, delay,
          timeout) = row
-        id_ = Tokens(
+        tokens = Tokens(
             cycle=cycle,
             task=name,
-        ).relative_id
+        )
+        id_ = tokens.relative_id
         try:
             # Extract type namedtuple variables from JSON strings
             ctx_key = json.loads(str(ctx_key_raw))
@@ -680,14 +684,13 @@ def load_db_task_action_timers(self, row_idx, row):
             itask.try_timers[ctx_key[1]] = TaskActionTimer(
                 ctx, delays, num, delay, timeout)
         elif ctx:
-            key1, submit_num = ctx_key
-            # Convert key1 to type tuple - JSON restores as type list
-            # and this will not previously have been converted back
-            if isinstance(key1, list):
-                key1 = tuple(key1)
-            key = (key1, cycle, name, submit_num)
+            (handler, event), submit_num = ctx_key
             self.task_events_mgr.add_event_timer(
-                key,
+                EventKey(
+                    handler,
+                    event,
+                    tokens.duplicate(job=submit_num),
+                ),
                 TaskActionTimer(
                     ctx, delays, num, delay, timeout
                 )
@@ -1091,7 +1094,7 @@ def can_stop(self, stop_mode):
             for itask in self.get_tasks()
         )
 
-    def warn_stop_orphans(self):
+    def warn_stop_orphans(self) -> None:
         """Log (warning) orphaned tasks on workflow stop."""
         orphans = []
         orphans_kill_failed = []
@@ -1118,11 +1121,12 @@ def warn_stop_orphans(self):
                 )
             )
 
-        for key1, point, name, submit_num in (
-                self.task_events_mgr._event_timers
-        ):
-            LOG.warning("%s/%s/%s: incomplete task event handler %s" % (
-                point, name, submit_num, key1))
+        for id_key in self.task_events_mgr._event_timers:
+            LOG.warning(
+                f"{id_key.tokens.relative_id}:"
+                " incomplete task event handler"
+                f" {(id_key.handler, id_key.event)}"
+            )
 
     def log_incomplete_tasks(self) -> bool:
         """Log finished but incomplete tasks; return True if there any."""
diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py
index 3db5c731ec7..0e7fe1d8868 100644
--- a/cylc/flow/task_proxy.py
+++ b/cylc/flow/task_proxy.py
@@ -20,7 +20,15 @@
 from copy import copy
 from fnmatch import fnmatchcase
 from typing import (
-    Any, Callable, Dict, List, Set, Tuple, Optional, TYPE_CHECKING
+    Any,
+    Callable,
+    Counter as TypingCounter,
+    Dict,
+    List,
+    Optional,
+    Set,
+    TYPE_CHECKING,
+    Tuple,
 )
 
 from metomi.isodatetime.timezone import get_local_time_zone
@@ -242,7 +250,7 @@ def __init__(
         self.poll_timer: Optional['TaskActionTimer'] = None
         self.timeout: Optional[float] = None
         self.try_timers: Dict[str, 'TaskActionTimer'] = {}
-        self.non_unique_events = Counter()  # type: ignore # TODO: figure out
+        self.non_unique_events: TypingCounter[str] = Counter()
 
         self.clock_trigger_times: Dict[str, int] = {}
         self.expire_time: Optional[float] = None
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index e9402ba058b..99dc91e975d 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -47,6 +47,7 @@
     from cylc.flow.cycling import PointBase
     from cylc.flow.scheduler import Scheduler
     from cylc.flow.task_pool import TaskPool
+    from cylc.flow.task_events_mgr import EventKey
 
 Version = Any
 # TODO: narrow down Any (should be str | int) after implementing type
@@ -385,16 +386,17 @@ def put_workflow_template_vars(
             for key, value in template_vars.items()
         )
 
-    def put_task_event_timers(self, task_events_mgr):
+    def put_task_event_timers(self, task_events_mgr) -> None:
         """Put statements to update the task_action_timers table."""
         if task_events_mgr.event_timers_updated:
             self.db_deletes_map[self.TABLE_TASK_ACTION_TIMERS].append({})
-            for key, timer in task_events_mgr._event_timers.items():
-                key1, point, name, submit_num = key
+            id_key: 'EventKey'
+            for id_key, timer in task_events_mgr._event_timers.items():
+                key1 = (id_key.handler, id_key.event)
                 self.db_inserts_map[self.TABLE_TASK_ACTION_TIMERS].append({
-                    "name": name,
-                    "cycle": point,
-                    "ctx_key": json.dumps((key1, submit_num,)),
+                    "name": id_key.tokens['task'],
+                    "cycle": id_key.tokens['cycle'],
+                    "ctx_key": json.dumps((key1, id_key.tokens['job'],)),
                     "ctx": self._namedtuple2json(timer.ctx),
                     "delays": json.dumps(timer.delays),
                     "num": timer.num,
diff --git a/tests/integration/events/test_task_events.py b/tests/integration/events/test_task_events.py
index 4d155c4a48d..2e7db7efe21 100644
--- a/tests/integration/events/test_task_events.py
+++ b/tests/integration/events/test_task_events.py
@@ -18,6 +18,9 @@
 
 from .test_workflow_events import TEMPLATES
 
+from cylc.flow.id import Tokens
+from cylc.flow.task_events_mgr import EventKey
+
 import pytest
 
 
@@ -48,7 +51,7 @@ async def test_mail_footer_template(
 
     # start the workflow and get it to send an email
     ctx = SimpleNamespace(mail_to=None, mail_from=None)
-    id_keys = [((None, 'failed'), '1', 'a', 1)]
+    id_keys = [EventKey('none', 'failed', Tokens('//1/a'))]
     async with start(mod_one) as one_log:
         mod_one.task_events_mgr._process_event_email(mod_one, ctx, id_keys)
 

From 01d6b8f74622245271e1cb0270665a956ff22789 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 12 Oct 2023 17:27:06 +0100
Subject: [PATCH 081/110] task_events_mgr: include event message with event
 emails

* Closes #5566
* Note, the event message is not stored in the database, so when events
  are restored from the DB on restart, the event message will default to
  the event name.
---
 changes.d/5769.feat.md                        |  1 +
 cylc/flow/id.py                               |  6 +++
 cylc/flow/task_events_mgr.py                  | 25 +++++++++--
 cylc/flow/task_pool.py                        |  3 ++
 tests/functional/events/09-task-event-mail.t  | 20 ++++-----
 .../functional/events/29-task-event-mail-1.t  | 16 ++++----
 .../functional/events/30-task-event-mail-2.t  | 41 +++++++++++--------
 tests/integration/events/test_task_events.py  | 29 ++++++++++++-
 8 files changed, 102 insertions(+), 39 deletions(-)
 create mode 100644 changes.d/5769.feat.md

diff --git a/changes.d/5769.feat.md b/changes.d/5769.feat.md
new file mode 100644
index 00000000000..e0bda65a22c
--- /dev/null
+++ b/changes.d/5769.feat.md
@@ -0,0 +1 @@
+Include task messages and workflow port as appropriate in emails configured by "mail events".
diff --git a/cylc/flow/id.py b/cylc/flow/id.py
index 5e2a7da7d6d..db6d5ca8d80 100644
--- a/cylc/flow/id.py
+++ b/cylc/flow/id.py
@@ -163,6 +163,12 @@ def __eq__(self, other):
             for key in self._KEYS
         )
 
+    def __lt__(self, other):
+        return self.id < other.id
+
+    def __gt__(self, other):
+        return self.id > other.id
+
     def __ne__(self, other):
         if not isinstance(other, self.__class__):
             return True
diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py
index 31514f88ca5..650f4da21fa 100644
--- a/cylc/flow/task_events_mgr.py
+++ b/cylc/flow/task_events_mgr.py
@@ -126,6 +126,13 @@ class EventKey(NamedTuple):
     """The task event."""
     event: str
 
+    """The task event message.
+
+    Warning: This information is not currently preserved in the DB so will be
+    lost on restart.
+    """
+    message: str
+
     """The job tokens."""
     tokens: 'Tokens'
 
@@ -897,7 +904,7 @@ def setup_event_handlers(self, itask, event, message):
             msg = message
         self._db_events_insert(itask, event, msg)
         self._setup_job_logs_retrieval(itask, event)
-        self._setup_event_mail(itask, event)
+        self._setup_event_mail(itask, event, message)
         self._setup_custom_event_handlers(itask, event, message)
 
     def _custom_handler_callback(
@@ -951,14 +958,18 @@ def _process_event_email(
                 subject = "[%d task events] %s" % (
                     len(id_keys), schd.workflow)
         cmd = ["mail", "-s", subject]
+
         # From: and To:
         cmd.append("-r")
         cmd.append(ctx.mail_from)
         cmd.append(ctx.mail_to)
+
         # STDIN for mail, tasks
         stdin_str = ""
         for id_key in sorted(id_keys):
-            stdin_str += f'{id_key.event}: {id_key.tokens.relative_id}\n'
+            stdin_str += f'job: {id_key.tokens.relative_id}\n'
+            stdin_str += f'event: {id_key.event}\n'
+            stdin_str += f'message: {id_key.message}\n\n'
 
         # STDIN for mail, event info + workflow detail
         stdin_str += "\n"
@@ -1491,6 +1502,7 @@ def _setup_job_logs_retrieval(self, itask, event) -> None:
         id_key = EventKey(
             self.HANDLER_JOB_LOGS_RETRIEVE,
             event,
+            event,
             itask.tokens.duplicate(job=itask.submit_num),
         )
         if id_key in self._event_timers:
@@ -1514,7 +1526,12 @@ def _setup_job_logs_retrieval(self, itask, event) -> None:
             )
         )
 
-    def _setup_event_mail(self, itask: 'TaskProxy', event: str) -> None:
+    def _setup_event_mail(
+        self,
+        itask: 'TaskProxy',
+        event: str,
+        message: str,
+    ) -> None:
         """Set up task event notification, by email."""
         if event not in self._get_events_conf(itask, "mail events", []):
             # event does not need to be processed
@@ -1523,6 +1540,7 @@ def _setup_event_mail(self, itask: 'TaskProxy', event: str) -> None:
         id_key = EventKey(
             self.HANDLER_MAIL,
             get_event_id(event, itask),
+            message,
             itask.tokens.duplicate(job=itask.submit_num),
         )
         if id_key in self._event_timers:
@@ -1571,6 +1589,7 @@ def _setup_custom_event_handlers(
             id_key = EventKey(
                 f'{self.HANDLER_CUSTOM}-{i:02d}',
                 get_event_id(event, itask),
+                message,
                 itask.tokens.duplicate(job=itask.submit_num),
             )
 
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index a7a2faa0a5a..7cefa96e2cd 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -689,6 +689,9 @@ def load_db_task_action_timers(self, row_idx, row) -> None:
                 EventKey(
                     handler,
                     event,
+                    # NOTE: the event "message" is not preserved in the DB so
+                    # we use the event as a placeholder
+                    event,
                     tokens.duplicate(job=submit_num),
                 ),
                 TaskActionTimer(
diff --git a/tests/functional/events/09-task-event-mail.t b/tests/functional/events/09-task-event-mail.t
index 9f6195feb20..f859b16c761 100755
--- a/tests/functional/events/09-task-event-mail.t
+++ b/tests/functional/events/09-task-event-mail.t
@@ -20,7 +20,7 @@
 if ! command -v mail 2>'/dev/null'; then
     skip_all '"mail" command not available'
 fi
-set_test_number 5
+set_test_number 6
 mock_smtpd_init
 OPT_SET=
 if [[ "${TEST_NAME_BASE}" == *-globalcfg ]]; then
@@ -49,17 +49,15 @@ run_ok "${TEST_NAME_BASE}-validate" \
 workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
-contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-retry: 1/t1/01
-succeeded: 1/t1/02
-see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
-__LOG__
+run_ok "${TEST_NAME_BASE}-grep-log-1" \
+    grep -Pizo 'job: 1/t1/01.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-grep-log-2" \
+    grep -Pizo 'job: 1/t1/02.*\n.*event: succeeded' "${TEST_SMTPD_LOG}"
 
-
-run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[1/t1/01 retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
-run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[1/t1/02 succeeded\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-grep-log-3" \
+    grep -Pizo "Subject: \[1/t1/01 retry\].*(\n)?.* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-grep-log-4" \
+    grep -Pizo "Subject: \[1/t1/02 succeeded\].*(\n)?.* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill
diff --git a/tests/functional/events/29-task-event-mail-1.t b/tests/functional/events/29-task-event-mail-1.t
index 7ca6e76a7c2..94669abc9be 100755
--- a/tests/functional/events/29-task-event-mail-1.t
+++ b/tests/functional/events/29-task-event-mail-1.t
@@ -20,7 +20,7 @@
 if ! command -v mail 2>'/dev/null'; then
     skip_all '"mail" command not available'
 fi
-set_test_number 4
+set_test_number 5
 
 mock_smtpd_init
 create_test_global_config "
@@ -37,13 +37,15 @@ run_ok "${TEST_NAME_BASE}-validate" \
 workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME"
 
-contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-retry: 1/t1/01
-see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
-__LOG__
+run_ok "${TEST_NAME_BASE}-grep-log-1" \
+    grep -Pizo "job: 1/t1/01.*\n.*event: retry.*\n.*" "${TEST_SMTPD_LOG}"
 
-run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -qPizo "Subject: \[1/t1/01 retry\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-grep-log-2" grep  \
+    "see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/" \
+    "${TEST_SMTPD_LOG}"
+
+run_ok "${TEST_NAME_BASE}-grep-log-2" \
+    grep -Pizo "Subject: \\[1/t1/01 retry\\].*(\n)?.*${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill
diff --git a/tests/functional/events/30-task-event-mail-2.t b/tests/functional/events/30-task-event-mail-2.t
index 85f6b301654..09d2618a515 100755
--- a/tests/functional/events/30-task-event-mail-2.t
+++ b/tests/functional/events/30-task-event-mail-2.t
@@ -20,7 +20,7 @@
 if ! command -v mail 2>'/dev/null'; then
     skip_all '"mail" command not available'
 fi
-set_test_number 5
+set_test_number 20
 mock_smtpd_init
 OPT_SET=
 if [[ "${TEST_NAME_BASE}" == *-globalcfg ]]; then
@@ -49,22 +49,29 @@ run_ok "${TEST_NAME_BASE}-validate" \
 workflow_run_fail "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
+# 1 - retry
+run_ok "${TEST_NAME_BASE}-t1-01" grep -Pizo 'job: 1/t1/01.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t2-01" grep -Pizo 'job: 1/t2/01.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t3-01" grep -Pizo 'job: 1/t3/01.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t4-01" grep -Pizo 'job: 1/t4/01.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t5-01" grep -Pizo 'job: 1/t5/01.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+
+# 2 - retry
+run_ok "${TEST_NAME_BASE}-t1-02" grep -Pizo 'job: 1/t1/02.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t2-02" grep -Pizo 'job: 1/t2/02.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t3-02" grep -Pizo 'job: 1/t3/02.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t4-02" grep -Pizo 'job: 1/t4/02.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t5-02" grep -Pizo 'job: 1/t5/02.*\n.*event: retry' "${TEST_SMTPD_LOG}"
+
+# 3 - fail
+run_ok "${TEST_NAME_BASE}-t1-03" grep -Pizo 'job: 1/t1/03.*\n.*event: failed' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t2-03" grep -Pizo 'job: 1/t2/03.*\n.*event: failed' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t3-03" grep -Pizo 'job: 1/t3/03.*\n.*event: failed' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t4-03" grep -Pizo 'job: 1/t4/03.*\n.*event: failed' "${TEST_SMTPD_LOG}"
+run_ok "${TEST_NAME_BASE}-t5-03" grep -Pizo 'job: 1/t5/03.*\n.*event: failed' "${TEST_SMTPD_LOG}"
+
+
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-retry: 1/t1/01
-retry: 1/t2/01
-retry: 1/t3/01
-retry: 1/t4/01
-retry: 1/t5/01
-retry: 1/t1/02
-retry: 1/t2/02
-retry: 1/t3/02
-retry: 1/t4/02
-retry: 1/t5/02
-failed: 1/t1/03
-failed: 1/t2/03
-failed: 1/t3/03
-failed: 1/t4/03
-failed: 1/t5/03
 see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
 
@@ -73,6 +80,6 @@ run_ok "${TEST_NAME_BASE}-grep-log" \
 run_ok "${TEST_NAME_BASE}-grep-log" \
     grep -qPizo "Subject: \[. tasks failed\]\n? ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
-    purge
+purge
 mock_smtpd_kill
 exit
diff --git a/tests/integration/events/test_task_events.py b/tests/integration/events/test_task_events.py
index 2e7db7efe21..3ae30c1fe73 100644
--- a/tests/integration/events/test_task_events.py
+++ b/tests/integration/events/test_task_events.py
@@ -51,7 +51,7 @@ async def test_mail_footer_template(
 
     # start the workflow and get it to send an email
     ctx = SimpleNamespace(mail_to=None, mail_from=None)
-    id_keys = [EventKey('none', 'failed', Tokens('//1/a'))]
+    id_keys = [EventKey('none', 'failed', 'failed', Tokens('//1/a'))]
     async with start(mod_one) as one_log:
         mod_one.task_events_mgr._process_event_email(mod_one, ctx, id_keys)
 
@@ -72,5 +72,32 @@ async def test_mail_footer_template(
     assert len(mail_calls) == 1
 
 
+async def test_event_email_body(
+    mod_one,
+    start,
+    capcall,
+):
+    """It should send an email with the event context."""
+    mail_calls = capcall(
+        'cylc.flow.task_events_mgr.TaskEventsManager._send_mail'
+    )
+
+    # start the workflow and get it to send an email
+    ctx = SimpleNamespace(mail_to=None, mail_from=None)
+    async with start(mod_one):
+        # send a custom task message with the warning severity level
+        id_keys = [EventKey('none', 'warning', 'warning message', Tokens('//1/a/01'))]
+        mod_one.task_events_mgr._process_event_email(mod_one, ctx, id_keys)
+
+    # test the email which would have been sent for this message
+    email_body = mail_calls[0][0][3]
+    assert 'event: warning'
+    assert 'job: 1/a/01' in email_body
+    assert 'message: warning message' in email_body
+    assert f'workflow: {mod_one.tokens["workflow"]}' in email_body
+    assert f'host: {mod_one.host}' in email_body
+    assert f'port: {mod_one.server.port}' in email_body
+    assert f'owner: {mod_one.owner}' in email_body
+
 # NOTE: we do not test custom event handlers here because these are tested
 # as a part of workflow validation (now also performed by cylc play)

From a4976e1e3c7bda092a2423c1a1d61238224ad375 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 12 Oct 2023 17:05:20 +0100
Subject: [PATCH 082/110] tests/f: fix restart/44

* Test had become broken by a positive behaviour change.
---
 tests/functional/restart/44-reinvoke.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/restart/44-reinvoke.t b/tests/functional/restart/44-reinvoke.t
index ed9c6905d69..d9043fa6c85 100755
--- a/tests/functional/restart/44-reinvoke.t
+++ b/tests/functional/restart/44-reinvoke.t
@@ -53,7 +53,7 @@ run_ok "${TEST_NAME_BASE}-cmd" \
 sed -n -i 's/CYLC_WORKFLOW_COMMAND=.*cylc //p' "${TEST_NAME_BASE}-cmd.stdout"
 
 # ensure the whole workflow ID is present in the command (including the run number)
-CMD="^play --pause ${WORKFLOW_NAME}/run1 --host=localhost$"
+CMD="^play --pause ${WORKFLOW_NAME}/run1 --host=localhost --color=never$"
 if grep "${CMD}" "${TEST_NAME_BASE}-cmd.stdout"; then
     ok "${TEST_NAME_BASE}-re-invoked-id"
 else

From 08adaeb9e9dd04f2c55333e37ab566c5b5ad8dfd Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 3 Jan 2024 16:53:13 +0000
Subject: [PATCH 083/110] coverage: exclude report-timings and set 90% as the
 lower limit

---
 .codecov.yml | 5 +++--
 .coveragerc  | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.codecov.yml b/.codecov.yml
index ad4d7104f80..5b3d02564ba 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -17,18 +17,19 @@ coverage:
     project:
       default:
         # commits below this threshold will be marked as failed
-        target: '85%'
+        target: '92%'
         # how much we allow the coverage to drop
         threshold: '2%'
     patch:
       default:
-        target: '95%'
+        target: '97%'
         threshold: '5%'
 
 # files to ignore
 ignore:
   - "tests/**"
   - "ws_messages_pb2.py"
+  - "cylc/flow/scripts/report_timings.py"
 
 flag_management:
   default_rules:
diff --git a/.coveragerc b/.coveragerc
index 1e2a2fb5157..5a4b396f19c 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -16,6 +16,7 @@ omit =
     tests/*
     */cylc/flow/*_pb2.py
     cylc/flow/etc/*
+    cylc/flow/scripts/report_timings.py
 parallel = True
 source = ./cylc
 timid = False

From 1c1e840fe2b8cae3eb7d765eaa89f5511b2258bb Mon Sep 17 00:00:00 2001
From: Mark Dawson <mark.dawson@metoffice.gov.uk>
Date: Thu, 4 Jan 2024 17:22:12 +0000
Subject: [PATCH 084/110] Added missing union type import (#5906)

* Added missing union type import

* moved functions back into log_level

---------

Co-authored-by: Mark Dawson <mark.dawson@metoffice.gov.ukpython>
---
 cylc/flow/log_level.py      |  4 +-
 cylc/flow/option_parsers.py | 93 -------------------------------------
 2 files changed, 2 insertions(+), 95 deletions(-)

diff --git a/cylc/flow/log_level.py b/cylc/flow/log_level.py
index 4b091154893..cda3c4493ad 100644
--- a/cylc/flow/log_level.py
+++ b/cylc/flow/log_level.py
@@ -39,7 +39,7 @@ def log_level_to_verbosity(lvl: int) -> int:
         >>> log_level_to_verbosity(logging.NOTSET)
         2
         >>> log_level_to_verbosity(logging.DEBUG)
-        1
+        2
         >>> log_level_to_verbosity(logging.INFO)
         0
         >>> log_level_to_verbosity(logging.WARNING)
@@ -47,7 +47,7 @@ def log_level_to_verbosity(lvl: int) -> int:
         >>> log_level_to_verbosity(logging.ERROR)
         -1
     """
-    if lvl < logging.DEBUG:
+    if lvl <= logging.DEBUG:
         return 2
     if lvl < logging.INFO:
         return 1
diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py
index 3dc789f2aed..b83ff45aab9 100644
--- a/cylc/flow/option_parsers.py
+++ b/cylc/flow/option_parsers.py
@@ -182,99 +182,6 @@ def format_help_headings(string):
     )
 
 
-def verbosity_to_log_level(verb: int) -> int:
-    """Convert Cylc verbosity to log severity level."""
-    if verb < 0:
-        return logging.WARNING
-    if verb > 0:
-        return logging.DEBUG
-    return logging.INFO
-
-
-def log_level_to_verbosity(lvl: int) -> int:
-    """Convert log severity level to Cylc verbosity.
-
-    Examples:
-        >>> log_level_to_verbosity(logging.NOTSET)
-        2
-        >>> log_level_to_verbosity(logging.DEBUG)
-        2
-        >>> log_level_to_verbosity(logging.INFO)
-        0
-        >>> log_level_to_verbosity(logging.WARNING)
-        -1
-        >>> log_level_to_verbosity(logging.ERROR)
-        -1
-    """
-    if lvl <= logging.DEBUG:
-        return 2
-    if lvl < logging.INFO:
-        return 1
-    if lvl == logging.INFO:
-        return 0
-    return -1
-
-
-def verbosity_to_opts(verb: int) -> List[str]:
-    """Convert Cylc verbosity to the CLI opts required to replicate it.
-
-    Examples:
-        >>> verbosity_to_opts(0)
-        []
-        >>> verbosity_to_opts(-2)
-        ['-q', '-q']
-        >>> verbosity_to_opts(2)
-        ['-v', '-v']
-
-    """
-    return [
-        '-q'
-        for _ in range(verb, 0)
-    ] + [
-        '-v'
-        for _ in range(0, verb)
-    ]
-
-
-def verbosity_to_env(verb: int) -> Dict[str, str]:
-    """Convert Cylc verbosity to the env vars required to replicate it.
-
-    Examples:
-        >>> verbosity_to_env(0)
-        {'CYLC_VERBOSE': 'false', 'CYLC_DEBUG': 'false'}
-        >>> verbosity_to_env(1)
-        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'false'}
-        >>> verbosity_to_env(2)
-        {'CYLC_VERBOSE': 'true', 'CYLC_DEBUG': 'true'}
-
-    """
-    return {
-        'CYLC_VERBOSE': str((verb > 0)).lower(),
-        'CYLC_DEBUG': str((verb > 1)).lower(),
-    }
-
-
-def env_to_verbosity(env: Union[Dict, os._Environ]) -> int:
-    """Extract verbosity from environment variables.
-
-    Examples:
-        >>> env_to_verbosity({})
-        0
-        >>> env_to_verbosity({'CYLC_VERBOSE': 'true'})
-        1
-        >>> env_to_verbosity({'CYLC_DEBUG': 'true'})
-        2
-        >>> env_to_verbosity({'CYLC_DEBUG': 'TRUE'})
-        2
-
-    """
-    return (
-        2 if env.get('CYLC_DEBUG', '').lower() == 'true'
-        else 1 if env.get('CYLC_VERBOSE', '').lower() == 'true'
-        else 0
-    )
-
-
 class CylcOption(Option):
     """Optparse option which adds a decrement action."""
 

From d4a9d0ad5e6c3bdf807af623342254e00869e8aa Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Mon, 8 Jan 2024 14:38:51 +1300
Subject: [PATCH 085/110] Apply suggestions from code review [skip ci]

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 tests/integration/reftests/test_pre_initial.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/integration/reftests/test_pre_initial.py b/tests/integration/reftests/test_pre_initial.py
index 1cf48cc4d7f..d1ed6a6e566 100644
--- a/tests/integration/reftests/test_pre_initial.py
+++ b/tests/integration/reftests/test_pre_initial.py
@@ -27,10 +27,6 @@ async def test_basic(flow, scheduler, reftest):
     })
     schd = scheduler(wid, paused_start=False)
 
-    # 20100101T0000Z/b -triggered off []
-    # 20100102T0000Z/b -triggered off []
-    # 20100101T0000Z/a -triggered off ['20091231T0000Z/a', '20100101T0000Z/b']
-    # 20100102T0000Z/a -triggered off ['20100101T0000Z/a', '20100102T0000Z/b']
     assert await reftest(schd) == {
         ('20100101T0000Z/b', None),
         ('20100102T0000Z/b', None),

From 22f41e250e54a92aea375e6692035002e9c7e7ec Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Fri, 12 Jan 2024 14:11:05 +0000
Subject: [PATCH 086/110] Update setup.cfg

---
 setup.cfg | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 3870c083545..7db9eb8bf4e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -121,7 +121,6 @@ tests =
     pytest-asyncio>=0.17,!=0.23.*
     pytest-cov>=2.8.0
     pytest-xdist>=2
-    pytest-env>=0.6.2
     pytest-mock>=3.7
     pytest>=6
     testfixtures>=6.11.0

From 6cec298280f21fc359bd998538bd1ca7babd73fb Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:41:57 +0000
Subject: [PATCH 087/110] Tests.move deprecations tests to integration battery
 (#5901)

tests/f: migrate to integration tests

* f/deprecations/04-simple-graph
* f/deprecations/00-pre-cylc8.t in
---
 tests/functional/deprecations/00-pre-cylc8.t  | 40 ------------
 .../deprecations/00-pre-cylc8/flow.cylc       | 20 ------
 .../functional/deprecations/04-simple-graph.t | 42 -------------
 .../scripts/test_validate_integration.py      | 63 +++++++++++++++++++
 tests/integration/utils/flow_tools.py         | 31 ++++++---
 5 files changed, 84 insertions(+), 112 deletions(-)
 delete mode 100755 tests/functional/deprecations/00-pre-cylc8.t
 delete mode 100644 tests/functional/deprecations/00-pre-cylc8/flow.cylc
 delete mode 100644 tests/functional/deprecations/04-simple-graph.t

diff --git a/tests/functional/deprecations/00-pre-cylc8.t b/tests/functional/deprecations/00-pre-cylc8.t
deleted file mode 100755
index 7058795bf3f..00000000000
--- a/tests/functional/deprecations/00-pre-cylc8.t
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test all current non-silent workflow obsoletions and deprecations.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 2
-#-------------------------------------------------------------------------------
-install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate -v "${WORKFLOW_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-cmp
-cylc validate -v "${WORKFLOW_NAME}" 2>&1 \
-    | sed  -n -e 's/^WARNING - \( \* (.*$\)/\1/p' > 'val.out'
-cmp_ok val.out <<__END__
- * (7.8.0) [runtime][foo, cat, dog][suite state polling]template - DELETED (OBSOLETE)
- * (7.8.1) [cylc][events]reset timer - DELETED (OBSOLETE)
- * (7.8.1) [cylc][events]reset inactivity timer - DELETED (OBSOLETE)
- * (7.8.1) [runtime][foo, cat, dog][events]reset timer - DELETED (OBSOLETE)
- * (8.0.0) [runtime][foo, cat, dog][suite state polling] -> [runtime][foo, cat, dog][workflow state polling] - value unchanged
- * (8.0.0) [cylc] -> [scheduler] - value unchanged
-__END__
-
-purge
diff --git a/tests/functional/deprecations/00-pre-cylc8/flow.cylc b/tests/functional/deprecations/00-pre-cylc8/flow.cylc
deleted file mode 100644
index 525ae7606ea..00000000000
--- a/tests/functional/deprecations/00-pre-cylc8/flow.cylc
+++ /dev/null
@@ -1,20 +0,0 @@
-# Test automatic deprecation and deletion of config items as specified
-# in cylc/flow/cfgspec/workflow.py.
-
-[cylc]
-    [[events]]
-        reset timer = 10
-        reset inactivity timer = 15
-[scheduling]
-    initial cycle point = 20150808T00
-    final cycle point = 20150808T00
-    [[graph]]
-        P1D = foo => cat & dog
-    [[special tasks]]
-        external-trigger = cat("meow available") # deprecated name
-[runtime]
-    [[foo, cat, dog]]
-        [[[suite state polling]]]
-            template = ""
-        [[[events]]]
-            reset timer = 20
diff --git a/tests/functional/deprecations/04-simple-graph.t b/tests/functional/deprecations/04-simple-graph.t
deleted file mode 100644
index c708c6e64f3..00000000000
--- a/tests/functional/deprecations/04-simple-graph.t
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test deprecation notice for Cylc 7 simple graph (no recurrence section)
-
-. "$(dirname "$0")/test_header"
-set_test_number 2
-
-init_workflow "${TEST_NAME_BASE}" << __FLOW__
-[scheduler]
-    allow implicit tasks = True
-[scheduling]
-    [[dependencies]]
-        graph = foo
-__FLOW__
-
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "$TEST_NAME" cylc validate -v "$WORKFLOW_NAME"
-
-TEST_NAME="${TEST_NAME_BASE}-cmp"
-cylc validate "$WORKFLOW_NAME" 2> 'val.out'
-cmp_ok val.out <<__END__
-WARNING - deprecated graph items were automatically upgraded in "workflow definition":
-${LOG_INDENT} * (8.0.0) [scheduling][dependencies][X]graph -> [scheduling][graph]X - for X in:
-${LOG_INDENT}       graph
-__END__
-
-purge
diff --git a/tests/integration/scripts/test_validate_integration.py b/tests/integration/scripts/test_validate_integration.py
index e271726dafb..b5b99c2efa7 100644
--- a/tests/integration/scripts/test_validate_integration.py
+++ b/tests/integration/scripts/test_validate_integration.py
@@ -94,3 +94,66 @@ async def test_validate_against_source_gets_old_tvars(
         flow_file.read_text().replace('P1Y = foo', 'P1Y = {{FOO}}'))
     with pytest.raises(Jinja2Error):
         validate(src_dir)
+
+
+def test_validate_simple_graph(flow, validate, caplog):
+    """Test deprecation notice for Cylc 7 simple graph (no recurrence section)
+    """
+    id_ = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {'dependencies': {'graph': 'foo'}}
+    })
+    validate(id_)
+    expect = (
+        'deprecated graph items were automatically upgraded '
+        'in "workflow definition":'
+        '\n * (8.0.0) [scheduling][dependencies][X]graph'
+        ' -> [scheduling][graph]X - for X in:\n       graph'
+    )
+    assert expect in caplog.messages
+
+
+def test_pre_cylc8(flow, validate, caplog):
+    """Test all current non-silent workflow obsoletions and deprecations.
+    """
+    id_ = flow({
+        'cylc': {
+            'events': {
+                'reset timer': 10,
+                'reset inactivity timer': 15,
+            }
+        },
+        "scheduling": {
+            "initial cycle point": "20150808T00",
+            "final cycle point": "20150808T00",
+            "graph": {
+                "P1D": "foo => cat & dog"
+            },
+            "special tasks": {
+                "external-trigger": 'cat("meow available")'
+            }
+        },
+        'runtime': {
+            'foo, cat, dog': {
+                'suite state polling': {'template': ''},
+                'events': {'reset timer': 20}
+            }
+        }
+    }, defaults=False)
+    validate(id_)
+    for warning in (
+        (
+            ' * (7.8.0) [runtime][foo, cat, dog][suite state polling]template'
+            ' - DELETED (OBSOLETE)'),
+        ' * (7.8.1) [cylc][events]reset timer - DELETED (OBSOLETE)',
+        ' * (7.8.1) [cylc][events]reset inactivity timer - DELETED (OBSOLETE)',
+        (
+            ' * (7.8.1) [runtime][foo, cat, dog][events]reset timer'
+            ' - DELETED (OBSOLETE)'),
+        (
+            ' * (8.0.0) [runtime][foo, cat, dog][suite state polling]'
+            ' -> [runtime][foo, cat, dog][workflow state polling]'
+            ' - value unchanged'),
+        ' * (8.0.0) [cylc] -> [scheduler] - value unchanged'
+    ):
+        assert warning in caplog.messages
diff --git a/tests/integration/utils/flow_tools.py b/tests/integration/utils/flow_tools.py
index 3b196c0113d..fef15e3e3dc 100644
--- a/tests/integration/utils/flow_tools.py
+++ b/tests/integration/utils/flow_tools.py
@@ -56,8 +56,16 @@ def _make_flow(
     conf: dict,
     name: Optional[str] = None,
     id_: Optional[str] = None,
+    defaults: Optional[bool] = True,
 ) -> str:
-    """Construct a workflow on the filesystem."""
+    """Construct a workflow on the filesystem.
+
+    Args:
+        defaults: Set up a common defaults.
+            * [scheduling]allow implicit tasks = true
+
+            Set false for Cylc 7 upgrader tests.
+    """
     if id_:
         flow_run_dir = (cylc_run_dir / id_)
     else:
@@ -66,15 +74,18 @@ def _make_flow(
         flow_run_dir = (test_dir / name)
     flow_run_dir.mkdir(parents=True, exist_ok=True)
     id_ = str(flow_run_dir.relative_to(cylc_run_dir))
-    # set the default simulation runtime to zero (can be overridden)
-    (
-        conf.setdefault('runtime', {})
-        .setdefault('root', {})
-        .setdefault('simulation', {})
-        .setdefault('default run length', 'PT0S')
-    )
-    # allow implicit tasks by default:
-    conf.setdefault('scheduler', {}).setdefault('allow implicit tasks', 'True')
+    if defaults:
+        # set the default simulation runtime to zero (can be overridden)
+        (
+            conf.setdefault('runtime', {})
+            .setdefault('root', {})
+            .setdefault('simulation', {})
+            .setdefault('default run length', 'PT0S')
+        )
+        # allow implicit tasks by default:
+        conf.setdefault('scheduler', {}).setdefault(
+            'allow implicit tasks', 'True')
+
     with open((flow_run_dir / WorkflowFiles.FLOW_FILE), 'w+') as flow_file:
         flow_file.write(flow_config_str(conf))
     return id_

From 615013b76e72fc68dcce0314107494d8386c3e90 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 16 Jan 2024 09:56:25 +0000
Subject: [PATCH 088/110] task message: provide message to scheduler mail
 footer

---
 cylc/flow/task_events_mgr.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py
index 650f4da21fa..2ff2d7e44d1 100644
--- a/cylc/flow/task_events_mgr.py
+++ b/cylc/flow/task_events_mgr.py
@@ -984,7 +984,11 @@ def _process_event_email(
         if self.mail_footer:
             stdin_str += process_mail_footer(
                 self.mail_footer,
-                get_workflow_template_variables(schd, id_keys[-1].event, ''),
+                get_workflow_template_variables(
+                    schd,
+                    id_keys[-1].event,
+                    id_keys[-1].message,
+                ),
             )
         self._send_mail(ctx, cmd, stdin_str, id_keys, schd)
 

From de85ac16eb374e3d589b996712076d630554bbc1 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 16 Jan 2024 17:38:43 +0000
Subject: [PATCH 089/110] tui: handle client timeout more elegantly and make it
 configurable

---
 cylc/flow/scripts/tui.py | 21 ++++++++++++++++++++-
 cylc/flow/tui/app.py     | 14 ++++++++++----
 cylc/flow/tui/updater.py | 23 ++++++++++++++++-------
 3 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/cylc/flow/scripts/tui.py b/cylc/flow/scripts/tui.py
index f8f0879a1e6..2d48e649ef2 100644
--- a/cylc/flow/scripts/tui.py
+++ b/cylc/flow/scripts/tui.py
@@ -61,6 +61,21 @@ def get_option_parser() -> COP:
         color=False
     )
 
+    parser.add_option(
+        '--comms-timeout',
+        metavar='SEC',
+        help=(
+            "Set a timeout for network connections"
+            " to the running workflow. The default is no timeout."
+            " For task messaging connections see"
+            " site/user config file documentation."
+        ),
+        action='store',
+        default=3,
+        dest='comms_timeout',
+        type=int,
+    )
+
     return parser
 
 
@@ -76,5 +91,9 @@ def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None:
         workflow_id = tokens.duplicate(user=getuser()).id
 
     # start Tui
-    with suppress_logging(), TuiApp().main(workflow_id):
+    with suppress_logging(), TuiApp().main(
+        workflow_id,
+        client_timeout=options.comms_timeout,
+    ):
+        # tui stops according to user input
         pass
diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py
index fff634ea894..d802a2d5738 100644
--- a/cylc/flow/tui/app.py
+++ b/cylc/flow/tui/app.py
@@ -200,7 +200,7 @@ def load_child_node(self, key):
 
 
 @contextmanager
-def updater_subproc(filters):
+def updater_subproc(filters, client_timeout):
     """Runs the Updater in its own process.
 
     The updater provides the data for Tui to render. Running the updater
@@ -209,7 +209,7 @@ def updater_subproc(filters):
     decoupling the application update logic from the data update logic.
     """
     # start the updater
-    updater = Updater()
+    updater = Updater(client_timeout=client_timeout)
     p = Process(target=updater.start, args=(filters,))
     try:
         p.start()
@@ -285,7 +285,13 @@ def __init__(self, screen=None):
         self.filters = get_default_filters()
 
     @contextmanager
-    def main(self, w_id=None, id_filter=None, interactive=True):
+    def main(
+        self,
+        w_id=None,
+        id_filter=None,
+        interactive=True,
+        client_timeout=3,
+    ):
         """Start the Tui app.
 
         With interactive=False, this does not start the urwid event loop to
@@ -299,7 +305,7 @@ def main(self, w_id=None, id_filter=None, interactive=True):
         """
         self.set_initial_filters(w_id, id_filter)
 
-        with updater_subproc(self.filters) as updater:
+        with updater_subproc(self.filters, client_timeout) as updater:
             self.updater = updater
 
             # pre-subscribe to the provided workflow if requested
diff --git a/cylc/flow/tui/updater.py b/cylc/flow/tui/updater.py
index f9bd0cf2c00..26a8614b1a3 100644
--- a/cylc/flow/tui/updater.py
+++ b/cylc/flow/tui/updater.py
@@ -93,9 +93,6 @@ class Updater():
 
     """
 
-    # the maximum time to wait for a workflow update
-    CLIENT_TIMEOUT = 2
-
     # the interval between workflow listing scans
     BASE_SCAN_INTERVAL = 20
 
@@ -105,7 +102,7 @@ class Updater():
     # the command signal used to tell the updater to shut down
     SIGNAL_TERMINATE = 'terminate'
 
-    def __init__(self):
+    def __init__(self, client_timeout=3):
         # Cylc comms clients for each workflow we're connected to
         self._clients = {}
 
@@ -124,6 +121,9 @@ def __init__(self):
         # queue for commands to the updater
         self._command_queue = Queue()
 
+        # the maximum time to wait for a workflow update
+        self.client_timeout = client_timeout
+
     def subscribe(self, w_id):
         """Subscribe to updates from a workflow."""
         self._command_queue.put((self._subscribe.__name__, w_id))
@@ -269,10 +269,17 @@ async def _update_workflow(self, w_id, client, data):
                     'id': w_id,
                     'status': 'stopped',
                 })
-        except (CylcError, ZMQError):
+        except (CylcError, ZMQError) as exc:
             # something went wrong :(
             # remove the client on any error, we'll reconnect next time
             self._clients[w_id] = None
+            for workflow in data['workflows']:
+                if workflow['id'] == w_id:
+                    workflow['_tui_data'] = (
+                        f'Error - {str(exc).splitlines()[0]}'
+                    )
+                    break
+
         else:
             # the data arrived, add it to the update
             workflow_data = workflow_update['workflows'][0]
@@ -288,7 +295,7 @@ def _connect(self, data):
                 try:
                     self._clients[w_id] = get_client(
                         Tokens(w_id)['workflow'],
-                        timeout=self.CLIENT_TIMEOUT
+                        timeout=self.client_timeout,
                     )
                 except WorkflowStopped:
                     for workflow in data['workflows']:
@@ -297,7 +304,9 @@ def _connect(self, data):
                 except (ZMQError, ClientError, ClientTimeout) as exc:
                     for workflow in data['workflows']:
                         if workflow['id'] == w_id:
-                            workflow['_tui_data'] = f'Error: {exc}'
+                            workflow['_tui_data'] = (
+                                f'Error - {str(exc).splitlines()[0]}'
+                            )
                             break
 
     async def _scan(self):

From d72f46013014a9842f0491481eb01e8e47251370 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 10 Jan 2024 14:19:59 +0000
Subject: [PATCH 090/110] resolvers: optimize "uniq" iteration

* Add a more efficient method for stripping duplicate items whilst
  maintaining iteration order.
---
 cylc/flow/network/resolvers.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/network/resolvers.py b/cylc/flow/network/resolvers.py
index c51946ff188..5af50b0a4f8 100644
--- a/cylc/flow/network/resolvers.py
+++ b/cylc/flow/network/resolvers.py
@@ -126,6 +126,27 @@ def uniq(iterable):
     return ret
 
 
+def iter_uniq(iterable):
+    """Iterate over an iterable omitting any duplicate entries.
+
+    Useful for unhashable things like dicts, relies on __eq__ for testing
+    equality.
+
+    Note:
+        More efficient than "uniq" for iteration use cases.
+
+    Examples:
+        >>> list(iter_uniq([1, 1, 2, 3, 5, 8, 1]))
+        [1, 2, 3, 5, 8]
+
+    """
+    cache = set()
+    for item in iterable:
+        if item not in cache:
+            cache.add(item)
+            yield item
+
+
 def workflow_ids_filter(workflow_tokens, items) -> bool:
     """Match id arguments with workflow attributes.
 
@@ -211,7 +232,7 @@ def node_ids_filter(tokens, state, items) -> bool:
                 or get_state_from_selectors(item) == state
             )
         )
-        for item in uniq(items)
+        for item in iter_uniq(items)
     )
 
 
@@ -292,7 +313,7 @@ def get_flow_data_from_ids(data_store, native_ids):
         )
     return [
         data_store[w_id]
-        for w_id in uniq(w_ids)
+        for w_id in iter_uniq(w_ids)
         if w_id in data_store
     ]
 

From efb3016cf96c6db2d3be2f0818cd661e9bc788b0 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Fri, 26 Jan 2024 08:51:47 +0000
Subject: [PATCH 091/110] Lint packaged workflows (#5873)

* prevent lines starting with % being marked as a problem.
* add a noqa comment to Cylc lint
* run tutoral linting via tutorial .validate scripts
---
 changes.d/5873.feat.md                        |  3 ++
 .../tutorial/consolidation-tutorial/.validate |  1 +
 .../cylc-forecasting-workflow/.validate       |  1 +
 .../etc/tutorial/retries-tutorial/.validate   |  1 +
 .../etc/tutorial/retries-tutorial/flow.cylc   |  4 +-
 .../tutorial/runtime-introduction/.validate   |  1 +
 cylc/flow/scripts/lint.py                     | 54 ++++++++++++++++---
 tests/unit/scripts/test_lint.py               | 17 ++++++
 8 files changed, 74 insertions(+), 8 deletions(-)
 create mode 100644 changes.d/5873.feat.md

diff --git a/changes.d/5873.feat.md b/changes.d/5873.feat.md
new file mode 100644
index 00000000000..0b718922607
--- /dev/null
+++ b/changes.d/5873.feat.md
@@ -0,0 +1,3 @@
+- Allow use of `#noqa: S001` comments to skip Cylc lint
+  checks for a single line.
+- Stop Cylc lint objecting to `%include <file>` syntax.
\ No newline at end of file
diff --git a/cylc/flow/etc/tutorial/consolidation-tutorial/.validate b/cylc/flow/etc/tutorial/consolidation-tutorial/.validate
index 4ecb762f179..2c73648700e 100755
--- a/cylc/flow/etc/tutorial/consolidation-tutorial/.validate
+++ b/cylc/flow/etc/tutorial/consolidation-tutorial/.validate
@@ -18,4 +18,5 @@
 
 set -eux
 
+cylc lint .
 cylc validate . --icp=2000
diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate
index 2292afd6aad..38d306164de 100755
--- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate
+++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate
@@ -18,6 +18,7 @@
 set -eux
 APIKEY="$(head --lines 1 ../api-keys)"
 FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+cylc lint .
 cylc install --workflow-name "$FLOW_NAME" --no-run-name
 sed -i "s/DATAPOINT_API_KEY/$APIKEY/" "$HOME/cylc-run/$FLOW_NAME/flow.cylc"
 cylc validate --check-circular --icp=2000 "$FLOW_NAME"
diff --git a/cylc/flow/etc/tutorial/retries-tutorial/.validate b/cylc/flow/etc/tutorial/retries-tutorial/.validate
index 1c532cd1120..1eb85986072 100755
--- a/cylc/flow/etc/tutorial/retries-tutorial/.validate
+++ b/cylc/flow/etc/tutorial/retries-tutorial/.validate
@@ -18,4 +18,5 @@
 
 set -eux
 
+cylc lint .
 cylc validate --check-circular . --icp=2000
diff --git a/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc b/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc
index d0e920fba0e..04b97c274c1 100644
--- a/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc
+++ b/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc
@@ -16,8 +16,8 @@
             DIE_2=$((RANDOM%6 + 1))
             echo "Rolled $DIE_1 and $DIE_2..."
             if (($DIE_1 == $DIE_2)); then
-               echo "doubles!"
+                echo "doubles!"
             else
-               exit 1
+                exit 1
             fi
         """
diff --git a/cylc/flow/etc/tutorial/runtime-introduction/.validate b/cylc/flow/etc/tutorial/runtime-introduction/.validate
index 94dace9fbbc..ec6980f05da 100755
--- a/cylc/flow/etc/tutorial/runtime-introduction/.validate
+++ b/cylc/flow/etc/tutorial/runtime-introduction/.validate
@@ -18,6 +18,7 @@
 set -eux
 FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
 SRC=$(cylc get-resources tutorial 2>&1 | head -n1 | awk '{print $NF}')
+cylc lint .
 cylc install "${SRC}/runtime-introduction" --workflow-name "$FLOW_NAME" --no-run-name
 cylc validate --check-circular --icp=2000 "$FLOW_NAME"
 cylc play --no-detach --abort-if-any-task-fails "$FLOW_NAME"
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index 636d611194f..f038d569d78 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -37,6 +37,12 @@
 This can be overridden by providing the "--exit-zero" flag.
 """
 
+NOQA = """
+Individual errors can be ignored using the ``# noqa`` line comment.
+It is good practice to specify specific errors you wish to ignore using
+``# noqa: S002 S007 U999``
+"""
+
 TOMLDOC = """
 pyproject.toml configuration:{}
    [cylc-lint]                     # any of {}
@@ -399,7 +405,7 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
         'short': 'Item not indented.',
         # Non-indented items should be sections:
         'url': STYLE_GUIDE + 'indentation',
-        FUNCTION: re.compile(r'^[^\{\[|\s]').findall
+        FUNCTION: re.compile(r'^[^%\{\[|\s]').findall
     },
     "S003": {
         'short': 'Top level sections should not be indented.',
@@ -1066,6 +1072,33 @@ def check_cylc_file(
                 pass
 
 
+def no_qa(line: str, index: str):
+    """This line has a no-qa comment.
+
+    Examples:
+        # No comment, no exception:
+        >>> no_qa('foo = bar', 'S001')
+        False
+
+        # Comment, no error codes, no checking:
+        >>> no_qa('foo = bar # noqa', 'S001')
+        True
+
+        # Comment, no relevent error codes, no checking:
+        >>> no_qa('foo = bar # noqa: S999, 997', 'S001')
+        False
+
+        # Comment, relevent error codes, checking:
+        >>> no_qa('foo = bar # noqa: S001 S003', 'S001')
+        True
+    """
+    NOQA = re.compile(r'.*#\s*[Nn][Oo][Qq][Aa]:?(.*)')
+    noqa = NOQA.findall(line)
+    if noqa and (noqa[0] == '' or index in noqa[0]):
+        return True
+    return False
+
+
 def lint(
     file_rel: Path,
     lines: Iterator[str],
@@ -1105,9 +1138,13 @@ def lint(
         # run lint checks against the current line
         for index, check_meta in checks.items():
             # Skip commented line unless check says not to.
+            index_str = get_index_str(check_meta, index)
             if (
-                line.strip().startswith('#')
-                and not check_meta.get('evaluate commented lines', False)
+                (
+                    line.strip().startswith('#')
+                    and not check_meta.get('evaluate commented lines', False)
+                )
+                or no_qa(line, index_str)
             ):
                 continue
 
@@ -1139,7 +1176,7 @@ def lint(
                     url = get_url(check_meta)
 
                     yield (
-                        f'# [{get_index_str(check_meta, index)}]: '
+                        f'# [{index_str}]: '
                         f'{msg}\n'
                         f'# - see {url}\n'
                     )
@@ -1147,7 +1184,7 @@ def lint(
                     # write a message to inform the user
                     write(
                         Fore.YELLOW +
-                        f'[{get_index_str(check_meta, index)}]'
+                        f'[{index_str}]'
                         f' {file_rel}:{line_no}: {msg}'
                     )
         if modify:
@@ -1278,7 +1315,11 @@ def target_version_check(
 
 def get_option_parser() -> COP:
     parser = COP(
-        COP_DOC + TOMLDOC.format('', str(LINT_SECTIONS)),
+        (
+            COP_DOC
+            + NOQA.replace('``', '"')
+            + TOMLDOC.format('', str(LINT_SECTIONS))
+        ),
         argdoc=[
             COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC)
         ],
@@ -1403,4 +1444,5 @@ def main(parser: COP, options: 'Values', target=None) -> None:
 
 # NOTE: use += so that this works with __import__
 # (docstring needed for `cylc help all` output)
+__doc__ += NOQA
 __doc__ += get_reference('all', 'rst')
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index db3ae052ca3..f3009a6ddb3 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -148,6 +148,8 @@
     [[and_another_thing]]
         [[[remote]]]
             host = `rose host-select thingy`
+
+%include foo.cylc
 """
 
 
@@ -649,3 +651,18 @@ def test_indents(spaces, expect):
         assert expect in result
     else:
         assert not result
+
+
+def test_noqa():
+    """Comments turn of checks.
+
+    """
+    output = lint_text(
+        'foo = bar#noqa\n'
+        'qux = baz # noqa: S002\n'
+        'buzz = food # noqa: S007\n'
+        'quixotic = foolish # noqa: S007, S992 S002\n',
+        ['style']
+    )
+    assert len(output.messages) == 1
+    assert 'flow.cylc:3' in output.messages[0]

From 08d9c531f1adc7cf83651dbb7befdadac5bd8aed Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Mon, 29 Jan 2024 16:28:54 +0000
Subject: [PATCH 092/110] add example workflows (#5773)

add example workflows

* Partially addresses https://github.com/cylc/cylc-doc/issues/627
* Add some examples of Cylc workflow implementation patterns.

---------

Co-authored-by: Hilary James Oliver <hilary.j.oliver@gmail.com>
Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 README.md                                     | 22 ++-----
 cylc/flow/etc/examples/README.md              | 27 +++++++++
 .../examples/converging-workflow/.validate    | 33 +++++++++++
 .../examples/converging-workflow/flow.cylc    | 35 ++++++++++++
 .../examples/converging-workflow/index.rst    | 39 +++++++++++++
 .../etc/examples/datetime-cycling/.validate   | 28 +++++++++
 .../etc/examples/datetime-cycling/flow.cylc   | 44 ++++++++++++++
 .../etc/examples/datetime-cycling/index.rst   | 16 ++++++
 .../examples/event-driven-cycling/.validate   | 47 +++++++++++++++
 .../examples/event-driven-cycling/README.rst  | 49 ++++++++++++++++
 .../examples/event-driven-cycling/bin/trigger | 32 +++++++++++
 .../examples/event-driven-cycling/flow.cylc   | 32 +++++++++++
 .../examples/event-driven-cycling/index.rst   | 14 +++++
 cylc/flow/etc/examples/hello-world/.validate  | 23 ++++++++
 cylc/flow/etc/examples/hello-world/flow.cylc  | 13 +++++
 cylc/flow/etc/examples/hello-world/index.rst  | 21 +++++++
 .../etc/examples/integer-cycling/.validate    | 23 ++++++++
 .../etc/examples/integer-cycling/flow.cylc    | 28 +++++++++
 .../etc/examples/integer-cycling/index.rst    | 16 ++++++
 .../inter-workflow-triggers/.validate         | 57 +++++++++++++++++++
 .../inter-workflow-triggers/README.rst        | 14 +++++
 .../downstream/flow.cylc                      | 25 ++++++++
 .../inter-workflow-triggers/index.rst         | 28 +++++++++
 .../upstream/flow.cylc                        | 27 +++++++++
 cylc/flow/resources.py                        | 57 +++++++++++++++----
 etc/bin/run-validate-tutorials                | 18 +++---
 26 files changed, 730 insertions(+), 38 deletions(-)
 create mode 100644 cylc/flow/etc/examples/README.md
 create mode 100755 cylc/flow/etc/examples/converging-workflow/.validate
 create mode 100644 cylc/flow/etc/examples/converging-workflow/flow.cylc
 create mode 100644 cylc/flow/etc/examples/converging-workflow/index.rst
 create mode 100755 cylc/flow/etc/examples/datetime-cycling/.validate
 create mode 100644 cylc/flow/etc/examples/datetime-cycling/flow.cylc
 create mode 100644 cylc/flow/etc/examples/datetime-cycling/index.rst
 create mode 100755 cylc/flow/etc/examples/event-driven-cycling/.validate
 create mode 100644 cylc/flow/etc/examples/event-driven-cycling/README.rst
 create mode 100755 cylc/flow/etc/examples/event-driven-cycling/bin/trigger
 create mode 100644 cylc/flow/etc/examples/event-driven-cycling/flow.cylc
 create mode 100644 cylc/flow/etc/examples/event-driven-cycling/index.rst
 create mode 100755 cylc/flow/etc/examples/hello-world/.validate
 create mode 100644 cylc/flow/etc/examples/hello-world/flow.cylc
 create mode 100644 cylc/flow/etc/examples/hello-world/index.rst
 create mode 100755 cylc/flow/etc/examples/integer-cycling/.validate
 create mode 100644 cylc/flow/etc/examples/integer-cycling/flow.cylc
 create mode 100644 cylc/flow/etc/examples/integer-cycling/index.rst
 create mode 100755 cylc/flow/etc/examples/inter-workflow-triggers/.validate
 create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/README.rst
 create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc
 create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/index.rst
 create mode 100644 cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc

diff --git a/README.md b/README.md
index bc28920d48f..e97a7731ab4 100644
--- a/README.md
+++ b/README.md
@@ -29,28 +29,14 @@ domains.
 # install cylc
 conda install cylc-flow
 
-# write your first workflow
-mkdir -p ~/cylc-src/example
-cat > ~/cylc-src/example/flow.cylc <<__CONFIG__
-[scheduling]
-    initial cycle point = 1
-    cycling mode = integer
-    [[graph]]
-        P1 = """
-            a => b => c & d
-            b[-P1] => b
-        """
-[runtime]
-    [[a, b, c, d]]
-        script = echo "Hello $CYLC_TASK_NAME"
-__CONFIG__
+# extract an example to run
+cylc get-resources examples/integer-cycling
 
 # install and run it
-cylc install example
-cylc play example
+cylc vip integer-cycling  # vip = validate, install and play
 
 # watch it run
-cylc tui example
+cylc tui integer-cycling
 ```
 
 ### The Cylc Ecosystem
diff --git a/cylc/flow/etc/examples/README.md b/cylc/flow/etc/examples/README.md
new file mode 100644
index 00000000000..ae4e1391d23
--- /dev/null
+++ b/cylc/flow/etc/examples/README.md
@@ -0,0 +1,27 @@
+# Examples
+
+These examples are intended to illustrate the major patterns for implementing
+Cylc workflows. The hope is that users can find a workflow which fits their
+pattern, make a copy and fill in the details. Keep the examples minimal and
+abstract. We aren't trying to document every Cylc feature here, just the
+major design patterns.
+
+These examples are auto-documented in cylc-doc which looks for an `index.rst`
+file in each example.
+
+Users can extract them using `cylc get-resources` which will put them into the
+configured Cylc source directory (`~/cylc-src` by default). They can then be
+run using the directory name, e.g. `cylc vip hello-world`.
+
+Files:
+
+* `index.rst`
+  This file is used to generate a page in the documentation for the example.
+  This file is excluded when the user extracts the example.
+* `.validate`
+  This is a test file, it gets detected and run automatically.
+  This file is excluded when the user extracts the example.
+* `README.rst`
+  Examples can include a README file, to save duplication, you can
+  `.. include::` this in the `index.rst` file (hence using ReStructuredText
+  rather than Markdown).
diff --git a/cylc/flow/etc/examples/converging-workflow/.validate b/cylc/flow/etc/examples/converging-workflow/.validate
new file mode 100755
index 00000000000..ad61b518ab2
--- /dev/null
+++ b/cylc/flow/etc/examples/converging-workflow/.validate
@@ -0,0 +1,33 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+
+# start the workflow
+cylc vip --check-circular --no-run-name --no-detach --workflow-name "$ID"
+
+# it should have reached the forth cycle
+test -d "${HOME}/cylc-run/${ID}/log/job/4"
+test ! -d "${HOME}/cylc-run/${ID}/log/job/5"
+
+# lint
+cylc lint "$ID"
+
+# clean up
+cylc clean "$ID"
diff --git a/cylc/flow/etc/examples/converging-workflow/flow.cylc b/cylc/flow/etc/examples/converging-workflow/flow.cylc
new file mode 100644
index 00000000000..b2069a3f479
--- /dev/null
+++ b/cylc/flow/etc/examples/converging-workflow/flow.cylc
@@ -0,0 +1,35 @@
+[meta]
+    title = Converging Workflow
+    description = """
+        A workflow which runs a pattern of tasks over and over until a
+        convergence condition has been met.
+    """
+
+[scheduling]
+    cycling mode = integer
+    initial cycle point = 1
+    [[graph]]
+        P1 = """
+            # run "increment" then check the convergence condition
+            check_convergence[-P1]:not_converged? => increment => check_convergence
+
+            # if the workflow has converged, then do nothing
+            check_convergence:converged?
+        """
+
+[runtime]
+    [[increment]]
+        # a task which evolves the data
+    [[check_convergence]]
+        # a task which checks whether the convergence condition has been met
+        script = """
+            if (( CYLC_TASK_CYCLE_POINT == 4 )); then
+                # for the purpose of example, assume convergence at cycle point 4
+                cylc message -- 'convergence condition met'
+            else
+                cylc message -- 'convergence condition not met'
+            fi
+        """
+        [[[outputs]]]
+            converged = 'convergence condition met'
+            not_converged = 'convergence condition not met'
diff --git a/cylc/flow/etc/examples/converging-workflow/index.rst b/cylc/flow/etc/examples/converging-workflow/index.rst
new file mode 100644
index 00000000000..27da079d58f
--- /dev/null
+++ b/cylc/flow/etc/examples/converging-workflow/index.rst
@@ -0,0 +1,39 @@
+Converging Workflow
+===================
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/converging-workflow
+
+A workflow which runs a pattern of tasks over and over until a convergence
+condition has been met.
+
+* The ``increment`` task runs some kind of model or process which increments
+  us toward the solution.
+* The ``check_convergence`` task, checks if the convergence condition has been
+  met.
+
+.. literalinclude:: flow.cylc
+   :language: cylc
+
+Run it with::
+
+   $ cylc vip converging-workflow
+
+.. admonition:: Example - Genetic algorithms
+   :class: hint
+
+   .. _genetic algorithm: https://en.wikipedia.org/wiki/Genetic_algorithm
+   
+   An example of a converging workflow might be a `genetic algorithm`_, where you
+   "breed" entities, then test their "fitness", and breed again, over and over
+   until you end up with an entity which is able to satisfy the requirement.
+   
+   .. digraph:: Example
+   
+      random_seed -> breed -> test_fitness
+      test_fitness -> breed
+      test_fitness -> stop
diff --git a/cylc/flow/etc/examples/datetime-cycling/.validate b/cylc/flow/etc/examples/datetime-cycling/.validate
new file mode 100755
index 00000000000..32c2a51908b
--- /dev/null
+++ b/cylc/flow/etc/examples/datetime-cycling/.validate
@@ -0,0 +1,28 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+cylc vip \
+    --check-circular \
+    --no-detach \
+    --no-run-name \
+    --final-cycle-point "$(isodatetime now --format 'CCYYMMDD')T00" \
+    --workflow-name "$ID"
+cylc lint "$ID"
+cylc clean "$ID"
diff --git a/cylc/flow/etc/examples/datetime-cycling/flow.cylc b/cylc/flow/etc/examples/datetime-cycling/flow.cylc
new file mode 100644
index 00000000000..76945349987
--- /dev/null
+++ b/cylc/flow/etc/examples/datetime-cycling/flow.cylc
@@ -0,0 +1,44 @@
+[meta]
+    title = Datetime Cycling
+    description = """
+        A basic cycling workflow which runs the same set of tasks over
+        and over. Each cycle will be given a datetime identifier.
+
+        The task "a" will wait until the real-world (or wallclock) time passes
+        the cycle time.
+
+        Try changing the "initial cycle point" to "previous(00T00) - P1D" to
+        see how this works.
+    """
+
+[scheduling]
+    # set the start of the graph to 00:00 this morning
+    initial cycle point = previous(T00)
+
+    [[graph]]
+        # repeat this with a "P"eriod of "1" "D"ay -> P1D
+        P1D = """
+            # this is the workflow we want to repeat:
+            a => b => c & d
+
+            # this is an "inter-cycle dependency", it makes the task "b"
+            # wait until its previous instance has successfully completed:
+            b[-P1D] => b
+
+            # this makes the task "a" wait until its cycle point matches
+            # the real world time - i.e. it prevents the workflow from getting
+            # ahead of the clock. If the workflow is running behind (e.g. after
+            # a delay, or from an earlier initial cycle point) it will catch
+            # until the clock-trigger constrains it again. To run entirely in
+            # "simulated time" remove this line:
+            @wall_clock => a
+        """
+
+[runtime]
+    [[root]]
+        # all tasks will "inherit" the configuration in the "root" section
+        script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!"
+    [[a]]
+    [[b]]
+    [[c]]
+    [[d]]
diff --git a/cylc/flow/etc/examples/datetime-cycling/index.rst b/cylc/flow/etc/examples/datetime-cycling/index.rst
new file mode 100644
index 00000000000..803a532551f
--- /dev/null
+++ b/cylc/flow/etc/examples/datetime-cycling/index.rst
@@ -0,0 +1,16 @@
+Datetime Cycling
+================
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/datetime-cycling
+
+.. literalinclude:: flow.cylc
+   :language: cylc
+
+Run it with::
+
+   $ cylc vip datetime-cycling
diff --git a/cylc/flow/etc/examples/event-driven-cycling/.validate b/cylc/flow/etc/examples/event-driven-cycling/.validate
new file mode 100755
index 00000000000..ef224cd9e2c
--- /dev/null
+++ b/cylc/flow/etc/examples/event-driven-cycling/.validate
@@ -0,0 +1,47 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+
+# start the workflow
+cylc vip --check-circular --no-run-name --workflow-name "$ID"
+sleep 1  # give it a reasonable chance to start up
+
+# kick off the first cycle
+./bin/trigger "$ID" WORLD=earth
+
+# wait for it to complete
+cylc workflow-state "$ID" \
+    --task=run \
+    --point=1 \
+    --status=succeeded \
+    --max-polls=60 \
+    --interval=1
+
+# check the job received the environment variable we provided
+grep 'Hello earth' "$HOME/cylc-run/$ID/log/job/1/run/NN/job.out"
+
+# stop the workflow
+cylc stop --kill --max-polls=10 --interval=2 "$ID"
+
+# lint
+cylc lint "$ID"
+
+# clean up
+cylc clean "$ID"
diff --git a/cylc/flow/etc/examples/event-driven-cycling/README.rst b/cylc/flow/etc/examples/event-driven-cycling/README.rst
new file mode 100644
index 00000000000..4fa4ddde241
--- /dev/null
+++ b/cylc/flow/etc/examples/event-driven-cycling/README.rst
@@ -0,0 +1,49 @@
+Cylc is good at orchestrating tasks to a schedule, e.g:
+
+* ``PT1H`` - every hour
+* ``P1D`` - every day
+* ``P1M`` - every month
+* ``PT1H ! (T00, T12)`` - every hour, except midnight and midday.
+
+But sometimes the things you want to run don't have a schedule.
+
+This example uses ``cylc ext-trigger`` to establish a pattern where Cylc waits
+for an external signal and starts a new cycle every time a signal is recieved.
+
+The signal can carry data using the ext-trigger ID, this example sets the ID
+as a file path containing some data that we want to make available to the tasks
+that run in the cycle it triggers.
+
+To use this example, first start the workflow as normal::
+
+   cylc vip event-driven-cycling
+
+Then, when you're ready, kick off a new cycle, specifying any 
+environment variables you want to configure this cycle with::
+
+   ./bin/trigger <workflow-id> WORLD=earth
+
+Replacing ``<workflow-id>`` with the ID you installed this workflow as.
+
+.. admonition:: Example - CI/CD
+   :class: hint
+
+   This pattern is good for CI/CD type workflows where you're waiting on
+   external events. This pattern is especially powerful when used with
+   sub-workflows where it provides a solution to two-dimensional cycling
+   problems.
+
+.. admonition:: Example - Polar satellite data processing
+   :class: hint
+
+   Polar satellites pass overhead at irregular intervals. This makes it tricky
+   to schedule data processing because you don't know when the satellite will
+   pass over the receiver station. With the event driven cycling approach you
+   could start a new cycle every time data arrives.
+
+.. note::
+
+   * The number of parallel cycles can be adjusted by changing the
+     :cylc:conf:`[scheduling]runahead limit`.
+   * To avoid hitting the runahead limit, ensure that failures are handled in
+     the graph.
diff --git a/cylc/flow/etc/examples/event-driven-cycling/bin/trigger b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger
new file mode 100755
index 00000000000..9f917f35f5c
--- /dev/null
+++ b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -eu
+
+if [[ $# -lt 1 ]]; then
+    echo 'Usage ./trigger WORKFLOW_ID [KEY=VALUE ...]' >&2
+    echo
+    echo 'Trigger a new cycle in the target workflow.'
+    echo 'Any environment variable KEY=VALUE pairs will be broadcasted to'
+    echo 'all tasks in the cycle.'
+    exit 1
+fi
+
+# determine the workflow
+WORKFLOW_ID="$1"
+shift
+WORKFLOW_RUN_DIR="${HOME}/cylc-run/${WORKFLOW_ID}"
+EXT_TRIGGER_DIR="${WORKFLOW_RUN_DIR}/triggers"
+mkdir -p "$EXT_TRIGGER_DIR"
+
+# pick a trigger-id
+TRIGGER_ID="$(isodatetime --print-format CCYYMMDDThhmmss)"
+
+# write environment variables to a broadcast file
+TRIGGER_FILE="${EXT_TRIGGER_DIR}/${TRIGGER_ID}.cylc"
+echo '[environment]' >"$TRIGGER_FILE"
+for env in "$@"; do
+    echo "    $env" >> "$TRIGGER_FILE"
+done
+
+# issue the xtrigger
+cylc ext-trigger "$WORKFLOW_ID" 'trigger' "$TRIGGER_ID"
diff --git a/cylc/flow/etc/examples/event-driven-cycling/flow.cylc b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc
new file mode 100644
index 00000000000..7711c2ba258
--- /dev/null
+++ b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc
@@ -0,0 +1,32 @@
+[scheduling]
+    cycling mode = integer
+    initial cycle point = 1
+    runahead limit = P5  # max number of cycles which can run in parallel
+    [[special tasks]]
+        # register the external trigger, it must be given a name,
+        # here, 'trigger' is used as a placeholder, the bash script will
+        # need to be updated if this is changed
+        external-trigger = configure("trigger")
+    [[graph]]
+        P1 = """
+            # use a "?" to prevent failures causing runahead stalls
+            configure? => run
+        """
+
+[runtime]
+    [[configure]]
+        # this task reads in the broadcast file the trigger wrote
+        # and broadcasts any variables set to all tasks in this cycle
+        script = """
+            echo "received new ext-trigger ID=$CYLC_EXT_TRIGGER_ID"
+            TRIGGER_FILE="${CYLC_WORKFLOW_RUN_DIR}/triggers/${CYLC_EXT_TRIGGER_ID}.cylc"
+            cylc broadcast "${CYLC_WORKFLOW_ID}" \
+                -p "${CYLC_TASK_CYCLE_POINT}" \
+                -F "${TRIGGER_FILE}"
+        """
+
+    [[run]]
+        # this task could be a sub-workflow
+        script = """
+            echo "Hello $WORLD!"
+        """
diff --git a/cylc/flow/etc/examples/event-driven-cycling/index.rst b/cylc/flow/etc/examples/event-driven-cycling/index.rst
new file mode 100644
index 00000000000..5980591f0fd
--- /dev/null
+++ b/cylc/flow/etc/examples/event-driven-cycling/index.rst
@@ -0,0 +1,14 @@
+Event Driven Cycling
+====================
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/event-driven-cycling
+
+.. include:: README.rst
+
+.. literalinclude:: flow.cylc
+   :language: cylc
diff --git a/cylc/flow/etc/examples/hello-world/.validate b/cylc/flow/etc/examples/hello-world/.validate
new file mode 100755
index 00000000000..968a7b349cb
--- /dev/null
+++ b/cylc/flow/etc/examples/hello-world/.validate
@@ -0,0 +1,23 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+cylc vip --check-circular --no-detach --no-run-name --workflow-name "$ID"
+cylc lint "$ID"
+cylc clean "$ID"
diff --git a/cylc/flow/etc/examples/hello-world/flow.cylc b/cylc/flow/etc/examples/hello-world/flow.cylc
new file mode 100644
index 00000000000..92510d1928b
--- /dev/null
+++ b/cylc/flow/etc/examples/hello-world/flow.cylc
@@ -0,0 +1,13 @@
+[meta]
+    title = Hello World
+    description = """
+        A simple workflow which runs a single task (hello_world) once.
+    """
+
+[scheduling]
+    [[graph]]
+        R1 = hello_world
+
+[runtime]
+    [[hello_world]]
+        script = echo "Hello World!"
diff --git a/cylc/flow/etc/examples/hello-world/index.rst b/cylc/flow/etc/examples/hello-world/index.rst
new file mode 100644
index 00000000000..a73afdf1e44
--- /dev/null
+++ b/cylc/flow/etc/examples/hello-world/index.rst
@@ -0,0 +1,21 @@
+Hello World
+-----------
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/hello-world
+
+In the time honoured tradition, this is the minimal Cylc workflow:
+
+.. literalinclude:: flow.cylc
+   :language: cylc
+
+It writes the phrase "Hello World!" to standard output (captured to the
+``job.out`` log file).
+
+Run it with::
+
+   $ cylc vip hello-world
diff --git a/cylc/flow/etc/examples/integer-cycling/.validate b/cylc/flow/etc/examples/integer-cycling/.validate
new file mode 100755
index 00000000000..054f2662862
--- /dev/null
+++ b/cylc/flow/etc/examples/integer-cycling/.validate
@@ -0,0 +1,23 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+cylc vip --check-circular --no-detach --no-run-name --final-cycle-point=1 --workflow-name "$ID"
+cylc lint "$ID"
+cylc clean "$ID"
diff --git a/cylc/flow/etc/examples/integer-cycling/flow.cylc b/cylc/flow/etc/examples/integer-cycling/flow.cylc
new file mode 100644
index 00000000000..b8d2f2f05ce
--- /dev/null
+++ b/cylc/flow/etc/examples/integer-cycling/flow.cylc
@@ -0,0 +1,28 @@
+[meta]
+    title = Integer Cycling
+    description = """
+        A basic cycling workflow which runs the same set of tasks over
+        and over. Each cycle will be given an integer number.
+    """
+
+[scheduling]
+    # tell Cylc to count cycles as numbers starting from the number 1
+    cycling mode = integer
+    initial cycle point = 1
+    [[graph]]
+        P1 = """
+            # this is the workflow we want to repeat:
+            a => b => c & d
+            # this is an "inter-cycle dependency", it makes the task "b"
+            # wait until its previous instance has completed:
+            b[-P1] => b
+        """
+
+[runtime]
+    [[root]]
+        # all tasks will "inherit" the configuration in the "root" section
+        script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!"
+    [[a]]
+    [[b]]
+    [[c]]
+    [[d]]
diff --git a/cylc/flow/etc/examples/integer-cycling/index.rst b/cylc/flow/etc/examples/integer-cycling/index.rst
new file mode 100644
index 00000000000..89be2bb0871
--- /dev/null
+++ b/cylc/flow/etc/examples/integer-cycling/index.rst
@@ -0,0 +1,16 @@
+Integer Cycling
+===============
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/integer-cycling
+
+.. literalinclude:: flow.cylc
+   :language: cylc
+
+Run it with::
+
+   $ cylc vip integer-cycling
diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/.validate b/cylc/flow/etc/examples/inter-workflow-triggers/.validate
new file mode 100755
index 00000000000..bdd414e275d
--- /dev/null
+++ b/cylc/flow/etc/examples/inter-workflow-triggers/.validate
@@ -0,0 +1,57 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+UPID=inter-workflow-triggers/upstream
+DOID=inter-workflow-triggers/downstream
+
+ICP="$(isodatetime now --format=CCYYMMDDThh --offset=-PT2H)"
+
+# run the workflows
+cylc vip \
+    --check-circular \
+    --no-run-name \
+    --final-cycle-point="$ICP" \
+    --workflow-name "$UPID" \
+    ./upstream
+cylc vip \
+    --check-circular \
+    --no-run-name \
+    --final-cycle-point="$ICP" \
+    --workflow-name "$DOID" \
+    ./downstream
+
+# wait for the first task in the downstream to succeed
+cylc workflow-state "$DOID" \
+    --task=process \
+    --point="$ICP" \
+    --status=succeeded \
+    --max-polls=60 \
+    --interval=1
+
+# stop the workflows
+cylc stop --kill --max-polls=10 --interval=2 "$UPID"
+cylc stop --kill --max-polls=10 --interval=2 "$DOID"
+
+# lint'em
+cylc lint "$UPID"
+cylc lint "$DOID"
+
+# clean up
+cylc clean "$UPID"
+cylc clean "$DOID"
diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/README.rst b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst
new file mode 100644
index 00000000000..8aa8d7d7b6f
--- /dev/null
+++ b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst
@@ -0,0 +1,14 @@
+This example shows how one workflow can "trigger off" of tasks in another
+workflow.
+
+In this example, there are two workflows:
+
+* "upstream" writes a file.
+* "downstream" reads this file.
+
+Run both workflows simultaneously to see this in action:
+
+.. code-block:: console
+
+   $ cylc vip inter-workflow-triggers/upstream
+   $ cylc vip inter-workflow-triggers/downstream
diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc
new file mode 100644
index 00000000000..115ecd94755
--- /dev/null
+++ b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc
@@ -0,0 +1,25 @@
+[meta]
+    title = Downstream Workflow
+    description = """
+        This workflow uses the data provided by the upstream workflow.
+    """
+
+[scheduling]
+    # start two hours before the current hour
+    initial cycle point = previous(T-00) - PT2H
+    [[xtriggers]]
+        # this is an "xtrigger" - it will wait for the task "b" in the same
+        # cycle from the workflow "upstream"
+        upstream = workflow_state(workflow="inter-workflow-triggers/upstream", task="b", point="%(point)s")
+    [[graph]]
+        PT1H = """
+            @upstream => process
+        """
+
+[runtime]
+    [[process]]
+        script = echo "The random number is: $(cat "$file")"
+        [[[environment]]]
+            # this is where the data should be written to in the upstream workflow
+            # Note: "runN" will point to the most recent run of a workflow
+            file = $HOME/cylc-run/upstream/runN/share/$CYLC_TASK_CYCLE_POINT
diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/index.rst b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst
new file mode 100644
index 00000000000..0fd70b17d21
--- /dev/null
+++ b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst
@@ -0,0 +1,28 @@
+Inter-Workflow Triggering
+=========================
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/inter-workflow-triggers
+
+.. include:: README.rst
+
+.. literalinclude:: upstream/flow.cylc
+   :language: cylc
+   :caption: Upstream Workflow
+
+.. literalinclude:: downstream/flow.cylc
+   :language: cylc
+   :caption: Downstream Workflow
+
+.. admonition:: Example - Decoupled workflows
+   :class: hint
+
+   This pattern is useful where you have workflows that you want to keep decoupled
+   from one another, but still need to exchange data. E.G. in operational
+   meteorology we might have a global model (covering the whole Earth) and a
+   regional model (just covering a little bit of it) where the regional model
+   obtains its boundary condition from the global model.
diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc
new file mode 100644
index 00000000000..145c3b323db
--- /dev/null
+++ b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc
@@ -0,0 +1,27 @@
+[meta]
+    title = Upstream Workflow
+    description = """
+        This is the workflow which is providing the data that the downstream
+        workflow wants to use.
+    """
+
+[scheduling]
+    # start two hours before the current time on the whole hour
+    initial cycle point = previous(T-00) - PT2H
+    [[graph]]
+        PT1H = """
+            # wait for the "real world" time before running "a":
+            @wall_clock => a
+
+            # then run task "b"
+            a => b
+        """
+
+[runtime]
+    [[a]]
+    [[b]]
+        # write a random number to ~/cylc-run/<workflow-id>/share/<cycle>
+        # for the downstream workflow to use
+        script = echo "$RANDOM" > "$file"
+        [[[environment]]]
+            file = ${CYLC_WORKFLOW_SHARE_DIR}/${CYLC_TASK_CYCLE_POINT}
diff --git a/cylc/flow/resources.py b/cylc/flow/resources.py
index 79884e27476..d81b7f7211b 100644
--- a/cylc/flow/resources.py
+++ b/cylc/flow/resources.py
@@ -16,6 +16,7 @@
 
 """Extract named resources from the cylc.flow package."""
 
+from contextlib import suppress
 from pathlib import Path
 from random import shuffle
 import shutil
@@ -31,6 +32,7 @@
 
 RESOURCE_DIR = Path(cylc.flow.__file__).parent / 'etc'
 TUTORIAL_DIR = RESOURCE_DIR / 'tutorial'
+EXAMPLE_DIR = RESOURCE_DIR / 'examples'
 
 
 # {resource: brief description}
@@ -52,6 +54,11 @@ def list_resources(write=print, headers=True):
         for path in TUTORIAL_DIR.iterdir()
         if path.is_dir()
     ]
+    examples = [
+        path.relative_to(RESOURCE_DIR)
+        for path in EXAMPLE_DIR.iterdir()
+        if path.is_dir()
+    ]
     if headers:
         write('Resources:')
     max_len = max(len(res) for res in RESOURCE_NAMES)
@@ -62,15 +69,21 @@ def list_resources(write=print, headers=True):
     for tutorial in tutorials:
         write(f'  {tutorial}')
     write(f'  {API_KEY}')
+    if headers:
+        write('\nExamples:')
+    for example in examples:
+        write(f'  {example}')
 
 
-def path_is_tutorial(src: Path) -> bool:
-    """Returns True if the src path is in the tutorial directory."""
-    try:
+def path_is_source_workflow(src: Path) -> bool:
+    """Returns True if the src path is a Cylc workflow."""
+    with suppress(ValueError):
         src.relative_to(TUTORIAL_DIR)
-    except ValueError:
-        return False
-    return True
+        return True
+    with suppress(ValueError):
+        src.relative_to(EXAMPLE_DIR)
+        return True
+    return False
 
 
 def get_resources(resource: str, tgt_dir: Optional[str]):
@@ -95,11 +108,11 @@ def get_resources(resource: str, tgt_dir: Optional[str]):
             '\nRun `cylc get-resources --list` for resource names.'
         )
 
-    is_tutorial = path_is_tutorial(src)
+    is_source_workflow = path_is_source_workflow(src)
 
     # get the target path
     if not tgt_dir:
-        if is_tutorial:
+        if is_source_workflow:
             # this is a tutorial => use the primary source dir
             _tgt_dir = Path(glbl_cfg().get(['install', 'source dirs'])[0])
         else:
@@ -113,8 +126,8 @@ def get_resources(resource: str, tgt_dir: Optional[str]):
     tgt = tgt.resolve()
 
     # extract resources
-    extract_resource(src, tgt, is_tutorial)
-    if is_tutorial:
+    extract_resource(src, tgt, is_source_workflow)
+    if is_source_workflow:
         set_api_key(tgt)
 
 
@@ -131,16 +144,32 @@ def _backup(tgt: Path) -> None:
     shutil.move(str(tgt), str(backup))
 
 
-def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None:
+def extract_resource(
+    src: Path,
+    tgt: Path,
+    is_source_workflow: bool = False,
+) -> None:
     """Extract src into tgt.
 
     NOTE: src can be a dir or a file.
     """
     LOG.info(f"Extracting {src.relative_to(RESOURCE_DIR)} to {tgt}")
-    if is_tutorial and tgt.exists():
+    if is_source_workflow and tgt.exists():
         # target exists, back up the old copy
         _backup(tgt)
 
+    # files to exclude
+    if is_source_workflow:
+        excludes = [
+            # test files
+            '.validate',
+            'reftest',
+            # documentation files
+            'index.rst',
+        ]
+    else:
+        excludes = []
+
     # create the target directory
     try:
         tgt.parent.mkdir(parents=True, exist_ok=True)
@@ -151,6 +180,10 @@ def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None:
             shutil.copytree(str(src), str(tgt))
         else:
             shutil.copyfile(str(src), str(tgt))
+        for exclude in excludes:
+            path = tgt / exclude
+            if path.exists():
+                path.unlink()
     except IsADirectoryError as exc:
         LOG.error(
             f'Cannot extract file {exc.filename} as there is an '
diff --git a/etc/bin/run-validate-tutorials b/etc/bin/run-validate-tutorials
index e9b7be1ecde..72bf164fe2a 100755
--- a/etc/bin/run-validate-tutorials
+++ b/etc/bin/run-validate-tutorials
@@ -17,13 +17,15 @@
 
 
 set -eu
-cd "$(dirname "$0")"
+cd "$(dirname "$0")/../../"
 
-for FILE in $(echo ../../cylc/flow/etc/tutorial/*/.validate) ; do
-    echo "running $FILE"
-    (
-        cd "$(dirname "$FILE")"
-        ./.validate
-    )
+for DIR in tutorial examples; do
+    echo "# Running tests for: $DIR"
+    for FILE in $(echo "cylc/flow/etc/$DIR/"*/.validate); do
+        echo "## Running test: $FILE"
+        (
+            cd "$(dirname "$FILE")"
+            ./.validate
+        )
+    done
 done
-

From 94fd7ac4129b8345a702382dbfdf6b55911eadf4 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Mon, 29 Jan 2024 17:28:13 +0000
Subject: [PATCH 093/110] docs: document exceptions (#5940)

---
 cylc/flow/exceptions.py                | 76 ++++++++++++++------------
 cylc/flow/jinja/filters/duration_as.py | 10 ++--
 cylc/flow/jinja/filters/pad.py         | 10 ++--
 cylc/flow/jinja/filters/strftime.py    | 14 +++--
 cylc/flow/network/client.py            |  8 +--
 cylc/flow/network/scan.py              | 32 +++++++----
 cylc/flow/parsec/exceptions.py         |  4 +-
 cylc/flow/workflow_files.py            | 23 +++++++-
 8 files changed, 111 insertions(+), 66 deletions(-)

diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py
index d1a459f996b..43591e7be9c 100644
--- a/cylc/flow/exceptions.py
+++ b/cylc/flow/exceptions.py
@@ -15,16 +15,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Exceptions for "expected" errors."""
 
-import errno
+
 from textwrap import wrap
 from typing import (
-    Callable,
     Dict,
     Iterable,
-    NoReturn,
     Optional,
-    Tuple,
-    Type,
     Union,
     TYPE_CHECKING,
 )
@@ -42,7 +38,6 @@ class CylcError(Exception):
     message to the user is more appropriate than traceback.
 
     CLI commands will catch this exception and exit with str(exception).
-
     """
 
 
@@ -50,10 +45,14 @@ class PluginError(CylcError):
     """Represents an error arising from a Cylc plugin.
 
     Args:
-        entry_point: The plugin entry point as defined in setup.cfg
+        entry_point:
+            The plugin entry point as defined in setup.cfg
             (e.g. 'cylc.main_loop')
-        plugin_name: Name of the plugin
-        exc: Original exception caught when trying to run the plugin
+        plugin_name:
+            Name of the plugin
+        exc:
+            Original exception caught when trying to run the plugin
+
     """
 
     def __init__(self, entry_point: str, plugin_name: str, exc: Exception):
@@ -78,12 +77,8 @@ class InputError(CylcError):
 
 
 class CylcConfigError(CylcError):
-    """Generic exception to handle an error in a Cylc configuration file.
-
-    TODO:
-        * reference the configuration element causing the problem
-
-    """
+    """Generic exception to handle an error in a Cylc configuration file."""
+    # TODO: reference the configuration element causing the problem
 
 
 class WorkflowConfigError(CylcConfigError):
@@ -142,19 +137,6 @@ class ContactFileExists(CylcError):
     """Workflow contact file exists."""
 
 
-def handle_rmtree_err(
-    function: Callable,
-    path: str,
-    excinfo: Tuple[Type[Exception], Exception, object]
-) -> NoReturn:
-    """Error handler for shutil.rmtree."""
-    exc = excinfo[1]
-    if isinstance(exc, OSError) and exc.errno == errno.ENOTEMPTY:
-        # "Directory not empty", likely due to filesystem lag
-        raise FileRemovalError(exc)
-    raise exc
-
-
 class FileRemovalError(CylcError):
     """Exception for errors during deletion of files/directories, which are
     probably the filesystem's fault, not Cylc's."""
@@ -269,6 +251,18 @@ def __str__(self):
 
 
 class ClientError(CylcError):
+    """Base class for errors raised by Cylc client instances.
+
+    For example, the workflow you are trying to connect to is stopped.
+
+    Attributes:
+        message:
+            The exception message.
+        traceback:
+            The underlying exception instance if available.
+        workflow:
+            The workflow ID if available.
+    """
 
     def __init__(
         self,
@@ -291,7 +285,7 @@ def __str__(self) -> str:
 
 
 class WorkflowStopped(ClientError):
-    """Special case of ClientError for a stopped workflow."""
+    """The Cylc scheduler you attempted to connect to is stopped."""
 
     def __init__(self, workflow):
         self.workflow = workflow
@@ -301,6 +295,16 @@ def __str__(self):
 
 
 class ClientTimeout(CylcError):
+    """The scheduler did not respond within the timeout.
+
+    This could be due to:
+    * Network issues.
+    * Scheduler issues.
+    * Insufficient timeout.
+
+    To increase the timeout, use the ``--comms-timeout`` option.
+
+    """
 
     def __init__(self, message: str, workflow: Optional[str] = None):
         self.message = message
@@ -311,7 +315,7 @@ def __str__(self) -> str:
 
 
 class CyclingError(CylcError):
-    pass
+    """Base class for errors in cycling configuration."""
 
 
 class CyclerTypeError(CyclingError):
@@ -446,10 +450,14 @@ class NoPlatformsError(PlatformLookupError):
     """None of the platforms of a given set were reachable.
 
     Args:
-        identity: The name of the platform group or install target
-        set_type: Whether the set of platforms is a platform group or an
-            install target
-        place: Where the attempt to get the platform failed.
+        identity:
+            The name of the platform group or install target.
+        set_type:
+            Whether the set of platforms is a platform group or an install
+            target.
+        place:
+            Where the attempt to get the platform failed.
+
     """
     def __init__(
         self, identity: str, set_type: str = 'group', place: str = ''
diff --git a/cylc/flow/jinja/filters/duration_as.py b/cylc/flow/jinja/filters/duration_as.py
index bc692a68017..7f2775facb9 100644
--- a/cylc/flow/jinja/filters/duration_as.py
+++ b/cylc/flow/jinja/filters/duration_as.py
@@ -15,6 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Filter for formatting ISO8601 duration strings."""
 
+from typing import Callable, Dict, Tuple
+
 from metomi.isodatetime.parsers import DurationParser
 
 SECONDS_PER_MINUTE = 60.0
@@ -26,7 +28,7 @@
 SECONDS_PER_DAY = SECONDS_PER_HOUR * HOURS_PER_DAY
 SECONDS_PER_WEEK = SECONDS_PER_DAY * DAYS_PER_WEEK
 
-CONVERSIONS = {
+CONVERSIONS: Dict[Tuple[str, str], Callable] = {
     ('s', 'seconds'): float,
     ('m', 'minutes'): lambda s: float(s) / SECONDS_PER_MINUTE,
     ('h', 'hours'): lambda s: float(s) / SECONDS_PER_HOUR,
@@ -35,7 +37,7 @@
 }
 
 
-def duration_as(iso8601_duration, units):
+def duration_as(iso8601_duration: str, units: str) -> float:
     """Format an :term:`ISO8601 duration` string as the specified units.
 
     Units for the conversion can be specified in a case-insensitive short or
@@ -57,8 +59,8 @@ def duration_as(iso8601_duration, units):
     - ``{{CYCLE_SUBINTERVAL | duration_as('s') | int}}`` - 1800
 
     Args:
-        iso8601_duration (str): Any valid ISO8601 duration as a string.
-        units (str): Destination unit for the duration conversion
+        iso8601_duration: Any valid ISO8601 duration as a string.
+        units: Destination unit for the duration conversion
 
     Return:
         The total number of the specified unit contained in the specified
diff --git a/cylc/flow/jinja/filters/pad.py b/cylc/flow/jinja/filters/pad.py
index 75c5e117dd3..1c013b22ce4 100644
--- a/cylc/flow/jinja/filters/pad.py
+++ b/cylc/flow/jinja/filters/pad.py
@@ -15,18 +15,20 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Filter for padding strings to a set number of chars."""
 
+from typing import Union
 
-def pad(value, length, fillchar=' '):
+
+def pad(value: str, length: Union[int, str], fillchar: str = ' '):
     """Pads a string to some length with a fill character
 
     Useful for generating task names and related values in ensemble workflows.
 
     Args:
-        value (str):
+        value:
             The string to pad.
-        length (int/str):
+        length:
             The length for the returned string.
-        fillchar (str - optional):
+        fillchar:
             The character to fill in surplus space (space by default).
 
     Returns:
diff --git a/cylc/flow/jinja/filters/strftime.py b/cylc/flow/jinja/filters/strftime.py
index d1fcb5d3eef..1af97d75abc 100644
--- a/cylc/flow/jinja/filters/strftime.py
+++ b/cylc/flow/jinja/filters/strftime.py
@@ -15,10 +15,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Filter for formatting ISO8601 datetime strings."""
 
+from typing import Optional
+
 from metomi.isodatetime.parsers import TimePointParser
 
 
-def strftime(iso8601_datetime, strftime_str, strptime_str=None):
+def strftime(
+    iso8601_datetime: str,
+    strftime_str: str,
+    strptime_str: Optional[str] = None,
+):
     """Format an :term:`ISO8601 datetime` string using an strftime string.
 
     .. code-block:: cylc
@@ -29,11 +35,11 @@ def strftime(iso8601_datetime, strftime_str, strptime_str=None):
     strptime string as the second argument.
 
     Args:
-        iso8601_datetime (str):
+        iso8601_datetime:
             Any valid ISO8601 datetime as a string.
-        strftime_str (str):
+        strftime_str:
             A valid strftime string to format the output datetime.
-        strptime_str (str - optional):
+        strptime_str:
             A valid strptime string defining the format of the provided
             iso8601_datetime.
 
diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py
index b3955d3b058..e7e26954d56 100644
--- a/cylc/flow/network/client.py
+++ b/cylc/flow/network/client.py
@@ -195,15 +195,15 @@ class WorkflowRuntimeClient(  # type: ignore[misc]
             the contact file.
 
     Attributes:
-        host (str):
+        host:
             Workflow host name.
-        port (int):
+        port:
             Workflow host port.
-        timeout_handler (function):
+        timeout_handler:
             Optional function which runs before ClientTimeout is raised.
             This provides an interface for raising more specific exceptions in
             the event of a communication timeout.
-        header (dict):
+        header:
             Request "header" data to attach to each request.
 
     Usage:
diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py
index c2202f3f31e..6de28ada517 100644
--- a/cylc/flow/network/scan.py
+++ b/cylc/flow/network/scan.py
@@ -48,7 +48,16 @@
 import asyncio
 from pathlib import Path
 import re
-from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union
+from typing import (
+    AsyncGenerator,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
 
 from packaging.version import parse as parse_version
 from packaging.specifiers import SpecifierSet
@@ -422,7 +431,7 @@ def format_query(fields, filters=None):
 
 
 @pipe(preproc=format_query)
-async def graphql_query(flow, fields, filters=None):
+async def graphql_query(flow: dict, fields: Iterable, filters=None):
     """Obtain information from a GraphQL request to the flow.
 
     Requires:
@@ -430,9 +439,9 @@ async def graphql_query(flow, fields, filters=None):
         * contact_info
 
     Args:
-        flow (dict):
+        flow:
             Flow information dictionary, provided by scan through the pipe.
-        fields (iterable):
+        fields:
             Iterable containing the fields to request e.g::
 
                ['id', 'name']
@@ -464,12 +473,15 @@ async def graphql_query(flow, fields, filters=None):
         LOG.warning(f'Workflow not running: {flow["name"]}')
         return False
     try:
-        ret = await client.async_request(
-            'graphql',
-            {
-                'request_string': query,
-                'variables': {}
-            }
+        ret = cast(
+            'dict',
+            await client.async_request(
+                'graphql',
+                {
+                    'request_string': query,
+                    'variables': {}
+                }
+            )
         )
     except WorkflowStopped:
         LOG.warning(f'Workflow not running: {flow["name"]}')
diff --git a/cylc/flow/parsec/exceptions.py b/cylc/flow/parsec/exceptions.py
index 3acfe31c820..690d583b885 100644
--- a/cylc/flow/parsec/exceptions.py
+++ b/cylc/flow/parsec/exceptions.py
@@ -78,8 +78,6 @@ class FileParseError(ParsecError):
             Classification of error (e.g. Jinja2Error).
         help_lines:
             Additional info to include in the exception.
-
-    Args (ways of providing exception context - TODO rationalise this!):
         lines:
             (preferred) Dictionary in the format
             {filename: [context_line, ..., error_line]}
@@ -140,7 +138,7 @@ def __str__(self) -> str:
 
 
 class TemplateVarLanguageClash(FileParseError):
-    ...
+    """Multiple workflow configuration templating engines configured."""
 
 
 class EmPyError(FileParseError):
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index de30bc1e7d2..2f03c8d4394 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -22,21 +22,25 @@
 
 """
 
+from enum import Enum
+import errno
 import os
+from pathlib import Path
 import re
 import shutil
-from enum import Enum
-from pathlib import Path
 from subprocess import (
     PIPE,
     Popen,
     TimeoutExpired,
 )
 from typing import (
+    Callable,
     Dict,
     Optional,
     Tuple,
     Union,
+    NoReturn,
+    Type,
 )
 
 import cylc.flow.flags
@@ -48,7 +52,7 @@
     InputError,
     ServiceFileError,
     WorkflowFilesError,
-    handle_rmtree_err,
+    FileRemovalError,
 )
 from cylc.flow.hostuserutil import (
     get_user,
@@ -64,6 +68,19 @@
 from cylc.flow.util import cli_format
 
 
+def handle_rmtree_err(
+    function: Callable,
+    path: str,
+    excinfo: Tuple[Type[Exception], Exception, object]
+) -> NoReturn:
+    """Error handler for shutil.rmtree."""
+    exc = excinfo[1]
+    if isinstance(exc, OSError) and exc.errno == errno.ENOTEMPTY:
+        # "Directory not empty", likely due to filesystem lag
+        raise FileRemovalError(exc)
+    raise exc
+
+
 class KeyType(Enum):
     """Used for authentication keys - public or private"""
 

From 82af1f3c59e8c2f87752928f3c659166983be765 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 30 Jan 2024 16:35:13 +0000
Subject: [PATCH 094/110] docs: fix sort order of the first few examples
 (#5948)

---
 cylc/flow/etc/examples/{hello-world => 1-hello-world}/.validate   | 0
 cylc/flow/etc/examples/{hello-world => 1-hello-world}/flow.cylc   | 0
 cylc/flow/etc/examples/{hello-world => 1-hello-world}/index.rst   | 0
 .../etc/examples/{integer-cycling => 2-integer-cycling}/.validate | 0
 .../etc/examples/{integer-cycling => 2-integer-cycling}/flow.cylc | 0
 .../etc/examples/{integer-cycling => 2-integer-cycling}/index.rst | 0
 .../examples/{datetime-cycling => 3-datetime-cycling}/.validate   | 0
 .../examples/{datetime-cycling => 3-datetime-cycling}/flow.cylc   | 0
 .../examples/{datetime-cycling => 3-datetime-cycling}/index.rst   | 0
 9 files changed, 0 insertions(+), 0 deletions(-)
 rename cylc/flow/etc/examples/{hello-world => 1-hello-world}/.validate (100%)
 rename cylc/flow/etc/examples/{hello-world => 1-hello-world}/flow.cylc (100%)
 rename cylc/flow/etc/examples/{hello-world => 1-hello-world}/index.rst (100%)
 rename cylc/flow/etc/examples/{integer-cycling => 2-integer-cycling}/.validate (100%)
 rename cylc/flow/etc/examples/{integer-cycling => 2-integer-cycling}/flow.cylc (100%)
 rename cylc/flow/etc/examples/{integer-cycling => 2-integer-cycling}/index.rst (100%)
 rename cylc/flow/etc/examples/{datetime-cycling => 3-datetime-cycling}/.validate (100%)
 rename cylc/flow/etc/examples/{datetime-cycling => 3-datetime-cycling}/flow.cylc (100%)
 rename cylc/flow/etc/examples/{datetime-cycling => 3-datetime-cycling}/index.rst (100%)

diff --git a/cylc/flow/etc/examples/hello-world/.validate b/cylc/flow/etc/examples/1-hello-world/.validate
similarity index 100%
rename from cylc/flow/etc/examples/hello-world/.validate
rename to cylc/flow/etc/examples/1-hello-world/.validate
diff --git a/cylc/flow/etc/examples/hello-world/flow.cylc b/cylc/flow/etc/examples/1-hello-world/flow.cylc
similarity index 100%
rename from cylc/flow/etc/examples/hello-world/flow.cylc
rename to cylc/flow/etc/examples/1-hello-world/flow.cylc
diff --git a/cylc/flow/etc/examples/hello-world/index.rst b/cylc/flow/etc/examples/1-hello-world/index.rst
similarity index 100%
rename from cylc/flow/etc/examples/hello-world/index.rst
rename to cylc/flow/etc/examples/1-hello-world/index.rst
diff --git a/cylc/flow/etc/examples/integer-cycling/.validate b/cylc/flow/etc/examples/2-integer-cycling/.validate
similarity index 100%
rename from cylc/flow/etc/examples/integer-cycling/.validate
rename to cylc/flow/etc/examples/2-integer-cycling/.validate
diff --git a/cylc/flow/etc/examples/integer-cycling/flow.cylc b/cylc/flow/etc/examples/2-integer-cycling/flow.cylc
similarity index 100%
rename from cylc/flow/etc/examples/integer-cycling/flow.cylc
rename to cylc/flow/etc/examples/2-integer-cycling/flow.cylc
diff --git a/cylc/flow/etc/examples/integer-cycling/index.rst b/cylc/flow/etc/examples/2-integer-cycling/index.rst
similarity index 100%
rename from cylc/flow/etc/examples/integer-cycling/index.rst
rename to cylc/flow/etc/examples/2-integer-cycling/index.rst
diff --git a/cylc/flow/etc/examples/datetime-cycling/.validate b/cylc/flow/etc/examples/3-datetime-cycling/.validate
similarity index 100%
rename from cylc/flow/etc/examples/datetime-cycling/.validate
rename to cylc/flow/etc/examples/3-datetime-cycling/.validate
diff --git a/cylc/flow/etc/examples/datetime-cycling/flow.cylc b/cylc/flow/etc/examples/3-datetime-cycling/flow.cylc
similarity index 100%
rename from cylc/flow/etc/examples/datetime-cycling/flow.cylc
rename to cylc/flow/etc/examples/3-datetime-cycling/flow.cylc
diff --git a/cylc/flow/etc/examples/datetime-cycling/index.rst b/cylc/flow/etc/examples/3-datetime-cycling/index.rst
similarity index 100%
rename from cylc/flow/etc/examples/datetime-cycling/index.rst
rename to cylc/flow/etc/examples/3-datetime-cycling/index.rst

From 870fb473399f6be909c041633c8af98691e4e08c Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Mon, 29 Jan 2024 15:12:40 +0000
Subject: [PATCH 095/110] stop after cycle point: support offsets

* Closes #5939
* Support offsets (measured from the ICP) for the `stop after cycle point`
  to mirror the behaviour of the `final cycle point`.
* Add integration test to lock-down `stop after cycle point`
  interactions.
---
 changes.d/5943.feat.md                        |   1 +
 cylc/flow/cfgspec/workflow.py                 |  21 ++-
 cylc/flow/config.py                           |  13 +-
 cylc/flow/parsec/validate.py                  |  28 ++++
 cylc/flow/scheduler.py                        |   1 +
 cylc/flow/task_pool.py                        |   2 +-
 .../functional/cylc-play/09-invalid-cp-opt.t  |   2 +-
 .../test_stop_after_cycle_point.py            | 132 ++++++++++++++++++
 tests/unit/test_config.py                     | 115 ++++++++-------
 9 files changed, 258 insertions(+), 57 deletions(-)
 create mode 100644 changes.d/5943.feat.md
 create mode 100644 tests/integration/test_stop_after_cycle_point.py

diff --git a/changes.d/5943.feat.md b/changes.d/5943.feat.md
new file mode 100644
index 00000000000..6db31d952be
--- /dev/null
+++ b/changes.d/5943.feat.md
@@ -0,0 +1 @@
+The `stop after cycle point` can now be specified as an offset from the inital cycle point.
diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
index de919c27c0f..7ae7cf014b2 100644
--- a/cylc/flow/cfgspec/workflow.py
+++ b/cylc/flow/cfgspec/workflow.py
@@ -539,7 +539,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
         ''')
         # NOTE: final cycle point is not a V_CYCLE_POINT to allow expressions
         # such as '+P1Y' (relative to initial cycle point)
-        Conf('final cycle point', VDR.V_STRING, desc='''
+        Conf('final cycle point', VDR.V_CYCLE_POINT_WITH_OFFSETS, desc='''
             The (optional) last cycle point at which tasks are run.
 
             Once all tasks have reached this cycle point, the
@@ -547,6 +547,12 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
             This item can be overridden on the command line using
             ``cylc play --final-cycle-point`` or ``--fcp``.
+
+            Examples:
+
+            - ``2000`` - Shorthand for ``2000-01-01T00:00``.
+            - ``+P1D`` - The initial cycle point plus one day.
+            - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day.
         ''')
         Conf('initial cycle point constraints', VDR.V_STRING_LIST, desc='''
             Rules to allow only some initial datetime cycle points.
@@ -599,7 +605,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
                {REPLACES}``[scheduling]hold after point``.
         ''')
-        Conf('stop after cycle point', VDR.V_CYCLE_POINT, desc='''
+        Conf('stop after cycle point', VDR.V_CYCLE_POINT_WITH_OFFSETS, desc='''
             Shut down the workflow after all tasks pass this cycle point.
 
             The stop cycle point can be overridden on the command line using
@@ -612,7 +618,18 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                choosing not to run that part of the graph. You can play
                the workflow and continue.
 
+            Examples:
+
+            - ``2000`` - Shorthand for ``2000-01-01T00:00``.
+            - ``+P1D`` - The initial cycle point plus one day.
+            - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day.
+
             .. versionadded:: 8.0.0
+
+            .. versionchanged:: 8.3.0
+
+               This now supports offsets (e.g. ``+P1D``) in the same way the
+               :cylc:conf:`[..]final cycle point` does.
         ''')
         Conf('cycling mode', VDR.V_STRING, Calendar.MODE_GREGORIAN,
              options=list(Calendar.MODES) + ['integer'], desc='''
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index d80456266bf..76c8b4e1980 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -851,9 +851,16 @@ def process_stop_cycle_point(self) -> None:
         if stopcp_str is None:
             stopcp_str = self.cfg['scheduling']['stop after cycle point']
 
-        if stopcp_str is not None:
-            self.stop_point = get_point(stopcp_str).standardise()
-            if self.final_point and (self.stop_point > self.final_point):
+        if stopcp_str:
+            self.stop_point = get_point_relative(
+                stopcp_str,
+                self.initial_point,
+            ).standardise()
+            if (
+                self.final_point is not None
+                and self.stop_point is not None
+                and self.stop_point > self.final_point
+            ):
                 LOG.warning(
                     f"Stop cycle point '{self.stop_point}' will have no "
                     "effect as it is after the final cycle "
diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py
index 4ef66e19862..6e0e604d0c5 100644
--- a/cylc/flow/parsec/validate.py
+++ b/cylc/flow/parsec/validate.py
@@ -657,6 +657,7 @@ class CylcConfigValidator(ParsecValidator):
     V_CYCLE_POINT = 'V_CYCLE_POINT'
     V_CYCLE_POINT_FORMAT = 'V_CYCLE_POINT_FORMAT'
     V_CYCLE_POINT_TIME_ZONE = 'V_CYCLE_POINT_TIME_ZONE'
+    V_CYCLE_POINT_WITH_OFFSETS = 'V_CYCLE_POINT_WITH_OFFSETS'
     V_INTERVAL = 'V_INTERVAL'
     V_INTERVAL_LIST = 'V_INTERVAL_LIST'
     V_PARAMETER_LIST = 'V_PARAMETER_LIST'
@@ -699,6 +700,30 @@ class CylcConfigValidator(ParsecValidator):
                 '-0830': 'UTC minus 8 hours and 30 minutes.'
             }
         ),
+        V_CYCLE_POINT_WITH_OFFSETS: (
+            'cycle point with support for offsets',
+            'An integer of date-time cycle point.',
+            {
+                '1': 'An integer cycle point.',
+                '1 +P5': (
+                    'An integer cycle point with offsets.',
+                    ' (this evaluates as ``6``)'
+                ),
+                '+P5': (
+                    'An integer cycle point offset.'
+                    ' This offset is added to the initial cycle point'
+                ),
+                '2000-01-01T00:00Z': 'A date-time cycle point.',
+                '2000-01-01T00:00Z +P1D +P1M': (
+                    'A date-time cycle point with offsets.'
+                    ' (this evaluates as ``2000-02-02T00:00Z``'
+                ),
+                '2000-01-01T00:00Z +P1D': (
+                    'A date-time offset.'
+                    ' This offset is added to the initial cycle point'
+                ),
+            }
+        ),
         V_INTERVAL: (
             'time interval',
             'An ISO8601 duration.',
@@ -751,6 +776,9 @@ def __init__(self):
             self.V_CYCLE_POINT: self.coerce_cycle_point,
             self.V_CYCLE_POINT_FORMAT: self.coerce_cycle_point_format,
             self.V_CYCLE_POINT_TIME_ZONE: self.coerce_cycle_point_time_zone,
+            # NOTE: This type exists for documentation reasons
+            # it doesn't actually process offsets, that happens later
+            self.V_CYCLE_POINT_WITH_OFFSETS: self.coerce_str,
             self.V_INTERVAL: self.coerce_interval,
             self.V_INTERVAL_LIST: self.coerce_interval_list,
             self.V_PARAMETER_LIST: self.coerce_parameter_list,
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 0e02f9384d9..ec3ba6f0c58 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -993,6 +993,7 @@ def command_stop(
             point = TaskID.get_standardised_point(cycle_point)
             if point is not None and self.pool.set_stop_point(point):
                 self.options.stopcp = str(point)
+                self.config.stop_point = point
                 self.workflow_db_mgr.put_workflow_stop_cycle_point(
                     self.options.stopcp)
         elif clock_time is not None:
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index a13ba34d3cb..daa4d8fe00e 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1026,7 +1026,7 @@ def set_stop_point(self, stop_point: 'PointBase') -> bool:
             LOG.info(f"Stop point unchanged: {stop_point}")
             return False
 
-        LOG.info("Setting stop point: {stop_point}")
+        LOG.info(f"Setting stop point: {stop_point}")
         self.stop_point = stop_point
 
         if (
diff --git a/tests/functional/cylc-play/09-invalid-cp-opt.t b/tests/functional/cylc-play/09-invalid-cp-opt.t
index 05fa6d2a2d6..ae633d5e75c 100644
--- a/tests/functional/cylc-play/09-invalid-cp-opt.t
+++ b/tests/functional/cylc-play/09-invalid-cp-opt.t
@@ -37,7 +37,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}"
 run_fail "${TEST_NAME_BASE}-run" \
     cylc play "${WORKFLOW_NAME}" --no-detach --stopcp='potato'
 
-grep_ok "ERROR - Workflow shutting down .* potato" "${TEST_NAME_BASE}-run.stderr"
+grep_ok "ERROR - Workflow shutting down .*potato" "${TEST_NAME_BASE}-run.stderr"
 
 # Check that we haven't got a database
 exists_ok "${WORKFLOW_RUN_DIR}/.service"
diff --git a/tests/integration/test_stop_after_cycle_point.py b/tests/integration/test_stop_after_cycle_point.py
new file mode 100644
index 00000000000..663e03649f8
--- /dev/null
+++ b/tests/integration/test_stop_after_cycle_point.py
@@ -0,0 +1,132 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test logic pertaining to the stop after cycle points.
+
+This may be defined in different ways:
+* In the workflow configuration.
+* On the command line.
+* Or loaded from the database.
+
+When the workflow hits the "stop after" point, it should be wiped (i.e. set
+to None).
+"""
+
+from typing import Optional
+
+from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.id import Tokens
+from cylc.flow.workflow_status import StopMode
+
+
+async def test_stop_after_cycle_point(
+    flow,
+    scheduler,
+    run,
+    reflog,
+    complete,
+):
+    """Test the stop after cycle point.
+
+    This ensures:
+    * The stop after point gets loaded from the config.
+    * The workflow stops when it hits this point.
+    * The point gets wiped when the workflow hits this point.
+    * The point is stored/retrieved from the DB as appropriate.
+
+    """
+    async def stops_after_cycle(schd) -> Optional[str]:
+        """Run the workflow until it stops and return the cycle point."""
+        triggers = reflog(schd)
+        await complete(schd, timeout=2)
+        assert len(triggers) == 1  # only one task (i.e. cycle) should be run
+        return Tokens(list(triggers)[0][0], relative=True)['cycle']
+
+    def get_db_value(schd) -> Optional[str]:
+        """Return the cycle point value stored in the DB."""
+        with schd.workflow_db_mgr.get_pri_dao() as pri_dao:
+            return dict(pri_dao.select_workflow_params())['stopcp']
+
+    config = {
+        'scheduling': {
+            'cycling mode': 'integer',
+            'initial cycle point': '1',
+            'stop after cycle point': '1',
+            'graph': {
+                'P1': 'a[-P1] => a',
+            },
+        },
+    }
+    id_ = flow(config)
+    schd = scheduler(id_, paused_start=False)
+    async with run(schd):
+        # the cycle point should be loaded from the workflow configuration
+        assert schd.config.stop_point == IntegerPoint('1')
+
+        # this value should *not* be written to the database
+        assert get_db_value(schd) is None
+
+        # the workflow should stop after cycle 1
+        assert await stops_after_cycle(schd) == '1'
+
+    # change the configured cycle point to "2"
+    config['scheduling']['stop after cycle point'] = '2'
+    id_ = flow(config, id_=id_)
+    schd = scheduler(id_, paused_start=False)
+    async with run(schd):
+        # the cycle point should be reloaded from the workflow configuration
+        assert schd.config.stop_point == IntegerPoint('2')
+
+        # this value should not be written to the database
+        assert get_db_value(schd) is None
+
+        # the workflow should stop after cycle 2
+        assert await stops_after_cycle(schd) == '2'
+
+    # override the configured value via the CLI option
+    schd = scheduler(id_, paused_start=False, **{'stopcp': '3'})
+    async with run(schd):
+        # the CLI should take precedence over the config
+        assert schd.config.stop_point == IntegerPoint('3')
+
+        # this value *should* be written to the database
+        assert get_db_value(schd) == '3'
+
+        # the workflow should stop after cycle 3
+        assert await stops_after_cycle(schd) == '3'
+
+    # once the workflow hits this point, it should get cleared
+    assert get_db_value(schd) is None
+
+    schd = scheduler(id_, paused_start=False)
+    async with run(schd):
+        # the workflow should fall back to the configured value
+        assert schd.config.stop_point == IntegerPoint('2')
+
+        # override this value whilst the workflow is running
+        schd.command_stop(
+            cycle_point=IntegerPoint('4'),
+            mode=StopMode.REQUEST_CLEAN,
+        )
+        assert schd.config.stop_point == IntegerPoint('4')
+
+    # the new *should* be written to the database
+    assert get_db_value(schd) == '4'
+
+    schd = scheduler(id_, paused_start=False)
+    async with run(schd):
+        # the workflow should stop after cycle 4
+        assert await stops_after_cycle(schd) == '4'
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 67f5f0f559a..57b06433181 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -114,7 +114,7 @@ def test_xfunction_imports(
         """
         flow_file.write_text(flow_config)
         workflow_config = WorkflowConfig(
-            workflow="name_a_tree", fpath=flow_file, options=Mock(spec=[]),
+            workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(spec=[]),
             xtrigger_mgr=xtrigger_mgr
         )
         assert 'tree' in workflow_config.xtrigger_mgr.functx_map
@@ -148,7 +148,7 @@ def test_xfunction_import_error(self, mock_glbl_cfg, tmp_path):
             WorkflowConfig(
                 workflow="caiman_workflow",
                 fpath=flow_file,
-                options=Mock(spec=[])
+                options=SimpleNamespace(spec=[])
             )
         assert "not found" in str(excinfo.value)
 
@@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path):
         flow_file.write_text(flow_config)
         with pytest.raises(XtriggerConfigError) as excinfo:
             WorkflowConfig(workflow="capybara_workflow", fpath=flow_file,
-                           options=Mock(spec=[]))
+                           options=SimpleNamespace(spec=[]))
         assert "not found" in str(excinfo.value)
 
     def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):
@@ -211,7 +211,7 @@ def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):
             WorkflowConfig(
                 workflow="workflow_with_not_callable",
                 fpath=flow_file,
-                options=Mock(spec=[])
+                options=SimpleNamespace(spec=[])
             )
         assert "callable" in str(excinfo.value)
 
@@ -378,14 +378,16 @@ def test_process_icp(
         expected_err: Exception class expected to be raised plus the message.
     """
     set_cycling_type(cycling_type, time_zone="+0530")
-    mocked_config = Mock(cycling_type=cycling_type)
-    mocked_config.cfg = {
-        'scheduling': {
-            'initial cycle point constraints': [],
-            **scheduling_cfg
-        }
-    }
-    mocked_config.options.icp = None
+    mocked_config = SimpleNamespace(
+        cycling_type=cycling_type,
+        options=SimpleNamespace(icp=None),
+        cfg={
+            'scheduling': {
+                'initial cycle point constraints': [],
+                **scheduling_cfg
+            },
+        },
+    )
     monkeypatch.setattr('cylc.flow.config.get_current_time_string',
                         lambda: '20050102T0615+0530')
 
@@ -461,9 +463,10 @@ def test_process_startcp(
         expected_err: Expected exception.
     """
     set_cycling_type(ISO8601_CYCLING_TYPE, time_zone="+0530")
-    mocked_config = Mock(initial_point='18990501T0000+0530')
-    mocked_config.options.startcp = startcp
-    mocked_config.options.starttask = starttask
+    mocked_config = SimpleNamespace(
+        initial_point='18990501T0000+0530',
+        options=SimpleNamespace(startcp=startcp, starttask=starttask),
+    )
     monkeypatch.setattr('cylc.flow.config.get_current_time_string',
                         lambda: '20050102T0615+0530')
     if expected_err is not None:
@@ -662,17 +665,20 @@ def test_process_fcp(
         expected_err: Exception class expected to be raised plus the message.
     """
     set_cycling_type(cycling_type, time_zone='+0530')
-    mocked_config = Mock(cycling_type=cycling_type)
-    mocked_config.cfg = {
-        'scheduling': {
-            'final cycle point constraints': [],
-            **scheduling_cfg
-        }
-    }
-    mocked_config.initial_point = loader.get_point(
-        scheduling_cfg['initial cycle point']).standardise()
-    mocked_config.final_point = None
-    mocked_config.options.fcp = options_fcp
+    mocked_config = SimpleNamespace(
+        cycling_type=cycling_type,
+        cfg={
+            'scheduling': {
+                'final cycle point constraints': [],
+                **scheduling_cfg,
+            },
+        },
+        initial_point=loader.get_point(
+            scheduling_cfg['initial cycle point']
+        ).standardise(),
+        final_point = None,
+        options=SimpleNamespace(fcp=options_fcp),
+    )
 
     if expected_err:
         err, msg = expected_err
@@ -692,24 +698,28 @@ def test_process_fcp(
     [
         pytest.param(
             None, None, None, None, None,
-            id="No stopcp"
+            id="no-stopcp"
         ),
         pytest.param(
             '1993', None, '1993', None, None,
-            id="From config by default"
+            id="stopcp"
         ),
         pytest.param(
             '1993', '1066', '1066', '1066', None,
-            id="From options"
+            id="stop-cp-and-cli-option"
         ),
         pytest.param(
             '1993', 'reload', '1993', None, None,
-            id="From cfg if --stopcp=reload on restart"
+            id="stop-cp-and-cli-reload-option"
         ),
         pytest.param(
             '3000', None, None, None,
             "will have no effect as it is after the final cycle point",
-            id="stopcp > fcp"
+            id="stopcp-beyond-fcp"
+        ),
+        pytest.param(
+            '+P12Y -P2Y', None, '2000', None, None,
+            id="stopcp-relative-to-icp"
         ),
     ]
 )
@@ -734,12 +744,13 @@ def test_process_stop_cycle_point(
     set_cycling_type(ISO8601_CYCLING_TYPE, dump_format='CCYY')
     caplog.set_level(logging.WARNING, CYLC_LOG)
     fcp = loader.get_point('2012').standardise()
-    mock_config = Mock(
+    mock_config = SimpleNamespace(
         cfg={
             'scheduling': {
                 'stop after cycle point': cfg_stopcp
             }
         },
+        initial_point=ISO8601Point('1990'),
         final_point=fcp,
         stop_point=None,
         options=RunOptions(stopcp=options_stopcp),
@@ -887,7 +898,7 @@ def test_prelim_process_graph(
             processing.
         expected_err: Exception class expected to be raised plus the message.
     """
-    mock_config = Mock(cfg={
+    mock_config = SimpleNamespace(cfg={
         'scheduling': scheduling_cfg
     })
 
@@ -913,13 +924,15 @@ def _test(utc_mode, expected, expected_warnings=0):
                 UTC mode = {utc_mode['glbl']}
             '''
         )
-        mock_config = Mock()
-        mock_config.cfg = {
-            'scheduler': {
-                'UTC mode': utc_mode['workflow']
-            }
-        }
-        mock_config.options.utc_mode = utc_mode['stored']
+        mock_config = SimpleNamespace(
+            cfg={
+                'scheduler': {
+                    'UTC mode': utc_mode['workflow']
+                }
+            },
+            options=SimpleNamespace(utc_mode=utc_mode['stored']),
+        )
+
         WorkflowConfig.process_utc_mode(mock_config)
         assert mock_config.cfg['scheduler']['UTC mode'] is expected
         assert get_utc_mode() is expected
@@ -963,13 +976,15 @@ def test_cycle_point_tz(caplog, monkeypatch):
 
     def _test(cp_tz, utc_mode, expected, expected_warnings=0):
         set_utc_mode(utc_mode)
-        mock_config = Mock()
-        mock_config.cfg = {
-            'scheduler': {
-                'cycle point time zone': cp_tz['workflow']
-            }
-        }
-        mock_config.options.cycle_point_tz = cp_tz['stored']
+        mock_config = SimpleNamespace(
+            cfg={
+                'scheduler': {
+                    'cycle point time zone': cp_tz['workflow'],
+                },
+            },
+            options=SimpleNamespace(cycle_point_tz=cp_tz['stored']),
+        )
+
         WorkflowConfig.process_cycle_point_tz(mock_config)
         assert mock_config.cfg['scheduler'][
             'cycle point time zone'] == expected
@@ -1148,7 +1163,7 @@ def test_process_runahead_limit(
     set_cycling_type: Callable
 ) -> None:
     set_cycling_type(cycling_type)
-    mock_config = Mock(cycling_type=cycling_type)
+    mock_config = SimpleNamespace(cycling_type=cycling_type)
     mock_config.cfg = {
         'scheduling': {
             'runahead limit': runahead_limit
@@ -1170,7 +1185,7 @@ def test_check_circular(opt, monkeypatch, caplog, tmp_flow_config):
     # ----- Setup -----
     caplog.set_level(logging.WARNING, CYLC_LOG)
 
-    options = Mock(spec=[], is_validate=True)
+    options = SimpleNamespace(spec=[], is_validate=True)
     if opt:
         setattr(options, opt, True)
 
@@ -1739,7 +1754,7 @@ def test_cylc_env_at_parsing(
 
     # Parse the workflow config then check the environment.
     WorkflowConfig(
-        workflow="name", fpath=flow_file, options=Mock(spec=[]),
+        workflow="name", fpath=flow_file, options=SimpleNamespace(spec=[]),
         run_dir=run_dir
     )
 

From a83df172a1a77efdc0938a777f66828fb1db6c6c Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 30 Jan 2024 17:06:28 +0000
Subject: [PATCH 096/110] doc: correct spelling

---
 cylc/flow/etc/examples/event-driven-cycling/README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/etc/examples/event-driven-cycling/README.rst b/cylc/flow/etc/examples/event-driven-cycling/README.rst
index 4fa4ddde241..d1e112374f1 100644
--- a/cylc/flow/etc/examples/event-driven-cycling/README.rst
+++ b/cylc/flow/etc/examples/event-driven-cycling/README.rst
@@ -8,7 +8,7 @@ Cylc is good at orchestrating tasks to a schedule, e.g:
 But sometimes the things you want to run don't have a schedule.
 
 This example uses ``cylc ext-trigger`` to establish a pattern where Cylc waits
-for an external signal and starts a new cycle every time a signal is recieved.
+for an external signal and starts a new cycle every time a signal is received.
 
 The signal can carry data using the ext-trigger ID, this example sets the ID
 as a file path containing some data that we want to make available to the tasks

From 2bcad6bd8ef6863e4cb19127a8378f2cdc6b80ef Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 31 Jan 2024 12:42:41 +0000
Subject: [PATCH 097/110] Apply suggestions from code review

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 cylc/flow/cfgspec/workflow.py |  4 ++--
 cylc/flow/parsec/validate.py  | 16 ++++++----------
 tests/unit/test_config.py     |  2 +-
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
index 7ae7cf014b2..6a951ae02fb 100644
--- a/cylc/flow/cfgspec/workflow.py
+++ b/cylc/flow/cfgspec/workflow.py
@@ -552,7 +552,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
             - ``2000`` - Shorthand for ``2000-01-01T00:00``.
             - ``+P1D`` - The initial cycle point plus one day.
-            - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day.
+            - ``2000 +P1D +P1Y`` - The year ``2000`` plus one day and one year.
         ''')
         Conf('initial cycle point constraints', VDR.V_STRING_LIST, desc='''
             Rules to allow only some initial datetime cycle points.
@@ -622,7 +622,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
             - ``2000`` - Shorthand for ``2000-01-01T00:00``.
             - ``+P1D`` - The initial cycle point plus one day.
-            - ``2000 +P1D +P1Y`` - The year ``2000`` plus one year and one day.
+            - ``2000 +P1D +P1Y`` - The year ``2000`` plus one day and one year.
 
             .. versionadded:: 8.0.0
 
diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py
index 6e0e604d0c5..2309a1cc4aa 100644
--- a/cylc/flow/parsec/validate.py
+++ b/cylc/flow/parsec/validate.py
@@ -702,25 +702,21 @@ class CylcConfigValidator(ParsecValidator):
         ),
         V_CYCLE_POINT_WITH_OFFSETS: (
             'cycle point with support for offsets',
-            'An integer of date-time cycle point.',
+            'An integer or date-time cycle point, with optional offset(s).',
             {
                 '1': 'An integer cycle point.',
                 '1 +P5': (
-                    'An integer cycle point with offsets.',
-                    ' (this evaluates as ``6``)'
+                    'An integer cycle point with an offset'
+                    ' (this evaluates as ``6``).'
                 ),
                 '+P5': (
                     'An integer cycle point offset.'
                     ' This offset is added to the initial cycle point'
                 ),
                 '2000-01-01T00:00Z': 'A date-time cycle point.',
-                '2000-01-01T00:00Z +P1D +P1M': (
-                    'A date-time cycle point with offsets.'
-                    ' (this evaluates as ``2000-02-02T00:00Z``'
-                ),
-                '2000-01-01T00:00Z +P1D': (
-                    'A date-time offset.'
-                    ' This offset is added to the initial cycle point'
+                '2000-02-29T00:00Z +P1D +P1M': (
+                    'A date-time cycle point with offsets'
+                    ' (this evaluates as ``2000-04-01T00:00Z``).'
                 ),
             }
         ),
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 57b06433181..fd07b2c8c18 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -114,7 +114,7 @@ def test_xfunction_imports(
         """
         flow_file.write_text(flow_config)
         workflow_config = WorkflowConfig(
-            workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(spec=[]),
+            workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(),
             xtrigger_mgr=xtrigger_mgr
         )
         assert 'tree' in workflow_config.xtrigger_mgr.functx_map

From a8988df0591bf77706547c427e215aa20b3291b3 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 31 Jan 2024 13:05:12 +0000
Subject: [PATCH 098/110] Tidy (#5886)

---
 cylc/flow/network/schema.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index f9ed95c7158..d076882ae73 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -349,8 +349,6 @@ async def get_nodes_all(root, info, **args):
 
     _, field_ids = process_resolver_info(root, info, args)
 
-    if hasattr(args, 'id'):
-        args['ids'] = [args.get('id')]
     if field_ids:
         if isinstance(field_ids, str):
             field_ids = [field_ids]
@@ -377,10 +375,8 @@ async def get_nodes_all(root, info, **args):
         else:
             # live objects can be represented by a universal ID
             args[arg] = [Tokens(n_id, relative=True) for n_id in args[arg]]
-    args['workflows'] = [
-        Tokens(w_id) for w_id in args['workflows']]
-    args['exworkflows'] = [
-        Tokens(w_id) for w_id in args['exworkflows']]
+    for arg in ('workflows', 'exworkflows'):
+        args[arg] = [Tokens(w_id) for w_id in args[arg]]
     resolvers = info.context.get('resolvers')
     return await resolvers.get_nodes_all(node_type, args)
 

From 85492ae3290e024cf65994f510c2aaa61abbf638 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 1 Feb 2024 09:49:46 +0000
Subject: [PATCH 099/110] build(deps): bump codecov/codecov-action from 3 to 4

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codecov/codecov-action/compare/v3...v4)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test_fast.yml       | 2 +-
 .github/workflows/test_functional.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
index 66f08ce61ac..5776185557e 100644
--- a/.github/workflows/test_fast.yml
+++ b/.github/workflows/test_fast.yml
@@ -140,7 +140,7 @@ jobs:
         uses: actions/download-artifact@v4
 
       - name: Codecov upload
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           name: ${{ github.workflow }}
           flags: fast-tests
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
index ed411c2eae8..f54f3b3d710 100644
--- a/.github/workflows/test_functional.yml
+++ b/.github/workflows/test_functional.yml
@@ -316,7 +316,7 @@ jobs:
         uses: actions/download-artifact@v4
 
       - name: Codecov upload
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           name: ${{ github.workflow }}
           flags: functional-tests

From 38a4e152a8eabea5a0243d6eeb265983c774c9d4 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 1 Feb 2024 14:53:06 +0000
Subject: [PATCH 100/110] cylc lint: deprecate `pyproject.toml` section
 `cylc-lint` in favour of `tool.cylc.lint`

---
 cylc/flow/scripts/lint.py                 | 67 ++++++++++------
 tests/functional/cylc-lint/01.lint-toml.t | 55 +++++++-------
 tests/unit/scripts/test_lint.py           | 93 ++++++++++++++++-------
 3 files changed, 136 insertions(+), 79 deletions(-)

diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index f038d569d78..a0f52da9f33 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -22,8 +22,6 @@
 # NOTE: docstring needed for `cylc help all` output
 # (if editing check this still comes out as expected)
 
-LINT_SECTIONS = ['cylc-lint', 'cylclint', 'cylc_lint']
-
 COP_DOC = """cylc lint [OPTIONS] ARGS
 
 Check .cylc and .rc files for code style, deprecated syntax and other issues.
@@ -44,12 +42,12 @@
 """
 
 TOMLDOC = """
-pyproject.toml configuration:{}
-   [cylc-lint]                     # any of {}
-       ignore = ['S001', 'S002']    # List of rules to ignore
-       exclude = ['etc/foo.cylc']  # List of files to ignore
-       rulesets = ['style', '728'] # Sets default rulesets to check
-       max-line-length = 130       # Max line length for linting
+pyproject.toml configuration:
+   [tool.cylc.lint]
+   ignore = ['S001', 'S002']    # List of rules to ignore
+   exclude = ['etc/foo.cylc']   # List of files to ignore
+   rulesets = ['style', '728']  # Sets default rulesets to check
+   max-line-length = 130        # Max line length for linting
 """
 from colorama import Fore
 import functools
@@ -77,6 +75,8 @@
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
+import cylc.flow.flags
+from cylc.flow.loggingutil import set_timestamps
 from cylc.flow.option_parsers import (
     CylcOptionParser as COP,
     WORKFLOW_ID_OR_PATH_ARG_DOC
@@ -90,6 +90,20 @@
 if TYPE_CHECKING:
     from optparse import Values
 
+LINT_TABLE = ['tool', 'cylc', 'lint']
+LINT_SECTION = '.'.join(LINT_TABLE)
+
+# BACK COMPAT: DEPR_LINT_SECTION
+# url:
+#     https://github.com/cylc/cylc-flow/issues/5811
+# from:
+#    8.1.0
+# to:
+#    8.3.0
+# remove at:
+#    8.4.0 ?
+DEPR_LINT_SECTION = 'cylc-lint'
+
 DEPRECATED_ENV_VARS = {
     'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST',
     'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER',
@@ -758,7 +772,7 @@ def validate_toml_items(tomldata):
         if key not in EXTRA_TOML_VALIDATION.keys():
             raise CylcError(
                 f'Only {[*EXTRA_TOML_VALIDATION.keys()]} '
-                f'allowed as toml sections but you used {key}'
+                f'allowed as toml sections but you used "{key}"'
             )
         if key != 'max-line-length':
             # Item should be a list...
@@ -780,26 +794,35 @@ def validate_toml_items(tomldata):
     return True
 
 
-def get_pyproject_toml(dir_):
+def get_pyproject_toml(dir_: Path) -> Dict[str, Any]:
     """if a pyproject.toml file is present open it and return settings.
     """
-    keys = ['rulesets', 'ignore', 'exclude', 'max-line-length']
-    tomlfile = Path(dir_ / 'pyproject.toml')
-    tomldata = {}
+    tomlfile = dir_ / 'pyproject.toml'
+    tomldata = {
+        'rulesets': [],
+        'ignore': [],
+        'exclude': [],
+        'max-line-length': None,
+    }
     if tomlfile.is_file():
         try:
             loadeddata = toml_loads(tomlfile.read_text())
         except TOMLDecodeError as exc:
             raise CylcError(f'pyproject.toml did not load: {exc}')
 
-        if any(
-            i in loadeddata for i in LINT_SECTIONS
-        ):
-            for key in keys:
-                tomldata[key] = loadeddata.get('cylc-lint').get(key, [])
-            validate_toml_items(tomldata)
-    if not tomldata:
-        tomldata = {key: [] for key in keys}
+        _tool, _cylc, _lint = LINT_TABLE
+        try:
+            data = loadeddata[_tool][_cylc][_lint]
+        except KeyError:
+            if DEPR_LINT_SECTION in loadeddata:
+                LOG.warning(
+                    f"The [{DEPR_LINT_SECTION}] section in pyproject.toml is "
+                    f"deprecated. Use [{LINT_SECTION}] instead."
+                )
+            data = loadeddata.get(DEPR_LINT_SECTION, {})
+        tomldata.update(data)
+        validate_toml_items(tomldata)
+
     return tomldata
 
 
@@ -1318,7 +1341,7 @@ def get_option_parser() -> COP:
         (
             COP_DOC
             + NOQA.replace('``', '"')
-            + TOMLDOC.format('', str(LINT_SECTIONS))
+            + TOMLDOC
         ),
         argdoc=[
             COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC)
diff --git a/tests/functional/cylc-lint/01.lint-toml.t b/tests/functional/cylc-lint/01.lint-toml.t
index 183b69beaed..fbdd243e859 100644
--- a/tests/functional/cylc-lint/01.lint-toml.t
+++ b/tests/functional/cylc-lint/01.lint-toml.t
@@ -58,21 +58,20 @@ named_grep_ok "it returns a 728 upgrade code" "^\[U" "${TESTOUT}"
 
 # Add a pyproject.toml file
 cat > pyproject.toml <<__HERE__
-[cylc-lint]
-    # Check against these rules
-    rulesets = [
-        "style"
-    ]
-    #  do not check for these errors
-    ignore = [
-        "S004"
-    ]
-    # do not lint files matching
-    # these globs:
-    exclude = [
-        "sites/*.cylc",
-    ]
-
+[tool.cylc.lint]
+# Check against these rules
+rulesets = [
+    "style"
+]
+#  do not check for these errors
+ignore = [
+    "S004"
+]
+# do not lint files matching
+# these globs:
+exclude = [
+    "sites/*.cylc",
+]
 __HERE__
 
 # Test that results are different:
@@ -102,19 +101,19 @@ named_grep_ok "${TEST_NAME}-line-too-long-message" \
 
 TEST_NAME="it_does_not_fail_if_max-line-length_set_but_ignored"
 cat > pyproject.toml <<__HERE__
-[cylc-lint]
-    # Check against these rules
-    rulesets = [
-        "style"
-    ]
-    #  do not check for these errors
-    ignore = [
-        "${LINE_LEN_NO}"
-    ]
-    exclude = [
-        "sites/*.cylc",
-    ]
-    max-line-length = 1
+[tool.cylc.lint]
+# Check against these rules
+rulesets = [
+    "style"
+]
+#  do not check for these errors
+ignore = [
+    "${LINE_LEN_NO}"
+]
+exclude = [
+    "sites/*.cylc",
+]
+max-line-length = 1
 __HERE__
 run_ok "${TEST_NAME}" cylc lint
 grep_ok "rules and found no issues" "${TEST_NAME}.stdout"
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index f3009a6ddb3..f5f4bb8cc98 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -16,15 +16,18 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Tests `cylc lint` CLI Utility."""
 
+import logging
 from pathlib import Path
 from pprint import pformat
 import re
+from textwrap import dedent
 from types import SimpleNamespace
 
 import pytest
 from pytest import param
 
 from cylc.flow.scripts.lint import (
+    LINT_SECTION,
     MANUAL_DEPRECATIONS,
     get_cylc_files,
     get_pyproject_toml,
@@ -230,7 +233,6 @@ def assert_contains(items, contains, instances=None):
     16: 3,
 }
 
-
 @pytest.mark.parametrize(
     # 11 won't be tested because there is no jinja2 shebang
     'number', set(range(1, len(MANUAL_DEPRECATIONS) + 1)) - {11}
@@ -473,52 +475,85 @@ def test_get_upg_info(fixture_get_deprecations, findme):
 
 
 @pytest.mark.parametrize(
-    'expect',
+    'settings, expected',
     [
-        param({
-            'rulesets': ['style'],
-            'ignore': ['S004'],
-            'exclude': ['sites/*.cylc']},
-            id="it returns what we want"
+        param(
+            """
+            rulesets = ['style']
+            ignore = ['S004']
+            exclude = ['sites/*.cylc']
+            """,
+            {
+                'rulesets': ['style'],
+                'ignore': ['S004'],
+                'exclude': ['sites/*.cylc'],
+                'max-line-length': None,
+            },
+            id="returns what we want"
         ),
-        param({
-            'northgate': ['sites/*.cylc'],
-            'mons-meg': 42},
-            id="it only returns requested sections"
+        param(
+            """
+            northgate = ['sites/*.cylc']
+            mons-meg = 42
+            """,
+            (CylcError, ".*northgate"),
+            id="invalid settings fail validation"
         ),
-        param({
-            'max-line-length': 22},
-            id='it sets max line length'
+        param(
+            "max-line-length = 22",
+            {
+                'exclude': [],
+                'ignore': [],
+                'rulesets': [],
+                'max-line-length': 22,
+            },
+            id='sets max line length'
         )
     ]
 )
-def test_get_pyproject_toml(tmp_path, expect):
+def test_get_pyproject_toml(tmp_path, settings, expected):
     """It returns only the lists we want from the toml file."""
-    tomlcontent = "[cylc-lint]"
-    permitted_keys = ['rulesets', 'ignore', 'exclude', 'max-line-length']
-
-    for section, value in expect.items():
-        tomlcontent += f'\n{section} = {value}'
+    tomlcontent = "[tool.cylc.lint]\n" + dedent(settings)
     (tmp_path / 'pyproject.toml').write_text(tomlcontent)
-    tomldata = get_pyproject_toml(tmp_path)
 
-    control = {}
-    for key in permitted_keys:
-        control[key] = expect.get(key, [])
-    assert tomldata == control
+    if isinstance(expected, tuple):
+        exc, match = expected
+        with pytest.raises(exc, match=match):
+            get_pyproject_toml(tmp_path)
+    else:
+        assert get_pyproject_toml(tmp_path) == expected
 
 
-@pytest.mark.parametrize('tomlfile', [None, '', '[cylc-lint]'])
+@pytest.mark.parametrize(
+    'tomlfile',
+    [None, '', '[tool.cylc.lint]', '[cylc-lint]']
+)
 def test_get_pyproject_toml_returns_blank(tomlfile, tmp_path):
     if tomlfile is not None:
         tfile = (tmp_path / 'pyproject.toml')
         tfile.write_text(tomlfile)
-    expect = {k: [] for k in {
-        'exclude', 'ignore', 'max-line-length', 'rulesets'
-    }}
+    expect = {
+        'exclude': [], 'ignore': [], 'max-line-length': None, 'rulesets': []
+    }
     assert get_pyproject_toml(tmp_path) == expect
 
 
+def test_get_pyproject_toml__depr(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+):
+    """It warns if the section is deprecated."""
+    file = tmp_path / 'pyproject.toml'
+    caplog.set_level(logging.WARNING)
+
+    file.write_text(f'[{LINT_SECTION}]\nmax-line-length=14')
+    assert get_pyproject_toml(tmp_path)['max-line-length'] == 14
+    assert not caplog.text
+
+    file.write_text('[cylc-lint]\nmax-line-length=17')
+    assert get_pyproject_toml(tmp_path)['max-line-length'] == 17
+    assert "[cylc-lint] section in pyproject.toml is deprecated" in caplog.text
+
+
 @pytest.mark.parametrize(
     'input_, error',
     [

From 810f3befa6c639b6a70ced3f32971459a9a5b755 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 2 Feb 2024 14:49:02 +0000
Subject: [PATCH 101/110] Apply suggestions from code review

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 tests/unit/test_config.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index fd07b2c8c18..629ca144739 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -148,7 +148,7 @@ def test_xfunction_import_error(self, mock_glbl_cfg, tmp_path):
             WorkflowConfig(
                 workflow="caiman_workflow",
                 fpath=flow_file,
-                options=SimpleNamespace(spec=[])
+                options=SimpleNamespace()
             )
         assert "not found" in str(excinfo.value)
 
@@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path):
         flow_file.write_text(flow_config)
         with pytest.raises(XtriggerConfigError) as excinfo:
             WorkflowConfig(workflow="capybara_workflow", fpath=flow_file,
-                           options=SimpleNamespace(spec=[]))
+                           options=SimpleNamespace()
         assert "not found" in str(excinfo.value)
 
     def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):
@@ -211,7 +211,7 @@ def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):
             WorkflowConfig(
                 workflow="workflow_with_not_callable",
                 fpath=flow_file,
-                options=SimpleNamespace(spec=[])
+                options=SimpleNamespace()
             )
         assert "callable" in str(excinfo.value)
 
@@ -1185,7 +1185,7 @@ def test_check_circular(opt, monkeypatch, caplog, tmp_flow_config):
     # ----- Setup -----
     caplog.set_level(logging.WARNING, CYLC_LOG)
 
-    options = SimpleNamespace(spec=[], is_validate=True)
+    options = SimpleNamespace(is_validate=True)
     if opt:
         setattr(options, opt, True)
 
@@ -1754,7 +1754,7 @@ def test_cylc_env_at_parsing(
 
     # Parse the workflow config then check the environment.
     WorkflowConfig(
-        workflow="name", fpath=flow_file, options=SimpleNamespace(spec=[]),
+        workflow="name", fpath=flow_file, options=SimpleNamespace(),
         run_dir=run_dir
     )
 

From 28878195d06e729912342e3cf68859e9678d4c4d Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Fri, 2 Feb 2024 16:28:13 +0000
Subject: [PATCH 102/110] Update tests/unit/test_config.py

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 tests/unit/test_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 629ca144739..494dd98e62b 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path):
         flow_file.write_text(flow_config)
         with pytest.raises(XtriggerConfigError) as excinfo:
             WorkflowConfig(workflow="capybara_workflow", fpath=flow_file,
-                           options=SimpleNamespace()
+                           options=SimpleNamespace())
         assert "not found" in str(excinfo.value)
 
     def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):

From 28e68fec1ec8cc0a0148d2da3884f40464b2acca Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 1 Feb 2024 15:08:53 +0000
Subject: [PATCH 103/110] Tidy

---
 cylc/flow/scripts/lint.py | 149 ++++++++++++++++++++------------------
 1 file changed, 77 insertions(+), 72 deletions(-)

diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index a0f52da9f33..0626a407f32 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -19,7 +19,7 @@
 Checks code style, deprecated syntax and other issues.
 
 """
-# NOTE: docstring needed for `cylc help all` output
+# NOTE: docstring needed for `cylc help all` output and docs
 # (if editing check this still comes out as expected)
 
 COP_DOC = """cylc lint [OPTIONS] ARGS
@@ -88,6 +88,10 @@
 from cylc.flow.terminal import cli_function
 
 if TYPE_CHECKING:
+    # BACK COMPAT: typing_extensions.Literal
+    # FROM: Python 3.7
+    # TO: Python 3.8
+    from typing_extensions import Literal
     from optparse import Values
 
 LINT_TABLE = ['tool', 'cylc', 'lint']
@@ -104,6 +108,11 @@
 #    8.4.0 ?
 DEPR_LINT_SECTION = 'cylc-lint'
 
+IGNORE = 'ignore'
+EXCLUDE = 'exclude'
+RULESETS = 'rulesets'
+MAX_LINE_LENGTH = 'max-line-length'
+
 DEPRECATED_ENV_VARS = {
     'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST',
     'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER',
@@ -602,7 +611,9 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
     },
     'U008': {
         'short': 'Suicide triggers are not required at Cylc 8.',
-        'url': '',
+        'url': (
+            'https://cylc.github.io/cylc-doc/stable/html/7-to-8'
+            '/major-changes/suicide-triggers.html'),
         'kwargs': True,
         FUNCTION: functools.partial(
             check_for_suicide_triggers,
@@ -616,9 +627,7 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
     },
     'U010': {
         'short': 'rose suite-hook is deprecated at Rose 2,',
-        'url': (
-            'https://cylc.github.io/cylc-doc/stable/html/7-to-8'
-            '/major-changes/suicide-triggers.html'),
+        'url': '',
         FUNCTION: lambda line: 'rose suite-hook' in line,
     },
     'U011': {
@@ -711,28 +720,34 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
             list_wrapper, check=CHECK_FOR_OLD_VARS.findall),
     },
 }
-RULESETS = ['728', 'style', 'all']
+ALL_RULESETS = ['728', 'style', 'all']
 EXTRA_TOML_VALIDATION = {
-    'ignore': {
+    IGNORE: {
         lambda x: re.match(r'[A-Z]\d\d\d', x):
             '{item} not valid: Ignore codes should be in the form X001',
         lambda x: x in parse_checks(['728', 'style']):
             '{item} is a not a known linter code.'
     },
-    'rulesets': {
-        lambda item: item in RULESETS:
+    RULESETS: {
+        lambda item: item in ALL_RULESETS:
             '{item} not valid: Rulesets can be '
             '\'728\', \'style\' or \'all\'.'
     },
-    'max-line-length': {
+    MAX_LINE_LENGTH: {
         lambda x: isinstance(x, int):
             'max-line-length must be an integer.'
     },
     # consider checking that item is file?
-    'exclude': {}
+    EXCLUDE: {}
 }
 
 
+def parse_ruleset_option(ruleset: str) -> List[str]:
+    if ruleset in {'all', ''}:
+        return ['728', 'style']
+    return [ruleset]
+
+
 def get_url(check_meta: Dict) -> str:
     """Get URL from check data.
 
@@ -774,7 +789,7 @@ def validate_toml_items(tomldata):
                 f'Only {[*EXTRA_TOML_VALIDATION.keys()]} '
                 f'allowed as toml sections but you used "{key}"'
             )
-        if key != 'max-line-length':
+        if key != MAX_LINE_LENGTH:
             # Item should be a list...
             if not isinstance(items, list):
                 raise CylcError(
@@ -798,11 +813,11 @@ def get_pyproject_toml(dir_: Path) -> Dict[str, Any]:
     """if a pyproject.toml file is present open it and return settings.
     """
     tomlfile = dir_ / 'pyproject.toml'
-    tomldata = {
-        'rulesets': [],
-        'ignore': [],
-        'exclude': [],
-        'max-line-length': None,
+    tomldata: Dict[str, Union[List[str], int, None]] = {
+        RULESETS: [],
+        IGNORE: [],
+        EXCLUDE: [],
+        MAX_LINE_LENGTH: None,
     }
     if tomlfile.is_file():
         try:
@@ -838,20 +853,14 @@ def merge_cli_with_tomldata(target: Path, options: 'Values') -> Dict[str, Any]:
     _merge_cli_with_tomldata to keep the testing of file-system touching
     and pure logic separate.
     """
-    ruleset_default = False
-    if options.linter == 'all':
-        options.linter = ['728', 'style']
-    elif options.linter == '':
-        options.linter = ['728', 'style']
-        ruleset_default = True
-    else:
-        options.linter = [options.linter]
+    ruleset_default = (options.ruleset == '')
+    options.ruleset = parse_ruleset_option(options.ruleset)
     tomlopts = get_pyproject_toml(target)
     return _merge_cli_with_tomldata(
         {
-            'exclude': [],
-            'ignore': options.ignores,
-            'rulesets': options.linter
+            EXCLUDE: [],
+            IGNORE: options.ignores,
+            RULESETS: options.ruleset
         },
         tomlopts,
         ruleset_default
@@ -886,30 +895,30 @@ def _merge_cli_with_tomldata(
     >>> result['exclude']
     ['*.bk']
     """
-    if isinstance(clidata['rulesets'][0], list):
-        clidata['rulesets'] = clidata['rulesets'][0]
+    if isinstance(clidata[RULESETS][0], list):
+        clidata[RULESETS] = clidata[RULESETS][0]
 
     output = {}
 
     # Combine 'ignore' sections:
-    output['ignore'] = sorted(set(clidata['ignore'] + tomldata['ignore']))
+    output[IGNORE] = sorted(set(clidata[IGNORE] + tomldata[IGNORE]))
 
-    # Replace 'rulesets from toml with those from CLI if they exist:
+    # Replace 'rulesets' from toml with those from CLI if they exist:
 
     if override_cli_default_rules:
-        output['rulesets'] = (
-            tomldata['rulesets'] if tomldata['rulesets']
-            else clidata['rulesets']
+        output[RULESETS] = (
+            tomldata[RULESETS] if tomldata[RULESETS]
+            else clidata[RULESETS]
         )
     else:
-        output['rulesets'] = (
-            clidata['rulesets'] if clidata['rulesets']
-            else tomldata['rulesets']
+        output[RULESETS] = (
+            clidata[RULESETS] if clidata[RULESETS]
+            else tomldata[RULESETS]
         )
 
     # Return 'exclude' and 'max-line-length' for the tomldata:
-    output['exclude'] = tomldata['exclude']
-    output['max-line-length'] = tomldata.get('max-line-length', None)
+    output[EXCLUDE] = tomldata[EXCLUDE]
+    output[MAX_LINE_LENGTH] = tomldata.get(MAX_LINE_LENGTH, None)
 
     return output
 
@@ -1239,29 +1248,18 @@ def get_cylc_files(
                 yield path
 
 
-REFERENCE_TEMPLATES = {
-    'section heading': '\n{title}\n{underline}\n',
-    'issue heading': {
-        'text': '\n{check}:\n    {summary}\n    {url}\n\n',
-        'rst': '\n{url}_\n{underline}\n{summary}\n\n',
-    },
-    'auto gen message': (
-        'U998 and U999 represent automatically generated'
-        ' sets of deprecations and upgrades.'
-    ),
-}
-
-
-def get_reference(linter, output_type):
+def get_reference(ruleset: str, output_type: 'Literal["text", "rst"]') -> str:
     """Fill out a template with all the issues Cylc Lint looks for.
     """
-    if linter in {'all', ''}:
-        rulesets = ['728', 'style']
-    else:
-        rulesets = [linter]
-    checks = parse_checks(rulesets, reference=True)
+    checks = parse_checks(
+        parse_ruleset_option(ruleset),
+        reference=True
+    )
 
-    issue_heading_template = REFERENCE_TEMPLATES['issue heading'][output_type]
+    issue_heading_template = (
+        '\n{url}_\n{underline}\n{summary}\n\n' if output_type == 'rst' else
+        '\n{check}:\n    {summary}\n    {url}\n\n'
+    )
     output = ''
     current_checkset = ''
     for index, meta in checks.items():
@@ -1270,11 +1268,15 @@ def get_reference(linter, output_type):
         if meta['purpose'] != current_checkset:
             current_checkset = meta['purpose']
             title = CHECKS_DESC[meta["purpose"]]
-            output += REFERENCE_TEMPLATES['section heading'].format(
-                title=title, underline="-" * len(title))
+            output += '\n{title}\n{underline}\n'.format(
+                title=title, underline="-" * len(title)
+            )
 
             if current_checkset == 'A':
-                output += REFERENCE_TEMPLATES['auto gen message']
+                output += (
+                    'U998 and U999 represent automatically generated'
+                    ' sets of deprecations and upgrades.'
+                )
 
         # Fill a template with info about the issue.
         if output_type == 'rst':
@@ -1321,18 +1323,18 @@ def target_version_check(
     disabled thatr with --exit-zero.
     """
     cylc8 = (target / 'flow.cylc').exists()
-    if not cylc8 and mergedopts['rulesets'] == ['728']:
+    if not cylc8 and mergedopts[RULESETS] == ['728']:
         LOG.error(
             f'{target} not a Cylc 8 workflow: '
             'Lint after renaming '
             '"suite.rc" to "flow.cylc"'
         )
         sys.exit(not quiet)
-    elif not cylc8 and '728' in mergedopts['rulesets']:
-        check_names = mergedopts['rulesets']
+    elif not cylc8 and '728' in mergedopts[RULESETS]:
+        check_names = mergedopts[RULESETS]
         check_names.remove('728')
     else:
-        check_names = mergedopts['rulesets']
+        check_names = mergedopts[RULESETS]
     return check_names
 
 
@@ -1364,7 +1366,7 @@ def get_option_parser() -> COP:
         ),
         default='',
         choices=["728", "style", "all", ''],
-        dest='linter'
+        dest='ruleset'
     )
     parser.add_option(
         '--list-codes',
@@ -1399,8 +1401,11 @@ def get_option_parser() -> COP:
 
 @cli_function(get_option_parser)
 def main(parser: COP, options: 'Values', target=None) -> None:
+    if cylc.flow.flags.verbosity < 2:
+        set_timestamps(LOG, False)
+
     if options.ref_mode:
-        print(get_reference(options.linter, 'text'))
+        print(get_reference(options.ruleset, 'text'))
         sys.exit(0)
 
     # If target not given assume we are looking at PWD:
@@ -1426,13 +1431,13 @@ def main(parser: COP, options: 'Values', target=None) -> None:
     # Get the checks object.
     checks = parse_checks(
         check_names,
-        ignores=mergedopts['ignore'],
-        max_line_len=mergedopts['max-line-length']
+        ignores=mergedopts[IGNORE],
+        max_line_len=mergedopts[MAX_LINE_LENGTH]
     )
 
     # Check each file matching a pattern:
     counter: Dict[str, int] = {}
-    for file in get_cylc_files(target, mergedopts['exclude']):
+    for file in get_cylc_files(target, mergedopts[EXCLUDE]):
         LOG.debug(f'Checking {file}')
         check_cylc_file(
             file,

From f301d547ae67a6946435da67a94b02863e04aa4b Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 1 Feb 2024 15:26:43 +0000
Subject: [PATCH 104/110] Changelog

---
 changes.d/5956.break.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes.d/5956.break.md

diff --git a/changes.d/5956.break.md b/changes.d/5956.break.md
new file mode 100644
index 00000000000..642c805814d
--- /dev/null
+++ b/changes.d/5956.break.md
@@ -0,0 +1 @@
+`cylc lint`: deprecated `[cylc-lint]` section in favour of `[tool.cylc.lint]` in `pyproject.toml`

From b1822a607fe1080cee9106c99394ae02e8140cb6 Mon Sep 17 00:00:00 2001
From: Mark Dawson <mark.dawson@metoffice.gov.uk>
Date: Thu, 8 Feb 2024 10:11:09 +0000
Subject: [PATCH 105/110] added suite state xtrigger for back-compat (#5864)

added suite state xtrigger for back commpat
extended back compat a unit test
added deprecation log message
Replace icky unit test with functional test (#4)

---------

Co-authored-by: Mark Dawson <mark.dawson@metoffice.gov.ukpython>
---
 changes.d/5864.feat.md                      |  1 +
 cylc/flow/xtriggers/suite_state.py          | 81 +++++++++++++++++++++
 setup.cfg                                   |  1 +
 tests/functional/xtriggers/04-suite_state.t | 50 +++++++++++++
 tests/unit/xtriggers/test_workflow_state.py | 10 ++-
 5 files changed, 142 insertions(+), 1 deletion(-)
 create mode 100644 changes.d/5864.feat.md
 create mode 100644 cylc/flow/xtriggers/suite_state.py
 create mode 100644 tests/functional/xtriggers/04-suite_state.t

diff --git a/changes.d/5864.feat.md b/changes.d/5864.feat.md
new file mode 100644
index 00000000000..905b6b9dadd
--- /dev/null
+++ b/changes.d/5864.feat.md
@@ -0,0 +1 @@
+Reimplemented the `suite-state` xtrigger for interoperability with Cylc 7.
diff --git a/cylc/flow/xtriggers/suite_state.py b/cylc/flow/xtriggers/suite_state.py
new file mode 100644
index 00000000000..dc8351a69df
--- /dev/null
+++ b/cylc/flow/xtriggers/suite_state.py
@@ -0,0 +1,81 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cylc.flow import LOG
+import cylc.flow.flags
+from cylc.flow.xtriggers.workflow_state import workflow_state
+
+if not cylc.flow.flags.cylc7_back_compat:
+    LOG.warning(
+        "The suite_state xtrigger is deprecated. "
+        "Please use the workflow_state xtrigger instead."
+    )
+
+
+def suite_state(suite, task, point, offset=None, status='succeeded',
+                message=None, cylc_run_dir=None, debug=False):
+    """Suite state xtrigger, required for interoperability with Cylc 7.
+
+    * The suite_state xtrigger was renamed to workflow_state, this breaks Cylc 7-8 interoperability.
+    * This suite_state xtrigger replicates workflow_state - ensuring back-support.
+
+    Arguments:
+        suite:
+            The workflow to interrogate.
+        task:
+            The name of the task to query.
+        point:
+            The cycle point.
+        offset:
+            The offset between the cycle this xtrigger is used in and the one
+            it is querying for as an ISO8601 time duration.
+            e.g. PT1H (one hour).
+        status:
+            The task status required for this xtrigger to be satisfied.
+        message:
+            The custom task output required for this xtrigger to be satisfied.
+            .. note::
+
+               This cannot be specified in conjunction with ``status``.
+
+        cylc_run_dir:
+            The directory in which the workflow to interrogate.
+
+            .. note::
+
+               This only needs to be supplied if the workflow is running in a
+               different location to what is specified in the global
+               configuration (usually ``~/cylc-run``).
+
+    Returns:
+        tuple: (satisfied, results)
+
+        satisfied:
+            True if ``satisfied`` else ``False``.
+        results:
+            Dictionary containing the args / kwargs which were provided
+            to this xtrigger.
+
+    """
+    return workflow_state(
+        workflow=suite,
+        task=task,
+        point=point,
+        offset=offset,
+        status=status,
+        message=message,
+        cylc_run_dir=cylc_run_dir
+    )
diff --git a/setup.cfg b/setup.cfg
index 7db9eb8bf4e..849f4ae71b8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -220,6 +220,7 @@ cylc.xtriggers =
     echo = cylc.flow.xtriggers.echo:echo
     wall_clock = cylc.flow.xtriggers.wall_clock:wall_clock
     workflow_state = cylc.flow.xtriggers.workflow_state:workflow_state
+    suite_state = cylc.flow.xtriggers.suite_state:suite_state
     xrandom = cylc.flow.xtriggers.xrandom:xrandom
 
 [bdist_rpm]
diff --git a/tests/functional/xtriggers/04-suite_state.t b/tests/functional/xtriggers/04-suite_state.t
new file mode 100644
index 00000000000..ece9cbe715e
--- /dev/null
+++ b/tests/functional/xtriggers/04-suite_state.t
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test that deprecation warnings are printed appropriately for the suite_state
+# xtrigger.
+
+. "$(dirname "$0")/test_header"
+
+set_test_number 4
+
+init_workflow "$TEST_NAME_BASE" << __FLOW_CONFIG__
+[scheduling]
+    initial cycle point = 2000
+    [[dependencies]]
+        [[[R1]]]
+            graph = @upstream => foo
+    [[xtriggers]]
+        upstream = suite_state(suite=thorin/oin/gloin, task=mithril, point=1)
+[runtime]
+    [[foo]]
+__FLOW_CONFIG__
+
+msg='WARNING - The suite_state xtrigger is deprecated'
+
+TEST_NAME="${TEST_NAME_BASE}-val"
+run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME"
+
+grep_ok "$msg" "${TEST_NAME}.stderr"
+
+# Rename flow.cylc to suite.rc:
+mv "${WORKFLOW_RUN_DIR}/flow.cylc" "${WORKFLOW_RUN_DIR}/suite.rc"
+
+TEST_NAME="${TEST_NAME_BASE}-val-2"
+run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME"
+
+grep_fail "$msg" "${TEST_NAME}.stderr"
diff --git a/tests/unit/xtriggers/test_workflow_state.py b/tests/unit/xtriggers/test_workflow_state.py
index b3d25737cc2..ed02750f04a 100644
--- a/tests/unit/xtriggers/test_workflow_state.py
+++ b/tests/unit/xtriggers/test_workflow_state.py
@@ -42,7 +42,7 @@ def test_inferred_run(tmp_run_dir: Callable, monkeymock: MonkeyMock):
     assert results['workflow'] == expected_workflow_id
 
 
-def test_back_compat(tmp_run_dir):
+def test_back_compat(tmp_run_dir, caplog):
     """Test workflow_state xtrigger backwards compatibility with Cylc 7
     database."""
     id_ = 'celebrimbor'
@@ -80,7 +80,15 @@ def test_back_compat(tmp_run_dir):
     finally:
         conn.close()
 
+    # Test workflow_state function
     satisfied, _ = workflow_state(id_, task='mithril', point='2012')
     assert satisfied
     satisfied, _ = workflow_state(id_, task='arkenstone', point='2012')
     assert not satisfied
+
+    # Test back-compat (old suite_state function)
+    from cylc.flow.xtriggers.suite_state import suite_state
+    satisfied, _ = suite_state(suite=id_, task='mithril', point='2012')
+    assert satisfied
+    satisfied, _ = suite_state(suite=id_, task='arkenstone', point='2012')
+    assert not satisfied

From 381c691d9d5ecd7f7c369ad1ee2833b4793e73fc Mon Sep 17 00:00:00 2001
From: Mark Dawson <mark.dawson@metoffice.gov.uk>
Date: Thu, 8 Feb 2024 12:34:24 +0000
Subject: [PATCH 106/110] fixed flake8 (#5962)

Co-authored-by: Mark Dawson <mark.dawson@metoffice.gov.ukpython>
---
 cylc/flow/xtriggers/suite_state.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/xtriggers/suite_state.py b/cylc/flow/xtriggers/suite_state.py
index dc8351a69df..45a8418a832 100644
--- a/cylc/flow/xtriggers/suite_state.py
+++ b/cylc/flow/xtriggers/suite_state.py
@@ -29,8 +29,10 @@ def suite_state(suite, task, point, offset=None, status='succeeded',
                 message=None, cylc_run_dir=None, debug=False):
     """Suite state xtrigger, required for interoperability with Cylc 7.
 
-    * The suite_state xtrigger was renamed to workflow_state, this breaks Cylc 7-8 interoperability.
-    * This suite_state xtrigger replicates workflow_state - ensuring back-support.
+    * The suite_state xtrigger was renamed to workflow_state -
+        this breaks Cylc 7-8 interoperability.
+    * This suite_state xtrigger replicates workflow_state -
+        ensuring back-support.
 
     Arguments:
         suite:

From 8ab57ad0943abeea5b297f7d3b19d3cd8a055175 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Tue, 13 Feb 2024 15:57:02 +0000
Subject: [PATCH 107/110] fix tui tests broken by urwid update (#5967)

---
 .../tui/screenshots/test_auto_expansion.later-time.html         | 2 +-
 .../tui/screenshots/test_auto_expansion.on-load.html            | 2 +-
 tests/integration/tui/screenshots/test_errors.open-error.html   | 2 +-
 tests/integration/tui/screenshots/test_job_logs.01-job.out.html | 2 +-
 tests/integration/tui/screenshots/test_job_logs.02-job.out.html | 2 +-
 .../screenshots/test_navigation.cursor-at-bottom-of-screen.html | 2 +-
 .../tui/screenshots/test_navigation.family-A-collapsed.html     | 2 +-
 tests/integration/tui/screenshots/test_navigation.on-load.html  | 2 +-
 .../tui/screenshots/test_navigation.workflow-expanded.html      | 2 +-
 .../test_offline_mutation.clean-mutation-selected.html          | 2 +-
 .../test_offline_mutation.stop-all-mutation-selected.html       | 2 +-
 .../test_online_mutation.hold-mutation-selected.html            | 2 +-
 .../tui/screenshots/test_online_mutation.task-selected.html     | 2 +-
 .../screenshots/test_restart_reconnect.1-workflow-running.html  | 2 +-
 .../screenshots/test_restart_reconnect.2-workflow-stopped.html  | 2 +-
 .../test_restart_reconnect.3-workflow-restarted.html            | 2 +-
 .../tui/screenshots/test_scheduler_logs.log-file-selection.html | 2 +-
 .../tui/screenshots/test_scheduler_logs.scheduler-log-file.html | 2 +-
 .../test_scheduler_logs.workflow-configuration-file.html        | 2 +-
 .../tui/screenshots/test_subscribe_unsubscribe.subscribed.html  | 2 +-
 .../screenshots/test_subscribe_unsubscribe.unsubscribed.html    | 2 +-
 .../tui/screenshots/test_task_logs.latest-job.err.html          | 2 +-
 .../tui/screenshots/test_task_logs.latest-job.out.html          | 2 +-
 .../tui/screenshots/test_tui_basics.test-rakiura-enter.html     | 2 +-
 .../tui/screenshots/test_tui_basics.test-rakiura.html           | 2 +-
 .../tui/screenshots/test_workflow_states.filter-active.html     | 2 +-
 .../screenshots/test_workflow_states.filter-starts-with-t.html  | 2 +-
 .../test_workflow_states.filter-stopped-or-paused.html          | 2 +-
 .../tui/screenshots/test_workflow_states.filter-stopped.html    | 2 +-
 .../tui/screenshots/test_workflow_states.unfiltered.html        | 2 +-
 30 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html
index f5a19fd428d..ca1a8a73f0c 100644
--- a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html
+++ b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b                                                                  </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 2                                                                     </span>
diff --git a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
index df3c9f5c41b..d8f3ca712b8 100644
--- a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
+++ b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
 <span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ a                                                               </span>
diff --git a/tests/integration/tui/screenshots/test_errors.open-error.html b/tests/integration/tui/screenshots/test_errors.open-error.html
index 142d0d88c72..31d842ca75b 100644
--- a/tests/integration/tui/screenshots/test_errors.open-error.html
+++ b/tests/integration/tui/screenshots/test_errors.open-error.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Error: Something went wrong :(                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
index c1c767b98cf..6f6f87fdce8 100644
--- a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
+++ b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/01                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job log                                                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
index 0eb94051201..e14a77b99e5 100644
--- a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
+++ b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/02                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job log                                                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
index bf5e3812008..c25e28ebd11 100644
--- a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
+++ b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
@@ -10,7 +10,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b12                                                          </span>
 <span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B2                                                              </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b21                                                          </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#e5e5e5;background:#000000"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b22                                                          </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b22                                                          </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
index cefab5264f4..4c988d41792 100644
--- a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
+++ b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
@@ -3,7 +3,7 @@
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
-<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#e5e5e5;background:#000000">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">+</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B                                                                  </span>
 <span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ B1                                                              </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ b11                                                          </span>
diff --git a/tests/integration/tui/screenshots/test_navigation.on-load.html b/tests/integration/tui/screenshots/test_navigation.on-load.html
index a0bd107742b..ea7d6577c9b 100644
--- a/tests/integration/tui/screenshots/test_navigation.on-load.html
+++ b/tests/integration/tui/screenshots/test_navigation.on-load.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
index 6b26ced563e..e91225fef23 100644
--- a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
+++ b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ A                                                                  </span>
 <span style="color:#000000;background:#e5e5e5">            </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ a1                                                              </span>
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
index f28cced0714..5590254a28d 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
@@ -4,7 +4,7 @@
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stop</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">lean                                   </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">lean                                   </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">log                                     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">play                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">reinstall-reload                        </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
index c2355597f78..eb04278317b 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
@@ -4,7 +4,7 @@
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stop</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">s</span><span style="color:#000000;background:#e5e5e5">top-all                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">s</span><span style="color:#000000;background:#e5e5e5">top-all                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
index 34be2ffa0ce..b8a802948f3 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -4,7 +4,7 @@
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paus</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1    </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">(cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ on</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">h</span><span style="color:#000000;background:#e5e5e5">old                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">h</span><span style="color:#000000;background:#e5e5e5">old                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">kill                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">log                                     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">poll                                    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
diff --git a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
index 7d94d5e43dd..df7a917ff60 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
@@ -3,7 +3,7 @@
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
-<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#e5e5e5;background:#000000"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
+<span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
index 74c02508239..aabddcc9cf6 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
index 09c3bbd7fb0..ee75b87558d 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">Workflow is not running                                                 </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
index 74c02508239..aabddcc9cf6 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
index f88e1b0124d..14be08c1066 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
@@ -14,7 +14,7 @@
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Select File                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">onfig/01-start-01.cylc       </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">c</span><span style="color:#000000;background:#e5e5e5">onfig/01-start-01.cylc       </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">config/flow-processed.cylc    </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">scheduler/01-start-01.log     </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                   </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                      </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
index 68dbcc10f9c..57d6e4066b1 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is the                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">scheduler log file                                                        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
index 011734bc410..c7ab1e925ec 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">[scheduling]                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">    [[graph]]                                                             </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
index 019184ec897..7d190cb6f4d 100644
--- a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
-<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
+<span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">      </span><span style="color:#000000;background:#e5e5e5">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ 1                                                                     </span>
 <span style="color:#000000;background:#e5e5e5">         </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">̿○ one                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
index 8fa0f4329a1..39c2f247c97 100644
--- a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
index 4814892df7a..481d427be6f 100644
--- a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/02                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job error                                                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
index 0eb94051201..e14a77b99e5 100644
--- a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
@@ -1,7 +1,7 @@
 <pre><span style="color:#000000;background:#e5e5e5">┌</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">───────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┐</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Host: myhost                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Path: mypath                                                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
-<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
+<span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">S</span><span style="color:#000000;background:#e5e5e5">elect File                                                           </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">job: 1/a/02                                                               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
 <span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">this is a job log                                                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span>
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
index d54d9538d26..34601749413 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
@@ -12,7 +12,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">id: ~cylc/root                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">Action                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#e5e5e5;background:#000000">(</span><span style="color:#000000;background:#e5e5e5">cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">(</span><span style="color:#000000;background:#e5e5e5">cancel)                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">&lt;</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">stop-all                                </span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">&gt;</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
index 7f80031804b..d507b83d799 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
index 282f76735ed..994d00e911e 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff00ff;background:#e5e5e5">stopping</span><span style="color:#000000;background:#e5e5e5">                                                             </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
index 8c26ce6ccc9..65b11fe4411 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
index 8c26ce6ccc9..65b11fe4411 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
index 1ff602df101..3913180f312 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
diff --git a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
index 0651eedec30..d8cf9ff39fd 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
@@ -1,6 +1,6 @@
 <pre><span style="color:#ff0000;background:#e5e5e5;font-weight:bold">C</span><span style="color:#ffff00;background:#e5e5e5;font-weight:bold">y</span><span style="color:#00ff00;background:#e5e5e5;font-weight:bold">l</span><span style="color:#5c5cff;background:#e5e5e5;font-weight:bold">c</span><span style="color:#000000;background:#e5e5e5;font-weight:bold"> Tui</span><span style="color:#7f7f7f;background:#e5e5e5">   workflows filtered (</span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">W</span><span style="color:#7f7f7f;background:#e5e5e5"> - edit, </span><span style="color:#7f7f7f;background:#e5e5e5;font-weight:bold">E</span><span style="color:#7f7f7f;background:#e5e5e5"> - reset)</span><span style="color:#000000;background:#e5e5e5">                             </span>
 <span style="color:#000000;background:#e5e5e5">                                                                                </span>
-<span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
+<span style="color:#000000;background:#e5e5e5"></span><span style="color:#e5e5e5;background:#000000">-</span><span style="color:#000000;background:#e5e5e5"></span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5">~cylc                                                                         </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">one</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff00ff;background:#e5e5e5">stopping</span><span style="color:#000000;background:#e5e5e5">                                                             </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">tre</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#ff0000;background:#e5e5e5">stopped</span><span style="color:#000000;background:#e5e5e5">                                                              </span>
 <span style="color:#000000;background:#e5e5e5">   </span><span style="color:#000000;background:#e5e5e5">+</span><span style="color:#000000;background:#e5e5e5"> </span><span style="color:#000000;background:#e5e5e5;font-weight:bold">two</span><span style="color:#000000;background:#e5e5e5"> - </span><span style="color:#cdcd00;background:#e5e5e5">paused</span><span style="color:#000000;background:#e5e5e5">                                                               </span>

From ca6c64d930fa28a24d0e86d3f50d35ffa815099f Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 15 Feb 2024 14:15:24 +0000
Subject: [PATCH 108/110] doc: link workflow event configs to user guide
 section (#5954)

---
 cylc/flow/cfgspec/globalcfg.py | 64 ++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 2d39ad74829..713db726660 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -165,9 +165,17 @@
 
         Template variables can be used to configure handlers. For a full list
         of supported variables see :ref:`workflow_event_template_variables`.
+
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
     ''',
     'handler events': '''
         Specify the events for which workflow event handlers should be invoked.
+
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
     ''',
     'mail events': '''
         Specify the workflow events for which notification emails should
@@ -176,6 +184,10 @@
     'startup handlers': f'''
         Handlers to run at scheduler startup.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``startup handler``.
@@ -183,6 +195,10 @@
     'shutdown handlers': f'''
         Handlers to run at scheduler shutdown.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``shutdown handler``.
@@ -191,6 +207,10 @@
         Handlers to run if the scheduler shuts down with error status due to
         a configured timeout or a fatal error condition.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``aborted handler``.
@@ -199,22 +219,38 @@
         Workflow timeout interval. The timer starts counting down at scheduler
         startup. It resets on workflow restart.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.0.0
     ''',
     'workflow timeout handlers': '''
         Handlers to run if the workflow timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.0.0
     ''',
     'abort on workflow timeout': '''
         Whether the scheduler should shut down immediately with error status if
         the workflow timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.0.0
     ''',
     'stall handlers': f'''
         Handlers to run if the scheduler stalls.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``stalled handler``.
@@ -222,6 +258,10 @@
     'stall timeout': f'''
         The length of a timer which starts if the scheduler stalls.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``timeout``.
@@ -229,6 +269,10 @@
     'stall timeout handlers': f'''
         Handlers to run if the stall timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``timeout handler``.
@@ -237,6 +281,10 @@
         Whether the scheduler should shut down immediately with error status if
         the stall timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``abort on timeout``.
@@ -245,6 +293,10 @@
         Scheduler inactivity timeout interval. The timer resets when any
         workflow activity occurs.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES} ``inactivity``.
@@ -252,6 +304,10 @@
     'inactivity timeout handlers': f'''
         Handlers to run if the inactivity timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``inactivity handler``.
@@ -260,6 +316,10 @@
         Whether the scheduler should shut down immediately with error status if
         the inactivity timer times out.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionchanged:: 8.0.0
 
            {REPLACES}``abort on inactivity``.
@@ -268,6 +328,10 @@
         How long to wait for intervention on restarting a completed workflow.
         The timer stops if any task is triggered.
 
+        .. seealso::
+
+           :ref:`user_guide.scheduler.workflow_events`
+
         .. versionadded:: 8.2.0
 
     '''

From 2c511843489774bad42c7adf2fee9fc817f7280c Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 16 Feb 2024 16:43:28 +0000
Subject: [PATCH 109/110] Fix tests for new urwid version

---
 .../tui/screenshots/test_show.success.html    |  2 +-
 .../test_tui_basics.test-rakiura-help.html    | 20 +++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/integration/tui/screenshots/test_show.success.html b/tests/integration/tui/screenshots/test_show.success.html
index afdcd1a73b4..5f9c192b04b 100644
--- a/tests/integration/tui/screenshots/test_show.success.html
+++ b/tests/integration/tui/screenshots/test_show.success.html
@@ -16,7 +16,7 @@
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">URL: (not given)                            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">state: waiting                              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">prerequisites: (None)                       </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
-<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">outputs: ('-': not completed)               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
+<span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">outputs: (&#x27;-&#x27;: not completed)               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo expired                           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo submitted                         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
 <span style="color:#000000;background:#e5e5e5">               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">  - 1/foo submit-failed                     </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">               </span>
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html
index 1795c586d9a..5b47a04d728 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-help.html
@@ -15,26 +15,26 @@
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                                                      </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                           _,@@@@@@.                  </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                         &lt;=@@@, `@@@@@.               </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                            `-@@@@@@@@@@@'            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                            `-@@@@@@@@@@@&#x27;            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                               :@@@@@@@@@@.           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                              (.@@@@@@@@@@@           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                             ( '@@@@@@@@@@@@.         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                             ( &#x27;@@@@@@@@@@@@.         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                            ;.@@@@@@@@@@@@@@@         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                          '@@@@@@@@@@@@@@@@@@,        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                        ,@@@@@@@@@@@@@@@@@@@@'        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                          &#x27;@@@@@@@@@@@@@@@@@@,        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                        ,@@@@@@@@@@@@@@@@@@@@&#x27;        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                      :.@@@@@@@@@@@@@@@@@@@@@.        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                    .@@@@@@@@@@@@@@@@@@@@@@@@.        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                  '@@@@@@@@@@@@@@@@@@@@@@@@@.         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                  &#x27;@@@@@@@@@@@@@@@@@@@@@@@@@.         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">                ;@@@@@@@@@@@@@@@@@@@@@@@@@@@          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">               .@@@@@@@@@@@@@@@@@@@@@@@@@@.           </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">              .@@@@@@@@@@@@@@@@@@@@@@@@@@,            </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">             .@@@@@@@@@@@@@@@@@@@@@@@@@'              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">            .@@@@@@@@@@@@@@@@@@@@@@@@'     ,          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">          :@@@@@@@@@@@@@@@@@@@@@..''';,,,;::-         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">         '@@@@@@@@@@@@@@@@@@@.        `.   `          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">             .@@@@@@@@@@@@@@@@@@@@@@@@@&#x27;              </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">            .@@@@@@@@@@@@@@@@@@@@@@@@&#x27;     ,          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">          :@@@@@@@@@@@@@@@@@@@@@..&#x27;&#x27;&#x27;;,,,;::-         </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">         &#x27;@@@@@@@@@@@@@@@@@@@.        `.   `          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">        .@@@@@@.: ,.@@@@@@@.            `             </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">      :@@@@@@@,         ;.@,                          </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
-<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">     '@@@@@@.              `@'                        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
+<span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">     &#x27;@@@@@@.              `@&#x27;                        </span><span style="color:#000000;background:#e5e5e5">  </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#000000;background:#e5e5e5">          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">                                                          </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5">          </span>
 <span style="color:#ffffff;background:#0000ee">quit: </span><span style="color:#00ffff;background:#0000ee">q</span><span style="color:#ffffff;background:#0000ee">  h</span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#000000;background:#e5e5e5"> q to close                                               </span><span style="color:#000000;background:#e5e5e5">│</span><span style="color:#00ffff;background:#0000ee">ome End </span><span style="color:#ffffff;background:#0000ee">  </span>
 <span style="color:#ffffff;background:#0000ee">filter tas</span><span style="color:#000000;background:#e5e5e5">└</span><span style="color:#000000;background:#e5e5e5">──────────────────────────────────────────────────────────</span><span style="color:#000000;background:#e5e5e5">┘</span><span style="color:#ffffff;background:#0000ee">          </span>

From 2b2fc7721f4bc0c05c7384170ddc82278aad5875 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 22 Feb 2024 08:17:53 +0000
Subject: [PATCH 110/110] Xtrig arg validate (#5955)

* Xtrigger function arg validation.
* Add integration tests for xtrigger validation, which provides simple examples for each of the built in xtriggers.
- Xrandom validate function
- Init test xrandom validate function
- Add unit tests for validation of built in xtriggers
- Automatically validate xtrigger function signature

---------

Co-authored-by: Hilary James Oliver <hilary.j.oliver@gmail.com>

* Apply suggestions from code review

Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>

* fix flake8

* Improve xtrigger validation (#60)

* Improve xtrigger validation

* wall_clock: use placeholder function for signature validation & autodocs

* Fix docstring for autodoc [skip ci]

---------

Co-authored-by: Hilary Oliver <hilary.j.oliver@gmail.com>
Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
---
 changes.d/5955.feat.md                        |   1 +
 cylc/flow/config.py                           |  42 ++--
 cylc/flow/exceptions.py                       |   7 +-
 cylc/flow/scripts/function_run.py             |  11 +-
 cylc/flow/subprocpool.py                      |  77 +++++---
 cylc/flow/xtrigger_mgr.py                     |  93 ++++++---
 cylc/flow/xtriggers/echo.py                   |  45 ++++-
 cylc/flow/xtriggers/wall_clock.py             |  46 ++++-
 cylc/flow/xtriggers/xrandom.py                |  55 ++++--
 setup.cfg                                     |  14 +-
 tests/functional/runahead/04-no-final-cycle.t |   2 +-
 tests/functional/runahead/no_final/flow.cylc  |   6 +-
 .../spawn-on-demand/15-stop-flow-3/flow.cylc  |   4 +-
 .../35-pass-special-tasks-non-word-names.t    |  40 ----
 tests/functional/xtriggers/03-sequence.t      |   4 +-
 tests/integration/test_config.py              | 179 +++++++++++++++++-
 tests/integration/test_xtrigger_mgr.py        |   1 +
 tests/unit/test_subprocpool.py                |  23 +--
 tests/unit/test_xtrigger_mgr.py               |  19 +-
 .../xtriggers/test_echo.py}                   |  32 ++--
 tests/unit/xtriggers/test_wall_clock.py       |  61 ++++++
 tests/unit/xtriggers/test_xrandom.py          |  39 ++++
 22 files changed, 602 insertions(+), 199 deletions(-)
 create mode 100644 changes.d/5955.feat.md
 delete mode 100755 tests/functional/validate/35-pass-special-tasks-non-word-names.t
 rename tests/{functional/validate/69-bare-clock-xtrigger.t => unit/xtriggers/test_echo.py} (55%)
 create mode 100644 tests/unit/xtriggers/test_wall_clock.py
 create mode 100644 tests/unit/xtriggers/test_xrandom.py

diff --git a/changes.d/5955.feat.md b/changes.d/5955.feat.md
new file mode 100644
index 00000000000..c9fc9b4d879
--- /dev/null
+++ b/changes.d/5955.feat.md
@@ -0,0 +1 @@
+Support xtrigger argument validation.
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index 76c8b4e1980..f871531adeb 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -57,6 +57,7 @@
 from cylc.flow.id import Tokens
 from cylc.flow.cycling.integer import IntegerInterval
 from cylc.flow.cycling.iso8601 import ingest_time, ISO8601Interval
+
 from cylc.flow.exceptions import (
     CylcError,
     InputError,
@@ -1718,28 +1719,25 @@ def generate_triggers(self, lexpression, left_nodes, right, seq,
                 if label != 'wall_clock':
                     raise WorkflowConfigError(f"xtrigger not defined: {label}")
                 else:
-                    # Allow "@wall_clock" in the graph as an undeclared
-                    # zero-offset clock xtrigger.
-                    xtrig = SubFuncContext(
-                        'wall_clock', 'wall_clock', [], {})
+                    # Allow "@wall_clock" in graph as implicit zero-offset.
+                    xtrig = SubFuncContext('wall_clock', 'wall_clock', [], {})
 
-            if xtrig.func_name == 'wall_clock':
-                if self.cycling_type == INTEGER_CYCLING_TYPE:
-                    raise WorkflowConfigError(
-                        "Clock xtriggers require datetime cycling:"
-                        f" {label} = {xtrig.get_signature()}"
-                    )
-                else:
-                    # Convert offset arg to kwarg for certainty later.
-                    if "offset" not in xtrig.func_kwargs:
-                        xtrig.func_kwargs["offset"] = None
-                        with suppress(IndexError):
-                            xtrig.func_kwargs["offset"] = xtrig.func_args[0]
+            if (
+                xtrig.func_name == 'wall_clock'
+                and self.cycling_type == INTEGER_CYCLING_TYPE
+            ):
+                raise WorkflowConfigError(
+                    "Clock xtriggers require datetime cycling:"
+                    f" {label} = {xtrig.get_signature()}"
+                )
 
-            if self.xtrigger_mgr is None:
-                XtriggerManager.validate_xtrigger(label, xtrig, self.fdir)
-            else:
+            # Generic xtrigger validation.
+            XtriggerManager.check_xtrigger(label, xtrig, self.fdir)
+
+            if self.xtrigger_mgr:
+                # (not available during validation)
                 self.xtrigger_mgr.add_trig(label, xtrig, self.fdir)
+
             self.taskdefs[right].add_xtrig_label(label, seq)
 
     def get_actual_first_point(self, start_point):
@@ -2427,10 +2425,10 @@ def upgrade_clock_triggers(self):
             # Derive an xtrigger label.
             label = '_'.join(('_cylc', 'wall_clock', task_name))
             # Define the xtrigger function.
-            xtrig = SubFuncContext(label, 'wall_clock', [], {})
-            xtrig.func_kwargs["offset"] = offset
+            args = [] if offset is None else [offset]
+            xtrig = SubFuncContext(label, 'wall_clock', args, {})
             if self.xtrigger_mgr is None:
-                XtriggerManager.validate_xtrigger(label, xtrig, self.fdir)
+                XtriggerManager.check_xtrigger(label, xtrig, self.fdir)
             else:
                 self.xtrigger_mgr.add_trig(label, xtrig, self.fdir)
             # Add it to the task, for each sequence that the task appears in.
diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py
index 43591e7be9c..3ad8c093757 100644
--- a/cylc/flow/exceptions.py
+++ b/cylc/flow/exceptions.py
@@ -241,13 +241,12 @@ class XtriggerConfigError(WorkflowConfigError):
 
     """
 
-    def __init__(self, label: str, trigger: str, message: str):
+    def __init__(self, label: str, message: str):
         self.label: str = label
-        self.trigger: str = trigger
         self.message: str = message
 
-    def __str__(self):
-        return f'[{self.label}] {self.message}'
+    def __str__(self) -> str:
+        return f'[@{self.label}] {self.message}'
 
 
 class ClientError(CylcError):
diff --git a/cylc/flow/scripts/function_run.py b/cylc/flow/scripts/function_run.py
index 7ada8805de3..63029a97782 100755
--- a/cylc/flow/scripts/function_run.py
+++ b/cylc/flow/scripts/function_run.py
@@ -18,10 +18,13 @@
 
 (This command is for internal use.)
 
-Run a Python function "<name>(*args, **kwargs)" in the process pool. It must be
-defined in a module of the same name. Positional and keyword arguments must be
-passed in as JSON strings. <src-dir> is the workflow source dir, needed to find
-local xtrigger modules.
+Run a Python xtrigger function "<name>(*args, **kwargs)" in the process pool.
+It must be in a module of the same name. Positional and keyword arguments must
+be passed in as JSON strings.
+
+Python entry points are the preferred way to make xtriggers available to the
+scheduler, but local xtriggers can be stored in <src-dir>.
+
 """
 import sys
 
diff --git a/cylc/flow/subprocpool.py b/cylc/flow/subprocpool.py
index 9ac3845bb54..1a49eed680d 100644
--- a/cylc/flow/subprocpool.py
+++ b/cylc/flow/subprocpool.py
@@ -41,7 +41,8 @@
 from cylc.flow.task_proxy import TaskProxy
 from cylc.flow.wallclock import get_current_time_string
 
-_XTRIG_FUNCS: dict = {}
+_XTRIG_MOD_CACHE: dict = {}
+_XTRIG_FUNC_CACHE: dict = {}
 
 
 def _killpg(proc, signal):
@@ -66,46 +67,58 @@ def _killpg(proc, signal):
     return True
 
 
-def get_func(func_name, src_dir):
-    """Find and return an xtrigger function from a module of the same name.
+def get_xtrig_mod(mod_name, src_dir):
+    """Find, cache, and return a named xtrigger module.
 
-    These locations are checked in this order:
-    - <src_dir>/lib/python/
-    - `$CYLC_PYTHONPATH`
-    - defined via a `cylc.xtriggers` entry point for an
-      installed Python package.
+    Locations checked in this order:
+    - <src_dir>/lib/python (prepend to sys.path)
+    - $CYLC_PYTHONPATH (already in sys.path)
+    - `cylc.xtriggers` entry point
 
-    Workflow source directory passed in as this is executed in an independent
-    process in the command pool and therefore doesn't know about the workflow.
+    (Check entry point last so users can override with local implementations).
+
+    Workflow source dir passed in - this executes in an independent subprocess.
+
+    Raises:
+        ImportError, if the module is not found
 
     """
-    if func_name in _XTRIG_FUNCS:
-        return _XTRIG_FUNCS[func_name]
+    if mod_name in _XTRIG_MOD_CACHE:
+        # Found and cached already.
+        return _XTRIG_MOD_CACHE[mod_name]
 
     # First look in <src-dir>/lib/python.
     sys.path.insert(0, os.path.join(src_dir, 'lib', 'python'))
-    mod_name = func_name
     try:
-        mod_by_name = __import__(mod_name, fromlist=[mod_name])
+        _XTRIG_MOD_CACHE[mod_name] = __import__(mod_name, fromlist=[mod_name])
     except ImportError:
-        # Look for xtriggers via entry_points for external sources.
-        # Do this after the lib/python and PYTHONPATH approaches to allow
-        # users to override entry_point definitions with local/custom
-        # implementations.
+        # Then entry point.
         for entry_point in iter_entry_points('cylc.xtriggers'):
-            if func_name == entry_point.name:
-                _XTRIG_FUNCS[func_name] = entry_point.load()
-                return _XTRIG_FUNCS[func_name]
-
+            if mod_name == entry_point.name:
+                _XTRIG_MOD_CACHE[mod_name] = entry_point.load()
+                return _XTRIG_MOD_CACHE[mod_name]
         # Still unable to find anything so abort
         raise
 
-    try:
-        _XTRIG_FUNCS[func_name] = getattr(mod_by_name, func_name)
-    except AttributeError:
-        # Module func_name has no function func_name, nor an entry_point entry.
-        raise
-    return _XTRIG_FUNCS[func_name]
+    return _XTRIG_MOD_CACHE[mod_name]
+
+
+def get_xtrig_func(mod_name, func_name, src_dir):
+    """Find, cache, and return a function from an xtrigger module.
+
+    Raises:
+        ImportError, if the module is not found
+        AttributeError, if the function is not found in the module
+
+    """
+    if (mod_name, func_name) in _XTRIG_FUNC_CACHE:
+        return _XTRIG_FUNC_CACHE[(mod_name, func_name)]
+
+    mod = get_xtrig_mod(mod_name, src_dir)
+
+    _XTRIG_FUNC_CACHE[(mod_name, func_name)] = getattr(mod, func_name)
+
+    return _XTRIG_FUNC_CACHE[(mod_name, func_name)]
 
 
 def run_function(func_name, json_args, json_kwargs, src_dir):
@@ -113,6 +126,8 @@ def run_function(func_name, json_args, json_kwargs, src_dir):
 
     func_name(*func_args, **func_kwargs)
 
+    The function is presumed to be in a module of the same name.
+
     Redirect any function stdout to stderr (and workflow log in debug mode).
     Return value printed to stdout as a JSON string - allows use of the
     existing process pool machinery as-is. src_dir is for local modules.
@@ -120,14 +135,18 @@ def run_function(func_name, json_args, json_kwargs, src_dir):
     """
     func_args = json.loads(json_args)
     func_kwargs = json.loads(json_kwargs)
+
     # Find and import then function.
-    func = get_func(func_name, src_dir)
+    func = get_xtrig_func(func_name, func_name, src_dir)
+
     # Redirect stdout to stderr.
     orig_stdout = sys.stdout
     sys.stdout = sys.stderr
     res = func(*func_args, **func_kwargs)
+
     # Restore stdout.
     sys.stdout = orig_stdout
+
     # Write function return value as JSON to stdout.
     sys.stdout.write(json.dumps(res))
 
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index 6ced0023a56..b891165dbd3 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -16,6 +16,7 @@
 
 from contextlib import suppress
 from enum import Enum
+from inspect import signature
 import json
 import re
 from copy import deepcopy
@@ -26,10 +27,11 @@
 from cylc.flow.exceptions import XtriggerConfigError
 import cylc.flow.flags
 from cylc.flow.hostuserutil import get_user
-from cylc.flow.subprocpool import get_func
-from cylc.flow.xtriggers.wall_clock import wall_clock
+from cylc.flow.subprocpool import get_xtrig_func
+from cylc.flow.xtriggers.wall_clock import _wall_clock
 
 if TYPE_CHECKING:
+    from inspect import BoundArguments
     from cylc.flow.broadcast_mgr import BroadcastMgr
     from cylc.flow.data_store_mgr import DataStoreMgr
     from cylc.flow.subprocctx import SubFuncContext
@@ -240,12 +242,16 @@ def __init__(
         self.do_housekeeping = False
 
     @staticmethod
-    def validate_xtrigger(
+    def check_xtrigger(
         label: str,
         fctx: 'SubFuncContext',
         fdir: str,
     ) -> None:
-        """Validate an Xtrigger function.
+        """Generic xtrigger validation: check existence, string templates and
+        function signature.
+
+        Xtrigger modules may also supply a specific `validate` function
+        which will be run here.
 
         Args:
             label: xtrigger label
@@ -259,29 +265,39 @@ def validate_xtrigger(
                 * If the function is not callable.
                 * If any string template in the function context
                   arguments are not present in the expected template values.
+                * If the arguments do not match the function signature.
 
         """
         fname: str = fctx.func_name
+
         try:
-            func = get_func(fname, fdir)
+            func = get_xtrig_func(fname, fname, fdir)
         except ImportError:
             raise XtriggerConfigError(
-                label,
-                fname,
-                f"xtrigger module '{fname}' not found",
+                label, f"xtrigger module '{fname}' not found",
             )
         except AttributeError:
             raise XtriggerConfigError(
-                label,
-                fname,
-                f"'{fname}' not found in xtrigger module '{fname}'",
+                label, f"'{fname}' not found in xtrigger module '{fname}'",
             )
+
         if not callable(func):
             raise XtriggerConfigError(
-                label,
-                fname,
-                f"'{fname}' not callable in xtrigger module '{fname}'",
+                label, f"'{fname}' not callable in xtrigger module '{fname}'",
+            )
+
+        # Validate args and kwargs against the function signature
+        sig_str = fctx.get_signature()
+        try:
+            bound_args = signature(func).bind(
+                *fctx.func_args, **fctx.func_kwargs
             )
+        except TypeError as exc:
+            raise XtriggerConfigError(label, f"{sig_str}: {exc}")
+        # Specific xtrigger.validate(), if available.
+        XtriggerManager.try_xtrig_validate_func(
+            label, fname, fdir, bound_args, sig_str
+        )
 
         # Check any string templates in the function arg values (note this
         # won't catch bad task-specific values - which are added dynamically).
@@ -297,9 +313,7 @@ def validate_xtrigger(
                     template_vars.add(TemplateVariables(match))
                 except ValueError:
                     raise XtriggerConfigError(
-                        label,
-                        fname,
-                        f"Illegal template in xtrigger: {match}",
+                        label, f"Illegal template in xtrigger: {match}",
                     )
 
         # check for deprecated template variables
@@ -315,16 +329,41 @@ def validate_xtrigger(
                 f' {", ".join(t.value for t in deprecated_variables)}'
             )
 
+    @staticmethod
+    def try_xtrig_validate_func(
+        label: str,
+        fname: str,
+        fdir: str,
+        bound_args: 'BoundArguments',
+        signature_str: str,
+    ):
+        """Call an xtrigger's `validate()` function if it is implemented.
+
+        Raise XtriggerConfigError if validation fails.
+        """
+        try:
+            xtrig_validate_func = get_xtrig_func(fname, 'validate', fdir)
+        except (AttributeError, ImportError):
+            return
+        bound_args.apply_defaults()
+        try:
+            xtrig_validate_func(bound_args.arguments)
+        except Exception as exc:  # Note: catch all errors
+            raise XtriggerConfigError(
+                label, f"{signature_str} validation failed: {exc}"
+            )
+
     def add_trig(self, label: str, fctx: 'SubFuncContext', fdir: str) -> None:
         """Add a new xtrigger function.
 
-        Check the xtrigger function exists here (e.g. during validation).
+        Call check_xtrigger before this, during validation.
+
         Args:
             label: xtrigger label
             fctx: function context
             fdir: function module directory
+
         """
-        self.validate_xtrigger(label, fctx, fdir)
         self.functx_map[label] = fctx
 
     def mutate_trig(self, label, kwargs):
@@ -395,20 +434,18 @@ def get_xtrig_ctx(
         args = []
         kwargs = {}
         if ctx.func_name == "wall_clock":
-            if "trigger_time" in ctx.func_kwargs:
+            if "trigger_time" in ctx.func_kwargs:  # noqa: SIM401 (readabilty)
                 # Internal (retry timer): trigger_time already set.
                 kwargs["trigger_time"] = ctx.func_kwargs["trigger_time"]
-            elif "offset" in ctx.func_kwargs:  # noqa: SIM106
+            else:
                 # External (clock xtrigger): convert offset to trigger_time.
                 # Datetime cycling only.
                 kwargs["trigger_time"] = itask.get_clock_trigger_time(
                     itask.point,
-                    ctx.func_kwargs["offset"]
-                )
-            else:
-                # Should not happen!
-                raise ValueError(
-                    "wall_clock function kwargs needs trigger time or offset"
+                    ctx.func_kwargs.get(
+                        "offset",
+                        ctx.func_args[0] if ctx.func_args else None
+                    )
                 )
         else:
             # Other xtrig functions: substitute template values.
@@ -440,7 +477,7 @@ def call_xtriggers_async(self, itask: 'TaskProxy'):
                 if sig in self.sat_xtrig:
                     # Already satisfied, just update the task
                     itask.state.xtriggers[label] = True
-                elif wall_clock(*ctx.func_args, **ctx.func_kwargs):
+                elif _wall_clock(*ctx.func_args, **ctx.func_kwargs):
                     # Newly satisfied
                     itask.state.xtriggers[label] = True
                     self.sat_xtrig[sig] = {}
diff --git a/cylc/flow/xtriggers/echo.py b/cylc/flow/xtriggers/echo.py
index e3c5880cab0..d200e2389bb 100644
--- a/cylc/flow/xtriggers/echo.py
+++ b/cylc/flow/xtriggers/echo.py
@@ -16,22 +16,49 @@
 
 """A Cylc xtrigger function."""
 
-from contextlib import suppress
+from typing import Any, Dict, Tuple
+from cylc.flow.exceptions import WorkflowConfigError
 
 
-def echo(*args, **kwargs):
-    """Prints args to stdout and return success only if kwargs['succeed']
-    is True.
+def echo(*args, **kwargs) -> Tuple:
+    """Print arguments to stdout, return kwargs['succeed'] and kwargs.
 
     This may be a useful aid to understanding how xtriggers work.
 
+    Args:
+        succeed: Set the success of failure of this xtrigger.
+        *args: Print to stdout.
+        **kwargs: Print to stdout, and return as output.
+
+    Examples:
+
+        >>> echo('Breakfast Time', succeed=True, egg='poached')
+        echo: ARGS: ('Breakfast Time',)
+        echo: KWARGS: {'succeed': True, 'egg': 'poached'}
+        (True, {'succeed': True, 'egg': 'poached'})
+
     Returns
-        tuple: (True/False, kwargs)
+        (True/False, kwargs)
 
     """
     print("echo: ARGS:", args)
     print("echo: KWARGS:", kwargs)
-    result = False
-    with suppress(KeyError):
-        result = kwargs["succeed"] is True
-    return result, kwargs
+
+    return kwargs["succeed"], kwargs
+
+
+def validate(all_args: Dict[str, Any]):
+    """
+    Validate the xtrigger function arguments parsed from the workflow config.
+
+    This is separate from the xtrigger to allow parse-time validation.
+
+    """
+    # NOTE: with (*args, **kwargs) pattern, all_args looks like:
+    # {
+    #     'args': (arg1, arg2, ...),
+    #     'kwargs': {kwarg1: val, kwarg2: val, ...}
+    # }
+    succeed = all_args['kwargs'].get("succeed")
+    if not isinstance(succeed, bool):
+        raise WorkflowConfigError("Requires 'succeed=True/False' arg")
diff --git a/cylc/flow/xtriggers/wall_clock.py b/cylc/flow/xtriggers/wall_clock.py
index 3c50d956f0a..d5d1a009154 100644
--- a/cylc/flow/xtriggers/wall_clock.py
+++ b/cylc/flow/xtriggers/wall_clock.py
@@ -17,13 +17,53 @@
 """xtrigger function to trigger off of a wall clock time."""
 
 from time import time
+from typing import Any, Dict
+from cylc.flow.cycling.iso8601 import interval_parse
+from cylc.flow.exceptions import WorkflowConfigError
 
 
-def wall_clock(trigger_time=None):
-    """Return True after the desired wall clock time, False.
+def wall_clock(offset: str = 'PT0S'):
+    """Trigger at a specific real "wall clock" time relative to the cycle point
+    in the graph.
+
+    Clock triggers, unlike other trigger functions, are executed synchronously
+    in the main process.
+
+    Args:
+        offset:
+            ISO 8601 interval to wait after the cycle point is reached in real
+            time before triggering. May be negative, in which case it will
+            trigger before the real time reaches the cycle point.
+    """
+    # NOTE: This is just a placeholder for the actual implementation.
+    # This is only used for validating the signature and for autodocs.
+    ...
+
+
+def _wall_clock(trigger_time: int) -> bool:
+    """Actual implementation of wall_clock.
+
+    Return True after the desired wall clock time, or False before.
 
     Args:
-        trigger_time (int):
+        trigger_time:
             Trigger time as seconds since Unix epoch.
     """
     return time() > trigger_time
+
+
+def validate(args: Dict[str, Any]):
+    """Validate and manipulate args parsed from the workflow config.
+
+    NOTE: the xtrigger signature is different to the function signature above
+
+    wall_clock()  # infer zero interval
+    wall_clock(PT1H)
+    wall_clock(offset=PT1H)
+
+    The offset must be a valid ISO 8601 interval.
+    """
+    try:
+        interval_parse(args["offset"])
+    except (ValueError, AttributeError):
+        raise WorkflowConfigError(f"Invalid offset: {args['offset']}")
diff --git a/cylc/flow/xtriggers/xrandom.py b/cylc/flow/xtriggers/xrandom.py
index 53d98865ef1..867035d46f8 100644
--- a/cylc/flow/xtriggers/xrandom.py
+++ b/cylc/flow/xtriggers/xrandom.py
@@ -16,39 +16,46 @@
 
 from random import random, randint
 from time import sleep
+from typing import Any, Dict, Tuple
+
+from cylc.flow.exceptions import WorkflowConfigError
 
 
 COLORS = ["red", "orange", "yellow", "green", "blue", "indigo", "violet"]
 SIZES = ["tiny", "small", "medium", "large", "huge", "humongous"]
 
 
-def xrandom(percent, secs=0, _=None):
+def xrandom(
+    percent: float, secs: int = 0, _: Any = None
+) -> Tuple[bool, Dict[str, str]]:
     """Random xtrigger, with configurable sleep and percent success.
 
-    Sleep for ``sec`` seconds, and report satisfied with ~``percent``
+    Sleep for ``sec`` seconds, and report satisfied with ``percent``
     likelihood.
 
     The ``_`` argument is not used in the function code, but can be used to
     specialize the function signature to cycle point or task.
 
     Args:
-        percent (float):
+        percent:
             Percent likelihood of passing.
-        secs (int):
+        secs:
             Seconds to sleep before starting the trigger.
-        _ (object):
+        _:
             Used to allow users to specialize the trigger with extra
             parameters.
 
     Examples:
         If the percent is zero, it returns that the trigger condition was
         not satisfied, and an empty dictionary.
+
         >>> xrandom(0, 0)
         (False, {})
 
         If the percent is not zero, but the random percent success is not met,
         then it also returns that the trigger condition was not satisfied,
         and an empty dictionary.
+
         >>> import sys
         >>> mocked_random = lambda: 0.3
         >>> sys.modules[__name__].random = mocked_random
@@ -57,7 +64,8 @@ def xrandom(percent, secs=0, _=None):
 
         Finally, if the percent is not zero, and the random percent success is
         met, then it returns that the trigger condition was satisfied, and a
-        dictionary containing random color and size as result.
+        dictionary containing random colour and size as result.
+
         >>> import sys
         >>> mocked_random = lambda: 0.9
         >>> sys.modules[__name__].random = mocked_random
@@ -69,23 +77,44 @@ def xrandom(percent, secs=0, _=None):
     Returns:
         tuple: (satisfied, results)
 
-        satisfied (bool):
+        satisfied:
             True if ``satisfied`` else ``False``.
-        results (dict):
+        results:
             A dictionary containing the following keys:
 
             ``COLOR``
-                A random color (e.g. red, orange, ...)
+                A random colour (e.g. red, orange, ...).
             ``SIZE``
-                A random size (e.g. small, medium, ...) as the result.
+                A random size (e.g. small, medium, ...).
 
     """
     sleep(float(secs))
     results = {}
-    satisfied = random() < float(percent) / 100  # nosec
+    satisfied = random() < float(percent) / 100  # nosec: B311
     if satisfied:
         results = {
-            'COLOR': COLORS[randint(0, len(COLORS) - 1)],  # nosec
-            'SIZE': SIZES[randint(0, len(SIZES) - 1)]  # nosec
+            'COLOR': COLORS[randint(0, len(COLORS) - 1)],  # nosec: B311
+            'SIZE': SIZES[randint(0, len(SIZES) - 1)]  # nosec: B311
         }
     return satisfied, results
+
+
+def validate(args: Dict[str, Any]):
+    """Validate and manipulate args parsed from the workflow config.
+
+    The rules for args are:
+    * percent: Must be 0 ≤ x ≤ 100
+    * secs: Must be an integer.
+    """
+    percent = args['percent']
+    if (
+        not isinstance(percent, (float, int))
+        or not (0 <= percent <= 100)
+    ):
+        raise WorkflowConfigError(
+            "'percent' should be a float between 0 and 100"
+        )
+
+    secs = args['secs']
+    if not isinstance(secs, int):
+        raise WorkflowConfigError("'secs' should be an integer")
diff --git a/setup.cfg b/setup.cfg
index 849f4ae71b8..bb90ebd1ed6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -215,13 +215,15 @@ cylc.main_loop =
 cylc.pre_configure =
 cylc.post_install =
     log_vc_info = cylc.flow.install_plugins.log_vc_info:main
-# NOTE: Built-in xtriggers
+# NOTE: Built-in xtrigger modules
+#   - must contain a function (the xtrigger) with the same name as the module
+#   - and may contain a "validate" function to check arguments
 cylc.xtriggers =
-    echo = cylc.flow.xtriggers.echo:echo
-    wall_clock = cylc.flow.xtriggers.wall_clock:wall_clock
-    workflow_state = cylc.flow.xtriggers.workflow_state:workflow_state
-    suite_state = cylc.flow.xtriggers.suite_state:suite_state
-    xrandom = cylc.flow.xtriggers.xrandom:xrandom
+    echo = cylc.flow.xtriggers.echo
+    wall_clock = cylc.flow.xtriggers.wall_clock
+    workflow_state = cylc.flow.xtriggers.workflow_state
+    suite_state = cylc.flow.xtriggers.suite_state
+    xrandom = cylc.flow.xtriggers.xrandom
 
 [bdist_rpm]
 requires =
diff --git a/tests/functional/runahead/04-no-final-cycle.t b/tests/functional/runahead/04-no-final-cycle.t
index 1c9609033df..ae1ecebc500 100644
--- a/tests/functional/runahead/04-no-final-cycle.t
+++ b/tests/functional/runahead/04-no-final-cycle.t
@@ -32,7 +32,7 @@ TEST_NAME=${TEST_NAME_BASE}-check-fail
 DB="$RUN_DIR/${WORKFLOW_NAME}/log/db"
 TASKS=$(sqlite3 "${DB}" 'select count(*) from task_states where status=="failed"')
 # manual comparison for the test
-if ((TASKS==4)); then
+if ((TASKS==2)); then
     ok "${TEST_NAME}"
 else
     fail "${TEST_NAME}"
diff --git a/tests/functional/runahead/no_final/flow.cylc b/tests/functional/runahead/no_final/flow.cylc
index 89e23f7af1c..3dd1d7e9734 100644
--- a/tests/functional/runahead/no_final/flow.cylc
+++ b/tests/functional/runahead/no_final/flow.cylc
@@ -1,11 +1,13 @@
-# Should shutdown stalled with 4 failed tasks.
+# Should shutdown stalled with 2 failed tasks due to P1 runahead limit
 [scheduler]
     cycle point time zone = Z
     [[events]]
+        abort on stall timeout = True
+        stall timeout = PT0S
         abort on inactivity timeout = True
         inactivity timeout = PT20S
 [scheduling]
-    runahead limit = P3
+    runahead limit = P1
     initial cycle point = 20100101T00
     [[xtriggers]]
         never = wall_clock(P100Y)
diff --git a/tests/functional/spawn-on-demand/15-stop-flow-3/flow.cylc b/tests/functional/spawn-on-demand/15-stop-flow-3/flow.cylc
index fb796f784f5..4731fee49ea 100644
--- a/tests/functional/spawn-on-demand/15-stop-flow-3/flow.cylc
+++ b/tests/functional/spawn-on-demand/15-stop-flow-3/flow.cylc
@@ -12,11 +12,11 @@
         abort on inactivity timeout = True
 [scheduling]
     [[xtriggers]]
-        x = xrandom("0", "10")  # never succeeds
+        x = xrandom(0, 10)  # never succeeds
     [[graph]]
         R1 = """
                @x => never
-               c => never_again 
+               c => never_again
              """
 [runtime]
     [[c]]
diff --git a/tests/functional/validate/35-pass-special-tasks-non-word-names.t b/tests/functional/validate/35-pass-special-tasks-non-word-names.t
deleted file mode 100755
index c523ea25b78..00000000000
--- a/tests/functional/validate/35-pass-special-tasks-non-word-names.t
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# Test validation of special tasks names with non-word characters
-. "$(dirname "$0")/test_header"
-set_test_number 1
-cat >'flow.cylc' <<'__FLOW_CONFIG__'
-[scheduling]
-    initial cycle point = 20200202
-    final cycle point = 20300303
-    [[special tasks]]
-        clock-trigger = t-1, t+1, t%1, t@1
-    [[graph]]
-        P1D = """
-            t-1
-            t+1
-            t%1
-            t@1
-        """
-
-[runtime]
-    [[t-1, t+1, t%1, t@1]]
-        script = true
-__FLOW_CONFIG__
-run_ok "${TEST_NAME_BASE}" cylc validate "${PWD}"
-exit
diff --git a/tests/functional/xtriggers/03-sequence.t b/tests/functional/xtriggers/03-sequence.t
index f45af9b1caf..63c360f66c5 100644
--- a/tests/functional/xtriggers/03-sequence.t
+++ b/tests/functional/xtriggers/03-sequence.t
@@ -31,7 +31,7 @@ init_workflow "${TEST_NAME_BASE}" << '__FLOW_CONFIG__'
    final cycle point = +P1Y
    [[xtriggers]]
        e1 = echo(name='bob', succeed=True)
-       e2 = echo(name='alice')
+       e2 = echo(name='alice', succeed=False)
    [[dependencies]]
       [[[R1]]]
           graph = "start"
@@ -55,7 +55,7 @@ cylc show "${WORKFLOW_NAME}//2026/foo" | grep -E '^  - xtrigger' > 2026.foo.log
 
 # 2026/foo should get only xtrigger e2.
 cmp_ok 2026.foo.log - <<__END__
-  - xtrigger "e2 = echo(name=alice)"
+  - xtrigger "e2 = echo(name=alice, succeed=False)"
 __END__
 
 cylc stop --now --max-polls=10 --interval=2 "${WORKFLOW_NAME}"
diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py
index acf24d17eaf..581ec4d83fb 100644
--- a/tests/integration/test_config.py
+++ b/tests/integration/test_config.py
@@ -16,12 +16,19 @@
 
 from pathlib import Path
 import sqlite3
+from typing import Any
 import pytest
 
-from cylc.flow.exceptions import ServiceFileError, WorkflowConfigError
+from cylc.flow.exceptions import (
+    ServiceFileError,
+    WorkflowConfigError,
+    XtriggerConfigError,
+)
 from cylc.flow.parsec.exceptions import ListValueError
 from cylc.flow.pathutil import get_workflow_run_pub_db_path
 
+Fixture = Any
+
 
 @pytest.mark.parametrize(
     'task_name,valid', [
@@ -341,3 +348,173 @@ def test_validate_incompatible_db(one_conf, flow, validate):
     finally:
         conn.close()
     assert tables == ['suite_params']
+
+
+def test_xtrig_validation_wall_clock(
+    flow: 'Fixture',
+    validate: 'Fixture',
+):
+    """If an xtrigger module has a `validate()` function is called.
+
+    https://github.com/cylc/cylc-flow/issues/5448
+    """
+    id_ = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {
+            'initial cycle point': '1012',
+            'xtriggers': {'myxt': 'wall_clock(offset=PT7MH)'},
+            'graph': {'R1': '@myxt => foo'},
+        }
+    })
+    with pytest.raises(WorkflowConfigError, match=(
+        r'\[@myxt\] wall_clock\(offset=PT7MH\) validation failed: '
+        r'Invalid offset: PT7MH'
+    )):
+        validate(id_)
+
+
+def test_xtrig_implicit_wall_clock(flow: Fixture, validate: Fixture):
+    """Test @wall_clock is allowed in graph without explicit
+    xtrigger definition.
+    """
+    wid = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {
+            'initial cycle point': '2024',
+            'graph': {'R1': '@wall_clock => foo'},
+        }
+    })
+    validate(wid)
+
+
+def test_xtrig_validation_echo(
+    flow: 'Fixture',
+    validate: 'Fixture',
+):
+    """If an xtrigger module has a `validate()` function is called.
+
+    https://github.com/cylc/cylc-flow/issues/5448
+    """
+    id_ = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {
+            'xtriggers': {'myxt': 'echo()'},
+            'graph': {'R1': '@myxt => foo'},
+        }
+    })
+    with pytest.raises(
+        WorkflowConfigError,
+        match=r'echo.* Requires \'succeed=True/False\' arg'
+    ):
+        validate(id_)
+
+
+def test_xtrig_validation_xrandom(
+    flow: 'Fixture',
+    validate: 'Fixture',
+):
+    """If an xtrigger module has a `validate()` function it is called.
+
+    https://github.com/cylc/cylc-flow/issues/5448
+    """
+    id_ = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {
+            'xtriggers': {'myxt': 'xrandom(200)'},
+            'graph': {'R1': '@myxt => foo'},
+        }
+    })
+    with pytest.raises(
+        XtriggerConfigError,
+        match=r"'percent' should be a float between 0 and 100"
+    ):
+        validate(id_)
+
+
+def test_xtrig_validation_custom(
+    flow: 'Fixture',
+    validate: 'Fixture',
+    monkeypatch: 'Fixture',
+):
+    """If an xtrigger module has a `validate()` function
+    an exception is raised if that validate function fails.
+
+    https://github.com/cylc/cylc-flow/issues/5448
+    """
+    # Rather than create our own xtrigger module on disk
+    # and attempt to trigger a validation failure we
+    # mock our own exception, xtrigger and xtrigger
+    # validation functions and inject these into the
+    # appropriate locations:
+    def kustom_xt(feature):
+        return True, {}
+
+    def kustom_validate(args):
+        raise Exception('This is only a test.')
+
+    # Patch xtrigger func & its validate func
+    monkeypatch.setattr(
+        'cylc.flow.xtrigger_mgr.get_xtrig_func',
+        lambda *args: kustom_validate if "validate" in args else kustom_xt
+    )
+
+    id_ = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {
+            'initial cycle point': '1012',
+            'xtriggers': {'myxt': 'kustom_xt(feature=42)'},
+            'graph': {'R1': '@myxt => foo'},
+        }
+    })
+
+    Path(id_)
+    with pytest.raises(XtriggerConfigError, match=r'This is only a test.'):
+        validate(id_)
+
+
+@pytest.mark.parametrize('xtrig_call, expected_msg', [
+    pytest.param(
+        'xrandom()',
+        r"xrandom.* missing a required argument: 'percent'",
+        id="missing-arg"
+    ),
+    pytest.param(
+        'wall_clock(alan_grant=1)',
+        r"wall_clock.* unexpected keyword argument 'alan_grant'",
+        id="unexpected-arg"
+    ),
+])
+def test_xtrig_signature_validation(
+    flow: 'Fixture', validate: 'Fixture',
+    xtrig_call: str, expected_msg: str
+):
+    """Test automatic xtrigger function signature validation."""
+    id_ = flow({
+        'scheduler': {'allow implicit tasks': True},
+        'scheduling': {
+            'initial cycle point': '2024',
+            'xtriggers': {'myxt': xtrig_call},
+            'graph': {'R1': '@myxt => foo'},
+        }
+    })
+    with pytest.raises(XtriggerConfigError, match=expected_msg):
+        validate(id_)
+
+
+def test_special_task_non_word_names(flow: Fixture, validate: Fixture):
+    """Test validation of special tasks names with non-word characters"""
+    wid = flow({
+        'scheduling': {
+            'initial cycle point': '2020',
+            'special tasks': {
+                'clock-trigger': 't-1, t+1, t%1, t@1',
+            },
+            'graph': {
+                'P1D': 't-1 & t+1 & t%1 & t@1',
+            },
+        },
+        'runtime': {
+            't-1, t+1, t%1, t@1': {'script': True},
+        },
+    })
+    validate(wid)
diff --git a/tests/integration/test_xtrigger_mgr.py b/tests/integration/test_xtrigger_mgr.py
index c116e6968a5..612049163cc 100644
--- a/tests/integration/test_xtrigger_mgr.py
+++ b/tests/integration/test_xtrigger_mgr.py
@@ -21,6 +21,7 @@
 
 from cylc.flow.pathutil import get_workflow_run_dir
 
+
 async def test_2_xtriggers(flow, start, scheduler, monkeypatch):
     """Test that if an itask has 2 wall_clock triggers with different
     offsets that xtrigger manager gets both of them.
diff --git a/tests/unit/test_subprocpool.py b/tests/unit/test_subprocpool.py
index 981ca8e75f9..9e2d4af212b 100644
--- a/tests/unit/test_subprocpool.py
+++ b/tests/unit/test_subprocpool.py
@@ -29,7 +29,7 @@
 from cylc.flow.cycling.iso8601 import ISO8601Point
 from cylc.flow.task_events_mgr import TaskJobLogsRetrieveContext
 from cylc.flow.subprocctx import SubProcContext
-from cylc.flow.subprocpool import SubProcPool, _XTRIG_FUNCS, get_func
+from cylc.flow.subprocpool import SubProcPool, _XTRIG_FUNC_CACHE, _XTRIG_MOD_CACHE, get_xtrig_func
 from cylc.flow.task_proxy import TaskProxy
 
 
@@ -155,7 +155,8 @@ def test_xfunction(self):
             with the_answer_file.open(mode="w") as f:
                 f.write("""the_answer = lambda: 42""")
                 f.flush()
-            fn = get_func("the_answer", temp_dir)
+                f_name = "the_answer"
+            fn = get_xtrig_func(f_name, f_name, temp_dir)
             result = fn()
             self.assertEqual(42, result)
 
@@ -166,19 +167,18 @@ def test_xfunction_cache(self):
             python_dir.mkdir(parents=True)
             amandita_file = python_dir / "amandita.py"
             with amandita_file.open(mode="w") as f:
-                f.write("""amandita = lambda: 'chocolate'""")
+                f.write("""choco = lambda: 'chocolate'""")
                 f.flush()
-            fn = get_func("amandita", temp_dir)
+            m_name = "amandita"  # module
+            f_name = "choco"  # function
+            fn = get_xtrig_func(m_name, f_name, temp_dir)
             result = fn()
             self.assertEqual('chocolate', result)
 
             # is in the cache
-            self.assertTrue('amandita' in _XTRIG_FUNCS)
+            self.assertTrue((m_name, f_name) in _XTRIG_FUNC_CACHE)
             # returned from cache
-            self.assertEqual(fn, get_func("amandita", temp_dir))
-            del _XTRIG_FUNCS['amandita']
-            # is not in the cache
-            self.assertFalse('amandita' in _XTRIG_FUNCS)
+            self.assertEqual(fn, get_xtrig_func(m_name, f_name, temp_dir))
 
     def test_xfunction_import_error(self):
         """Test for error on importing a xtrigger function.
@@ -189,7 +189,7 @@ def test_xfunction_import_error(self):
         """
         with TemporaryDirectory() as temp_dir:
             with self.assertRaises(ModuleNotFoundError):
-                get_func("invalid-module-name", temp_dir)
+                get_xtrig_func("invalid-module-name", "func-name", temp_dir)
 
     def test_xfunction_attribute_error(self):
         """Test for error on looking for an attribute in a xtrigger script."""
@@ -200,8 +200,9 @@ def test_xfunction_attribute_error(self):
             with the_answer_file.open(mode="w") as f:
                 f.write("""the_droid = lambda: 'excalibur'""")
                 f.flush()
+            f_name = "the_sword"
             with self.assertRaises(AttributeError):
-                get_func("the_sword", temp_dir)
+                get_xtrig_func(f_name, f_name, temp_dir)
 
 
 @pytest.fixture
diff --git a/tests/unit/test_xtrigger_mgr.py b/tests/unit/test_xtrigger_mgr.py
index bba77c1f7f1..78f8fe6d969 100644
--- a/tests/unit/test_xtrigger_mgr.py
+++ b/tests/unit/test_xtrigger_mgr.py
@@ -68,7 +68,7 @@ def test_add_xtrigger_with_params(xtrigger_mgr):
     assert xtrig == xtrigger_mgr.functx_map["xtrig"]
 
 
-def test_add_xtrigger_with_unknown_params(xtrigger_mgr):
+def test_check_xtrigger_with_unknown_params(xtrigger_mgr):
     """Test for adding an xtrigger with an unknown parameter.
 
     The XTriggerManager contains a list of specific parameters that are
@@ -84,22 +84,25 @@ def test_add_xtrigger_with_unknown_params(xtrigger_mgr):
         label="echo",
         func_name="echo",
         func_args=[1, "name", "%(what_is_this)s"],
-        func_kwargs={"location": "soweto"}
+        func_kwargs={"succeed": True}
     )
-    with pytest.raises(XtriggerConfigError):
-        xtrigger_mgr.add_trig("xtrig", xtrig, 'fdir')
+    with pytest.raises(
+        XtriggerConfigError,
+        match="Illegal template in xtrigger: what_is_this"
+    ):
+        xtrigger_mgr.check_xtrigger("xtrig", xtrig, 'fdir')
 
 
-def test_add_xtrigger_with_deprecated_params(xtrigger_mgr, caplog):
+def test_check_xtrigger_with_deprecated_params(xtrigger_mgr, caplog):
     """It should flag deprecated template variables."""
     xtrig = SubFuncContext(
         label="echo",
         func_name="echo",
         func_args=[1, "name", "%(suite_name)s"],
-        func_kwargs={"location": "soweto"}
+        func_kwargs={"succeed": True}
     )
     caplog.set_level(logging.WARNING, CYLC_LOG)
-    xtrigger_mgr.add_trig("xtrig", xtrig, 'fdir')
+    xtrigger_mgr.check_xtrigger("xtrig", xtrig, 'fdir')
     assert caplog.messages == [
         'Xtrigger "xtrig" uses deprecated template variables: suite_name'
     ]
@@ -139,7 +142,6 @@ def test_housekeeping_nothing_satisfied(xtrigger_mgr):
 def test_housekeeping_with_xtrigger_satisfied(xtrigger_mgr):
     """The housekeeping method makes sure only satisfied xtrigger function
     are kept."""
-    xtrigger_mgr.validate_xtrigger = lambda *a, **k: True  # Ignore validation
     xtrig = SubFuncContext(
         label="get_name",
         func_name="get_name",
@@ -171,7 +173,6 @@ def test_housekeeping_with_xtrigger_satisfied(xtrigger_mgr):
 
 def test__call_xtriggers_async(xtrigger_mgr):
     """Test _call_xtriggers_async"""
-    xtrigger_mgr.validate_xtrigger = lambda *a, **k: True  # Ignore validation
     # the echo1 xtrig (not satisfied)
     echo1_xtrig = SubFuncContext(
         label="echo1",
diff --git a/tests/functional/validate/69-bare-clock-xtrigger.t b/tests/unit/xtriggers/test_echo.py
similarity index 55%
rename from tests/functional/validate/69-bare-clock-xtrigger.t
rename to tests/unit/xtriggers/test_echo.py
index 4408a064dba..7b1f4d140ee 100644
--- a/tests/functional/validate/69-bare-clock-xtrigger.t
+++ b/tests/unit/xtriggers/test_echo.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
 # Copyright (C) NIWA & British Crown (Met Office) & Contributors.
 #
@@ -15,18 +14,25 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Test that undeclared zero-offset clock xtriggers are allowed.
-. "$(dirname "$0")/test_header"
+from cylc.flow.exceptions import WorkflowConfigError
+from cylc.flow.xtriggers.echo import validate
+import pytest
+from pytest import param
 
-set_test_number 1
 
-cat >'flow.cylc' <<'__FLOW_CONFIG__'
-[scheduler]
-    allow implicit tasks = True
-[scheduling]
-    initial cycle point = now
-    [[graph]]
-        T00 = "@wall_clock => foo"
-__FLOW_CONFIG__
+def test_validate_good():
+    validate({
+        'args': (),
+        'kwargs': {'succeed': False, 'egg': 'fried', 'potato': 'baked'}
+    })
 
-run_ok "${TEST_NAME_BASE}-val" cylc validate .
+
+@pytest.mark.parametrize(
+    'all_args', (
+        param({'args': (False,), 'kwargs': {}}, id='no-kwarg'),
+        param({'args': (), 'kwargs': {'spud': 'mashed'}}, id='no-succeed-kwarg'),
+    )
+)
+def test_validate_exceptions(all_args):
+    with pytest.raises(WorkflowConfigError, match='^Requires'):
+        validate(all_args)
diff --git a/tests/unit/xtriggers/test_wall_clock.py b/tests/unit/xtriggers/test_wall_clock.py
new file mode 100644
index 00000000000..44bba2619fc
--- /dev/null
+++ b/tests/unit/xtriggers/test_wall_clock.py
@@ -0,0 +1,61 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cylc.flow.xtriggers.wall_clock import _wall_clock, validate
+from metomi.isodatetime.parsers import DurationParser
+import pytest
+from pytest import param
+
+
+@pytest.mark.parametrize('trigger_time, expected', [
+    (499, True),
+    (500, False),
+])
+def test_wall_clock(
+    monkeypatch: pytest.MonkeyPatch, trigger_time: int, expected: bool
+):
+    monkeypatch.setattr(
+        'cylc.flow.xtriggers.wall_clock.time', lambda: 500
+    )
+    assert _wall_clock(trigger_time) == expected
+
+
+@pytest.fixture
+def monkeypatch_interval_parser(monkeypatch):
+    """Interval parse only works normally if a WorkflowSpecifics
+    object identify the parser to be used.
+    """
+    monkeypatch.setattr(
+        'cylc.flow.xtriggers.wall_clock.interval_parse',
+        DurationParser().parse
+    )
+
+
+def test_validate_good(monkeypatch_interval_parser):
+    validate({'offset': 'PT1H'})
+
+
+@pytest.mark.parametrize(
+    'args, err', (
+        param({'offset': 1}, "^Invalid", id='invalid-offset-int'),
+        param({'offset': 'Zaphod'}, "^Invalid", id='invalid-offset-str'),
+    )
+)
+def test_validate_exceptions(
+    monkeypatch_interval_parser, args, err
+):
+    with pytest.raises(Exception, match=err):
+        validate(args)
diff --git a/tests/unit/xtriggers/test_xrandom.py b/tests/unit/xtriggers/test_xrandom.py
new file mode 100644
index 00000000000..21a9fdb776d
--- /dev/null
+++ b/tests/unit/xtriggers/test_xrandom.py
@@ -0,0 +1,39 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+from pytest import param
+
+from cylc.flow.xtriggers.xrandom import validate
+from cylc.flow.exceptions import WorkflowConfigError
+
+
+def test_validate_good():
+    validate({'percent': 1, 'secs': 0, '_': 'HelloWorld'})
+
+
+@pytest.mark.parametrize(
+    'args, err', (
+        param({'percent': 'foo'}, r"'percent", id='percent-not-numeric'),
+        param({'percent': 101}, r"'percent", id='percent>100'),
+        param({'percent': -1}, r"'percent", id='percent<0'),
+        param({'percent': 100, 'secs': 1.1}, r"'secs'", id='secs-not-int'),
+    )
+)
+def test_validate_exceptions(args, err):
+    """Illegal args and kwargs cause a WorkflowConfigError raised."""
+    with pytest.raises(WorkflowConfigError, match=f'^{err}'):
+        validate(args)