diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000000..9079951da7 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,33 @@ +/** + * Copyright (C) NIWA & British Crown (Met Office) & Contributors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module.exports = { + root: true, + env: { + browser: true + }, + extends: [ + 'standard', + 'eslint:recommended' + ], + rules: { + 'operator-linebreak': ['error', 'before'] + }, + globals: { + '$': 'readonly' + } +} diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index ac5ee369f7..0000000000 --- a/.eslintrc.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "eslint:recommended", - "parserOptions": { - "ecmaVersion": 6 - }, - "env": { - "browser": true, - "es6": true, - "jquery": true - } -} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..7c415317c4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,227 @@ +name: test + +on: + pull_request: + workflow_dispatch: + inputs: + rose_ref: + description: The Rose branch to test against + required: true + fcm_ref: + description: The FCM branch to test against + required: false + fcm_repo: + description: The FCM repo to test against + required: false + cylc_ref: + description: The Cylc branch to test against + required: false + cylc_repo: + description: The Cylc repo to test against + required: false + +defaults: + run: + shell: bash # macos default shell is zsh + +jobs: + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest'] + python-version: ['3.7', '3.8', '3.9'] + # TODO: re-enable macos testing + # currently (in the absence of a wrapper script) rose cannot be run + # from within cylc jobs in situations where the output of the following + # commands differ: + # bash -c 'which python' + # bash -l -c 'which python' + # include: + # - os: 'macos-latest' + # python-version: '3.7' + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.rose_ref || github.sha }} + path: rose + + - name: Configure Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Cylc + env: + cylc_repo: ${{ github.event.inputs.cylc_repo || 'cylc/cylc-flow' }} + cylc_branch: ${{ github.event.inputs.cylc_ref || 'master' }} + run: | + pip install "git+https://github.com/${cylc_repo}@${cylc_branch}" + + - name: Brew Install + if: startsWith(matrix.os, 'macos') + run: | + # install system deps + brew install bash coreutils gnu-sed shellcheck sqlite3 subversion + + # add GNU coreutils and sed to the user PATH (for actions steps) + # (see instructions in brew install output) + echo \ + "$(brew --prefix)/opt/coreutils/libexec/gnubin" \ + >> "${GITHUB_PATH}" + echo \ + "/usr/local/opt/gnu-sed/libexec/gnubin" \ + >> "${GITHUB_PATH}" + + # add GNU coreutils and sed to the user PATH (for Cylc jobs) + cat >> "$HOME/.bashrc" <<__HERE__ + PATH="/usr/local/opt/coreutils/libexec/gnubin:\$PATH" + PATH="/usr/local/opt/gnu-sed/libexec/gnubin:\$PATH" + PATH="$pythonLocation:\$PATH" + export PATH + # see NOTE in t/rosie-lookup/00-basic.t + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + __HERE__ + cat "$HOME/.bashrc" + + - name: Apt-Get Install + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y shellcheck sqlite3 at + + - name: MacOS DNS Patch + if: startsWith(matrix.os, 'macos') + run: | + # apply DNS patch + hostuserutil="$(python3 -c ' + import cylc.flow.hostuserutil + print(cylc.flow.hostuserutil.__file__) + ')" + patch "${hostuserutil}" < rose/etc/conf/macos-patch + + - name: Install Rose + working-directory: rose + run: | + pip install ."[all]" + pip install --no-deps git+https://github.com/cylc/cylc-rose.git + yarn install + + - name: Checkout FCM + if: startsWith(matrix.os, 'ubuntu') + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.fcm_repo || 'metomi/fcm' }} + ref: ${{ github.event.inputs.fcm_ref || 'master' }} + path: 'fcm' + + - name: Install FCM + if: startsWith(matrix.os, 'ubuntu') + run: | + # install FCM deps + sudo apt-get install -y \ + subversion \ + build-essential \ + gfortran \ + libxml-parser-perl \ + libconfig-inifiles-perl \ + libdbi-perl \ + libdbd-sqlite3-perl + + # install wandisco + sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu \ + `lsb_release -cs` svn19" \ + >> /etc/apt/sources.list.d/subversion19.list' + sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- \ + | sudo apt-key add - + + # prepend FCM bin to $PATH + FCM_PATH="$GITHUB_WORKSPACE/fcm/bin" + # the github actions way (needed for cylc jobs) + echo "$FCM_PATH" >> "${GITHUB_PATH}" + # the bashrc wat (needed for subsequent gh action steps) + echo "export PATH=\"$FCM_PATH:\$PATH\"" >> "$HOME/.bashrc" + + - name: Style + working-directory: rose + run: | + flake8 + etc/bin/shellchecker + yarn run lint + + - name: Unit Tests + working-directory: rose + run: | + pytest + + - name: Functional Tests + timeout-minutes: 30 + id: functest + working-directory: rose + env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: YES + run: | + # rose tests should pass first time around + etc/bin/rose-test-battery -j 4 --state=save + + - name: Re-Run Fails + if: failure() && steps.functest.outcome == 'failure' + working-directory: rose + env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: YES + run: | + # so we only re-run for debug purposes + cylc scan --state=all --color=never + etc/bin/rose-test-battery -j 1 -v --state=save,failed + + - name: Upload + if: failure() && steps.functest.outcome == 'failure' + uses: actions/upload-artifact@v2 + with: + name: cylc-run ${{ matrix.os }} + path: ~/cylc-run/ + + docs: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.rose_ref || github.sha }} + + - name: Configure Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: install graphviz + run: | + sudo apt-get update + sudo apt-get install -y graphviz pkg-config libgraphviz-dev + pip install pygraphviz + + - name: install + run: | + pip install -e .[docs] + + - name: build (html) + run: | + make -C sphinx/ html SPHINXOPTS='-Wn' + + - name: build (slides) + run: | + make -C sphinx/ slides SPHINXOPTS='-Wn' + + - name: build (linkcheck) + run: | + make -C sphinx/ linkcheck SPHINXOPTS='-Wn' + + - name: debug + if: failure() + run: | + cat /tmp/sphinx-err* >&2 || true diff --git a/.gitignore b/.gitignore index 901998cab0..1794ebeac4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,11 +2,11 @@ *.swp etc/rose.conf etc/opt -lib/bash/rose_init_site doc venv metomi_rose.egg-info dist +node_modules # coverage .coverage diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f7c9d572cd..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Configuration for running Rose test battery on Travis CI -# See https://travis-ci.org/ for more info. - ---- -language: python -python: - - 3.7 -dist: xenial - -before_install: - - export PATH="$PWD/.travis:$PATH" - -after_success: - - now report coverage - -after_failure: - - now report error - -jobs: - include: - - name: "Unit Tests" - install: - - now install coverage linters pytest cylc rose - script: - - now test style - - now test units - - - name: "Test Battery" - before_install: - - export PATH="$PWD/.travis:$PATH" - - now install coverage fcm tut_suite - install: - - now install cylc rose - script: - - now test battery - - - name: "Documentation" - install: - - now install coverage rose sphinx tut_suite - script: - - now test docs diff --git a/.travis/cover.py b/.travis/cover.py deleted file mode 100644 index c88e58b372..0000000000 --- a/.travis/cover.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- - -import sys - -from subprocess import call - - -def main(): - command = ['etc/bin/rose-test-battery', '-j', '5'] - if call(command + ['--state=save']): - # Non-zero return code - sys.stderr.write('\n\nRerunning Failed Tests...\n\n') - # Exit with final return code - sys.exit(call(command + ['--state=save,failed', '-v'])) - - -if __name__ == '__main__': - main() diff --git a/.travis/now b/.travis/now deleted file mode 100755 index c32b4f1026..0000000000 --- a/.travis/now +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- - -# Bash script for use with Travis-CI builds. -# -# usage: now command [args...] -# -# commands: -# build -# install -# report_coverage -# test - -# source the .bashrc because for some reason this doesn't get done for us -# do this before the set -eu to avoid bailing on the build for hardcoded -# bashrc issues -if [[ -f "${HOME}/.bashrc" ]]; then - source "${HOME}/.bashrc" -fi - - -set -eu - -APT=() -NPM=() -PIP=() -GIT=() -PY_PATH=() -RC_PATH=("${HOME}") -RI_PATH=() -WANDISCO=false - - -_build_docs () { - etc/bin/rose-make-docs --strict clean html slides latexpdf -} - -_gh_extract () { # extract project from GitHub to $HOME - IFS='|' read -ra SPEC <<< "$1" - local USR="${SPEC[0]}" - local REPO="${SPEC[1]}" - local BRANCH="${SPEC[2]}" - local URL="https://github.com/${USR}/${REPO}/archive/${BRANCH}.tar.gz" - local DEST="${HOME}/${REPO}-${BRANCH}" - - if [[ -d "${DEST}" ]]; then - # already installed - return - fi - - # download + unpack - wget "${URL}" -O - | tar -xz -C "${HOME}" - - # in-place installation - if [[ -f "${DEST}/setup.py" ]]; then - pip install -e "${DEST}" - fi -} - -_install_coverage () { - PIP+=(coverage pytest-cov) - PY_PATH+=("./.travis") -} - -_install_rose () { - pip install -e . - PY_PATH+=("./metomi") - RC_PATH+=("./bin") -} - -_install_cylc () { - APT+=(at) -} - -_install_fcm () { - APT+=(subversion build-essential gfortran libxml-parser-perl \ - libconfig-inifiles-perl libdbi-perl libdbd-sqlite3-perl) - GIT+=('metomi|fcm|master') - RC_PATH+=("${HOME}/fcm-master/bin") - WANDISCO=true -} - -_install_pytest () { - # pytest and its extensions - PIP+=(pytest) -} - -_install_rosie () { - PIP+=(requests tornado sqlalchemy) - RI_PATH+=("$(_path_for python)" "$(_path_for rose)") -} - -_install_sphinx () { - # sphinx documentation and its extensions - APT+=(latexmk texlive texlive-generic-extra texlive-latex-extra \ - texlive-fonts-recommended graphviz) - pip install -e .[docs] -} - -_install_tut_suite () { - # cylc tutorial suite - PIP+=(pillow) -} - -_join () { - local IFS="$1"; - shift; - echo "$*"; -} - -_install_linters () { - APT+=(shellcheck) - PIP+=(pycodestyle) - NPM+=(eslint@6) -} - -_path_for () { - COMMAND="$1" - dirname "$(command -v "${COMMAND}")" -} - -_test_units () { - pytest --cov-append metomi/rose/tests/* -} - -_test_style () { - pycodestyle - eslint . - "$(dirname "$0")/shellchecker" -} - -_test_battery () { - cp "./.travis/sitecustomize.py" ./lib/python - coverage run .travis/cover.py -} - -_test_docs () { - etc/bin/rose-make-docs --strict clean linkcheck doctest -} - -_wandisco_configure () { # extract Wandisco stuff - # shellcheck disable=SC1004,SC2016 - sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu \ - `lsb_release -cs` svn19" >> /etc/apt/sources.list.d/subversion19.list' - sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | \ - sudo apt-key add - -} - -build () { - for arg in "$@"; do - "_build_${arg}" - done -} - - # shellcheck disable=SC2032 -install () { - for arg in "$@"; do - "_install_${arg}" - done - - if ${WANDISCO}; then - _wandisco_configure - fi - - if [[ ${#PIP[@]} -gt 0 ]]; then - pip install "${PIP[@]}" & - fi - - if [[ ${#NPM[@]} -gt 0 ]]; then - npm install -g "${NPM[@]}" & - fi - - if [[ ${#APT[@]} -gt 0 ]]; then - sudo apt-get update - # shellcheck disable=SC155,2033 - sudo apt-get install -y "${APT[@]}" - fi - - if [[ ${#GIT[@]} -gt 0 ]]; then - # wrapping the for loop to avoid unbound variable "GIT[@]" error - for gh_project in "${GIT[@]}"; do - _gh_extract "${gh_project}" - done - fi - - wait - - # .bashrc - cat >"${HOME}/.bashrc" \ - <<<"export PATH=\"$(_join ':' "${RC_PATH[@]}"):\$PATH\";" - cat >>"${HOME}/.bashrc" \ - <<<"export PYTHONPATH=\"$(_join ':' "${PY_PATH[@]}"):\$PYTHONPATH\";" - - # rose_init_site - cat >"./lib/bash/rose_init_site" \ - <<<" - if [[ -z \${HOME+x} ]]; then - export PATH=\"$(_join ':' "${RI_PATH[@]}"):\${PATH:-}\"; - fi - " -} - -_report_coverage () { - coverage combine --append - coverage xml --ignore-errors - bash <(curl -s https://codecov.io/bash) -} - -_report_error() { - # don't bail out on error - set +eu - - printenv PATH PYTHONPATH - rose check-software - cat /tmp/sphinx-err* >&2 # sphinx traceback -} - -report () { - for arg in "$@"; do - "_report_${arg}" - done -} - -test () { - PS1='$' . "${HOME}/.bashrc" - export COVERAGE_PROCESS_START="./.coveragerc" - for arg in "$@"; do - "_test_${arg}" - done -} - -# do this here so we only trace the commands we are interested in -set -o xtrace - -# run the specified function -"$@" diff --git a/ACKNOWLEDGEMENT.md b/ACKNOWLEDGEMENT.md index bc7458284b..824c733bbf 100644 --- a/ACKNOWLEDGEMENT.md +++ b/ACKNOWLEDGEMENT.md @@ -3,69 +3,36 @@ Licences for non-Rose works included in this distribution can be found in the licences/ directory. -doc/rose-icon.png, etc/images/rose-icon.png, etc/images/rose-icon.svg, etc/images/rose-icon-trim.png, etc/images/rose-icon-trim.svg, etc/images/rose-logo.png, -etc/images/rose-splash-logo.png, etc/images/rosie-icon.png, etc/images/rosie-icon.svg, etc/images/rosie-icon-trim.png, etc/images/rosie-icon-trim.svg, -etc/metadata/all/etc/images/icon.png: +metomi/rose/etc/rose-all/etc/images/icon.png, +metomi/rose/etc/rose-meta/rose-all/etc/images/icon.png * These icons are all derived from the public domain image at . -etc/images/rose-config-edit/gnome_add.png, -etc/images/rose-config-edit/gnome_package_system.png: -* These icons are part of the GNOME icon theme 2.28.0, licensed under - GPLv2. This theme was developed by: - * Ulisse Perusin - * Riccardo Buzzotta - * Josef Vybíral - * Hylke Bons - * Ricardo González - * Lapo Calamandrei - * Rodney Dawes - * Luca Ferretti - * Tuomas Kuosmanen - * Andreas Nilsson - * Jakub Steiner - -etc/images/rose-config-edit/gnome_add_???, -etc/images/rose-config-edit/gnome_package_system_???: -* These icons are derivative works produced from - * etc/images/rose-config-edit/gnome_add.png or - * etc/images/rose-config-edit/gnome_package_system.png, and are - distributed in their preferred PNG form. - -lib/html/static/css/bootstrap-???, -lib/html/static/images/glyphicons-halflings-???, -lib/html/static/js/bootstrap.min.js: +metomi/rosie/lib/html/static/css/bootstrap.min.css +metomi/rosie/lib/html/static/js/bootstrap.min.js +metomi/rosie/lib/html/static/fonts/glyphicons-halflings*, * Unmodified external software library copyright 2013 Twitter Inc released under the Apache 2.0 license. See . -lib/html/static/css/rose-bush.css -* Contains modified code from Twitter Bootstrap v2.3.1 released - under the Apache 2.0 licence. - See . - -lib/html/static/css/jquery.dataTables.???: -lib/html/static/images/sort_???: -lib/html/static/js/jquery.dataTables.???: +metomi/rosie/lib/html/static/css/jquery.dataTables.css +metomi/rosie/lib/html/static/images/sort_* * Unmodified external software library released under GPLv2 and BSD. See . -lib/html/static/js/livestamp.min.js: +metomi/rosie/lib/html/static/js/livestamp.min.js * Unmodified external software library released under the MIT license. See . -lib/html/static/js/moment.min.js: +metomi/rosie/lib/html/static/js/moment.min.js * Unmodified external software library released under the MIT license. See - -lib/python/rose/tests/test_ancils/unicode.txt: -* Markus Kuhn - 2015-08-28 - CC BY 4.0 diff --git a/CHANGES.md b/CHANGES.md index f5a0e0fd2d..5b02a0532f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,8 @@ It is able to run most existing Rose Suites and has been ported to Python3. **This is the first Python3 version of Rose** +[#2446](https://github.com/metomi/rose/pull/2446): Host select: change implementation to psutil for portability. + [#2288](https://github.com/metomi/rose/pull/2288): Rosie & Rosa: migrate to Python 3(.6-.7) & Tornado diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..294cd4596f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include metomi/rose/etc/ * +recursive-include metomi/rosie/lib/ * diff --git a/bin/rose b/bin/rose index 7c9b1bd028..6b766356eb 100755 --- a/bin/rose +++ b/bin/rose @@ -28,6 +28,7 @@ # rose help # Print help and list available utilities # rose help UTIL ... # Print help for UTIL ... # rose version # Print version information +# rose version --long # Print more version information # # DESCRIPTION # Simple launcher for utilities in the "rose", "rosie" or "rosa" @@ -37,10 +38,55 @@ if ${ROSE_DEBUG:-false}; then set -x fi -# shellcheck source=lib/bash/rose_init -# shellcheck source=lib/bash/rose_usage -. rose_init -rose_init +set -eu + +ROSE_NS=$(basename "$0") +ROSE_HOME_BIN="$(dirname "$0")" +ROSE_VERSION="$(python3 -c "import metomi.rose; print(metomi.rose.__version__)")" +export ROSE_NS ROSE_HOME_BIN ROSE_VERSION + +# NOTE: cannot use associative array due to bash version requirement +DEAD_ENDS=( + rose-config-edit + rose-edit + rose-suite-clean + rose-suite-cmp-vc + rose-suite-gcontrol + rose-sgc + rose-suite-hook + rose-task-hook + rose-suite-log-view + rose-suite-log + rose-slv + rose-suite-restart + rose-suite-run + rose-suite-init + rose-suite-scan + rose-suite-shutdown + rose-suite-stop +) + +# NOTE: indicies must match DEAD_ENDS array +# (be very careful not to jumble them) +DEAD_END_MSGS=( + 'The Rose configuration editor has been removed, use the Cylc GUI.' + 'The Rose configuration editor has been removed, use the Cylc GUI.' + 'This command has been replaced by: "cylc clean".' + 'This command is awaiting re-implementation in Cylc8' + 'This command has been removed: use the Cylc GUI.' + 'This command has been removed: use the Cylc GUI.' + 'Command obsolete, use Cylc event handlers' + 'Command obsolete, use Cylc event handlers' + 'This command has been removed: use the Cylc GUI.' + 'This command has been removed: use the Cylc GUI' + 'This command has been removed: use the Cylc GUI' + 'This command has been replaced by: "cylc restart".' + 'This command has been replaced by: "cylc install".' + 'This command has been replaced by: "cylc install".' + 'This command has been replaced by: "cylc scan".' + 'This command has been replaced by: "cylc stop".' + 'This command has been replaced by: "cylc stop".' +) # Print actual command of a command alias get_alias() { @@ -49,6 +95,7 @@ get_alias() { "$ROSE_HOME_BIN/$ROSE_NS-$NAME" } + # Print help for a given utility help_util() { local NAME=$1 @@ -102,7 +149,13 @@ path_lead() { # Print Rose version function print_version() { - echo "Rose $ROSE_VERSION ($ROSE_HOME_BIN)" + for arg in "$@"; do + if [[ "$arg" == '--long' ]]; then + echo "Rose $ROSE_VERSION ($ROSE_HOME_BIN)" + return + fi + done + echo "$ROSE_VERSION" } #------------------------------------------------------------------------------- @@ -118,6 +171,8 @@ help|h|?|--help|-h) if (($# == 0)); then { print_version + # shellcheck source=metomi/rose/etc/lib/bash/rose_usage + . "$(rose resource lib/bash/rose_usage)" rose_usage echo echo "$ROSE_NS provides the following utilities:" @@ -129,8 +184,16 @@ help|h|?|--help|-h) echo " (=$ALIAS)" else echo " $NAME" - sed '1,/^# DESCRIPTION$/d;{s/^# / /;q;}' \ - "$ROSE_HOME_BIN/$U" + if [[ $U == 'rose-date' ]]; then + # hack an exception for this lone Python command + # shellcheck disable=SC2026 + echo ' '\ + 'Parse and print 1. a date time point '\ + 'or 2. a duration.' + else + sed '1,/^# DESCRIPTION$/d;{s/^# / /;q;}' \ + "$ROSE_HOME_BIN/$U" + fi fi done } | ${PAGER:-less} @@ -146,7 +209,7 @@ help|h|?|--help|-h) exit $RC :;; version|--version|-V) - print_version + print_version "$@" exit :;; doc) @@ -176,6 +239,14 @@ doc) :;; esac +CMD="${ROSE_NS}-${UTIL}" +for i in $(seq 0 $(( ${#DEAD_ENDS[@]} - 1 ))); do + if [[ "${DEAD_ENDS[$i]}" == "$CMD" ]]; then + echo "${DEAD_END_MSGS[$i]}" >&2 + exit 42 + fi +done + COMMAND="$(dirname "$0")/$ROSE_NS-$UTIL" if [[ ! -f $COMMAND || ! -x $COMMAND ]]; then echo "$ROSE_NS: $UTIL: unknown utility. Abort." >&2 @@ -188,4 +259,5 @@ if (($# > 0)) && [[ $1 == '--help' || $1 == '-h' ]]; then fi ROSE_UTIL=$UTIL export ROSE_UTIL + exec "$COMMAND" "$@" diff --git a/bin/rose-config-edit b/bin/rose-config-edit deleted file mode 100755 index 8ffe233cca..0000000000 --- a/bin/rose-config-edit +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -#----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#----------------------------------------------------------------------------- -# NAME -# rose config-edit -# -# SYNOPSIS -# rose config-edit [OPTIONS]... [PAGE_PATH]... -# -# DESCRIPTION -# Launch the GTK+ GUI to edit a suite or application configuration. -# -# If a suite contains more than 10 applications then they will only be -# loaded after startup, on demand. -# -# OPTIONS -# --config=DIR, -C DIR -# A path to either: -# -# 1. a directory containing a suite with a file named -# `suite.rc` and a directory called `app` containing -# subdirectories with files named `rose-app.conf`, -# in the format specified in the Rose pages. -# 2. a directory containing a single 'application' - a file named -# `rose-app.conf` and an optional subdirectory called `file` -# with other application files. -# -# --load-all-apps -# Force loading of all applications on startup. -# --load-no-apps -# Load applications in the suite on demand. -# --meta-path=PATH, -M PATH -# Prepend `PATH` to the search path for metadata (look here first). -# This option can be used repeatedly to load multiple paths. -# --new -# Launch, ignoring any configuration. -# --no-metadata -# Launch with metadata switched off. -# --no-warn=WARNING-TYPE -# Suppress warnings of the provided type. `WARNING-TYPE` may be: -# -# `version` -# Suppress "Could not find metadata for app/version, using app/HEAD" -# warnings. -# -# ARGUMENTS -# PAGE_PATH -# One or more paths to open on load, pages may be full or partial -# namespaces e.g. `foo/bar/env` or `env`. -# -# NOTE: Opens the shortest namespace that matches the provided string. -# -# ENVIRONMENT VARIABLES -# optional ROSE_META_PATH -# Prepend `$ROSE_META_PATH` to the metadata search path. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.config_editor.main "$@" diff --git a/bin/rose-date b/bin/rose-date index 83f75ab438..616f5ef394 100755 --- a/bin/rose-date +++ b/bin/rose-date @@ -182,6 +182,11 @@ from metomi.isodatetime.main import main as iso_main def main(): """Implement rose date.""" + print( + 'WARNING: "rose date" is depreacted, use the "isodatetime" command.', + file=sys.stderr + ) + # Handle Legacy Rose-date -c functionality if '-c' in sys.argv or '--use-task-cycle-time' in sys.argv: if os.getenv('ROSE_TASK_CYCLE_TIME'): diff --git a/bin/rose-edit b/bin/rose-edit deleted file mode 100755 index 80ec62fb6b..0000000000 --- a/bin/rose-edit +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Alias of "rose config-edit". -#------------------------------------------------------------------------------- -exec "$(dirname "$0")/rose-config-edit" "$@" diff --git a/bin/rose-host-select b/bin/rose-host-select index e3b7b6db56..d3a342d5cc 100755 --- a/bin/rose-host-select +++ b/bin/rose-host-select @@ -27,9 +27,6 @@ # Select a host from a set of groups or names by load, by free memory # or by random. # -# Use settings in `$ROSE_HOME/etc/rose.conf` and `$HOME/.metomi/rose.conf` -# to determine the ranking method. -# # Print the selected host name. # # OPTIONS @@ -75,8 +72,9 @@ # # CONFIGURATION # The command reads its settings from the `[rose-host-select]` section in -# `$ROSE_HOME/etc/rose.conf` and `$HOME/.metomi/rose.conf`. All settings -# are optional. Type `rose config rose-host-select` to print settings. +# the Rose configuration. All settings are optional. Type +# `rose config rose-host-select` to print settings. +# # Valid settings are: # # default = GROUP/HOST ... diff --git a/bin/rose-host-select-client b/bin/rose-host-select-client new file mode 100644 index 0000000000..c5f97eade0 --- /dev/null +++ b/bin/rose-host-select-client @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) 2012-2019 British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# NAME +# rose host-select-client +# +# SYNOPSIS +# rose host-select-client +# +# DESCRIPTION +# Internal command for obtaining information about hosts using the +# Python psutil module. +# +# The names of psutil methods are passed to stdin along with any arguments +# as a JSON list. +# +# Examples: +# # return the virtual memory and cpu usage +# [["virual_memory"], ["cpu_percent"]] +# +# # return disk usage for the filesystem mounted at "/" +# [["disk_usage", "/"]] +# +# The input json should be preceded by \n**start**\n +# and followed by \n**end**\n to avoid issues with user profile scripts +# and stdin deadlock. +# +# $ rose host-select-client <<'__HERE__' +# > **start** +# > [["virtual_memory"]] +# > **end** +# > __HERE__ +# [{"total": 17179869184, "available": 6276612096, "percent": 63.5, ...}] +# +#------------------------------------------------------------------------------- +exec python3 -m metomi.rose.host_select_client "$@" diff --git a/bin/rose-metadata-graph b/bin/rose-metadata-graph index 2d4bd83421..bb703f8cfc 100755 --- a/bin/rose-metadata-graph +++ b/bin/rose-metadata-graph @@ -50,4 +50,5 @@ # optional ROSE_META_PATH # Prepend `$ROSE_META_PATH` to the metadata search path. #------------------------------------------------------------------------------- -exec python3 -m metomi.rose.metadata_graph "$@" +echo 'This command has been removed pending re-implementation' >&2 +exit 1 diff --git a/bin/rose-mpi-launch b/bin/rose-mpi-launch index ad534df38b..d035d29dfa 100755 --- a/bin/rose-mpi-launch +++ b/bin/rose-mpi-launch @@ -54,7 +54,8 @@ # # CONFIGURATION # The command reads from the `[rose-mpi-launch]` section in -# `$ROSE_HOME/etc/rose.conf` and `$HOME/.metomi/rose.conf`. +# the Rose configuration. +# # Valid settings are: # # launcher-list=LIST @@ -111,9 +112,13 @@ # DIAGNOSTICS # Return 0 on success, 1 or exit code of the launcher program on failure. #------------------------------------------------------------------------------- -# shellcheck source=lib/bash/rose_init -. rose_init -rose_init rose_log +set -eu +# shellcheck source=metomi/rose/etc/lib/bash/rose_log +. "$(rose resource lib/bash/rose_log)" +# shellcheck source=metomi/rose/etc/lib/bash/rose_usage +. "$(rose resource lib/bash/rose_usage)" + +ROSE_CMD="${ROSE_NS}-${ROSE_UTIL}" # ------------------------------------------------------------------------------ ROSE_COMMAND= @@ -200,7 +205,7 @@ else fi ROSE_LAUNCHER_LIST=${ROSE_LAUNCHER_LIST:-$( \ - rose config --default= "$ROSE_NS" launcher-list)} + rose config --default= "$ROSE_CMD" launcher-list)} export NPROC=${NPROC:-1} #------------------------------------------------------------------------------- @@ -244,7 +249,7 @@ if [[ -n $ROSE_COMMAND_FILE ]]; then fi ROSE_LAUNCHER_FILEOPTS=${ROSE_LAUNCHER_FILEOPTS:-$( \ rose config --default= \ - "$ROSE_NS" "launcher-fileopts.$ROSE_LAUNCHER_BASE")} + "$ROSE_CMD" "launcher-fileopts.$ROSE_LAUNCHER_BASE")} eval "info 2 exec $ROSE_LAUNCHER $ROSE_LAUNCHER_FILEOPTS $*" eval "exec $ROSE_LAUNCHER $ROSE_LAUNCHER_FILEOPTS $*" else @@ -255,9 +260,9 @@ else fi # Options ROSE_LAUNCHER_PREOPTS=${ROSE_LAUNCHER_PREOPTS:-$(rose config -E \ - --default= "$ROSE_NS" "launcher-preopts.$ROSE_LAUNCHER_BASE")} + --default= "$ROSE_CMD" "launcher-preopts.$ROSE_LAUNCHER_BASE")} ROSE_LAUNCHER_POSTOPTS=${ROSE_LAUNCHER_POSTOPTS:-$(rose config -E \ - --default= "$ROSE_NS" "launcher-postopts.$ROSE_LAUNCHER_BASE")} + --default= "$ROSE_CMD" "launcher-postopts.$ROSE_LAUNCHER_BASE")} else ROSE_LAUNCH_INNER= ROSE_LAUNCHER_PREOPTS= diff --git a/bin/rose-suite-scan b/bin/rose-resource similarity index 67% rename from bin/rose-suite-scan rename to bin/rose-resource index fdaba4d0e4..ac1be09419 100755 --- a/bin/rose-suite-scan +++ b/bin/rose-resource @@ -18,12 +18,23 @@ # along with Rose. If not, see . #------------------------------------------------------------------------------- # NAME -# rose suite-scan +# rose resource # # SYNOPSIS -# rose suite-scan [...] +# rose resource RESOURCE_PATH # # DESCRIPTION -# Run `cylc scan [...]`. +# Display the path of resources in the Rose Python installation. +# +# * If the requested resource exists and is a file its path is printed. +# * If the requested resource exists and is a directory it is listed. +# +# Provide no arguments to see a list of top-level resources. +# +# OPTIONS +# --quiet, -q +# Decrement verbosity. +# --verbose, -v +# Increment verbosity. #------------------------------------------------------------------------------- -exec cylc scan "$@" +exec python3 -m metomi.rose.resource "$@" diff --git a/bin/rose-slv b/bin/rose-slv deleted file mode 100755 index 847b85b0a9..0000000000 --- a/bin/rose-slv +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Alias of "rose suite-log". -#------------------------------------------------------------------------------- -exec "$(dirname "$0")/rose-suite-log" "$@" diff --git a/bin/rose-stem b/bin/rose-stem index c88d5a76cb..1059dbdf85 100755 --- a/bin/rose-stem +++ b/bin/rose-stem @@ -82,4 +82,5 @@ # is intended to specify the revision of `fcm-make` config files. # #------------------------------------------------------------------------------- -exec python3 -m metomi.rose.stem "$@" +echo 'Awaiting re-implementation in cylc-rose.' >&2 +exit 1 diff --git a/bin/rose-suite-clean b/bin/rose-suite-clean deleted file mode 100755 index d310e066e9..0000000000 --- a/bin/rose-suite-clean +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-clean -# -# SYNOPSIS -# rose suite-clean [OPTIONS] [SUITE-NAME [...]] -# -# DESCRIPTION -# Remove items created by "rose suite-run". -# -# If no argument is specified, use the base-name of `$PWD` as suite name. -# -# Correctly remove runtime directories created by `rose suite-run` -# including those on remote job hosts. -# -# OPTIONS -# --name=NAME, -n NAME -# Append `NAME` to the argument list. -# --non-interactive, --yes, -y -# Switch off interactive prompting (=answer yes to everything) -# --only=ITEM -# If one or more `--only=ITEM` option is specified, only files and/or -# directories matching an `ITEM` will be removed. An `ITEM` should be a -# glob pattern (bash extglob) for matching a relative path in the run -# directory of the suite(s). -# --quiet, -q -# Decrement verbosity. -# --verbose, -v -# Increment verbosity. -# -# DIAGNOSTICS -# Return the difference between the number of arguments and number of -# successfully cleaned suites, i.e. 0 if all successful. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.suite_clean "$@" diff --git a/bin/rose-suite-cmp-vc b/bin/rose-suite-cmp-vc deleted file mode 100755 index 277ac9cc87..0000000000 --- a/bin/rose-suite-cmp-vc +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-cmp-vc -# -# SYNOPSIS -# rose suite-cmp-vc NAME -# rose suite-cmp-vc --name=NAME -# -# # If CYLC_SUITE_NAME is exported, compare source info of the current -# # suite from within a suite task if no name is specified. -# rose suite-cmp-vc -# -# DESCRIPTION -# Compare VCS information of a suite source between installation and now. -# -# Version control system information of a suite is installed under -# log/rose-suite-run.version file. This command attempts to regenerate the -# information from the recorded source, and compare the original file with -# the latest information. -# -# Return 0 if no difference. Print unified diff and return 1 if difference -# found. Return 2 on other errors. -# -# ARGUMENTS -# Specify the suite NAME. -# -# OPTIONS -# --name=NAME, -n NAME -# Specify the suite NAME. -# --quiet, -q -# Decrement verbosity. -# --verbose, -v -# Increment verbosity. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.cmp_source_vc "$@" diff --git a/bin/rose-suite-hook b/bin/rose-suite-hook deleted file mode 100755 index cb1e5b8c74..0000000000 --- a/bin/rose-suite-hook +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-hook -# -# SYNOPSIS -# # Cylc interface -# rose suite-hook [OPTIONS] EVENT SUITE MSG -# rose task-hook [OPTIONS] EVENT SUITE TASK_ID MSG -# -# DESCRIPTION -# Deprecated. Use cylc built-in event handlers instead. -# -# Provide a common event hook for cylc suites and tasks. -# -# * (Task event only) Retrieve remote task logs back to server host. -# * Email user if `--mail` specified. -# * Shutdown suite if `--shutdown` specified. -# -# OPTIONS -# --debug -# Switch on debug mode. -# --mail-cc=LIST -# Only useful if the `--mail` option is specified. Specify a -# comma separated list of additional addresses to email. -# --mail -# Trigger an email notification to the user. -# --retrieve-job-logs -# Retrieve remote task job logs. -# --shutdown -# Trigger a shutdown of the suite. -# --quiet, -q -# Decrement verbosity. -# --verbose, -v -# Increment verbosity. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.suite_hook "$@" diff --git a/bin/rose-suite-init b/bin/rose-suite-init deleted file mode 100755 index 0ee6bc98d0..0000000000 --- a/bin/rose-suite-init +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Deprecated. -# Alias of "rose suite-run". -#------------------------------------------------------------------------------- -exec "$(dirname "$0")/rose-suite-run" "$@" diff --git a/bin/rose-suite-log b/bin/rose-suite-log deleted file mode 100755 index f27e50b44e..0000000000 --- a/bin/rose-suite-log +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-log -# -# SYNOPSIS -# 1. rose suite-log [--view] -# 2. rose suite-log --update [ITEM ...] -# rose suite-log --update '*' # all task jobs -# 3. rose suite-log --archive CYCLE ... -# rose suite-log --archive '*' # all cycles -# -# DESCRIPTION -# View or update suite log. -# -# 1. Launch web browser to view suite log. If "rose bush" is not -# configured, the command will offer to start it. -# 2. Pull back task job logs from any remote hosts for specified cycle -# times or task names or IDs. -# 3. Archive (tar-gzip) job logs at the specified cycle time. -# -# If `--name=SUITE-NAME` is not specified, the name will be determined by -# locating a `rose-suite.conf` file in `$PWD` or its nearest parent -# directories. In a normal suite, the basename of the (nearest parent) -# directory containing the `rose-suite.conf` file is assumed to be the -# suite name. In a project containing a rose stem suite, the basename of -# the (nearest parent) directory containing the `rose-stem/rose-suite.conf` -# file is assumed to be the suite name. -# -# OPTIONS -# --archive -# Archive (tar-gzip) job logs at specified cycle times. Implies -# `--update`. -# --force, -f -# Same as `rose suite-log --update '*'`. -# --name=SUITE-NAME, -n SUITE-NAME -# Specify the suite name, instead of using basename of `$PWD`. -# --prune-remote -# If specified, remove job logs from remote hosts after pulling them to -# suite host. -# --tidy-remote -# Deprecated. Use `--prune-remote` instead. -# --update, -U -# Update job logs for items specified in arguments. -# --user=USER-NAME, -u USER-NAME -# View mode only. View logs of a suite of a different user. -# --view -# Launch web browser to view suite log. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.suite_log "$@" diff --git a/bin/rose-suite-log-view b/bin/rose-suite-log-view deleted file mode 100755 index 847b85b0a9..0000000000 --- a/bin/rose-suite-log-view +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Alias of "rose suite-log". -#------------------------------------------------------------------------------- -exec "$(dirname "$0")/rose-suite-log" "$@" diff --git a/bin/rose-suite-restart b/bin/rose-suite-restart deleted file mode 100755 index b05d670570..0000000000 --- a/bin/rose-suite-restart +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-restart -# -# SYNOPSIS -# rose suite-restart [OPTIONS] [[--] CYLC-RESTART-ARGS] -# -# # Restart cylc suite with name equal to basename of $PWD. -# rose suite-restart -# -# # Restart cylc suite NAME -# rose suite-restart --name=NAME -# -# # Restart cylc suite NAME with a given state dump -# rose suite-restart --name=NAME -- state.20141118T161121.195326Z -# -# # Restart cylc suite NAME on given host -# rose suite-restart --name=NAME --host=my-suite-host -# -# DESCRIPTION -# Restart a shutdown suite from its last known state without reinstalling -# it. -# -# If re-installation is required, use `rose suite-run --restart`. -# -# ARGUMENTS -# Arguments (and options after `--`) are passed to `cylc restart`. -# -# OPTIONS -# --host=HOST -# Specify a host for restarting the suite. -# --name=NAME, -n NAME -# Specify the suite `NAME` in cylc, instead of using the basename of -# the current working directory. -# --quiet, -q -# Decrement verbosity. -# --verbose, -v -# Increment verbosity. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.suite_restart "$@" diff --git a/bin/rose-suite-run b/bin/rose-suite-run deleted file mode 100755 index 47b5175904..0000000000 --- a/bin/rose-suite-run +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-run -# -# SYNOPSIS -# rose suite-run [OPTIONS] [[--] CYLC-RUN-ARGS] -# -# # Install and run a Cylc suite in $PWD. -# rose suite-run -# -# # As above, but start the suite in simulation mode. -# rose suite-run -- --mode=simulation -# -# # Install and run the suite in $PWD, and register it as "my.suite". -# rose suite-run -n my.suite -# -# # Install and run suite in "/dir/to/my.suite". -# # Equivalent to (cd /dir/to/my.suite && rose suite-run). -# rose suite-run -C /dir/to/my.suite -# -# DESCRIPTION -# Install and run a Cylc suite. -# -# Install a suite (in `$PWD`), register it in Cylc using the basename of -# the configuration directory or the option specified in `--name=NAME`. -# Invoke `cylc run` on it. Arguments (and options after `--`) are passed -# to `cylc run`. -# -# OPTIONS -# --config=DIR, -C DIR -# Specify the configuration directory of the suite. (default=`$PWD`) -# --define=[SECTION]KEY=VALUE, -D [SECTION]KEY=VALUE -# Each of these overrides the `[SECTION]KEY` setting with a given -# `VALUE`. Can be used to disable a setting using the syntax -# `--define=[SECTION]!KEY` or even `--define=[!SECTION]`. -# See also `--define-suite`. -# --define-suite=KEY=VALUE, -S KEY=VALUE -# As `--define`, but with an implicit `[SECTION]` for suite variables. -# --host=HOST -# Specify a host for running the suite. -# --install-only, -i -# Install the suite. Do not run it. -# --local-install-only, -l -# Install the suite locally. Do not install to job hosts. Do not run -# it. -# --validate-suite-only -# Install the suite in a temporary location. Do not setup any directories. -# Do not run any jobs. Overrides --install-only and -# --local-install-only. -# --log-keep=DAYS -# Specify the number of days to keep log directories/archives. -# Do not housekeep if not specified. Named log directories (created by -# `--log-name=NAME` in previous runs) will not be housekept. -# --log-name=NAME -# Specify a name for the log directory of the current run. If -# specified, it will create a symbolic link `log.NAME` to point to the -# log directory of the current run. Named log directories will not be -# automatically archived or housekept. Only works with `--run=run`. -# --name=NAME, -n NAME -# Specify the suite `NAME` in Cylc, instead of using the basename of -# the configuration directory. -# --new, -N -# (Re-)create suite runtime locations. This option is equivalent to -# running `rose suite-clean -y && rose suite-run` and will remove -# previous runs. Users may want to take extra care when using this -# option. -# --no-log-archive -# Do not archive (tar-gzip) old log directories. -# --no-overwrite -# Do not overwrite existing files. -# --no-strict -# Do not validate (suite engine configuration) in strict mode. -# --opt-conf-key=KEY, -O KEY, --opt-conf-key='(KEY)', -O '(KEY)' -# Each of these switches on an optional configuration identified by -# `KEY`. The configurations are applied first-to-last. The `(KEY)` -# syntax denotes an optional configuration that can be missing. -# Otherwise, the optional configuration must exist. -# --quiet, -q -# Decrement verbosity. -# --reload -# Shorthand for `--run=reload`. -# --restart -# Shorthand for `--run=restart`. -# --run=reload|restart|run -# Invoke `cylc reload|restart|run` according to this option. -# (default=`run`) -# --verbose, -v -# Increment verbosity. -# -# ENVIRONMENT VARIABLES -# `rose suite-run` provides the following environment variables to the suite. -# -# ROSE_ORIG_HOST -# The name of the host where the `rose suite-run` command was invoked. -# optional ROSE_SUITE_OPT_CONF_KEYS -# Each `KEY` in this space delimited list switches on an optional -# configuration. The configurations are applied first-to-last. -# -# SUITE VARIABLES -# `rose suite-run` provides the following variables to the suite as -# template variables (i.e. Jinja2 or EmPy). -# -# ROSE_ORIG_HOST -# The name of the host where the `rose suite-run` command was invoked. -# ROSE_SUITE_VARIABLES -# Dictionary containing all suite variables inserted via -# `[jinja2:suite.rc]` or `[empy:suite.rc]` section. -# -# SEE ALSO -# * `cylc help gui` -# * `cylc help register` -# * `cylc help run` -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.suite_run "$@" diff --git a/bin/rose-suite-shutdown b/bin/rose-suite-shutdown deleted file mode 100755 index 4cfd3ca1a1..0000000000 --- a/bin/rose-suite-shutdown +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose suite-shutdown -# -# SYNOPSIS -# rose suite-shutdown [OPTIONS] [--name=SUITE-NAME] [[--] EXTRA-ARGS ...] -# -# DESCRIPTION -# Shutdown a running suite. -# -# If `--name=SUITE-NAME` is not specified, the name will be determined by -# locating a `rose-suite.conf` file in `$PWD` or its nearest parent -# directories. In a normal suite, the basename of the (nearest parent) -# directory containing the `rose-suite.conf` file is assumed to be the -# suite name. In a project containing a rose stem suite, the basename of -# the (nearest parent) directory containing the `rose-stem/rose-suite.conf` -# file is assumed to be the suite name. -# -# This wrapper also deals with the use case where a suite may be running on -# dedicated servers at a site. The wrapper will make an attempt to detect -# where the suite is running or last run. -# -# OPTIONS -# --all -# Shutdown all running suites. You will be prompted to confirm shutting -# down each affected suite. -# --name=SUITE-NAME -# Specify the suite name. -# --non-interactive, --yes, -y -# Switch off interactive prompting (=answer yes to everything) -# --quiet, -q -# Decrement verbosity. -# --verbose, -v -# Increment verbosity. -# -- --EXTRA-ARGS -# See the cylc documentation, cylc shutdown for options and details on -# `EXTRA-ARGS`. -#------------------------------------------------------------------------------- -exec python3 -m metomi.rose.suite_control shutdown "$@" diff --git a/bin/rose-suite-stop b/bin/rose-suite-stop deleted file mode 100755 index 30a76f43ff..0000000000 --- a/bin/rose-suite-stop +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Alias of "rose suite-shutdown". -#------------------------------------------------------------------------------- -exec "$(dirname "$0")/rose-suite-shutdown" "$@" diff --git a/bin/rose-task-env b/bin/rose-task-env index 0f6fa472ec..ad53feebfe 100755 --- a/bin/rose-task-env +++ b/bin/rose-task-env @@ -136,7 +136,7 @@ # # USAGE IN SUITES # rose `task-env` can be used to make environment variables available to a -# suite by defining its `suite.rc` `env-script` option as +# suite by defining its `flow.cylc` `env-script` option as # `env-script = eval $(rose task-env)`. # #------------------------------------------------------------------------------- diff --git a/bin/rose-task-hook b/bin/rose-task-hook deleted file mode 100755 index 34934cee3e..0000000000 --- a/bin/rose-task-hook +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Alias of "rose suite-hook". -#------------------------------------------------------------------------------- -exec "$(dirname "$0")/rose-suite-hook" "$@" diff --git a/bin/rose-tutorial b/bin/rose-tutorial index 7a41346187..fdb8b4b795 100755 --- a/bin/rose-tutorial +++ b/bin/rose-tutorial @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -30,8 +30,7 @@ # Make a copy of one of the tutorial SUITE in the cylc-run directory. # #------------------------------------------------------------------------------- -# shellcheck source=lib/bash/rose_init -. rose_init +set -eu mkdir -p "$HOME/cylc-run" usage () { @@ -86,7 +85,7 @@ fi # Copy files. echo -n "Copying tutorial files to ${DIRECTORY} ... " -rose_init 'rose_log' + mkdir -p "$(dirname "${DIRECTORY}")" if run rsync -rLptgoD "${ROSE_HOME}/etc/tutorial/$1/" "${DIRECTORY}" \ --exclude='.validate'; then diff --git a/bin/rosie-id b/bin/rosie-id index d6db8e61fd..3ce8f23827 100755 --- a/bin/rosie-id +++ b/bin/rosie-id @@ -29,9 +29,6 @@ # # Print the local location of a given suite ID # rosie id --to-local-copy mo1-abc45 # -# # Print the output directory of a given suite ID -# rosie id --to-output mo1-abc45 -# # # Print the web URL of a given suite ID # rosie id --to-web mo1-abc45 # @@ -63,8 +60,6 @@ # Print the local location of a given suite ID # --to-origin # Print the repository URL of a given suite ID -# --to-output -# Print the output directory of a given suite ID # --to-web # Print the web URL of a given suite ID # --next diff --git a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/badenvswitch.py b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/badenvswitch.py index 59380b8893..b0d2599c38 100644 --- a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/badenvswitch.py +++ b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/badenvswitch.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro +import metomi.rose.macro -class InvalidValueTransformer(rose.macro.MacroBase): +class InvalidValueTransformer(metomi.rose.macro.MacroBase): """Test class to return an invalid value.""" diff --git a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/envswitch.py b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/envswitch.py index c144c819a3..3e5bd8605d 100644 --- a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/envswitch.py +++ b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/envswitch.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro +import metomi.rose.macro -class LogicalTransformer(rose.macro.MacroBase): +class LogicalTransformer(metomi.rose.macro.MacroBase): """Test class to change the value of a boolean environment variable.""" @@ -17,10 +15,10 @@ def transform(self, config, meta_config=None): """Perform the transform operation on the env switch.""" if config.get(["env", "TRANSFORM_SWITCH"]) is not None: value = config.get(["env", "TRANSFORM_SWITCH"]).value - if value == rose.TYPE_BOOLEAN_VALUE_FALSE: - new_value = rose.TYPE_BOOLEAN_VALUE_TRUE + if value == metomi.rose.TYPE_BOOLEAN_VALUE_FALSE: + new_value = metomi.rose.TYPE_BOOLEAN_VALUE_TRUE else: - new_value = rose.TYPE_BOOLEAN_VALUE_FALSE + new_value = metomi.rose.TYPE_BOOLEAN_VALUE_FALSE config.set(["env", "TRANSFORM_SWITCH"], new_value) info = self.WARNING_CHANGED_VALUE.format(value, new_value) self.add_report("env", "TRANSFORM_SWITCH", value, info) diff --git a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_add_remove.py b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_add_remove.py index c62773d19f..3210f17aad 100644 --- a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_add_remove.py +++ b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_add_remove.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro +import metomi.rose.macro -class NamelistAdderRemover(rose.macro.MacroBase): +class NamelistAdderRemover(metomi.rose.macro.MacroBase): """Test class to add and remove a section.""" diff --git a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_ignore.py b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_ignore.py index cbc427c900..fe0c075266 100644 --- a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_ignore.py +++ b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/nl_ignore.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro +import metomi.rose.macro -class NamelistIgnorer(rose.macro.MacroBase): +class NamelistIgnorer(metomi.rose.macro.MacroBase): """Test class to ignore and enable a section.""" @@ -16,15 +14,14 @@ class NamelistIgnorer(rose.macro.MacroBase): def transform(self, config, meta_config=None): """Perform the transform operation on the section.""" - change_list = [] section = "namelist:ignore_nl" node = config.get([section]) if node is not None: if node.state: - node.state = rose.config.ConfigNode.STATE_NORMAL + node.state = metomi.rose.config.ConfigNode.STATE_NORMAL info = self.WARNING_ENABLED.format(section) else: - node.state = rose.config.ConfigNode.STATE_USER_IGNORED + node.state = metomi.rose.config.ConfigNode.STATE_USER_IGNORED info = self.WARNING_IGNORED.format(section) self.add_report(section, None, None, info) return config, self.reports diff --git a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/null.py b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/null.py index fad4c9baf0..7809e61bb2 100644 --- a/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/null.py +++ b/demo/rose-config-edit/demo_meta/app/04-transform/meta/lib/python/macros/null.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro -import rose.variable +import metomi.rose.macro -class NullTransformer(rose.macro.MacroBase): +class NullTransformer(metomi.rose.macro.MacroBase): """Class to report changes for missing or null settings.""" diff --git a/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/fib.py b/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/fib.py index 3482750e15..774525bf37 100644 --- a/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/fib.py +++ b/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/fib.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro -import rose.variable +import metomi.rose.macro +import metomi.rose.variable -class FibonacciChecker(rose.macro.MacroBase): +class FibonacciChecker(metomi.rose.macro.MacroBase): """Class to check if an array matches a Fibonacci sequence.""" @@ -21,14 +19,13 @@ def validate(self, config, meta_config, starts_at_zero=False): seq = [0, 1] else: seq = [1, 1] - problem_list = [] section = "env" option = "INVALID_SEQUENCE" node = config.get([section, option]) if node is None: return [] value = node.value - elems = rose.variable.array_split(value) + elems = metomi.rose.variable.array_split(value) if all([w.isdigit() for w in elems]) and len(elems) > 1: int_elems = [int(w) for w in elems] if len(int_elems) >= 2 and int_elems[:2] != seq: diff --git a/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/null.py b/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/null.py index cc33318b15..d63a78e0fa 100644 --- a/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/null.py +++ b/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/null.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- -import rose.macro -import rose.variable +import metomi.rose.macro -class NullChecker(rose.macro.MacroBase): +class NullChecker(metomi.rose.macro.MacroBase): """Class to report errors for missing or null settings.""" diff --git a/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/url.py b/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/url.py index 887c59d9b5..c96349863c 100644 --- a/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/url.py +++ b/demo/rose-config-edit/demo_meta/app/05-validate/meta/lib/python/macros/url.py @@ -1,15 +1,13 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- import http.client -import rose.macro +import metomi.rose.macro -class URLChecker(rose.macro.MacroBase): +class URLChecker(metomi.rose.macro.MacroBase): """Class to check if a URL is valid.""" @@ -18,8 +16,6 @@ class URLChecker(rose.macro.MacroBase): def validate(self, config, meta_config): """Validate a string containing a URL.""" self.reports = [] - seq = [1, 1] - problem_list = [] for section in config.value.keys(): node = config.get([section]) if not isinstance(node.value, dict): diff --git a/demo/rose-config-edit/demo_meta/suite.rc b/demo/rose-config-edit/demo_meta/flow.cylc similarity index 100% rename from demo/rose-config-edit/demo_meta/suite.rc rename to demo/rose-config-edit/demo_meta/flow.cylc diff --git a/etc/bin/rose-make-docs b/etc/bin/rose-make-docs deleted file mode 100755 index 1594ebbacd..0000000000 --- a/etc/bin/rose-make-docs +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env bash -#----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#----------------------------------------------------------------------------- -# NAME -# rose make-docs -# -# SYNOPSIS -# rose make-docs [OPTIONS] [BUILD...] -# -# DESCRIPTION -# Build the rose documentation in the requested `BUILD` format(s). -# -# OPTIONS -# --venv -# Build virtualenv for temporarilly installing python dependencies -# if necessary. -# --dev -# Development mode, don't remove virtualenv after build. -# --strict -# Disable cache forcing a complete re-build and turn warnings into -# errors. -# --debug -# Run `make` with the --debug option. -# --default-version=VERSION -# By default the current version is symlinked as the default version, -# provide an alternative version to override this. -# -# BUILD -# The format(s) to build the documentation in - default html. -# Avaliable formats are listed in the sphinx documentation -# (http://www.sphinx-doc.org/en/stable/builders.html). -# The most commonly used formats are: -# -# `html` -# For building standalone HTML files. -# `singlehtml` -# For building a single page HTML document. -# `latexpdf` -# For building PDF documentation. -# `clean` -# Removes all built documentation for the current rose version. -# (use `rm -rf doc` to remove all documentation). -# -# DEVELOPMENT BUILDS -# For development purposes use the following BUILDs: -# -# `doctest` -# Runs any doctest examples present in documented python modules. -# `linkcheck` -# Checks external links. -# -# SOFTWARE ENVIRONMENT -# The software dependencies can be found by running the command:: -# -# $ rose check-software --docs -# -# Rose provides two ways of installing the Python dependencies for the Rose -# documentation builder: -# -# Virtual Environment: -# Rose will automatically build a Python virtual environment -# when the --venv flag is used with this command. The virtual -# environment will be created in rose/venv and will be destroyed -# after use. Use the --dev flag to prevent the environment being -# destroyed for future use. Example:: -# -# $ rose make-docs --venv --dev html # create and keep a virtualenv -# -# Conda -# An environment file for creating a conda env can be found in -# etc/rose-docs-env.yaml. Example usage:: -# -# $ conda env create -f etc/rose-docs-env.yaml # create conda env -# $ source activate rose-docs # activate conda env -# $ rose make-docs html # make docs as normal -#----------------------------------------------------------------------------- -set -e -set -o pipefail -shopt -s extglob - -# Move into rose directory -cd "$(dirname "$0")/../.." -ROSE_HOME=$PWD -# Path for virtualenv. -VENV_PATH='venv' # Note: update above docs when changing this value -# Set to `true` when the virtualenv is being used. -USING_VENV=false -# Path to the sphinx directory. -SPHINX_PATH=sphinx -# pick up official rose version -ROSE_VERSION="$(python3 -c "import metomi.rose; print(metomi.rose.__version__)")" -# documentation root output directory -DOCS_DIR="${ROSE_HOME}/doc" - -# is the virtualenv command available -if command -v virtualenv >/dev/null 2>&1; then - VENV_COMPLIANT=true -else - VENV_COMPLIANT=false -fi - -# Parse command line args. -VENV_MODE=false -FORCE=false -DEV_MODE=false -DEBUG='' -BUILDS=() -SPHINX_OPTS=() -DEFAULT_ALIAS='doc' -DEFAULT_VERSION= -while [[ $# -gt 0 ]]; do - case $1 in - --venv) - VENV_MODE=true - shift - ;; - --dev) - DEV_MODE=true - shift - ;; - --force) - FORCE=true - shift - ;; - --strict) - SPHINX_OPTS+=('SPHINXOPTS=-aEW') - shift - ;; - --debug) - DEBUG='--debug' - shift - ;; - --default-version) - DEFAULT_VERSION="$2" - shift - shift - ;; - *) - BUILDS+=("$1") - shift - ;; - esac -done -if [[ "${#BUILDS}" == 0 ]]; then - BUILDS=('html') -fi - - -venv-activate () { - USING_VENV=true - . "${VENV_PATH}/bin/activate" -} - -venv-install () { - venv-destroy - virtualenv --python=python3.7 "${VENV_PATH}" - venv-activate - pip install 'sphinx' - pip install 'sphinx_rtd_theme' - pip install 'sphinxcontrib-httpdomain' - pip install 'hieroglyph' -} - -venv-deactivate () { - deactivate >/dev/null 2>&1 || true -} - -venv-destroy () { - venv-deactivate - rm -rf "${VENV_PATH}" -} - -version_file () { - # output the dictionary {"version": ["build", ...]} in JSON format - DOCS_DIR="$1" - ret='{' - - # scrape filesystem for list of rose versions which have built docs - for version_dir in "${DOCS_DIR}/"*.*.*; do - version=$(basename "$version_dir") - # scrape filesystem to get list of formats this version is available in - ret+="\n \"$version\": [" - for format_dir in "${version_dir}/"!(doctrees|*.html); do - format=$(basename "$format_dir") - ret+="\"$format\", " - done - ret="${ret:0:-2}]," - done - - ret="${ret:0:-1}\n}" - echo -e "$ret" -} - -html_redirect () { - # write an html file to $2 which auto-redirects to the relative path $1 - SRC="$1" - DEST="$2" - - cat >"$DEST" << __HTML__ - - - - Rose Documentation - - - -

If not automatically redirected, please click - Rose Documentation.

- - -__HTML__ -} - - -# Use virtualenv if present and requested or if we are in development mode. -if "${DEV_MODE}" || "${VENV_MODE}"; then - if ! "${VENV_COMPLIANT}"; then - echo 'The virtualenv command is required for the --venv option.' - exit 1 - fi - if [[ -d "${VENV_PATH}" ]]; then - venv-activate - fi -fi - -# Check core (sphinx) builder. -if ! rose-check-software --doc >/dev/null; then - if "${VENV_MODE}"; then - venv-install - elif ! "${FORCE}"; then - echo 'Software required by the rose documentation builder is not -present (run "rose check-software --doc" for details). - -For information on building a Python environment for the Rose documentation -builder see the "Software Dependencies" section of the help page for this -command.' >&2 - exit 1 - fi -fi - -# makefile argument to set the output directory for this build -SPHINX_OPTS+=("BUILDDIR=${DOCS_DIR}/${ROSE_VERSION}") - -# run sphinx-build -if make ${DEBUG} -C "${SPHINX_PATH}" "${BUILDS[@]}" "${SPHINX_OPTS[@]}"; then - RET=0 - # output file containing details of all versions and formats the - # documentation has been built in (locally) for the version / format - # switching pane - version_file "${DOCS_DIR}" > "${DOCS_DIR}/versions.json" - # symlink this version as the default - ( - cd "${DOCS_DIR}" - rm "${DEFAULT_ALIAS}" 2>/dev/null || true - ln -s "${DEFAULT_VERSION:-$ROSE_VERSION}" "${DEFAULT_ALIAS}" - ) - # symlink landing pages - html_redirect "${DEFAULT_ALIAS}/html/index.html" 'doc/index.html' - html_redirect "html/index.html" "doc/${ROSE_VERSION}/index.html" - # support legacy doc/rose.html url - mkdir 'doc/doc' 2>/dev/null || true - html_redirect "html/index.html" "doc/${ROSE_VERSION}/rose.html" -else - RET=1 -fi - -# Remove virtualenv if used and if we are not in development mode. -if "${USING_VENV}"; then - if ! "${DEV_MODE}"; then - venv-destroy - fi -fi - -exit ${RET} diff --git a/etc/bin/rose-test-battery b/etc/bin/rose-test-battery index 79b09d20b7..a2aebb2b67 100755 --- a/etc/bin/rose-test-battery +++ b/etc/bin/rose-test-battery @@ -48,19 +48,18 @@ # SEE ALSO # * `prove(1)` #------------------------------------------------------------------------------- -# shellcheck source=lib/bash/rose_init -# shellcheck source=lib/bash/rose_log -. rose_init -. rose_log - set -eu - -rose_init +# shellcheck source=metomi/rose/etc/lib/bash/rose_log +. "$(rose resource lib/bash/rose_log)" # Move to folder in which prove command should be run. TESTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$TESTDIR/../../" mkdir -p ~/.metomi +mkdir -p "${HOME}/cylc-run" + +ROSE_TEST_TIME_INIT="$(date -u +'%Y%m%dT%H%M%SZ')" +export ROSE_TEST_TIME_INIT # Recompile *.pyc files to ensure we are running the current code. # @TODO Consider if this is appropriate in new version diff --git a/.travis/shellchecker b/etc/bin/shellchecker similarity index 95% rename from .travis/shellchecker rename to etc/bin/shellchecker index f6857d954b..96e3b55da1 100755 --- a/.travis/shellchecker +++ b/etc/bin/shellchecker @@ -21,7 +21,7 @@ # Wrapper for running `shellcheck` over projects. set -e -cd "$(dirname "$0")/../" +cd "$(dirname "$0")/../../" # find shell files under the specified directory find_files () { @@ -77,9 +77,10 @@ main () { default () { # run a strict check on all "functional" scripts main . \ - --exclude etc/rose-bash-completion \ --exclude t \ - -- -e SC1090 -e SC2119 -e SC2001 + --exclude node_modules \ + --exclude .git \ + -- -e SC1090 -e SC2119 -e SC2001 \ # run a lenient check on all test scripts main t -- -S error -e SC1090 diff --git a/etc/conf/macos-patch b/etc/conf/macos-patch new file mode 100644 index 0000000000..51ef158d55 --- /dev/null +++ b/etc/conf/macos-patch @@ -0,0 +1,13 @@ +diff --git a/cylc/flow/hostuserutil.py b/cylc/flow/hostuserutil.py +index 21e51735e..17917b8fc 100644 +--- a/cylc/flow/hostuserutil.py ++++ b/cylc/flow/hostuserutil.py +@@ -113,7 +113,7 @@ class HostUtil: + """Return the extended info of the current host.""" + if target not in self._host_exs: + if target is None: +- target = socket.getfqdn() ++ target = socket.gethostname() + try: + self._host_exs[target] = socket.gethostbyname_ex(target) + except IOError as exc: diff --git a/etc/deploy-docs b/etc/deploy-docs index 26deecfabf..1392c083c9 100755 --- a/etc/deploy-docs +++ b/etc/deploy-docs @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/etc/images/rose-config-edit/change_icon.xpm b/etc/images/rose-config-edit/change_icon.xpm deleted file mode 100644 index 8e731df3ba..0000000000 --- a/etc/images/rose-config-edit/change_icon.xpm +++ /dev/null @@ -1,78 +0,0 @@ -/* XPM */ -static char * change_icon_xpm[] = { -"12 19 56 1", -" c None", -". c #0E2171", -"+ c #1C3282", -"@ c #445CA3", -"# c #5268AC", -"$ c #435AA2", -"% c #1D3486", -"& c #1D3284", -"* c #647CBA", -"= c #97ABD4", -"- c #9EB1D8", -"; c #96AAD4", -"> c #637AB9", -", c #253E90", -"' c #122778", -") c #344E9E", -"! c #88A0D0", -"~ c #95ABD5", -"{ c #93A9D4", -"] c #859ECF", -"^ c #354E9F", -"/ c #485FA7", -"( c #14297A", -"_ c #334DA0", -": c #6985C3", -"< c #6E89C5", -"[ c #6D89C5", -"} c #6A87C4", -"| c #6481C1", -"1 c #304A9D", -"2 c #1B3285", -"3 c #132879", -"4 c #274198", -"5 c #3B56A8", -"6 c #415DAD", -"7 c #3F5BAC", -"8 c #3752A5", -"9 c #253E95", -"0 c #1B3185", -"a c #1D358A", -"b c #253F96", -"c c #29439A", -"d c #29449B", -"e c #223B92", -"f c #1D368B", -"g c #162D81", -"h c #1C348B", -"i c #1F388F", -"j c #1E368D", -"k c #1A3187", -"l c #1E3589", -"m c #273E8E", -"n c #20388C", -"o c #182E83", -"p c #1C3389", -"q c #22398A", -" ", -" ", -" ", -" ", -" ", -" . ", -" +@#$% ", -" &*=-;>, ", -" ')!~~{]^/ ", -" (_:<[}|12 ", -" 345667890 ", -" abcd4ef ", -" ghiijkl ", -" mnopq ", -" ", -" ", -" ", -" ", -" "}; diff --git a/etc/images/rose-config-edit/error_icon.xpm b/etc/images/rose-config-edit/error_icon.xpm deleted file mode 100644 index 993ed1bde2..0000000000 --- a/etc/images/rose-config-edit/error_icon.xpm +++ /dev/null @@ -1,96 +0,0 @@ -/* XPM */ -static char * error_icon_xpm[] = { -"14 20 73 1", -" c None", -". c #9D7A77", -"+ c #9D7773", -"@ c #B0A8A6", -"# c #F66662", -"$ c #F52121", -"% c #B4A4A2", -"& c #BB847D", -"* c #FF2127", -"= c #FF1D23", -"- c #BB3D3D", -"; c #AA9794", -"> c #F85E5A", -", c #B31919", -"' c #B22020", -") c #F73D3D", -"! c #AA8988", -"~ c #D1736B", -"{ c #FD2D2D", -"] c #1A0606", -"^ c #1B0909", -"/ c #FD5955", -"( c #CF504D", -"_ c #AA7F7C", -": c #FF4747", -"< c #FF3939", -"[ c #2A0F0E", -"} c #2B1817", -"| c #FE716A", -"1 c #FE6966", -"2 c #A67370", -"3 c #E65E5A", -"4 c #FF3131", -"5 c #FF4545", -"6 c #3D1917", -"7 c #3F2B27", -"8 c #FE807A", -"9 c #E6726C", -"0 c #AD6461", -"a c #FF3333", -"b c #FF524D", -"c c #A24A46", -"d c #A4655E", -"e c #FF8E85", -"f c #FF978F", -"g c #AD6A67", -"h c #F64040", -"i c #FF2929", -"j c #FF3D3D", -"k c #FF5551", -"l c #49201F", -"m c #4A3732", -"n c #FF9C91", -"o c #FEACA2", -"p c #F68F88", -"q c #AD4D4B", -"r c #FF191F", -"s c #FF5753", -"t c #F8716A", -"u c #F78F87", -"v c #FEAEA4", -"w c #FEB0A6", -"x c #FF9E93", -"y c #AC6764", -"z c #BF7171", -"A c #BF7979", -"B c #BF7F7F", -"C c #BF8785", -"D c #BF8F8D", -"E c #BF9693", -"F c #BF9D99", -"G c #BFA19D", -"H c #BFA39F", -" ", -" ", -" ", -" ", -" ", -" .+ ", -" @#$% ", -" &*=- ", -" ;>,')! ", -" ~{]^/( ", -" _:<[}|12 ", -" 34567889 ", -" 0a. -#------------------------------------------------------------------------------- -# NAME -# rose-bash-completion -# -# SYNOPSIS -# . $ROSE_HOME/etc/rose-bash-completion -# -# DESCRIPTION -# Set Bash auto-completion for rose/rosie commands. -# -# Users should source this file in their ~/.bashrc, using something -# like this: -# if [[ $- =~ i && -f /path/to/rose-bash-completion ]]; then -# . /path/to/rose-bash-completion -# fi -# where /path/to/rose-bash-completion is replaced by the path to -# this file. -# -# Administrators may want to place this file in the -# /etc/bash_completion.d/ (or equivalent) directory. -#------------------------------------------------------------------------------- - -_rose() { - local current possible_subcommand rose_command ROSE_DIR ok_subcommands - local subcommand current_option_line current_option_arg_index - local current_option current_option_arg current_index suboption_lines - local suboption_sep_lines suboptions sub_args custom_function_name metavar - local formal_current_option - COMPREPLY=() - current="${COMP_WORDS[COMP_CWORD]}" - possible_subcommand="${COMP_WORDS[1]:-}" - rose_command="${COMP_WORDS[0]}" - rose --version 1>/dev/null 2>&1 || return 1 - if grep -q "bin/ros[^/]*$" <<<"$rose_command"; then - ROSE_DIR=$(dirname "$rose_command") - eval ROSE_DIR="$ROSE_DIR" - ROSE_DIR=$(dirname $(cd $ROSE_DIR && pwd -P)) - rose_command=$(basename "$rose_command") - else - ROSE_DIR=$(__rose_get_rose_dir) - fi - ok_subcommands=$(cd $ROSE_DIR/ && ls $rose_command-* | \ - sed "s/^$rose_command-//g") - - # Determine the subcommand (e.g. app-run for rose app-run) - subcommand= - if [[ -n $possible_subcommand ]]; then - for ok_subcommand in $ok_subcommands; do - if [[ $ok_subcommand == $possible_subcommand ]]; then - subcommand=$possible_subcommand - break - fi - done - fi - if [[ -n $subcommand && -L $ROSE_DIR/bin/rose-$subcommand ]]; then - subcommand=$(readlink $subcommand) - fi - - # Determine option or option argument completion. - current_option_line=$(printf "%s\n" "${COMP_WORDS[@]}" | grep -n "^-" | \ - tail -1) - if [[ -n $current_option_line ]]; then - current_option_arg_index=$( \ - grep -o "^[0-9][0-9]*" <<<"$current_option_line") - current_option=$(grep -o '\-.*$' <<<"$current_option_line") - current_option_arg="${COMP_WORDS[$current_option_arg_index]:-}" - if [[ $current_option_arg == "=" ]]; then - current_option_arg_index=$(( current_option_arg_index + 1 )) - current_option_arg="${COMP_WORDS[$current_option_arg_index]:-}" - fi - else - current_option_arg_index= - current_option_arg= - current_option= - fi - current_index=${#COMP_WORDS[@]} - current_index=$(( current_index - 1 )) - - # Supply any completion information if we can. - if [[ -n $subcommand ]]; then - suboption_lines=$(__${rose_command}_help $subcommand | \ - sed -n '/^ -/p') - suboption_sep_lines=$( \ - sed 's/^ *//g; s/, \(-\)/\n\1/g' <<<"$suboption_lines") - suboptions=$(sed 's/=/ /g' <<<"$suboption_sep_lines") - if grep -q '^'$current_option'$' <<<"$suboptions"; then - # The current option does not take an argument - forget about it. - current_option= - fi - if [[ -z $current_option ]] || \ - [[ $current_index -gt $current_option_arg_index && \ - -n $current_option_arg ]]; then - # No relevant current option - supply options and arguments. - custom_function_name="_${rose_command}_"${subcommand//-/_}_ARGS - sub_args= - if type -t $custom_function_name 1>/dev/null 2>&1; then - sub_args=$($custom_function_name "$current") - fi - suboptions=$( \ - sed 's/=.*$/=/g; s/ *.*$//g; s/ *$//g;' \ - <<<"$suboption_sep_lines") - COMPREPLY=( $(compgen -W "$suboptions $sub_args" -- "$current") ) - return 0 - fi - if grep -q "^$current_option\>" <<<"$suboptions" && \ - [[ $current != $current_option ]]; then - # The current string must be an argument to the option. - current_option_args= - if [[ $current != "=" ]]; then - current_option_args=$current - fi - metavar=$(sed -n "s/^$current_option \([^ ]*\)/\1/p" \ - <<<"$suboptions") - if [[ $metavar == "DIR" || $metavar == "PATH" || \ - $metavar = "FILE" ]]; then - return 0 - fi - formal_current_option=$( \ - sed -n "s/^ *\(-[^ =]*\).*, $current_option\b.*/\1/p" \ - <<<"$suboption_lines") - if [[ -n $formal_current_option ]]; then - current_option=$formal_current_option - fi - custom_function_name="_${rose_command}_"${subcommand//-/_} - custom_function_name+=${current_option//-/_} - if type -t $custom_function_name 1>/dev/null 2>&1; then - $custom_function_name "$current_option_args" - return $? - fi - COMPREPLY= - return 1 - fi - suboptions=$(sed 's/=.*$/=/g; s/ *.*$//g; s/ *$//g;' \ - <<<"$suboption_sep_lines") - COMPREPLY=( $(compgen -W "$suboptions" -- "$current") ) - return 0 - fi - # There is an incomplete subcommand or none at all - supply a list of them. - COMPREPLY=( $(compgen -W "$ok_subcommands" -- "$current") ) - return 0 -} - -_rose_app_run__app_mode() { - COMPREPLY=( $(compgen -W "fcm_make rose_ana rose_arch rose_prune" -- $1) ) -} - -_rose_app_run__command_key() { - local config_dir command_keys - config_dir=$(__rose_get_config_dir) - if [[ -d $config_dir ]]; then - command_keys=$(rose config --keys --file=$config_dir/rose-app.conf command 2>/dev/null) - COMPREPLY=( $(compgen -W "$command_keys" -- $1) ) - return 0 - fi - return 1 -} - -_rose_app_run__opt_conf_key() { - local config_dir opt_conf_keys - config_dir=$(__rose_get_config_dir) - if [[ -d $config_dir/opt ]]; then - opt_conf_keys=$(cd $config_dir/opt/ && ls rose-app-*.conf | \ - sed "s/rose-app-\(.*\).conf/\1/g") - COMPREPLY=( $(compgen -W "$opt_conf_keys" -- $1) ) - return 0 - fi - return 1 -} - -_rose_app_upgrade_ARGS() { - local config_dir meta_path meta_path_option all_versions_option - config_dir=$(__rose_get_config_dir) - meta_path=$(__rose_get_meta_path) - all_versions_option= - meta_path_option="--meta-path=$meta_path" - if [[ -z $meta_path ]]; then - meta_path_option= - fi - if printf "%s " "${COMP_WORDS[@]}" | \ - grep -q " --all-versions\| -a "; then - all_versions_option="--all-versions" - fi - rose app-upgrade $all_versions_option $meta_path_option \ - --config=$config_dir 2>/dev/null | sed 's/^. \(.*\)$/\1/g' -} - -_rose_host_select__rank_method() { - COMPREPLY=( $(compgen -W "load fs mem random" $1) ) -} - -_rose_macro_ARGS() { - local config_dir meta_path meta_path_option - config_dir=$(__rose_get_config_dir) - meta_path=$(__rose_get_meta_path) - meta_path_option="--meta-path=$meta_path" - if [[ -z $meta_path ]]; then - meta_path_option= - fi - rose macro -q $meta_path_option --config=$config_dir 2>/dev/null | \ - sed "s/^\[.*\] //g" -} - -_rose_metadata_graph_ARGS() { - local config_dir meta_path - config_dir=$(__rose_get_config_dir) - meta_path=$(__rose_get_meta_path) - if [[ -f $config_dir/rose-app.conf ]]; then - PATH=$PATH:$meta_path rose config --meta --keys \ - --file=$config_dir/rose-app.conf 2>/dev/null | \ - sed '/=/d' - else - rose config --keys --file=$config_dir/rose-meta.conf 2>/dev/null | \ - sed '/=/d' - fi -} - -_rose_metadata_graph__property() { - COMPREPLY=( $(compgen -W "trigger" -- $1) ) - return 0 -} - -_rose_stem__group() { - local config_dir groups - config_dir=$(__rose_stem_get_config_dir) - if [[ ! -e $config_dir/meta/rose-meta.conf ]]; then - return 1 - fi - groups=$(rose config --file=$config_dir/meta/rose-meta.conf \ - jinja2:suite.rc=RUN_NAMES widget[rose-config-edit] | \ - grep -o "\-\-choices.[^ ]*" | sed "s/--choices.//g;" | \ - tr '\n' ',' | tr ',' ' ') - tasks=$(rose config --file=$config_dir/meta/rose-meta.conf \ - jinja2:suite.rc=RUN_NAMES widget[rose-config-edit] | \ - tr ' ' '\n' | sed -n '/rose.config_editor.*/d; /^\([^-].*\)/p') - COMPREPLY=( $(compgen -W "$groups $tasks" -- $1) ) - return 0 -} - -# Should we separate this from _rose_stem__group? -_rose_stem__task() { - _rose_stem__group "$@" -} - -_rose_suite_hook__mail_cc() { - local users - prev_users=$1 - user_tail=${prev_users##*,} - user_head=${prev_users%,*} - users=$(__rose_get_users) - if [[ -n $user_tail ]] && grep -q "\<$user_tail\>" <<<"$users"; then - # A complete user name, now need a comma. - users=$(sed "s/^/$prev_users,/; s/ / $prev_users,/g" <<<"$users") - COMPREPLY=( $(compgen -W "$users" -- $prev_users) ) - return - fi - if [[ -n $user_head && $user_head != $prev_users ]]; then - users=$(sed "s/^/$user_head,/; s/ / $user_head,/g" <<<"$users") - fi - COMPREPLY=( $(compgen -W "$users" -- $prev_users) ) -} - -_rose_suite_log__name() { - local names - names=$(cylc print -xy 2>/dev/null) - COMPREPLY=( $(compgen -W "$names" -- $1) ) -} - -_rose_suite_log__user() { - local users - users=$(__rose_get_users) - COMPREPLY=( $(compgen -W "$users" -- $1) ) -} - -_rose_suite_run__host() { - local hosts - hosts=$(rose config rose-suite-run hosts 2>/dev/null) - hosts=$(__rose_get_expanded_hosts "$hosts") - COMPREPLY=( $(compgen -W "$hosts" -- $1) ) -} - -_rose_suite_run__opt_conf_key() { - local config_dir opt_conf_keys - config_dir=$(__rose_get_config_dir) - if [[ -d $config_dir/opt ]]; then - opt_conf_keys=$(cd $config_dir/opt/ && ls rose-suite-*.conf | \ - sed "s/rose-suite-\(.*\).conf/\1/g") - COMPREPLY=( $(compgen -W "$opt_conf_keys" -- $1) ) - return 0 - fi - return 1 -} - -_rose_suite_run__run() { - COMPREPLY=( $(compgen -W "reload restart run" -- $1) ) -} - -_rose_suite_shutdown__name() { - local names - names=$(cylc print -xy 2>/dev/null) - COMPREPLY=( $(compgen -W "$names" -- $1) ) -} - -_rose_suite_stop__name() { - _rose_suite_shutdown__name "$@" - return $? -} - -_rose_task_run__app_mode() { - _rose_app_run__app_mode "$@" - return $? -} - -_rose_task_run__command_key() { - _rose_app_run__command_key "$@" - return $? -} - -_rose_task_run__opt_conf_key() { - _rose_app_run__opt_conf_key "$@" - return $? -} - -_rose_test_battery_ARGS() { - cd $ROSE_DIR/t && ls -d ros* -} - -_rosie_create__prefix() { - __rosie_get_prefixes "$@" - return $? -} - -_rosie_go__prefix() { - __rosie_get_prefixes "$@" - return $? -} - -_rosie_lookup__prefix() { - __rosie_get_prefixes "$@" - return $? -} - -_rosie_ls__prefix() { - __rosie_get_prefixes "$@" - return $? -} - -__rose_get_config_dir() { - local config_dir - config_dir=$( \ - printf "%s " "${COMP_WORDS[@]}" | \ - sed -n "s/.*--config = \([^ ]*\).*/\1/p; s/.*-C *\([^ ]*\).*/\1/p") - if [[ -z $config_dir ]]; then - config_dir=$PWD - fi - eval config_dir="$config_dir" - cd $config_dir 1>/dev/null 2>&1 && pwd -P -} - -__rose_stem_get_config_dir() { - local config_dir - config_dir=$(__rose_get_config_dir) - if [[ ! -e $config_dir/rose-stem/suite.rc ]]; then - while [[ ! -d $config_dir/rose-stem ]] && \ - svn info $config_dir >/dev/null 2>&1; do - new_config_dir=$(cd $config_dir/.. && pwd -P) - if [[ $new_config_dir == $config_dir ]]; then - return 1 - fi - config_dir=$new_config_dir - done - if [[ ! -e $config_dir/rose-stem/suite.rc ]]; then - return 1 - fi - fi - config_dir=$config_dir/rose-stem - cd $config_dir 1>/dev/null 2>&1 && pwd -P -} - -__rose_get_expanded_hosts() { - for host in "$@"; do - rose config rose-host-select group{$host} || echo $host - done -} - -__rose_get_meta_path() { - local meta_path - meta_path=$( \ - printf "%s " "${COMP_WORDS[@]}" | \ - sed -n "s/.*--meta-path = \([^ ]*\).*/\1/p; s/.*-M *\([^ ]*\).*/\1/p") - if [[ -n $meta_path ]]; then - eval meta_path="$meta_path" - cd $meta_path 1>/dev/null 2>&1 && pwd -P - fi -} - -__rose_get_rose_dir() { - rose --version | sed "s/.*(\(.*\))/\1/" -} - -__rose_get_users() { - getent passwd | cut -d: -f1 | LANG=C sort -u -} - -__rose_help() { - local subcommand - subcommand=$1 - if type -t _rose_help__${subcommand//-/_}; then - _rose_help__${subcommand//-/_} | sed -n '/^OPTIONS$/,/^[^ ]$/p;' - return - fi - rose help $subcommand 2>/dev/null -} - -__rosie_get_prefixes() { - local prefixes - prefixes=$(rose config --keys rosie-id | \ - sed -n "s/^prefix-location\.\(.*\)/\1/p") - COMPREPLY=( $(compgen -W "$prefixes" -- $1) ) -} - -__rosie_help() { - local subcommand - subcommand=$1 - if type -t _rosie_help__${subcommand//-/_}; then - _rosie_help__${subcommand//-/_} - return - fi - rosie help $subcommand -} - -_rose_help__stem() { - rose help stem - rose help suite-run -} - -_rose_help__task_run() { - rose help task-run - rose help app-run - rose help task-env -} - -complete -o bashdefault -o default -o nospace -F _rose rose rosie diff --git a/etc/rose-config-edit/.gtkrc-2.0 b/etc/rose-config-edit/.gtkrc-2.0 deleted file mode 100644 index f8bd295494..0000000000 --- a/etc/rose-config-edit/.gtkrc-2.0 +++ /dev/null @@ -1 +0,0 @@ -gtk-icon-sizes = "gtk-large-toolbar=20,20:gtk-small-toolbar=20,20:panel-menu=16,16:gtk-button=16,16" diff --git a/etc/rose.conf.example b/etc/rose.conf.example index 3d0e279649..983362ab7f 100644 --- a/etc/rose.conf.example +++ b/etc/rose.conf.example @@ -1,7 +1,8 @@ # The :rose:file:`rose.conf` file can be installed in: # -# * ``$ROSE_HOME/etc/`` for *site* configuration. -# * ``$HOME/.metomi/rose/`` for *user* configuration. +# * ``/etc/rose.conf`` +# * ``$ROSE_SITE_CONF_PATH/rose.conf`` +# * ``$HOME/.metomi/rose.conf`` # #.. This file documents the settings and their default values. # Values of settings in this file are mostly placeholders, @@ -111,42 +112,6 @@ notification-from=EMAIL-ADDRESS user-tool=ldap|passwd -# Configuration for Rose Bush server. -[rose-bush] -# :default: 100 -# -# Cycles list view: default number of cycles per page. -cycles-per-page=NUMBER -# :default: The server's host name. -# -# An alternative host name. -host=NAME -# :default: 15 -# -# Job list view: default number of jobs per page. -jobs-per-page=NUMBER -# :default: 300 -# -# Job list view: maximum number of jobs per page. -jobs-per-page-max=NUMBER -# Image logo attributes, can be any HTML ```` tag attributes -# e.g: ``logo=src="http://server/my-rose-bush-logo.png" alt="My Rose Bush -# Logo"``. -logo=HTML-IMG-ATTRIBUTE ... -# :default: 100 -# -# Suites list view: default number of suites per page. -suites-per-page=NUMBER -# :default: "Rose Bush" -# -# An alternative service title. -title=TITLE -# :default: 10485760 -# -# File view: maximum viewable file size in bytes. -view-size-max=BYTES - - # Configuration specific to :ref:`command-rose-config-diff`. [rose-config-diff] # :default: "title,ns,description,help" @@ -158,20 +123,6 @@ properties=title,ns,description,help ignore{foo}=REGEX - -# Configuration specific to :ref:`command-rose-config-edit`. -[rose-config-edit] -# :default: /opt/cylc/images/icon.svg -# -# Path to an image containing the suite engine icon. -# See :py:mod:`rose.config_editor` for detail. -icon-path-scheduler=PATH -# :default: https://github.com/metomi/rose/ -# -# Hyperlink to the Rose project page. -project-url=URL - - # Configuration related to :ref:`command-rose-host-select`. [rose-host-select] # The default arguments to use for this command e.g. ``default=hpc``. @@ -206,17 +157,11 @@ timeout=FLOAT [rose-stem] # Automatic options. These are added as if the user added them with # ``--define-suite`` on the command line and can be accessed as Jinja2 -# variables in the ``suite.rc`` file. E.g ``automatic-options=TEA=earl_grey`` +# variables in the ``flow.cylc`` file. E.g ``automatic-options=TEA=earl_grey`` # would set the Jinja2 variable ``TEA`` to be ``earl_grey``. automatic-options=VARIABLE=VALUE -# Configuration related to :ref:`command-rose-suite-log`. -[rose-suite-log] -# URL to the site's Rose Bush web service. -rose-bush=URL - - # Configuration related to :ref:`command-rose-mpi-launch`. [rose-mpi-launch] # Specify a list of launcher commands e.g:: @@ -243,77 +188,6 @@ launcher-preopts.LAUNCHER=OPTIONS-TEMPLATE launcher-postopts.LAUNCHER=OPTIONS-TEMPLATE -# Configuration related to :ref:`command-rose-suite-run`. -[rose-suite-run] -# Hosts in the :rose:conf:`[rose-host-select]` section that can be used to -# run a suite e.g:: -# -# hosts=rose-vm -hosts=HOST-GROUP|HOST ... -# Hosts in the :rose:conf:`[rose-host-select]` section that can be scanned -# by rose utilities e.g:: -# -# scan-hosts=localhost rose-vm -scan-hosts=HOST-GROUP|HOST ... -# :default: false -# -# Don't use login shell to invoke ``rose suite-run --remote`` -# where ``HOST-GLOB`` is a glob for matching host names e.g:: -# -# remote-no-login-shell=myhpc*=true -# mycluster*=true -remote-no-login-shell=HOST-GLOB=true|false -# :default: rose -# -# Path to ``rose`` executable on remote hosts where: -# -# * ``HOST-GLOB`` is a glob for matching host names. -# * ``ROSE-BIN-PATH`` is the path to the ``rose`` executable. -# -# E.g:: -# -# remote-rose-bin=myhpc*=/opt/rose/bin/rose -# mycluster*=/usr/local/bin/rose -remote-rose-bin=HOST-GLOB=ROSE-BIN-PATH -# :default: $HOME -# -# Root location of a suite run directory where: -# -# * ``HOST-GLOB`` is a glob for matching host names. -# * ``HOST-DIR`` is the value of the root location for matching hosts. -# -# E.g:: -# -# root-dir=hpc*=$DATADIR -# =*=$HOME -root-dir=HOST-GLOB=$HOST-DIR -# :default: $HOME -# -# Root location of a suite run's ``share/`` directory. Syntax is the same -# as :rose:conf:`root-dir`. -# Multiple pairs can be specified by a new-line separated list. -# -# .. warning:: -# -# If a suite has previously been run changes to any of the ``root-dir`` -# settings will take effect on the next clean re-installation i.e: -# -# $ rose suite-run --new -root-dir{share}=HOST-GLOB=$HOST-DIR -# :default: $HOME -# -# Root location of a suite run's ``share/cycle/`` directory. Syntax is -# the same as :rose:conf:`root-dir`. -# Multiple pairs can be specified by a new-line separated list. -root-dir{share/cycle}=HOST-GLOB=$HOST-DIR -# :default: $HOME -# -# Root location of a suite run's ``work/`` directory. Syntax is the same -# as :rose:conf:`root-dir`. -# Multiple pairs can be specified by a new-line separated list. -root-dir{work}=HOST-GLOB=$HOST-DIR - - # Configuration related to :ref:`command-rose-task-run`. [rose-task-run] # Items to prepend to the ``PATH`` environment variable. @@ -333,9 +207,9 @@ method-path=/path/1 /path2 # `. kgo-database=.true. # Limits the number of lines printed when using the -# :py:mod:`rose.apps.ana_builtin.grepper` analysis class. +# :py:mod:`metomi.rose.apps.ana_builtin.grepper` analysis class. grepper-report-limit=42 -# Causes the :py:mod:`rose.apps.ana_builtin.grepper` class to pass +# Causes the :py:mod:`metomi.rose.apps.ana_builtin.grepper` class to pass # if all files to be compared are missing. skip-if-all-files-missing=.true. @@ -367,7 +241,6 @@ title=TITLE # Configuration related to the :ref:`command-rosie-go` GUI. -# See :py:mod:`rosie.browser` for detail. [rosie-go] # :default: /opt/cylc/images/icon.svg # @@ -437,6 +310,8 @@ access-list-default=USER-ID ... # Tool to compare two files when there are differences. difftool=COMMAND ... # List of selectable host groups. +# +# These must be defined in :rose:conf:`[rose-host-select]group{NAME}`. host-groups=GROUP ... # A remote host that does not share its ``HOME`` directory with ``localhost``. job-host=HOST diff --git a/etc/tutorial/consolidation-tutorial/suite.rc b/etc/tutorial/consolidation-tutorial/flow.cylc similarity index 100% rename from etc/tutorial/consolidation-tutorial/suite.rc rename to etc/tutorial/consolidation-tutorial/flow.cylc diff --git a/etc/tutorial/cylc-forecasting-suite/.validate b/etc/tutorial/cylc-forecasting-suite/.validate index 9d13a3b35d..3d0b183322 100644 --- a/etc/tutorial/cylc-forecasting-suite/.validate +++ b/etc/tutorial/cylc-forecasting-suite/.validate @@ -1,3 +1,3 @@ rose tutorial "$(basename $TUT_DIR)" "${CYLC_RUN_DIR}/${REG}" -sed -i '1s;^;[cylc]\n abort if any task fails = True\n;' "${CYLC_RUN_DIR}/${REG}/suite.rc" -cylc run --no-detach "${REG}" 2>&1 +sed -i '1s;^;[cylc]\n abort if any task fails = True\n;' "${CYLC_RUN_DIR}/${REG}/flow.cylc" +cylc play --no-detach "${REG}" 2>&1 diff --git a/etc/tutorial/cylc-forecasting-suite/bin/get-rainfall b/etc/tutorial/cylc-forecasting-suite/bin/get-rainfall index 4dd421cc44..12c7bddcb9 100755 --- a/etc/tutorial/cylc-forecasting-suite/bin/get-rainfall +++ b/etc/tutorial/cylc-forecasting-suite/bin/get-rainfall @@ -34,7 +34,7 @@ URL = ('http://datapoint.metoffice.gov.uk/public/data/layer/wxobs/' 'RADAR_UK_Composite_Highres/png?TIME={time}&key={api_key}') -class Rainfall(object): +class Rainfall: """Class for holding rainfall data. Args: diff --git a/etc/tutorial/cylc-forecasting-suite/suite.rc b/etc/tutorial/cylc-forecasting-suite/flow.cylc similarity index 100% rename from etc/tutorial/cylc-forecasting-suite/suite.rc rename to etc/tutorial/cylc-forecasting-suite/flow.cylc diff --git a/etc/tutorial/cylc-forecasting-suite/lib/python/mecator.py b/etc/tutorial/cylc-forecasting-suite/lib/python/mecator.py index 35e6a14365..2812e737dc 100644 --- a/etc/tutorial/cylc-forecasting-suite/lib/python/mecator.py +++ b/etc/tutorial/cylc-forecasting-suite/lib/python/mecator.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors - GNU V3+. # This is illustrative code developed for tutorial purposes, it is not # intended for scientific use and is not guarantied to be accurate or correct. diff --git a/etc/tutorial/cylc-forecasting-suite/lib/python/util.py b/etc/tutorial/cylc-forecasting-suite/lib/python/util.py index 4bf2c83618..eb9f7fa3a2 100644 --- a/etc/tutorial/cylc-forecasting-suite/lib/python/util.py +++ b/etc/tutorial/cylc-forecasting-suite/lib/python/util.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors - GNU V3+. # This is illustrative code developed for tutorial purposes, it is not # intended for scientific use and is not guarantied to be accurate or correct. @@ -213,7 +211,7 @@ def get_grid_coordinates(lng, lat, domain, resolution): length_y - int((abs(lat - domain['lat1'])) // resolution)) -class SurfaceFitter(object): +class SurfaceFitter: """A 2D interpolation for random points. A standin for scipy.interpolate.interp2d diff --git a/etc/tutorial/inheritance-tutorial/suite.rc b/etc/tutorial/inheritance-tutorial/flow.cylc similarity index 100% rename from etc/tutorial/inheritance-tutorial/suite.rc rename to etc/tutorial/inheritance-tutorial/flow.cylc diff --git a/etc/tutorial/queues-tutorial/suite.rc b/etc/tutorial/queues-tutorial/flow.cylc similarity index 100% rename from etc/tutorial/queues-tutorial/suite.rc rename to etc/tutorial/queues-tutorial/flow.cylc diff --git a/etc/tutorial/retries-tutorial/suite.rc b/etc/tutorial/retries-tutorial/flow.cylc similarity index 100% rename from etc/tutorial/retries-tutorial/suite.rc rename to etc/tutorial/retries-tutorial/flow.cylc diff --git a/etc/tutorial/rose-stem/.validate b/etc/tutorial/rose-stem/.validate index 736af32a69..adf657b18f 100644 --- a/etc/tutorial/rose-stem/.validate +++ b/etc/tutorial/rose-stem/.validate @@ -1,4 +1,4 @@ mkdir "${CYLC_RUN_DIR}/${REG}" -echo -e '#!Jinja2\n{% set RUN_NAMES=["command_spaceship"] %}' > "${CYLC_RUN_DIR}/${REG}/suite.rc" -cat "$TUT_DIR/rose-stem/suite.rc" >> "${CYLC_RUN_DIR}/${REG}/suite.rc" +echo -e '#!Jinja2\n{% set RUN_NAMES=["command_spaceship"] %}' > "${CYLC_RUN_DIR}/${REG}/flow.cylc" +cat "$TUT_DIR/rose-stem/flow.cylc" >> "${CYLC_RUN_DIR}/${REG}/flow.cylc" cylc validate "${CYLC_RUN_DIR}/${REG}" -s "SOURCE_SPACESHIP=foo" diff --git a/etc/tutorial/rose-stem/rose-stem/suite.rc b/etc/tutorial/rose-stem/rose-stem/flow.cylc similarity index 100% rename from etc/tutorial/rose-stem/rose-stem/suite.rc rename to etc/tutorial/rose-stem/rose-stem/flow.cylc diff --git a/etc/tutorial/rose-suite-tutorial/suite.rc b/etc/tutorial/rose-suite-tutorial/flow.cylc similarity index 100% rename from etc/tutorial/rose-suite-tutorial/suite.rc rename to etc/tutorial/rose-suite-tutorial/flow.cylc diff --git a/etc/tutorial/rose-weather-forecasting-suite/suite.rc b/etc/tutorial/rose-weather-forecasting-suite/flow.cylc similarity index 100% rename from etc/tutorial/rose-weather-forecasting-suite/suite.rc rename to etc/tutorial/rose-weather-forecasting-suite/flow.cylc diff --git a/etc/tutorial/runtime-introduction/suite.rc b/etc/tutorial/runtime-introduction/flow.cylc similarity index 100% rename from etc/tutorial/runtime-introduction/suite.rc rename to etc/tutorial/runtime-introduction/flow.cylc diff --git a/etc/tutorial/runtime-tutorial/suite.rc b/etc/tutorial/runtime-tutorial/flow.cylc similarity index 100% rename from etc/tutorial/runtime-tutorial/suite.rc rename to etc/tutorial/runtime-tutorial/flow.cylc diff --git a/etc/tutorial/widget/meta/lib/python/widget/username.py b/etc/tutorial/widget/meta/lib/python/widget/username.py index b2062c3cc3..d381db4397 100644 --- a/etc/tutorial/widget/meta/lib/python/widget/username.py +++ b/etc/tutorial/widget/meta/lib/python/widget/username.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ This module contains value widgets for helping enter usernames. @@ -10,9 +9,9 @@ from functools import partial -import gobject import pygtk pygtk.require('2.0') +# flake8: noqa: E402 import gtk diff --git a/lib/bash/rose_init b/lib/bash/rose_init deleted file mode 100755 index 0c219bd4e5..0000000000 --- a/lib/bash/rose_init +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose_init -# -# SYNOPSIS -# . $ROSE_HOME/lib/bash/rose_init -# rose_init [FUNC ...] -# -# DESCRIPTION -# Initialise a bash script with the following: -# * "set -eu". -# * load the "rose_usage" function. -# * load any FUNC specified in the argument list. -#------------------------------------------------------------------------------- -# shellcheck source=lib/bash/rose_usage -. rose_usage - -function rose_init() { - set -eu - ROSE_NS=$(basename "$0") - ROSE_LIB="$(dirname "$(python -c "import metomi.rose; print(metomi.rose.__file__)")")" - ROSE_HOME_BIN=$(dirname "$(command -v rose)") - ROSE_VERSION="$(python3 -c "import metomi.rose; print(metomi.rose.__version__)")" - export ROSE_LIB ROSE_HOME_BIN ROSE_NS ROSE_VERSION - local LIB=$ROSE_LIB - if [[ -f $LIB/${FUNCNAME[0]}_site ]]; then - . "$LIB/${FUNCNAME[0]}_site" - fi - - local ITEM= - for ITEM in rose_usage "$@"; do - local FILE= - for FILE in $ITEM; do - . "$FILE" - done - done -} diff --git a/lib/bash/rose_init_site.example b/lib/bash/rose_init_site.example deleted file mode 100644 index a569cb26b6..0000000000 --- a/lib/bash/rose_init_site.example +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# NAME -# rose_init_site -# -# SYNOPSIS -# . $ROSE_HOME/lib/bash/rose_init_site -# -# DESCRIPTION -# Site specified PATH, PYTHONPATH, etc. -#------------------------------------------------------------------------------- -#PYTHONPATH=/path/to/site/specific/python/lib:$PYTHONPATH -#PATH=/path/to/site/specific/bin:$PATH diff --git a/metomi/rose/__init__.py b/metomi/rose/__init__.py index 70bebfb1ed..a944af6af5 100644 --- a/metomi/rose/__init__.py +++ b/metomi/rose/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -56,15 +53,6 @@ "=type"] SUB_CONFIG_DEFAULT_META_IDS = ["=file-install-root", "=meta", "=mode", "=opts", "command", "file:", "poll"] -TOP_CONFIG_DEFAULT_META_IDS = [ - "file:", - "jinja2:suite.rc", - "=meta", - "=opts", - "=root-dir", - "=root-dir{share}", - "=root-dir{share/cycle}", - "=root-dir{work}"] CONFIG_SETTING_INDEX_DEFAULT = "1" diff --git a/metomi/rose/app_run.py b/metomi/rose/app_run.py index c9ed262a42..3195f5f7fb 100644 --- a/metomi/rose/app_run.py +++ b/metomi/rose/app_run.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -115,7 +112,7 @@ def __str__(self): ok_str, strftime("%Y-%m-%dT%H:%M:%S", localtime(sec)), test) -class Poller(object): +class Poller: """Handle the [poll] functionality for AppRunner.""" @@ -287,7 +284,7 @@ def _poll_file(self, file_, poll_file_test): return is_done -class BuiltinApp(object): +class BuiltinApp: """An abstract base class for a builtin application. diff --git a/metomi/rose/apps/__init__.py b/metomi/rose/apps/__init__.py index 8460776f05..f7b71f8284 100644 --- a/metomi/rose/apps/__init__.py +++ b/metomi/rose/apps/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/apps/ana_builtin/grepper.py b/metomi/rose/apps/ana_builtin/grepper.py index 5504ab5929..a7efb9ae7f 100644 --- a/metomi/rose/apps/ana_builtin/grepper.py +++ b/metomi/rose/apps/ana_builtin/grepper.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/apps/comparisons/cumf.py b/metomi/rose/apps/comparisons/cumf.py index 718b98b7ed..93195dc0c1 100644 --- a/metomi/rose/apps/comparisons/cumf.py +++ b/metomi/rose/apps/comparisons/cumf.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -30,7 +27,7 @@ HEADER = "differ, however the data fields are identical" -class Cumf(object): +class Cumf: """Analyse the output from the UM small exec cumf""" @@ -54,7 +51,7 @@ def run(self, task): return task -class CumfWarnHeader(object): +class CumfWarnHeader: """As cumf, but issue a warning if only the header has changed""" @@ -74,7 +71,7 @@ def run(self, task): return task -class CumfComparisonFailure(object): +class CumfComparisonFailure: """Class used if a cumf comparison fails.""" @@ -97,7 +94,7 @@ def __repr__(self): __str__ = __repr__ -class CumfComparisonSuccess(object): +class CumfComparisonSuccess: """Class used if a cumf comparison succeeds""" @@ -114,7 +111,7 @@ def __repr__(self): __str__ = __repr__ -class CumfComparisonHeaderWarning(object): +class CumfComparisonHeaderWarning: """Class used if cumf reports just the header of a file is different""" @@ -131,7 +128,7 @@ def __repr__(self): __str__ = __repr__ -class CumfSummaryNotFoundFailure(object): +class CumfSummaryNotFoundFailure: """Class used if there is a problem finding a cumf summary file""" @@ -148,7 +145,7 @@ def __repr__(self): __str__ = __repr__ -class CumfDiffNotFoundFailure(object): +class CumfDiffNotFoundFailure: """Class used if there is a problem finding a cumf diff file""" diff --git a/metomi/rose/apps/comparisons/exact.py b/metomi/rose/apps/comparisons/exact.py index 2ab46fb3f9..d1df06cefb 100644 --- a/metomi/rose/apps/comparisons/exact.py +++ b/metomi/rose/apps/comparisons/exact.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -26,10 +23,9 @@ FAIL = "!=" -class Exact(object): +class Exact: def run(self, task): """Perform an exact comparison between the result and the KGO data""" - failures = 0 if len(task.resultdata) != len(task.kgo1data): raise DataLengthError(task) location = 0 @@ -46,7 +42,7 @@ def run(self, task): return task -class ExactComparisonFailure(object): +class ExactComparisonFailure: """Class used if results do not match the KGO""" @@ -79,7 +75,7 @@ def __repr__(self): __str__ = __repr__ -class ExactComparisonSuccess(object): +class ExactComparisonSuccess: """Class used if results match the KGO""" diff --git a/metomi/rose/apps/comparisons/mandatory.py b/metomi/rose/apps/comparisons/mandatory.py index aabc159914..f3616f38ee 100644 --- a/metomi/rose/apps/comparisons/mandatory.py +++ b/metomi/rose/apps/comparisons/mandatory.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -24,10 +21,9 @@ FAIL = "!~" -class Mandatory(object): +class Mandatory: def run(self, task): """Perform an exact comparison between the result and the KGO data""" - failures = 0 if len(task.resultdata) == 0: task.set_failure(MandatoryStringResult(task, FAIL)) else: @@ -35,7 +31,7 @@ def run(self, task): return task -class MandatoryStringResult(object): +class MandatoryStringResult: """Result of mandatory text examination.""" diff --git a/metomi/rose/apps/comparisons/output_grepper.py b/metomi/rose/apps/comparisons/output_grepper.py index 5c42a97c08..7a0ccb247f 100644 --- a/metomi/rose/apps/comparisons/output_grepper.py +++ b/metomi/rose/apps/comparisons/output_grepper.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -28,7 +25,7 @@ } -class OutputGrepper(object): +class OutputGrepper: def run(self, task, variable): """Return a list of values matching a regular expression.""" filevar = variable + "file" diff --git a/metomi/rose/apps/comparisons/prohibited.py b/metomi/rose/apps/comparisons/prohibited.py index 7b45f731e8..82a796dc66 100644 --- a/metomi/rose/apps/comparisons/prohibited.py +++ b/metomi/rose/apps/comparisons/prohibited.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -24,10 +21,9 @@ FAIL = "~~" -class Prohibited(object): +class Prohibited: def run(self, task): """Perform an exact comparison between the result and the KGO data""" - failures = 0 if len(task.resultdata) == 0: task.set_pass(ProhibitedStringResult(task, PASS)) else: @@ -35,7 +31,7 @@ def run(self, task): return task -class ProhibitedStringResult(object): +class ProhibitedStringResult: """Result of prohibited text examination.""" diff --git a/metomi/rose/apps/comparisons/within.py b/metomi/rose/apps/comparisons/within.py index d88f484124..c95412ba50 100644 --- a/metomi/rose/apps/comparisons/within.py +++ b/metomi/rose/apps/comparisons/within.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -34,10 +31,9 @@ FAIL = ">" -class Within(object): +class Within: def run(self, task): """Check that the results are within a specified tolerance.""" - failures = 0 if len(task.resultdata) != len(task.kgo1data): raise DataLengthError(task) val_num = 0 @@ -63,7 +59,7 @@ def run(self, task): return task -class WithinComparisonFailure(object): +class WithinComparisonFailure: """Class used if results are not within a certain amount of the KGO""" @@ -108,7 +104,7 @@ def __repr__(self): __str__ = __repr__ -class WithinComparisonSuccess(object): +class WithinComparisonSuccess: """Class used if results are within a certain amount of the KGO""" diff --git a/metomi/rose/apps/fcm_make.py b/metomi/rose/apps/fcm_make.py index f6098152ef..06502be45c 100644 --- a/metomi/rose/apps/fcm_make.py +++ b/metomi/rose/apps/fcm_make.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -127,7 +124,13 @@ def _invoke_fcm_make(self, app_runner, conf_tree, opts, args, uuid, task, if fast_root: # N.B. Name in "little endian", like cycle task ID prefix = ".".join([ - task.task_name, task.task_cycle_time, task.suite_name]) + task.task_name, + task.task_cycle_time, + # suite_name may be a hierarchical registration which + # isn't a safe prefix + task.suite_name.replace(os.sep, '_') + ]) + os.makedirs(fast_root, exist_ok=True) dest = mkdtemp(prefix=prefix, dir=fast_root) # N.B. Don't use app_runner.popen.get_cmd("rsync") as we are using # "rsync" for a local copy. @@ -173,6 +176,7 @@ def _run_orig(self, app_runner, conf_tree, opts, args, uuid, task, # Determine the name of the continuation task task_name_cont = task.task_name.replace( orig_cont_map[ORIG], orig_cont_map[CONT]) + # TODO: get_task_auth currently does nothing auth = app_runner.suite_engine_proc.get_task_auth( task.suite_name, task_name_cont) if auth is not None: diff --git a/metomi/rose/apps/rose_ana.py b/metomi/rose/apps/rose_ana.py index 0aae1cae0d..07eafb5757 100644 --- a/metomi/rose/apps/rose_ana.py +++ b/metomi/rose/apps/rose_ana.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -50,7 +47,7 @@ def timestamp(): return time.strftime("%H:%M:%S") -class KGODatabase(object): +class KGODatabase: """ KGO Database object, stores comparison information for metomi.rose_ana apps. @@ -173,14 +170,14 @@ class AnalysisTask(object, metaclass=abc.ABCMeta): self.config: A dictionary containing any Rose Ana configuration options. self.reporter: - A reference to the :py:class:`rose.reporter.Reporter` instance used - by the parent app (for printing to stderr/stdout). + A reference to the :py:class:`metomi.rose.reporter.Reporter` + instance used by the parent app (for printing to stderr/stdout). self.kgo_db: A reference to the KGO database object created by the parent app (for adding entries to the database). self.popen: - A reference to the :py:class:`rose.popen.RosePopener` instance - used by the parent app (for spawning subprocesses). + A reference to the :py:class:`metomi.rose.popen.RosePopener` + instance used by the parent app (for spawning subprocesses). """ diff --git a/metomi/rose/apps/rose_ana_v1.py b/metomi/rose/apps/rose_ana_v1.py index f55fe3e16e..6859c87fa3 100644 --- a/metomi/rose/apps/rose_ana_v1.py +++ b/metomi/rose/apps/rose_ana_v1.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -49,7 +46,7 @@ USRCOMPARISON_EXT = ".py" -class KGODatabase(object): +class KGODatabase: """ KGO Database object, stores comparison information for rose_ana apps. """ @@ -239,7 +236,7 @@ def __repr__(self): __str__ = __repr__ -class Analyse(object): +class Analyse: """A comparison engine for Rose.""" @@ -388,7 +385,7 @@ def load_tasks(self): tasks = [] for task in self.config.value.keys(): - if task is "env": + if task == "env": continue if task.startswith("file:"): continue @@ -497,7 +494,7 @@ def write_config(self, filename, tasks): metomi.rose.config.dump(config, filename) -class AnalysisTask(object): +class AnalysisTask: """Class to completely describe an analysis task. diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index c31553b2b6..66edab7426 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -404,7 +401,7 @@ def _get_conf(self, r_node, t_node, key, compulsory=False, default=None): return value -class RoseArchTarget(object): +class RoseArchTarget: """An archive target.""" @@ -435,7 +432,7 @@ def __ne__(self, other): return not self.__eq__(other) -class RoseArchSource(object): +class RoseArchSource: """An archive source.""" @@ -457,7 +454,7 @@ def __ne__(self, other): return not self.__eq__(other) -class RoseArchDAO(object): +class RoseArchDAO: """Data access object for incremental mode.""" diff --git a/metomi/rose/apps/rose_arch_compressions/__init__.py b/metomi/rose/apps/rose_arch_compressions/__init__.py index 8460776f05..f7b71f8284 100644 --- a/metomi/rose/apps/rose_arch_compressions/__init__.py +++ b/metomi/rose/apps/rose_arch_compressions/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py index 7826021181..c79bd9eac4 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -23,7 +20,7 @@ import os -class RoseArchGzip(object): +class RoseArchGzip: """Compress archive sources in gzip.""" diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 8a3f869039..dfdfd64bdb 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -24,7 +21,7 @@ from tempfile import mkstemp -class RoseArchTarGzip(object): +class RoseArchTarGzip: """Compress archive sources in tar.""" diff --git a/metomi/rose/apps/rose_bunch.py b/metomi/rose/apps/rose_bunch.py index a7dbc6f44e..64091b6700 100644 --- a/metomi/rose/apps/rose_bunch.py +++ b/metomi/rose/apps/rose_bunch.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -20,7 +17,6 @@ """Builtin application: rose_bunch: run multiple commands in parallel. """ - import itertools import os import shlex @@ -372,7 +368,7 @@ def run(self, app_runner, conf_tree, opts, args, uuid, work_files): return 0 -class RoseBunchCmd(object): +class RoseBunchCmd: """A command instance to run.""" OUTPUT_TEMPLATE = "bunch.%s.%s" @@ -406,7 +402,7 @@ def get_log_prefix(self): return self.name -class RoseBunchDAO(object): +class RoseBunchDAO: """Database object for rose_bunch""" TABLE_COMMANDS = "commands" @@ -441,7 +437,7 @@ def connect(self): def create_tables(self): """Create tables as appropriate""" existing = [] - first_run = os.environ.get("CYLC_TASK_SUBMIT_NUMBER") == "1" + first_run = os.environ.get("CYLC_TASK_TRY_NUMBER") == "1" for row in self.conn.execute("SELECT name FROM sqlite_master " + "WHERE type=='table'"): @@ -543,7 +539,15 @@ def same_prev_config(self, current): unchanged = True current = self.flatten_config(current) for key, value in self.conn.execute(s_stmt): - if key in current: + if key == 'env_PATH': + # due to re-invocation the PATH may have changed in-between + # runs - only re-run jobs if the PATH has changed in a way + # that could actually make a difference + if simplify_path(current[key]) != simplify_path(value): + break + else: + current.pop(key) + elif key in current: if current[key] != value: break else: @@ -554,3 +558,29 @@ def same_prev_config(self, current): if current: unchanged = False return unchanged + + +def simplify_path(path): + """Removes duplication in paths whilst maintaining integrity. + + If duplicate items are present in a path this keeps the first item and + removes any subsequent duplicates. + + Examples: + >>> simplify_path('') + '' + >>> simplify_path('a') + 'a' + >>> simplify_path('a:a:a') + 'a' + >>> simplify_path('a:b:a') + 'a:b' + >>> simplify_path('a:b:b:a') + 'a:b' + >>> simplify_path('a:b:a:b:c:d:a:b:c:d:e') + 'a:b:c:d:e' + + """ + return ':'.join( + dict.fromkeys(path.split(':')).keys() + ) diff --git a/metomi/rose/apps/rose_prune.py b/metomi/rose/apps/rose_prune.py index 7da776a488..15481f1ff1 100644 --- a/metomi/rose/apps/rose_prune.py +++ b/metomi/rose/apps/rose_prune.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/bush_dao.py b/metomi/rose/bush_dao.py deleted file mode 100644 index fbe45abf9f..0000000000 --- a/metomi/rose/bush_dao.py +++ /dev/null @@ -1,696 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Rose Bush: data access to cylc suite runtime databases.""" - -from fnmatch import fnmatch -from glob import glob -import os -import re -import tarfile - -from metomi.rose.suite_engine_procs.cylc import CylcProcessor, CylcSuiteDAO - - -class RoseBushDAO(object): - - """Rose Bush: data access to cylc suite runtime databases.""" - - CYCLE_ORDERS = {"time_desc": " DESC", "time_asc": " ASC"} - JOB_ORDERS = { - "time_desc": "time DESC, submit_num DESC, name DESC, cycle DESC", - "time_asc": "time ASC, submit_num ASC, name ASC, cycle ASC", - "cycle_desc_name_asc": "cycle DESC, name ASC, submit_num DESC", - "cycle_desc_name_desc": "cycle DESC, name DESC, submit_num DESC", - "cycle_asc_name_asc": "cycle ASC, name ASC, submit_num DESC", - "cycle_asc_name_desc": "cycle ASC, name DESC, submit_num DESC", - "name_asc_cycle_asc": "name ASC, cycle ASC, submit_num DESC", - "name_desc_cycle_asc": "name DESC, cycle ASC, submit_num DESC", - "name_asc_cycle_desc": "name ASC, cycle DESC, submit_num DESC", - "name_desc_cycle_desc": "name DESC, cycle DESC, submit_num DESC", - "time_submit_desc": ( - "time_submit DESC, submit_num DESC, name DESC, cycle DESC"), - "time_submit_asc": ( - "time_submit ASC, submit_num DESC, name DESC, cycle DESC"), - "time_run_desc": ( - "time_run DESC, submit_num DESC, name DESC, cycle DESC"), - "time_run_asc": ( - "time_run ASC, submit_num DESC, name DESC, cycle DESC"), - "time_run_exit_desc": ( - "time_run_exit DESC, submit_num DESC, name DESC, cycle DESC"), - "time_run_exit_asc": ( - "time_run_exit ASC, submit_num DESC, name DESC, cycle DESC"), - "duration_queue_desc": ( - "(CAST(strftime('%s', time_run) AS NUMERIC) -" + - " CAST(strftime('%s', time_submit) AS NUMERIC)) DESC, " + - "submit_num DESC, name DESC, cycle DESC"), - "duration_queue_asc": ( - "(CAST(strftime('%s', time_run) AS NUMERIC) -" + - " CAST(strftime('%s', time_submit) AS NUMERIC)) ASC, " + - "submit_num DESC, name DESC, cycle DESC"), - "duration_run_desc": ( - "(CAST(strftime('%s', time_run_exit) AS NUMERIC) -" + - " CAST(strftime('%s', time_run) AS NUMERIC)) DESC, " + - "submit_num DESC, name DESC, cycle DESC"), - "duration_run_asc": ( - "(CAST(strftime('%s', time_run_exit) AS NUMERIC) -" + - " CAST(strftime('%s', time_run) AS NUMERIC)) ASC, " + - "submit_num DESC, name DESC, cycle DESC"), - "duration_queue_run_desc": ( - "(CAST(strftime('%s', time_run_exit) AS NUMERIC) -" + - " CAST(strftime('%s', time_submit) AS NUMERIC)) DESC, " + - "submit_num DESC, name DESC, cycle DESC"), - "duration_queue_run_asc": ( - "(CAST(strftime('%s', time_run_exit) AS NUMERIC) -" + - " CAST(strftime('%s', time_submit) AS NUMERIC)) ASC, " + - "submit_num DESC, name DESC, cycle DESC"), - } - JOB_STATUS_COMBOS = { - "all": "", - "submitted": "submit_status == 0 AND time_run IS NULL", - "submitted,running": "submit_status == 0 AND run_status IS NULL", - "submission-failed": "submit_status == 1", - "submission-failed,failed": "submit_status == 1 OR run_status == 1", - "running": "time_run IS NOT NULL AND run_status IS NULL", - "running,succeeded,failed": "time_run IS NOT NULL", - "succeeded": "run_status == 0", - "succeeded,failed": "run_status IS NOT NULL", - "failed": "run_status == 1", - } - REC_CYCLE_QUERY_OP = re.compile(r"\A(before |after |[<>]=?)(.+)\Z") - REC_SEQ_LOG = re.compile(r"\A(.+\.)([^\.]+)(\.[^\.]+)\Z") - SUITE_CONF = CylcProcessor.SUITE_CONF - SUITE_DIR_REL_ROOT = CylcProcessor.SUITE_DIR_REL_ROOT - TASK_STATUS_GROUPS = { - "active": [ - "ready", "queued", "submitting", "submitted", "submit-retrying", - "running", "retrying"], - "fail": ["submission failed", "failed"], - "success": ["expired", "succeeded"]} - TASK_STATUSES = ( - "runahead", "waiting", "held", "queued", "ready", "expired", - "submitted", "submit-failed", "submit-retrying", "running", - "succeeded", "failed", "retrying") - - def __init__(self): - self.daos = {} - - def get_suite_broadcast_states(self, user_name, suite_name): - """Return broadcast states of a suite. - - [[point, name, key, value], ...] - - """ - # Check if "broadcast_states" table is available or not - if not self._db_has_table(user_name, suite_name, "broadcast_states"): - return - - broadcast_states = [] - for row in self._db_exec( - user_name, suite_name, - "SELECT point,namespace,key,value FROM broadcast_states" + - " ORDER BY point ASC, namespace ASC, key ASC"): - point, namespace, key, value = row - broadcast_states.append([point, namespace, key, value]) - return broadcast_states - - def get_suite_broadcast_events(self, user_name, suite_name): - """Return broadcast events of a suite. - - [[time, change, point, name, key, value], ...] - - """ - # Check if "broadcast_events" table is available or not - if not self._db_has_table(user_name, suite_name, "broadcast_events"): - return {} - - broadcast_events = [] - for row in self._db_exec( - user_name, suite_name, - "SELECT time,change,point,namespace,key,value" + - " FROM broadcast_events" + - " ORDER BY time DESC, point DESC, namespace DESC, key DESC"): - time_, change, point, namespace, key, value = row - broadcast_events.append( - (time_, change, point, namespace, key, value)) - return broadcast_events - - @staticmethod - def get_suite_dir_rel(suite_name, *paths): - """Return the relative path to the suite running directory. - - paths -- if specified, are added to the end of the path. - """ - return CylcProcessor.get_suite_dir_rel(suite_name, *paths) - - def get_suite_job_entries( - self, user_name, suite_name, cycles, tasks, task_status, - job_status, order, limit, offset): - """Query suite runtime database to return a listing of task jobs. - - user -- A string containing a valid user ID - suite -- A string containing a valid suite ID - cycles -- If specified, display only task jobs matching these cycles. - A value in the list can be a cycle, the string "before|after - CYCLE", or a glob to match cycles. - tasks -- If specified, display only jobs with task names matching - these names. Values can be a valid task name or a glob like - pattern for matching valid task names. - task_status -- If specified, it should be a list of task statuses. - Display only jobs in the specified list. If not - specified, display all jobs. - job_status -- If specified, must be a string matching a key in - RoseBushDAO.JOB_STATUS_COMBOS. Select jobs by their - statuses. - order -- Order search in a predetermined way. A valid value is one of - the keys in RoseBushDAO.ORDERS. - limit -- Limit number of returned entries - offset -- Offset entry number - - Return (entries, of_n_entries) where: - entries -- A list of matching entries - of_n_entries -- Total number of entries matching query - - Each entry is a dict: - {"cycle": cycle, "name": name, "submit_num": submit_num, - "events": [time_submit, time_init, time_exit], - "task_status": task_status, - "logs": {"script": {"path": path, "path_in_tar", path_in_tar, - "size": size, "mtime": mtime}, - "out": {...}, - "err": {...}, - ...}} - """ - where_expr, where_args = self._get_suite_job_entries_where( - cycles, tasks, task_status, job_status) - - # Get number of entries - of_n_entries = 0 - stmt = ("SELECT COUNT(*)" + - " FROM task_jobs JOIN task_states USING (name, cycle)" + - where_expr) - for row in self._db_exec(user_name, suite_name, stmt, where_args): - of_n_entries = row[0] - break - else: - self._db_close(user_name, suite_name) - return ([], 0) - - # Get entries - entries = [] - entry_of = {} - stmt = ("SELECT" + - " task_states.time_updated AS time," + - " cycle, name," + - " task_jobs.submit_num AS submit_num," + - " task_states.submit_num AS submit_num_max," + - " task_states.status AS task_status," + - " time_submit, submit_status," + - " time_run, time_run_exit, run_signal, run_status," + - " user_at_host, batch_sys_name, batch_sys_job_id" + - " FROM task_jobs JOIN task_states USING (cycle, name)" + - where_expr + - " ORDER BY " + - self.JOB_ORDERS.get(order, self.JOB_ORDERS["time_desc"])) - limit_args = [] - if limit: - stmt += " LIMIT ? OFFSET ?" - limit_args = [limit, offset] - for row in self._db_exec( - user_name, suite_name, stmt, where_args + limit_args): - ( - cycle, name, submit_num, submit_num_max, task_status, - time_submit, submit_status, - time_run, time_run_exit, run_signal, run_status, - user_at_host, batch_sys_name, batch_sys_job_id - ) = row[1:] - entry = { - "cycle": cycle, - "name": name, - "submit_num": submit_num, - "submit_num_max": submit_num_max, - "events": [time_submit, time_run, time_run_exit], - "task_status": task_status, - "submit_status": submit_status, - "run_signal": run_signal, - "run_status": run_status, - "host": user_at_host, - "submit_method": batch_sys_name, - "submit_method_id": batch_sys_job_id, - "logs": {}, - "seq_logs_indexes": {}} - entries.append(entry) - entry_of[(cycle, name, submit_num)] = entry - self._db_close(user_name, suite_name) - if entries: - self._get_job_logs(user_name, suite_name, entries, entry_of) - return (entries, of_n_entries) - - def _get_suite_job_entries_where( - self, cycles, tasks, task_status, job_status): - """Helper for get_suite_job_entries. - - Get query's "WHERE" expression and its arguments. - """ - where_exprs = [] - where_args = [] - if cycles: - cycle_where_exprs = [] - for cycle in cycles: - match = self.REC_CYCLE_QUERY_OP.match(cycle) - if match: - operator, operand = match.groups() - where_args.append(operand) - if operator == "before ": - cycle_where_exprs.append("cycle <= ?") - elif operator == "after ": - cycle_where_exprs.append("cycle >= ?") - else: - cycle_where_exprs.append("cycle %s ?" % operator) - else: - where_args.append(cycle) - cycle_where_exprs.append("cycle GLOB ?") - where_exprs.append(" OR ".join(cycle_where_exprs)) - if tasks: - where_exprs.append(" OR ".join(["name GLOB ?"] * len(tasks))) - where_args += tasks - if task_status: - task_status_where_exprs = [] - for item in task_status: - task_status_where_exprs.append("task_states.status == ?") - where_args.append(item) - where_exprs.append(" OR ".join(task_status_where_exprs)) - try: - job_status_where = self.JOB_STATUS_COMBOS[job_status] - except KeyError: - pass - else: - if job_status_where: - where_exprs.append(job_status_where) - if where_exprs: - return (" WHERE (" + ") AND (".join(where_exprs) + ")", where_args) - else: - return ("", where_args) - - def _get_job_logs(self, user_name, suite_name, entries, entry_of): - """Helper for "get_suite_job_entries". Get job logs. - - Recent job logs are likely to be in the file system, so we can get a - listing of the relevant "log/job/CYCLE/NAME/SUBMI_NUM/" directory. - Older job logs may be archived in "log/job-CYCLE.tar.gz", we should - only open each relevant TAR file once to obtain a listing for all - relevant entries of that cycle. - - Modify each entry in entries. - """ - prefix = "~" - if user_name: - prefix += user_name - user_suite_dir = os.path.expanduser(os.path.join( - prefix, self.get_suite_dir_rel(suite_name))) - try: - fs_log_cycles = os.listdir( - os.path.join(user_suite_dir, "log", "job")) - except OSError: - fs_log_cycles = [] - targzip_log_cycles = [] - for name in glob(os.path.join(user_suite_dir, "log", "job-*.tar.gz")): - targzip_log_cycles.append(os.path.basename(name)[4:-7]) - - relevant_targzip_log_cycles = [] - for entry in entries: - if entry["cycle"] in fs_log_cycles: - pathd = "log/job/%(cycle)s/%(name)s/%(submit_num)02d" % entry - try: - filenames = os.listdir(os.path.join(user_suite_dir, pathd)) - except OSError: - continue - for filename in filenames: - try: - stat = os.stat( - os.path.join(user_suite_dir, pathd, filename)) - except OSError: - pass - else: - entry["logs"][filename] = { - "path": "/".join([pathd, filename]), - "path_in_tar": None, - "mtime": int(stat.st_mtime), # int precise enough - "size": stat.st_size, - "exists": True, - "seq_key": None} - continue - if entry["cycle"] in targzip_log_cycles: - if entry["cycle"] not in relevant_targzip_log_cycles: - relevant_targzip_log_cycles.append(entry["cycle"]) - - for cycle in relevant_targzip_log_cycles: - path = os.path.join("log", "job-%s.tar.gz" % cycle) - tar = tarfile.open(os.path.join(user_suite_dir, path), "r:gz") - for member in tar.getmembers(): - # member.name expected to be "job/cycle/task/submit_num/*" - if not member.isfile(): - continue - try: - cycle_str, name, submit_num_str = ( - member.name.split("/", 4)[1:4]) - entry = entry_of[(cycle_str, name, int(submit_num_str))] - except (KeyError, ValueError): - continue - entry["logs"][os.path.basename(member.name)] = { - "path": path, - "path_in_tar": member.name, - "mtime": int(member.mtime), # too precise otherwise - "size": member.size, - "exists": True, - "seq_key": None} - - # Sequential logs - for entry in entries: - for filename, filename_items in entry["logs"].items(): - seq_log_match = self.REC_SEQ_LOG.match(filename) - if not seq_log_match: - continue - head, index_str, tail = seq_log_match.groups() - seq_key = head + "*" + tail - filename_items["seq_key"] = seq_key - if seq_key not in entry["seq_logs_indexes"]: - entry["seq_logs_indexes"][seq_key] = {} - entry["seq_logs_indexes"][seq_key][index_str] = filename - for seq_key, indexes in entry["seq_logs_indexes"].items(): - # Only one item, not a sequence - if len(indexes) <= 1: - entry["seq_logs_indexes"].pop(seq_key) - # All index_str are numbers, convert key to integer so - # the template can sort them as numbers - try: - int_indexes = {} - for index_str, filename in indexes.items(): - int_indexes[int(index_str)] = filename - entry["seq_logs_indexes"][seq_key] = int_indexes - except ValueError: - pass - for filename, log_dict in entry["logs"].items(): - # Unset seq_key for singular items - if log_dict["seq_key"] not in entry["seq_logs_indexes"]: - log_dict["seq_key"] = None - - def get_suite_logs_info(self, user_name, suite_name): - """Return the information of the suite logs. - - Return a tuple that looks like: - ("cylc-run", - {"err": {"path": "log/suite/err", "mtime": mtime, "size": size}, - "log": {"path": "log/suite/log", "mtime": mtime, "size": size}, - "out": {"path": "log/suite/out", "mtime": mtime, "size": size}}) - - """ - logs_info = {} - prefix = "~" - if user_name: - prefix += user_name - d_rel = self.get_suite_dir_rel(suite_name) - dir_ = os.path.expanduser(os.path.join(prefix, d_rel)) - # Get cylc files. - cylc_files = ["cylc-suite-env", "suite.rc", "suite.rc.processed"] - for key in cylc_files: - f_name = os.path.join(dir_, key) - if os.path.isfile(f_name): - f_stat = os.stat(f_name) - logs_info[key] = {"path": key, - "mtime": f_stat.st_mtime, - "size": f_stat.st_size} - # Get cylc suite log files. - log_files = ["log/suite/err", "log/suite/log", "log/suite/out"] - for key in log_files: - f_name = os.path.join(dir_, key) - if os.path.isfile(f_name): - try: - link_path = os.readlink(f_name) - except OSError: - link_path = f_name - old_logs = [] # Old log naming system. - new_logs = [] # New log naming system. - # TODO: Post migration to cylc this logic can be replaced by: - # `from cylc.suite_logging import get_logs` (superior) - for log in glob(f_name + '.*'): - log_name = os.path.basename(log) - if log_name == link_path: - continue - if len(log_name.split('.')[1]) > 3: - new_logs.append(os.path.join("log", "suite", log_name)) - else: - old_logs.append(os.path.join("log", "suite", log_name)) - new_logs.sort(reverse=True) - old_logs.sort() - f_stat = os.stat(f_name) - logs_info[key] = {"path": key, - "paths": [key] + new_logs + old_logs, - "mtime": f_stat.st_mtime, - "size": f_stat.st_size} - return ("cylc", logs_info) - - def get_suite_cycles_summary( - self, user_name, suite_name, order, limit, offset): - """Return a the state summary (of each cycle) of a user's suite. - - user -- A string containing a valid user ID - suite -- A string containing a valid suite ID - limit -- Limit number of returned entries - offset -- Offset entry number - - Return (entries, of_n_entries), where entries is a data structure that - looks like: - [ { "cycle": cycle, - "n_states": { - "active": N, "success": M, "fail": L, "job_fails": K, - }, - "max_time_updated": T2, - }, - # ... - ] - where: - * cycle is a date-time cycle label - * N, M, L, K are the numbers of tasks in given states - * T2 is the time when last update time of (a task in) the cycle - - and of_n_entries is the total number of entries. - - """ - of_n_entries = 0 - stmt = ("SELECT COUNT(DISTINCT cycle) FROM task_states WHERE " + - "submit_num > 0") - for row in self._db_exec(user_name, suite_name, stmt): - of_n_entries = row[0] - break - if not of_n_entries: - return ([], 0) - - # Not strictly correct, if cycle is in basic date-only format, - # but should not matter for most cases - integer_mode = False - stmt = "SELECT cycle FROM task_states LIMIT 1" - for row in self._db_exec(user_name, suite_name, stmt): - integer_mode = row[0].isdigit() - break - - prefix = "~" - if user_name: - prefix += user_name - user_suite_dir = os.path.expanduser(os.path.join( - prefix, self.get_suite_dir_rel(suite_name))) - targzip_log_cycles = [] - try: - for item in os.listdir(os.path.join(user_suite_dir, "log")): - if item.startswith("job-") and item.endswith(".tar.gz"): - targzip_log_cycles.append(item[4:-7]) - except OSError: - pass - - states_stmt = {} - for key, names in self.TASK_STATUS_GROUPS.items(): - states_stmt[key] = " OR ".join( - ["status=='%s'" % (name) for name in names]) - stmt = ( - "SELECT" + - " cycle," + - " max(time_updated)," + - " sum(" + states_stmt["active"] + ") AS n_active," + - " sum(" + states_stmt["success"] + ") AS n_success," - " sum(" + states_stmt["fail"] + ") AS n_fail" - " FROM task_states" + - " GROUP BY cycle") - if integer_mode: - stmt += " ORDER BY cast(cycle as number)" - else: - stmt += " ORDER BY cycle" - stmt += self.CYCLE_ORDERS.get(order, self.CYCLE_ORDERS["time_desc"]) - stmt_args = [] - if limit: - stmt += " LIMIT ? OFFSET ?" - stmt_args += [limit, offset] - entry_of = {} - entries = [] - for row in self._db_exec(user_name, suite_name, stmt, stmt_args): - cycle, max_time_updated, n_active, n_success, n_fail = row - if n_active or n_success or n_fail: - entry_of[cycle] = { - "cycle": cycle, - "has_log_job_tar_gz": cycle in targzip_log_cycles, - "max_time_updated": max_time_updated, - "n_states": { - "active": n_active, - "success": n_success, - "fail": n_fail, - "job_active": 0, - "job_success": 0, - "job_fail": 0, - }, - } - entries.append(entry_of[cycle]) - - # Check if "task_jobs" table is available or not. - # Note: A single query with a JOIN is probably a more elegant solution. - # However, timing tests suggest that it is cheaper with 2 queries. - # This 2nd query may return more results than is necessary, but should - # be a very cheap query as it does not have to do a lot of work. - if self._db_has_table(user_name, suite_name, "task_jobs"): - stmt = ( - "SELECT cycle," + - " sum(" + self.JOB_STATUS_COMBOS["submitted,running"] + - ") AS n_job_active," + - " sum(" + self.JOB_STATUS_COMBOS["succeeded"] + - ") AS n_job_success," + - " sum(" + self.JOB_STATUS_COMBOS["submission-failed,failed"] + - ") AS n_job_fail" + - " FROM task_jobs GROUP BY cycle") - else: - fail_events_stmt = " OR ".join( - ["event=='%s'" % (name) - for name in self.TASK_STATUS_GROUPS["fail"]]) - stmt = ( - "SELECT cycle," + - " sum(" + fail_events_stmt + ") AS n_job_fail" + - " FROM task_events GROUP BY cycle") - for cycle, n_job_active, n_job_success, n_job_fail in self._db_exec( - user_name, suite_name, stmt): - try: - entry_of[cycle]["n_states"]["job_active"] = n_job_active - entry_of[cycle]["n_states"]["job_success"] = n_job_success - entry_of[cycle]["n_states"]["job_fail"] = n_job_fail - except KeyError: - pass - else: - del entry_of[cycle] - if not entry_of: - break - self._db_close(user_name, suite_name) - - return entries, of_n_entries - - def get_suite_state_summary(self, user_name, suite_name): - """Return a the state summary of a user's suite. - - Return {"is_running": b, "is_failed": b, "server": s} - where: - * is_running is a boolean to indicate if the suite is running - * is_failed: a boolean to indicate if any tasks (submit) failed - * server: host:port of server, if available - - """ - ret = { - "is_running": False, - "is_failed": False, - "server": None} - dao = self._db_init(user_name, suite_name) - if not os.access(dao.db_f_name, os.F_OK | os.R_OK): - return ret - - port_file_path = os.path.expanduser( - os.path.join( - "~" + user_name, "cylc-run", suite_name, ".service", - "contact")) - try: - host = None - port_str = None - for line in open(port_file_path): - key, value = [item.strip() for item in line.split("=", 1)] - if key == "CYLC_SUITE_HOST": - host = value - elif key == "CYLC_SUITE_PORT": - port_str = value - except (IOError, ValueError): - pass - else: - if host and port_str: - ret["is_running"] = True - ret["server"] = host.split(".", 1)[0] + ":" + port_str - - stmt = "SELECT status FROM task_states WHERE status GLOB ? LIMIT 1" - stmt_args = ["*failed"] - for _ in self._db_exec(user_name, suite_name, stmt, stmt_args): - ret["is_failed"] = True - break - self._db_close(user_name, suite_name) - - return ret - - @staticmethod - def is_conf(path): - """Return "cylc-suite-rc" if path is a Cylc suite.rc file.""" - if fnmatch(os.path.basename(path), "suite*.rc*"): - return "cylc-suite-rc" - - @classmethod - def parse_job_log_rel_path(cls, f_name): - """Return (cycle, task, submit_num, ext).""" - return CylcProcessor.parse_job_log_rel_path(f_name) - - def _db_close(self, user_name, suite_name): - """Close a named database connection.""" - key = (user_name, suite_name) - if self.daos.get(key) is not None: - self.daos[key].close() - - def _db_exec(self, user_name, suite_name, stmt, stmt_args=None): - """Execute a query on a named database connection.""" - daos = self._db_init(user_name, suite_name) - return daos.execute(stmt, stmt_args) - - def _db_has_table(self, user_name, suite_name, table_name): - """Return True if table_name exists in the suite database.""" - cursor = self._db_exec( - user_name, suite_name, - "SELECT name FROM sqlite_master WHERE name==?", [table_name]) - return cursor.fetchone() is not None - - def _db_init(self, user_name, suite_name): - """Initialise a named database connection.""" - key = (user_name, suite_name) - if key not in self.daos: - prefix = "~" - if user_name: - prefix += user_name - for name in [os.path.join("log", "db"), "cylc-suite.db"]: - db_f_name = os.path.expanduser(os.path.join( - prefix, self.get_suite_dir_rel(suite_name, name))) - self.daos[key] = CylcSuiteDAO(db_f_name) - if os.path.exists(db_f_name): - break - return self.daos[key] diff --git a/metomi/rose/c3.py b/metomi/rose/c3.py index d11d6ad708..bfa216d7bb 100644 --- a/metomi/rose/c3.py +++ b/metomi/rose/c3.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -105,7 +102,7 @@ def mro(target_name, get_base_names, *args, **kwargs): return results[target_name] -class _Test(object): +class _Test: """Self tests. Print results in TAP format. diff --git a/metomi/rose/checksum.py b/metomi/rose/checksum.py index 264e43edc3..4d74216675 100644 --- a/metomi/rose/checksum.py +++ b/metomi/rose/checksum.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/cmp_source_vc.py b/metomi/rose/cmp_source_vc.py deleted file mode 100644 index e08180995c..0000000000 --- a/metomi/rose/cmp_source_vc.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Tool to determine whether source of an installed suite has changed.""" - -from difflib import unified_diff -import os -from io import StringIO -import sys -import traceback - -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopener -from metomi.rose.reporter import Reporter -from metomi.rose.run_source_vc import write_source_vc_info -from metomi.rose.suite_engine_proc import SuiteEngineProcessor - - -class SuiteVCComparator(object): - """Tool to determine whether source of an installed suite has changed.""" - - def __init__(self, event_handler=None): - self.event_handler = event_handler - self.popen = RosePopener(self.event_handler) - self.suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=self.event_handler, popen=self.popen) - - def cmp_source_vc_info(self, suite_name): - """Compare source VC with installed "log/rose-suite-run.version". - - Return (list): Result in unified diff format or None if irrelevant. - - Args: - suite_name (str): suite name. - """ - rund = self.suite_engine_proc.get_suite_dir(suite_name) - old_info_file_name = self.suite_engine_proc.get_suite_dir( - suite_name, 'log', 'rose-suite-run.version') - try: - old_info = open(old_info_file_name).read().splitlines() - except IOError: # Cannot find/read version file - return None - else: - if len(old_info) <= 1: # No VC information - return None - handle = StringIO() - write_source_vc_info(old_info[0], handle, self.popen) - new_info = handle.getvalue().splitlines() - return unified_diff( - old_info, new_info, - "installed @ %s" % rund, "latest @ %s" % old_info[0]) - - def handle_event(self, *args, **kwargs): - """Handle event.""" - if callable(self.event_handler): - self.event_handler(*args, **kwargs) - - -def main(): - """Launcher for the CLI.""" - opt_parser = RoseOptionParser() - opt_parser.add_my_options('name') - opts, args = opt_parser.parse_args(sys.argv[1:]) - event_handler = Reporter(opts.verbosity - opts.quietness) - suite_vc_cmp = SuiteVCComparator(event_handler) - suite_name = opts.name - if not suite_name and args: - suite_name = args[0] - if not suite_name: - suite_name = os.getenv(suite_vc_cmp.suite_engine_proc.SUITE_NAME_ENV) - if not suite_name: - opt_parser.print_usage(sys.stderr) - sys.exit(2) - try: - lines = suite_vc_cmp.cmp_source_vc_info(suite_name=suite_name) - except Exception as exc: - event_handler(exc) - traceback.print_exc() - sys.exit(2) - else: - if lines is None: - event_handler( - '%s: rose-suite-run.version: VC info not found' % ( - suite_name), - kind=Reporter.KIND_ERR, level=Reporter.FAIL) - sys.exit(2) - lines = list(line for line in lines) - for line in lines: - event_handler('%s\n' % line, prefix='') - if lines: - sys.exit(1) - else: - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/config.py b/metomi/rose/config.py index 2999deed24..8ae578af12 100644 --- a/metomi/rose/config.py +++ b/metomi/rose/config.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -114,7 +111,7 @@ OPT_CONFIG_SETTING_COMMENT = " setting from opt config \"%s\" (%s)" -class ConfigNode(object): +class ConfigNode: """Represent a node in a configuration file. @@ -302,10 +299,10 @@ def get(self, keys=None, no_ignore=False): """Return a node at the position of keys, if any. Args: - keys (list, optional): A list defining a hierarchy of + keys (list): A list defining a hierarchy of node.value 'keys'. If an entry in keys is the null string, it is skipped. - no_ignore (bool, optional): If True any ignored nodes will + no_ignore (bool): If True any ignored nodes will not be returned. Returns: @@ -384,12 +381,12 @@ def get_value(self, keys=None, default=None): If the node does not exist or is ignored, return None. Args: - keys (list, optional): A list defining a hierarchy of node.value + keys (list): A list defining a hierarchy of node.value 'keys'. If an entry in keys is the null string, it is skipped. - default (obj, optional): Return default if the value is not set. + default (object): Return default if the value is not set. Returns: - obj: The value of this ConfigNode at the position of keys or + object: The value of this ConfigNode at the position of keys or default if not set. Examples: @@ -428,7 +425,7 @@ def set(self, keys=None, value=None, state=None, comments=None): Arguments: keys (list): A list defining a hierarchy of node.value 'keys'. If an entry in keys is the null string, it is skipped. - value (obj): The node.value property to set at this position. + value (object): The node.value property to set at this position. state (str): The node.state property to set at this position. If None, the node.state property is unchanged. comments (str): The node.comments property to set at this position. @@ -619,7 +616,8 @@ def __sub__(self, other_config_node): """Produce a ConfigNodeDiff from another ConfigNode. Arguments: - other_config_node - The ConfigNode to be applied to this ConfigNode + other_config_node (ConfigNode): + The ConfigNode to be applied to this ConfigNode to produce the ConfigNodeDiff. Returns: @@ -661,7 +659,7 @@ def __setstate__(self, state): self.comments = state["comments"] -class ConfigNodeDiff(object): +class ConfigNodeDiff: """Represent differences between two ConfigNode instances. @@ -792,9 +790,12 @@ def set_added_setting(self, keys, data): """Set a config setting to be "added" in this ConfigNodeDiff. Args: - keys (list/tuple): The position of the setting to add. - data (obj, str, str): A tuple (value, state, comments) for the - setting to add. + keys (list, tuple): + The position of the setting to add. + data (tuple): + A tuple of the form + ``(value: object, state: string, comments: string)`` + for the setting to add. Examples: >>> config_node_diff = ConfigNodeDiff() @@ -823,11 +824,14 @@ def set_modified_setting(self, keys, old_data, data): None then no change will be made to any pre-existing value. Args: - keys (list/tuple): The position of the setting to add. - old_data (obj, str, str): A tuple (value, state, comments) for - the "current" properties of the setting to modify. - data (obj, str, str): A tuple (value, state, comments) for "new" - properties to change this setting to. + keys (list, tuple): + The position of the setting to add. + old_data (tuple): + A tuple ``(value: object, state: str, comments: str)`` + for the "current" properties of the setting to modify. + data (object): + A tuple ``(value: object, state: str, comments: str)`` + for "new" properties to change this setting to. Examples: >>> # Create a ConfigNodeDiff. @@ -853,8 +857,10 @@ def set_removed_setting(self, keys, data): """Set a config setting to be "removed" in this ConfigNodeDiff. Arguments: - keys (list): The position of the setting to add. - data (obj, str, str): A tuple (value, state, comments) of the + keys (list): + The position of the setting to add. + data (tuple): + A tuple ``(value: object, state: str, comments: str)`` of the properties for the setting to remove. Example: @@ -929,10 +935,10 @@ def get_removed(self): set to None for sections. Returns: - list - A list of the form [(keys, data), ...]: - - keys - The position of an added setting. - - data - Tuple of the form (value, state, comments) of the - properties of the removed setting. + list: A list of the form ``[(keys, data), ...]``: + - keys - The position of an added setting. + - data - Tuple of the form (value, state, comments) of the + properties of the removed setting. Examples: >>> config_node_diff = ConfigNodeDiff() @@ -1007,7 +1013,7 @@ def delete_removed(self): self._data[self.KEY_REMOVED] = {} -class ConfigDumper(object): +class ConfigDumper: """Dumper of a ConfigNode object in Rose INI format. @@ -1040,17 +1046,17 @@ def dump(self, root, target=sys.stdout, sort_sections=None, Args: root (ConfigNode): The root config node. - target (str/file): An open file handle or a string containing a + target (object): An open file handle or a string containing a file path. If not specified, the result is written to sys.stdout. - sort_sections (fcn - optional): An optional argument that should be + sort_sections (Callable): An optional argument that should be a function for sorting a list of section keys. - sort_option_items (fcn - optional): An optional argument that + sort_option_items (Callable): An optional argument that should be a function for sorting a list of option (key, value) tuples in string values. - env_escape_ok (bool - optional): An optional argument to indicate + env_escape_ok (bool): An optional argument to indicate that $NAME and ${NAME} syntax in values should be escaped. - concat_mode (bool - optional): Switch on concatenation mode. If + concat_mode (bool): Switch on concatenation mode. If True, add [] before root level options. """ @@ -1148,7 +1154,7 @@ def _comment_format(cls, comment): return "#%s\n" % (comment) -class ConfigLoader(object): +class ConfigLoader: """Loader of an INI format configuration into a ConfigNode object. @@ -1210,25 +1216,25 @@ def load_with_opts(self, source, node=None, more_keys=None, Arguments: source (str): A file path. - node (ConfigNode - optional): A ConfigNode object if specified, + node (ConfigNode): A ConfigNode object if specified, otherwise one is created. - more_keys (list - optional): A list of additional optional + more_keys (list): A list of additional optional configuration names. If source is "rose-${TYPE}.conf", the file of each name should be "opt/rose-${TYPE}-${NAME}.conf". - used_keys (list - optional): If defined, it should be a list for + used_keys (list): If defined, it should be a list for this method to append to. The key of each successfully loaded optional configuration will be appended to the list (unless the key is already in the list). Missing optional configurations that are specified in more_keys will not raise an error. If not defined, any missing optional configuration will trigger an OSError. - mark_opt_configs (bool - optional): if True, add comments above any + mark_opt_configs (bool): if True, add comments above any settings which have been loaded from an optional config. - return_config_map (bool - optional): If True, construct and return + return_config_map (bool): If True, construct and return a dict (config_map) containing config names vs their uncombined nodes. Optional configurations use their opt keys as keys, and the main configuration uses 'None'. - defines (list - optional): A list of [SECTION]KEY=VALUE overrides. + defines (list): A list of [SECTION]KEY=VALUE overrides. Returns: tuple: node or (node, config_map): diff --git a/metomi/rose/config_cli.py b/metomi/rose/config_cli.py index 4617118d50..864afb9dcd 100644 --- a/metomi/rose/config_cli.py +++ b/metomi/rose/config_cli.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/config_diff.py b/metomi/rose/config_diff.py index cd11f7d6d6..03d1da6693 100644 --- a/metomi/rose/config_diff.py +++ b/metomi/rose/config_diff.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -36,7 +33,7 @@ import metomi.rose.run -class ConfigDiffDefaults(object): +class ConfigDiffDefaults: """Store default settings for the rose config-diff command.""" diff --git a/metomi/rose/config_dump.py b/metomi/rose/config_dump.py index 48feaf0cba..3718b53794 100644 --- a/metomi/rose/config_dump.py +++ b/metomi/rose/config_dump.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/config_processor.py b/metomi/rose/config_processor.py index c29abe61f5..133cbbc9a8 100644 --- a/metomi/rose/config_processor.py +++ b/metomi/rose/config_processor.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -67,7 +64,7 @@ def __str__(self): return "%s: %s" % (setting_str, e_str) -class ConfigProcessorBase(object): +class ConfigProcessorBase: """Base class for a config processor.""" diff --git a/metomi/rose/config_processors/__init__.py b/metomi/rose/config_processors/__init__.py index 8460776f05..f7b71f8284 100644 --- a/metomi/rose/config_processors/__init__.py +++ b/metomi/rose/config_processors/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/config_processors/empy.py b/metomi/rose/config_processors/empy.py deleted file mode 100644 index c14cd1a3e7..0000000000 --- a/metomi/rose/config_processors/empy.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Process a section in a metomi.rose.config.ConfigNode into a EmPy template. -""" - - -from metomi.rose.config_processors.jinja2 import ConfigProcessorForJinja2 - - -class ConfigProcessorForEmPy(ConfigProcessorForJinja2): - - """Processor for [empy:FILE] sections in a runtime configuration.""" - - SCHEME = "empy" - ASSIGN_TEMPL = "@{%s=%s}@\n" - COMMENT_TEMPL = "@# %s\n" - - -del ConfigProcessorForJinja2 # avoid loading it more than once diff --git a/metomi/rose/config_processors/env.py b/metomi/rose/config_processors/env.py index 11bcd9deb6..0e9cb4310c 100644 --- a/metomi/rose/config_processors/env.py +++ b/metomi/rose/config_processors/env.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/config_processors/fileinstall.py b/metomi/rose/config_processors/fileinstall.py index 5f366635d8..2e2ef2202f 100644 --- a/metomi/rose/config_processors/fileinstall.py +++ b/metomi/rose/config_processors/fileinstall.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -20,27 +17,34 @@ """Process "file:*" sections in node of a metomi.rose.config_tree.ConfigTree. """ +import aiofiles from fnmatch import fnmatch from glob import glob +from io import BytesIO import os +import shlex +from shutil import rmtree +import sqlite3 +import sys +from tempfile import mkdtemp +from typing import Any, Optional +from urllib.parse import urlparse + from metomi.rose.checksum import ( - get_checksum, get_checksum_func, guess_checksum_algorithm) -from metomi.rose.config_processor import (ConfigProcessError, - ConfigProcessorBase) + get_checksum, + get_checksum_func, + guess_checksum_algorithm +) +from metomi.rose.config_processor import ( + ConfigProcessError, + ConfigProcessorBase, +) from metomi.rose.env import env_var_process, UnboundEnvironmentVariableError from metomi.rose.fs_util import FileSystemUtil from metomi.rose.job_runner import JobManager, JobProxy, JobRunner from metomi.rose.popen import RosePopener from metomi.rose.reporter import Event from metomi.rose.scheme_handler import SchemeHandlersManager -import shlex -from shutil import rmtree -import sqlite3 -from io import BytesIO -import sys -from tempfile import mkdtemp -from urllib.parse import urlparse -import aiofiles class ConfigProcessorForFile(ConfigProcessorBase): @@ -256,19 +260,25 @@ def _process(self, conf_tree, nodes, loc_dao, **kwargs): # See if any sources have changed names. if not target.is_out_of_date: conn = loc_dao.get_conn() - try: - prev_dep_locs = conn.execute( - "SELECT * FROM dep_names WHERE name=?", [target.name] - ).fetchall() - prev_dep_locs = [i[1] for i in prev_dep_locs] - prev_dep_locs = [loc_dao.select(i) for i in prev_dep_locs] - if ( - [i.name for i in prev_dep_locs] != - [i.name for i in target.dep_locs] - ): - target.is_out_of_date = True - finally: - conn.close() + prev_dep_locs = conn.execute( + """ + SELECT * + FROM dep_names + WHERE name=? + ORDER BY ROWID + """, + [target.name] + ).fetchall() + prev_dep_locs = [i[1] for i in prev_dep_locs] + prev_dep_locs = [ + loc_dao.select(i) + for i in prev_dep_locs + ] + if ( + [i.name for i in prev_dep_locs] != + [i.name for i in target.dep_locs] + ): + target.is_out_of_date = True # See if any sources out of date if not target.is_out_of_date: for dep_loc in target.dep_locs: @@ -486,7 +496,7 @@ def __str__(self): return str(self.args[0]) -class Loc(object): +class Loc: """Represent a location. @@ -576,13 +586,22 @@ def __str__(self): return "%s <= %s, expected %s, got %s" % self.args -class LocSubPath(object): - """Represent a sub-path in a location.""" +class LocSubPath: + """Represent a sub-path in a location. + + Attrs: + name: + Path name. + checksum: + Computed checksum value. + access_mode: + File type and mode bits (see os.stat_result:st_mode). + """ def __init__(self, name, checksum=None, access_mode=None): - self.name = name - self.checksum = checksum - self.access_mode = access_mode + self.name: str = name + self.checksum: Any = checksum + self.access_mode: Optional[int] = access_mode def __lt__(self, other): return ( @@ -603,7 +622,7 @@ def __str__(self): return self.name -class LocDAO(object): +class LocDAO: """DAO for information for incremental updates.""" FILE_NAME = ".rose-config_processors-file.db" diff --git a/metomi/rose/config_processors/jinja2.py b/metomi/rose/config_processors/jinja2.py deleted file mode 100644 index b9a964b749..0000000000 --- a/metomi/rose/config_processors/jinja2.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Process a section in a metomi.rose.config.ConfigNode into a Jinja2 template. -""" - -import filecmp -from metomi.rose.config_processor import ( - ConfigProcessError, ConfigProcessorBase) -from metomi.rose.env import env_var_process, UnboundEnvironmentVariableError -from metomi.rose.fs_util import FileSystemEvent -import os -from tempfile import NamedTemporaryFile - - -class ConfigProcessorForJinja2(ConfigProcessorBase): - - """Processor for [jinja2:FILE] sections in a runtime configuration.""" - - SCHEME = "jinja2" - ASSIGN_TEMPL = "{%% set %s=%s %%}\n" - COMMENT_TEMPL = "{# %s #}\n" - SCHEME_TEMPL = "#!%s\n" - MSG_DONE = "Rose Configuration Insertion: Done" - MSG_INIT = "Rose Configuration Insertion: Init" - - def process(self, conf_tree, item, orig_keys=None, orig_value=None, - **kwargs): - """Process [jinja2:*] in "conf_tree.node". - - Arguments: - conf_tree: - The relevant metomi.rose.config_tree.ConfigTree object with the - full configuration. - item: The current configuration item to process. - orig_keys: - The keys for locating the originating setting in conf_tree in a - recursive processing. None implies a top level call. - orig_value: The value of orig_keys in conf_tree. - **kwargs: - environ (dict): suite level environment variables. - """ - for s_key, s_node in sorted(conf_tree.node.value.items()): - if (s_node.is_ignored() or - not s_key.startswith(self.PREFIX) or - not s_node.value): - continue - target = s_key[len(self.PREFIX):] - source = os.path.join(conf_tree.files[target], target) - if not os.access(source, os.F_OK | os.R_OK): - continue - scheme_ln = self.SCHEME_TEMPL % self.SCHEME - msg_init_ln = self.COMMENT_TEMPL % self.MSG_INIT - msg_done_ln = self.COMMENT_TEMPL % self.MSG_DONE - tmp_file = NamedTemporaryFile() - tmp_file.write(scheme_ln.encode('UTF-8')) - tmp_file.write(msg_init_ln.encode('UTF-8')) - suite_variables = ['{'] - for key, node in sorted(s_node.value.items()): - if node.is_ignored(): - continue - try: - value = env_var_process(node.value) - except UnboundEnvironmentVariableError as exc: - raise ConfigProcessError([s_key, key], node.value, exc) - tmp_file.write( - (self.ASSIGN_TEMPL % (key, value)).encode('UTF-8')) - suite_variables.append(" '%s': %s," % (key, key)) - suite_variables.append('}') - suite_variables = self.ASSIGN_TEMPL % ('ROSE_SUITE_VARIABLES', - '\n'.join(suite_variables)) - tmp_file.write(suite_variables.encode('UTF-8')) - environ = kwargs.get("environ") - if environ: - tmp_file.write('[cylc]\n'.encode('UTF-8')) - tmp_file.write(' [[environment]]\n'.encode('UTF-8')) - for key, value in sorted(environ.items()): - tmp_file.write( - (' %s=%s\n' % (key, value)).encode('UTF-8')) - tmp_file.write(msg_done_ln.encode('UTF-8')) - line_n = 0 - is_in_old_insert = False - for line in open(source): - line_n += 1 - if line_n == 1 and line.strip().lower() == scheme_ln.strip(): - continue - elif line_n == 2 and line == msg_init_ln: - is_in_old_insert = True - continue - elif is_in_old_insert and line == msg_done_ln: - is_in_old_insert = False - continue - elif is_in_old_insert: - continue - tmp_file.write(line.encode('UTF-8')) - tmp_file.seek(0) - if os.access(target, os.F_OK | os.R_OK): - if filecmp.cmp(target, tmp_file.name): # identical - tmp_file.close() - continue - else: - self.manager.fs_util.delete(target) - # Write content to target - target_file = open(target, "w") - for line in tmp_file: - try: - target_file.write(line) - except TypeError: - target_file.write(line.decode()) - event = FileSystemEvent(FileSystemEvent.INSTALL, target) - self.manager.handle_event(event) - tmp_file.close() diff --git a/metomi/rose/config_tree.py b/metomi/rose/config_tree.py index 725ac86e98..ca12e070f7 100644 --- a/metomi/rose/config_tree.py +++ b/metomi/rose/config_tree.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -38,7 +35,7 @@ def __str__(self): return "Bad optional configuration key(s): " + ", ".join(self.args[0]) -class ConfigTree(object): +class ConfigTree: """A run time Rose configuration with linearised inheritance. @@ -77,7 +74,7 @@ def get_file_locs_of(self, key): os.path.join(file_loc, key) for file_loc in self.file_locs[key]] -class ConfigTreeLoader(object): +class ConfigTreeLoader: """Load a Rose configuration with inheritance.""" @@ -186,7 +183,7 @@ def _search(cls, conf_dir, conf_dir_paths): return os.path.abspath(os.path.join(conf_dir_paths[0], conf_dir)) -class _Test(object): +class _Test: """Self tests. Print results in TAP format.""" diff --git a/metomi/rose/date.py b/metomi/rose/date.py index df511c5789..0a24c8b0a6 100644 --- a/metomi/rose/date.py +++ b/metomi/rose/date.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -39,7 +36,7 @@ def __str__(self): return "%s: bad offset value" % self.args[0] -class RoseDateTimeOperator(object): +class RoseDateTimeOperator: """A class to parse and print date string with an offset.""" diff --git a/metomi/rose/env.py b/metomi/rose/env.py index cb9c3f4068..f879017448 100644 --- a/metomi/rose/env.py +++ b/metomi/rose/env.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/env_cat.py b/metomi/rose/env_cat.py index 903bd7c404..dd157df461 100644 --- a/metomi/rose/env_cat.py +++ b/metomi/rose/env_cat.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/lib/bash/rose_log b/metomi/rose/etc/lib/bash/rose_log similarity index 98% rename from lib/bash/rose_log rename to metomi/rose/etc/lib/bash/rose_log index 034d26c2f9..ebfb721990 100644 --- a/lib/bash/rose_log +++ b/metomi/rose/etc/lib/bash/rose_log @@ -21,7 +21,7 @@ # rose_log # # SYNOPSIS -# . $ROSE_HOME/lib/bash/rose_log +# . rose_log # info 1 "Hello world" # info 2 "Hello world" # echo "Hello world" | out diff --git a/lib/bash/rose_usage b/metomi/rose/etc/lib/bash/rose_usage similarity index 95% rename from lib/bash/rose_usage rename to metomi/rose/etc/lib/bash/rose_usage index 47e1640f4b..761a6bcf0c 100644 --- a/lib/bash/rose_usage +++ b/metomi/rose/etc/lib/bash/rose_usage @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,7 +21,7 @@ # rose_usage # # SYNOPSIS -# . $ROSE_HOME/lib/bash/rose_usage +# . $rose_usage # rose_usage [CODE] # # DESCRIPTION @@ -45,7 +45,7 @@ function rose_usage() { } } } - }' "${ROSE_USAGE_FILE:-$0}" >&"$FD" + }' "$0" >&"$FD" if [[ -n $CODE ]]; then exit "$CODE" fi diff --git a/metomi/rose/etc/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py b/metomi/rose/etc/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py index 236963af89..99f9f03be1 100644 --- a/metomi/rose/etc/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py +++ b/metomi/rose/etc/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -23,9 +20,6 @@ """ -import re -import subprocess - import metomi.rose.macro diff --git a/metomi/rose/etc/rose-demo-upgrade-null/versions.py b/metomi/rose/etc/rose-demo-upgrade-null/versions.py index bc94de545c..b4d4a97f7f 100644 --- a/metomi/rose/etc/rose-demo-upgrade-null/versions.py +++ b/metomi/rose/etc/rose-demo-upgrade-null/versions.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- """Module containing test upgrade macros""" -import rose.upgrade +import metomi.rose.upgrade -class UpgradeNull01(rose.upgrade.MacroUpgrade): +class UpgradeNull01(metomi.rose.upgrade.MacroUpgrade): """Upgrade nothing...""" diff --git a/metomi/rose/etc/rose-demo-upgrade/versions.py b/metomi/rose/etc/rose-demo-upgrade/versions.py index 33d66a7887..62cecf036d 100644 --- a/metomi/rose/etc/rose-demo-upgrade/versions.py +++ b/metomi/rose/etc/rose-demo-upgrade/versions.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- """Module containing example macros for using rose app-upgrade. @@ -9,10 +7,10 @@ """ -import rose.upgrade +import metomi.rose.upgrade -class UpgradeGarden01(rose.upgrade.MacroUpgrade): +class UpgradeGarden01(metomi.rose.upgrade.MacroUpgrade): """'We want... a shrubbery!'""" @@ -28,7 +26,7 @@ def upgrade(self, config, meta_config=None): return config, self.reports -class UpgradeGarden02(rose.upgrade.MacroUpgrade): +class UpgradeGarden02(metomi.rose.upgrade.MacroUpgrade): """'...there is one small problem...'""" @@ -51,7 +49,7 @@ def upgrade(self, config, meta_config=None): return config, self.reports -class UpgradeGarden03(rose.upgrade.MacroUpgrade): +class UpgradeGarden03(metomi.rose.upgrade.MacroUpgrade): """'You must find... another shrubbery!'""" @@ -81,7 +79,7 @@ def _get_shrub_num(self, config): return shrub_num -class UpgradeGarden041(rose.upgrade.MacroUpgrade): +class UpgradeGarden041(metomi.rose.upgrade.MacroUpgrade): """'...the two-level effect with a little path running down the middle'""" @@ -97,7 +95,7 @@ def upgrade(self, config, meta_config=None): return config, self.reports -class UpgradeGarden09(rose.upgrade.MacroUpgrade): +class UpgradeGarden09(metomi.rose.upgrade.MacroUpgrade): """'cut down the mightiest tree in the forest... with... a herring!'""" diff --git a/metomi/rose/etc/rose-meta/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py b/metomi/rose/etc/rose-meta/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py index 236963af89..99f9f03be1 100644 --- a/metomi/rose/etc/rose-meta/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py +++ b/metomi/rose/etc/rose-meta/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -23,9 +20,6 @@ """ -import re -import subprocess - import metomi.rose.macro diff --git a/metomi/rose/etc/rose-meta/rose-demo-upgrade-null/versions.py b/metomi/rose/etc/rose-meta/rose-demo-upgrade-null/versions.py index bc94de545c..b4d4a97f7f 100644 --- a/metomi/rose/etc/rose-meta/rose-demo-upgrade-null/versions.py +++ b/metomi/rose/etc/rose-meta/rose-demo-upgrade-null/versions.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- """Module containing test upgrade macros""" -import rose.upgrade +import metomi.rose.upgrade -class UpgradeNull01(rose.upgrade.MacroUpgrade): +class UpgradeNull01(metomi.rose.upgrade.MacroUpgrade): """Upgrade nothing...""" diff --git a/metomi/rose/etc/rose-meta/rose-demo-upgrade/versions.py b/metomi/rose/etc/rose-meta/rose-demo-upgrade/versions.py index 33d66a7887..62cecf036d 100644 --- a/metomi/rose/etc/rose-meta/rose-demo-upgrade/versions.py +++ b/metomi/rose/etc/rose-meta/rose-demo-upgrade/versions.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- """Module containing example macros for using rose app-upgrade. @@ -9,10 +7,10 @@ """ -import rose.upgrade +import metomi.rose.upgrade -class UpgradeGarden01(rose.upgrade.MacroUpgrade): +class UpgradeGarden01(metomi.rose.upgrade.MacroUpgrade): """'We want... a shrubbery!'""" @@ -28,7 +26,7 @@ def upgrade(self, config, meta_config=None): return config, self.reports -class UpgradeGarden02(rose.upgrade.MacroUpgrade): +class UpgradeGarden02(metomi.rose.upgrade.MacroUpgrade): """'...there is one small problem...'""" @@ -51,7 +49,7 @@ def upgrade(self, config, meta_config=None): return config, self.reports -class UpgradeGarden03(rose.upgrade.MacroUpgrade): +class UpgradeGarden03(metomi.rose.upgrade.MacroUpgrade): """'You must find... another shrubbery!'""" @@ -81,7 +79,7 @@ def _get_shrub_num(self, config): return shrub_num -class UpgradeGarden041(rose.upgrade.MacroUpgrade): +class UpgradeGarden041(metomi.rose.upgrade.MacroUpgrade): """'...the two-level effect with a little path running down the middle'""" @@ -97,7 +95,7 @@ def upgrade(self, config, meta_config=None): return config, self.reports -class UpgradeGarden09(rose.upgrade.MacroUpgrade): +class UpgradeGarden09(metomi.rose.upgrade.MacroUpgrade): """'cut down the mightiest tree in the forest... with... a herring!'""" diff --git a/etc/rose-conf-mode.el b/metomi/rose/etc/syntax/rose-conf-mode.el similarity index 96% rename from etc/rose-conf-mode.el rename to metomi/rose/etc/syntax/rose-conf-mode.el index b99ceb93ad..febdc4936c 100644 --- a/etc/rose-conf-mode.el +++ b/metomi/rose/etc/syntax/rose-conf-mode.el @@ -21,10 +21,9 @@ ;; An emacs syntax highlighting mode for the various rose .conf files ;; ;; = Instructions = -;; Place this file in a directory on your emacs load path (or symlink it) -;; e.g. -;; mkdir -p ~/.emacs.d/lisp -;; ln -s $ROSE_HOME/etc/rose-conf-mode.el ~/.emacs.d/lisp/ +;; Place this file in a directory on your emacs load path. e.g: +;; +;; ~/.emacs.d/lisp ;; ;; and in your .emacs file: ;; diff --git a/etc/rose-conf.lang b/metomi/rose/etc/syntax/rose-conf.lang similarity index 100% rename from etc/rose-conf.lang rename to metomi/rose/etc/syntax/rose-conf.lang diff --git a/etc/rose-conf.vim b/metomi/rose/etc/syntax/rose-conf.vim similarity index 100% rename from etc/rose-conf.vim rename to metomi/rose/etc/syntax/rose-conf.vim diff --git a/etc/rose-conf.xml b/metomi/rose/etc/syntax/rose-conf.xml similarity index 100% rename from etc/rose-conf.xml rename to metomi/rose/etc/syntax/rose-conf.xml diff --git a/metomi/rose/external.py b/metomi/rose/external.py index f646c44995..5fc887f31d 100644 --- a/metomi/rose/external.py +++ b/metomi/rose/external.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/formats/__init__.py b/metomi/rose/formats/__init__.py index 81dfa29fb6..02b8f84c4f 100644 --- a/metomi/rose/formats/__init__.py +++ b/metomi/rose/formats/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -21,4 +18,5 @@ such as namelists. To add a new format, place it in this directory and add an import statement below. """ +# flake8: noqa: F401 from . import namelist diff --git a/metomi/rose/formats/namelist.py b/metomi/rose/formats/namelist.py index 0d267db58a..46fa2a99ef 100644 --- a/metomi/rose/formats/namelist.py +++ b/metomi/rose/formats/namelist.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -41,11 +38,11 @@ def _rec(exp): # Matches namelist literals for intrinsic types RE_INTEGER = r"[\+\-]?(?:" + RE_NATURAL + r")" REC_INTEGER = _rec(r"\A(?:" + RE_INTEGER + r")\Z") -RE_REAL = r"(?i)[\+\-]?(?:" + RE_FLOAT + r")(?:[de][\+\-]?\d+)?" +RE_REAL = r"[\+\-]?(?:" + RE_FLOAT + r")(?:[deDE][\+\-]?\d+)?" REC_REAL = _rec(r"\A(?:" + RE_REAL + r")\Z") RE_COMPLEX = r"\(\s*" + RE_REAL + r"\s*" + RE_SEP + r"\s*" + RE_REAL + r"\s*\)" REC_COMPLEX = _rec(r"\A(?:" + RE_COMPLEX + r")\Z") -RE_LOGICAL = r"(?i)\.(?:true|false)\." +RE_LOGICAL = r"\.(?:[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee])\." REC_LOGICAL = _rec(r"\A(?:" + RE_LOGICAL + r")\Z") RE_CHARACTER = r"'(?:[^']|'')*'|\"(?:[^\"]|\"\")*\"" REC_CHARACTER = _rec(r"\A(?:" + RE_CHARACTER + r")\Z") @@ -95,7 +92,7 @@ def _rec(exp): [_rec(r"^([+-])0+(\d)"), r"\1\2"]] # +02.0 => +2.0, -000.5 => -0.5 -class NamelistGroup(object): +class NamelistGroup: """Represent a namelist group. It has the following attributes: @@ -120,7 +117,7 @@ def __repr__(self): return "&%s\n%s\n/\n" % (self.name, "\n".join(object_strings)) -class NamelistObject(object): +class NamelistObject: """Represent an object in a namelist group. An object can be an assignment or a key=value pair in a @@ -187,7 +184,7 @@ def get_rhs_as_string(self, min_repeats=5, wrapped=False, max_len=60): return "\n".join(lines) -class NamelistValue(object): +class NamelistValue: """Represent a value in a namelist object.""" def __init__(self, value_init, quote=False): @@ -232,7 +229,7 @@ def _tidy_real(self, value): return value -class _ParseContext(object): +class _ParseContext: """Convenient object for storing the parser's state.""" def __init__(self): diff --git a/metomi/rose/fs_util.py b/metomi/rose/fs_util.py index 830abf71e2..46c2ab9616 100644 --- a/metomi/rose/fs_util.py +++ b/metomi/rose/fs_util.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -54,7 +51,7 @@ def __str__(self): return "%s: %s" % (self.action, target) -class FileSystemUtil(object): +class FileSystemUtil: """File system utilities with event reporting.""" diff --git a/metomi/rose/host_select.py b/metomi/rose/host_select.py index caaaea325b..fc7c2a2f7e 100644 --- a/metomi/rose/host_select.py +++ b/metomi/rose/host_select.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -19,20 +16,29 @@ # ----------------------------------------------------------------------------- """Select an available host machine by load or by random.""" +from collections import namedtuple +from functools import lru_cache +import json import os from random import choice, random, shuffle -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopener -from metomi.rose.reporter import Reporter, Event -from metomi.rose.resource import ResourceLocator import shlex import signal from socket import ( - getaddrinfo, gethostbyname_ex, gethostname, getfqdn, error as SocketError) + getaddrinfo, + gethostbyname_ex, + gethostname, + getfqdn, + error as SocketError +) import sys from time import sleep, time import traceback +from metomi.rose.opt_parse import RoseOptionParser +from metomi.rose.popen import RosePopener +from metomi.rose.reporter import Reporter, Event +from metomi.rose.resource import ResourceLocator + class NoHostError(Exception): @@ -50,14 +56,23 @@ def __str__(self): return "No hosts selected." -class DeadHostEvent(Event): +class HostSelectCommandFailedEvent(Event): - """An error raised when a host is not contactable.""" + """A remote host select command failed.""" KIND = Event.KIND_ERR + def __init__(self, return_code: int, host: str): + self.return_code = return_code + self.host = host + Event.__init__(self) + def __str__(self): - return self.args[0] + ": (ssh failed)" + if self.return_code == 255: + msg = 'ssh failed' + else: + msg = f'failed {self.return_code}' + return f'{self.host}: ({msg})' class HostThresholdNotMetEvent(Event): @@ -116,7 +131,7 @@ def __str__(self): return self.args[0] + ": (timed out)" -class HostSelector(object): +class HostSelector: """Select an available host machine by load of by random.""" @@ -342,7 +357,11 @@ def select(self, names=None, rank_method=None, thresholds=None, proc.wait() self.handle_event(TimedOutHostEvent(host_name)) elif proc.wait(): - self.handle_event(DeadHostEvent(host_name)) + self.handle_event( + HostSelectCommandFailedEvent( + proc.returncode, host_name + ) + ) else: return [(host_name, 1)] else: @@ -351,40 +370,65 @@ def select(self, names=None, rank_method=None, thresholds=None, # ssh to each host to return its score(s). host_proc_dict = {} for host_name in sorted(host_names): + # build host-select-client command command = [] if not self.is_local_host(host_name): command_args = [] command_args.append(host_name) command = self.popen.get_cmd("ssh", *command_args) - command.append("bash") - stdin = rank_conf.get_command() + command.extend(["rose", "host-select-client"]) + + # build list of metrics to obtain for each host + metrics = rank_conf.get_command() for threshold_conf in threshold_confs: - stdin += threshold_conf.get_command() - stdin += "exit\n" - proc = self.popen.run_bg(*command, stdin=stdin, - preexec_fn=os.setpgrp) + for metric in threshold_conf.get_command(): + if metric not in metrics: + metrics.append(metric) + + # convert metrics list to JSON stdin + stdin = ( + '\n***start**\n' + + json.dumps(metrics) + + '\n**end**\n' + ) + + # fire off host-select-client processes + proc = self.popen.run_bg( + *command, + stdin=stdin, + preexec_fn=os.setpgrp + ) proc.stdin.write(stdin.encode('UTF-8')) proc.stdin.flush() - host_proc_dict[host_name] = proc + host_proc_dict[host_name] = (proc, metrics) # Retrieve score for each host name host_score_list = [] time0 = time() while host_proc_dict: sleep(self.SSH_CMD_POLL_DELAY) - for host_name, proc in list(host_proc_dict.items()): + for host_name, (proc, metrics) in list(host_proc_dict.items()): if proc.poll() is None: score = None elif proc.wait(): - self.handle_event(DeadHostEvent(host_name)) + stdout, stderr = (f.decode() for f in proc.communicate()) + self.handle_event( + HostSelectCommandFailedEvent( + proc.returncode, host_name + ) + ) host_proc_dict.pop(host_name) else: - out = proc.communicate()[0] + out = proc.communicate()[0].decode() + out = _deserialise(metrics, json.loads(out.strip())) + host_proc_dict.pop(host_name) for threshold_conf in threshold_confs: try: - is_bad = threshold_conf.check_threshold(out) - score = threshold_conf.command_out_parser(out) + score = threshold_conf.command_out_parser( + out, metrics + ) + is_bad = threshold_conf.check_threshold(score) except ValueError: is_bad = True score = None @@ -394,7 +438,7 @@ def select(self, names=None, rank_method=None, thresholds=None, break else: try: - score = rank_conf.command_out_parser(out) + score = rank_conf.command_out_parser(out, metrics) host_score_list.append((host_name, score)) except ValueError: score = None @@ -404,7 +448,7 @@ def select(self, names=None, rank_method=None, thresholds=None, break # Report timed out hosts - for host_name, proc in sorted(host_proc_dict.items()): + for host_name, (proc, _) in sorted(host_proc_dict.items()): self.handle_event(TimedOutHostEvent(host_name)) os.killpg(proc.pid, signal.SIGTERM) proc.wait() @@ -419,7 +463,41 @@ def select(self, names=None, rank_method=None, thresholds=None, __call__ = select -class ScorerConf(object): +@lru_cache() +def _tuple_factory(name, params): + """Wrapper to namedtuple which caches results to prevent duplicates.""" + return namedtuple(name, params) + + +def _deserialise(metrics, data): + """Convert dict to named tuples. + + Examples: + >>> _deserialise( + ... [ + ... ['foo', 'bar'], + ... ['baz'] + ... ], + ... [ + ... {'a': 1, 'b': 2, 'c': 3}, + ... [1, 2, 3] + ... ] + ... ) + [foo(a=1, b=2, c=3), [1, 2, 3]] + + """ + for index, (metric, datum) in enumerate(zip(metrics, data)): + if isinstance(datum, dict): + data[index] = _tuple_factory( + metric[0], + tuple(datum.keys()) + )( + *datum.values() + ) + return data + + +class ScorerConf: """Wrap a threshold/ranking scorer + extra configuration.""" @@ -433,18 +511,25 @@ def get_command(self): """Return a shell command to get the info for scoring a host.""" return self.scorer.get_command(self.method_arg) - def check_threshold(self, out): + def check_threshold(self, score): """Parse command output. Return True if threshold not met.""" - score = self.command_out_parser(out) return (float(score) * self.scorer.SIGN > float(self.value) * self.scorer.SIGN) - def command_out_parser(self, out): + def command_out_parser(self, out, metrics): """Parse command output to return a numeric score.""" - return self.scorer.command_out_parser(out, self.method_arg) + results = self.get_results(out, metrics) + return self.scorer.command_out_parser(results, self.method_arg) + + def get_results(self, out, metrics): + """Return list of results for the requested metrics.""" + return [ + out[metrics.index(metric)] + for metric in self.scorer.get_command(self.method_arg) + ] -class RandomScorer(object): +class RandomScorer: """Base class for threshold/ranking scorer. @@ -454,17 +539,13 @@ class RandomScorer(object): ARG = None KEY = "random" - CMD = "true\n" + CMD = ['cpu_count'] # fetch an arbitrary metric CMD_IS_FORMAT = False SIGN = 1 # Positive def get_command(self, method_arg=None): """Return a shell command to get the info for scoring a host.""" - - if self.CMD_IS_FORMAT: - return self.CMD % {"method_arg": method_arg} - else: - return self.CMD + return list(self.CMD) @classmethod def command_out_parser(cls, out, method_arg=None): @@ -474,7 +555,6 @@ def command_out_parser(cls, out, method_arg=None): returned by the command run on the remote host. Otherwise, this method returns a random number. """ - return random() @@ -484,24 +564,13 @@ class LoadScorer(RandomScorer): ARG = "15" KEY = "load" - INDEX_OF = {"1": 1, "5": 2, "15": 3} - CMD = ("echo nproc=$((cat /proc/cpuinfo || lscfg) | grep -ic processor)\n" - "echo uptime=$(uptime)\n") + VALUES = ('1', '5', '15') # 1, 5, 15 min average values + CMD = [["getloadavg"], ["cpu_count"]] def command_out_parser(self, out, method_arg=None): - if method_arg is None: - method_arg = self.ARG - nprocs = None - load = None - for line in out.splitlines(): - if line.startswith(b"nproc="): - nprocs = line.split(b"=", 1)[1] - elif line.startswith(b"uptime="): - idx = self.INDEX_OF[method_arg] - load = line.rsplit(None, 3)[idx].rstrip(b",") - if load is None or not nprocs: - return None - return float(load) / float(nprocs) + load = out[0][self.VALUES.index(method_arg or self.ARG)] + cpus = out[1] + return load / cpus class MemoryScorer(RandomScorer): @@ -509,17 +578,11 @@ class MemoryScorer(RandomScorer): """Score host by amount of free memory""" KEY = "mem" - CMD = """echo mem=$(free -m | sed '3!d; s/^.* \\. +# ----------------------------------------------------------------------------- +import json +from pathlib import Path +import sys + +import psutil + + +def main(): + # read metrics from stdin + started = False + line = True + metrics = '' + while True: + line = sys.stdin.readline().strip() + if '**start**' in line: + started = True + continue + elif not started: + continue + elif '**end**' in line: + break + metrics += f'\n{line}' + metrics = json.loads(metrics) + + # extract metrics using psutil + ret = [ + getattr(psutil, key)( + *args + if key != 'disk_usage' + else [ + # expand ~ in paths for disk usage queries + Path(arg).expanduser() + for arg in args + ] + ) + for key, *args in metrics + ] + + # serialise results + for ind, item in enumerate(ret): + if hasattr(item, '_asdict'): + ret[ind] = item._asdict() + + # output results as json + print(json.dumps(ret)) + + +if __name__ == '__main__': + main() diff --git a/metomi/rose/job_runner.py b/metomi/rose/job_runner.py index 09b5dd6dee..0f0dfb18df 100644 --- a/metomi/rose/job_runner.py +++ b/metomi/rose/job_runner.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -35,7 +32,7 @@ def __str__(self): return str(self.args[0]) -class JobManager(object): +class JobManager: """Manage a set of JobProxy objects and their states.""" def __init__(self, jobs, names=None): @@ -110,7 +107,7 @@ def put_job(self, job_proxy): return job -class JobProxy(object): +class JobProxy: """Represent the state of the job.""" ST_DONE = "ST_DONE" @@ -148,7 +145,7 @@ def update(self, other): self.context.update(other.context) -class JobRunner(object): +class JobRunner: """Runs JobProxy objects with pool of workers.""" def __init__(self, job_processor, nproc=None): diff --git a/metomi/rose/loc_handlers/__init__.py b/metomi/rose/loc_handlers/__init__.py index ea8bb3646f..94490716c3 100644 --- a/metomi/rose/loc_handlers/__init__.py +++ b/metomi/rose/loc_handlers/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/loc_handlers/fs.py b/metomi/rose/loc_handlers/fs.py index ab9434da48..48da4ba8d9 100644 --- a/metomi/rose/loc_handlers/fs.py +++ b/metomi/rose/loc_handlers/fs.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -24,7 +21,7 @@ import os -class FileSystemLocHandler(object): +class FileSystemLocHandler: """Handler of file system locations.""" diff --git a/metomi/rose/loc_handlers/namelist.py b/metomi/rose/loc_handlers/namelist.py index b21f355cc4..0274bb826e 100644 --- a/metomi/rose/loc_handlers/namelist.py +++ b/metomi/rose/loc_handlers/namelist.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -37,7 +34,7 @@ class NamelistEvent(Event): LEVEL = Event.VV -class NamelistLocHandler(object): +class NamelistLocHandler: """Handler of namelists.""" SCHEME = "namelist" diff --git a/metomi/rose/loc_handlers/rsync.py b/metomi/rose/loc_handlers/rsync.py index a50988bd63..a1ffd0e666 100644 --- a/metomi/rose/loc_handlers/rsync.py +++ b/metomi/rose/loc_handlers/rsync.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -19,8 +16,6 @@ # ----------------------------------------------------------------------------- """A handler of locations on remote hosts.""" -from pathlib import Path -from tempfile import TemporaryFile from time import sleep, time from metomi.rose.popen import RosePopenError @@ -29,7 +24,7 @@ ) -class RsyncLocHandler(object): +class RsyncLocHandler: """Handler of locations on remote hosts.""" SCHEME = "rsync" @@ -67,10 +62,8 @@ def parse(self, loc, _): host, path = loc.name.split(":", 1) cmd = self.manager.popen.get_cmd( "ssh", host, "python3", "-", path, loc.TYPE_BLOB, loc.TYPE_TREE) - temp_file = TemporaryFile() - temp_file.write(Path(rsync_remote_check_file).read_bytes()) - temp_file.seek(0) - out = self.manager.popen(*cmd, stdin=temp_file)[0].decode() + with open(rsync_remote_check_file, 'rb') as stdin: + out = self.manager.popen(*cmd, stdin=stdin)[0].decode() lines = out.splitlines() if not lines or lines[0] not in [loc.TYPE_BLOB, loc.TYPE_TREE]: raise ValueError(loc.name) @@ -80,7 +73,7 @@ def parse(self, loc, _): access_mode, mtime, size, name = line.split(None, 3) fake_sum = "source=%s:mtime=%s:size=%s" % ( name, mtime, size) - loc.add_path(loc.BLOB, fake_sum, int(access_mode, base=8)) + loc.add_path(loc.BLOB, fake_sum, int(access_mode)) else: # if loc.loc_type == loc.TYPE_TREE: for line in lines: access_mode, mtime, size, name = line.split(None, 3) diff --git a/metomi/rose/loc_handlers/rsync_remote_check.py b/metomi/rose/loc_handlers/rsync_remote_check.py index 8ab7e85a83..3f6de15365 100644 --- a/metomi/rose/loc_handlers/rsync_remote_check.py +++ b/metomi/rose/loc_handlers/rsync_remote_check.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # # This file is part of Rose, a framework for meteorological suites. @@ -23,12 +21,18 @@ This is a Python file but we read it and pass it to stdin to avoid reliance on remote platforms having rose installed. + +Warning: + This script will not necessarily be run in the Rose Python environment. + It should not have any dependencies outside of the stdlib and should be + compatible with as wide a range of Python3 versions as possible. + """ import os import sys -def main(): +def main(path, str_blob, str_tree): """Check file exists and print some info: 1. Octal protection bits. @@ -36,7 +40,6 @@ def main(): 3. Filesize. 4. Path, which has been checked. """ - path, str_blob, str_tree = sys.argv[1:] if os.path.isdir(path): print(str_tree) os.chdir(path) @@ -46,19 +49,19 @@ def main(): if not dirname.startswith("."): good_dirnames.append(dirname) name = os.path.join(dirpath, dirname) - print(("-", "-", "-", name)) + print("-", "-", "-", name) dirnames[:] = good_dirnames for filename in filenames: if filename.startswith("."): continue name = os.path.join(dirpath, filename) stat = os.stat(name) - print((oct(stat.st_mode), stat.st_mtime, stat.st_size, name)) + print(stat.st_mode, stat.st_mtime, stat.st_size, name) elif os.path.isfile(path): print(str_blob) stat = os.stat(path) - print(oct(stat.st_mode), stat.st_mtime, stat.st_size, path) + print(stat.st_mode, stat.st_mtime, stat.st_size, path) if __name__ == '__main__': - main() + main(*sys.argv[1:]) diff --git a/metomi/rose/loc_handlers/svn.py b/metomi/rose/loc_handlers/svn.py index 92502f43d6..872b865665 100644 --- a/metomi/rose/loc_handlers/svn.py +++ b/metomi/rose/loc_handlers/svn.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -24,7 +21,7 @@ import xml.parsers.expat -class SvnLocHandler(object): +class SvnLocHandler: """Handler of Subversion locations.""" FCM = "fcm" @@ -73,7 +70,7 @@ async def pull(self, loc, conf_tree): "svn", "export", "-q", loc.real_name, loc.cache) -class SvnInfoXMLParser(object): +class SvnInfoXMLParser: """An XML parser tailored for a single entry of "svn info --xml".""" def __init__(self): diff --git a/metomi/rose/macro.py b/metomi/rose/macro.py index 5342a0dd96..69809bba30 100644 --- a/metomi/rose/macro.py +++ b/metomi/rose/macro.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -173,7 +170,7 @@ def __str__(self): return ERROR_LOAD_CONF_META_NODE -class MacroBase(object): +class MacroBase: """Base class for macros for validating or transforming configurations. @@ -341,7 +338,7 @@ def standard_format_config(self, config): def add_report(self, *args, **kwargs): """Add a metomi.rose.macro.MacroReport. - See :class:`rose.macro.MacroReport` for details of arguments. + See :class:`metomi.rose.macro.MacroReport` for details of arguments. Examples: >>> # An example validator macro which adds a report to the setting @@ -497,7 +494,7 @@ def transform(self, config, meta_config=None): return config, self.reports -class MacroReport(object): +class MacroReport: """Class to hold information about a macro issue. @@ -505,7 +502,7 @@ class MacroReport(object): section (str): The name of the section to attach this report to. option (str): The name of the option (within the section) to attach this report to. - value (obj): The value of the configuration associated with this + value (object): The value of the configuration associated with this report. info (str): Text information describing the nature of the report. is_warning (bool): If True then this report will be logged as a @@ -554,7 +551,9 @@ def add_site_meta_paths(): for path in path.split(os.pathsep): path = os.path.expanduser(os.path.expandvars(path)) sys.path.insert(0, os.path.abspath(path)) - sys.path.append(os.path.join(os.getenv("ROSE_LIB"), "etc/rose-meta")) + sys.path.append( + metomi.rose.resource.ResourceLocator.default().locate('rose-meta') + ) def add_env_meta_paths(): @@ -648,7 +647,7 @@ def load_meta_path(config=None, directory=None, is_upgrade=False, if is_upgrade: path = meta_key try: - meta_path = locator.locate(path) + meta_path = str(locator.locate(path)) except metomi.rose.resource.ResourceError: continue else: @@ -690,7 +689,7 @@ def load_meta_config_tree(config, directory=None, config_type=None, meta_config = metomi.rose.config.ConfigNode() for meta_key in meta_list: try: - meta_path = locator.locate(meta_key) + meta_path = str(locator.locate(meta_key)) except metomi.rose.resource.ResourceError: if not ignore_meta_error: error_handler(text=ERROR_LOAD_META_PATH.format(meta_key)) @@ -1633,7 +1632,6 @@ def main(): sys.exit(1) # Path manipulation. - sys.path.append(os.getenv("ROSE_LIB")) add_opt_meta_paths(opts.meta_path) # Run macros for each config. diff --git a/metomi/rose/macros/__init__.py b/metomi/rose/macros/__init__.py index c6301abb3a..00975571ea 100644 --- a/metomi/rose/macros/__init__.py +++ b/metomi/rose/macros/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/macros/compulsory.py b/metomi/rose/macros/compulsory.py index fe09fe092a..cc82a990de 100644 --- a/metomi/rose/macros/compulsory.py +++ b/metomi/rose/macros/compulsory.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/macros/duplicate.py b/metomi/rose/macros/duplicate.py index a4248c530d..c6345c77c6 100644 --- a/metomi/rose/macros/duplicate.py +++ b/metomi/rose/macros/duplicate.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/macros/format.py b/metomi/rose/macros/format.py index f201662fd6..db860c2160 100644 --- a/metomi/rose/macros/format.py +++ b/metomi/rose/macros/format.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # ----------------------------------------------------------------------------- diff --git a/metomi/rose/macros/rule.py b/metomi/rose/macros/rule.py index 3254663aae..9cc506faaa 100644 --- a/metomi/rose/macros/rule.py +++ b/metomi/rose/macros/rule.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/macros/trigger.py b/metomi/rose/macros/trigger.py index 38c965dcc9..d14d641bf6 100644 --- a/metomi/rose/macros/trigger.py +++ b/metomi/rose/macros/trigger.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/macros/value.py b/metomi/rose/macros/value.py index 5ee7038fa3..4837b3fd55 100644 --- a/metomi/rose/macros/value.py +++ b/metomi/rose/macros/value.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/meta_type.py b/metomi/rose/meta_type.py index 2d234206ad..12e81deedc 100644 --- a/metomi/rose/meta_type.py +++ b/metomi/rose/meta_type.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -26,7 +23,7 @@ REC_CHARACTER = re.compile(r"'(?:[^']|'')*'$") -class MetaType(object): +class MetaType: KEY = None meta_type_classes = {} diff --git a/metomi/rose/metadata_check.py b/metomi/rose/metadata_check.py index 7deec314d3..a935ec3929 100644 --- a/metomi/rose/metadata_check.py +++ b/metomi/rose/metadata_check.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/metadata_gen.py b/metomi/rose/metadata_gen.py index 919f25c00b..70c6541320 100644 --- a/metomi/rose/metadata_gen.py +++ b/metomi/rose/metadata_gen.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/metadata_graph.py b/metomi/rose/metadata_graph.py index 11b556cf1d..a84b10cb01 100644 --- a/metomi/rose/metadata_graph.py +++ b/metomi/rose/metadata_graph.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -267,8 +264,6 @@ def main(): if opts.conf_dir: os.chdir(opts.conf_dir) opts.conf_dir = os.getcwd() - sys.path.append( - metomi.rose.resource.ResourceLocator.default().get_util_home()) metomi.rose.macro.add_opt_meta_paths(opts.meta_path) config_file_path = os.path.join(opts.conf_dir, metomi.rose.SUB_CONFIG_NAME) diff --git a/metomi/rose/namelist_dump.py b/metomi/rose/namelist_dump.py index 0f4b35bee4..5e11015883 100644 --- a/metomi/rose/namelist_dump.py +++ b/metomi/rose/namelist_dump.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/opt_parse.py b/metomi/rose/opt_parse.py index 139aa95374..dcbf878634 100644 --- a/metomi/rose/opt_parse.py +++ b/metomi/rose/opt_parse.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -20,7 +17,7 @@ """Common option parser for Rose command utilities.""" from optparse import OptionParser -from metomi.rose.resource import ResourceLocator +import metomi.rose.resource class RoseOptionParser(OptionParser): @@ -616,10 +613,6 @@ class RoseOptionParser(OptionParser): ["--to-origin"], {"action": "store_true", "help": "Convert ID to the origin URL"}], - "to_output": [ - ["--to-output"], - {"action": "store_true", - "help": "Get the ID output directory"}], "to_web": [ ["--to-web"], {"action": "store_true", @@ -686,9 +679,15 @@ class RoseOptionParser(OptionParser): def __init__(self, *args, **kwargs): if hasattr(kwargs, "prog"): namespace, util = kwargs["prog"].split(None, 1) - resource_loc = ResourceLocator(namespace=namespace, util=util) + resource_loc = ( + metomi.rose.resource.ResourceLocator( + namespace=namespace, util=util + ) + ) else: - resource_loc = ResourceLocator.default() + resource_loc = ( + metomi.rose.resource.ResourceLocator.default() + ) kwargs["prog"] = resource_loc.get_util_name() if not hasattr(kwargs, "usage"): kwargs["usage"] = resource_loc.get_synopsis() diff --git a/metomi/rose/popen.py b/metomi/rose/popen.py index 63b15ce104..c4f2cb2244 100644 --- a/metomi/rose/popen.py +++ b/metomi/rose/popen.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -19,16 +16,17 @@ # ----------------------------------------------------------------------------- """Wraps Python's subprocess.Popen.""" -import os import asyncio +import os import re -import io -from metomi.rose.reporter import Event -from metomi.rose.resource import ResourceLocator +import select import shlex from subprocess import Popen, PIPE import sys +from metomi.rose.reporter import Event +from metomi.rose.resource import ResourceLocator + class RosePopenError(Exception): @@ -69,21 +67,44 @@ def __str__(self): ret = command else: ret = RosePopener.list_to_shell_str(self.command) - if isinstance(self.stdin, str): - ret += " <<'__STDIN__'\n" + self.stdin + "\n'__STDIN__'" - elif isinstance(self.stdin, io.IOBase): - try: - # FIXME: Is this safe? - pos = self.stdin.tell() - ret += " <<'__STDIN__'\n" +\ - self.stdin.read().decode() + "\n'__STDIN__'" - self.stdin.seek(pos) - except IOError: - pass + + try: + # real file or real stream + self.stdin.fileno() + # ask select if it is readable (real files can hang) + readable = bool(select.select([self.stdin], [], [], 0.0)[0]) + except (AttributeError, IOError): + # file like + readable = True + + if self.stdin: + if isinstance(self.stdin, str): + # string + stdin = self.stdin + elif isinstance(self.stdin, bytes): + # byte string + stdin = self.stdin.decode() + elif readable: + # file like + try: + pos = self.stdin.tell() + stdin = self.stdin.read() + self.stdin.seek(pos) + if not isinstance(stdin, str): + stdin = stdin.decode() + except Exception: + # purposefully vague for safety (catch any exception) + stdin = '' + else: + stdin = '' + + if stdin: + ret += f" <<'__STDIN__'\n{stdin}\n__STDIN__" + return ret -class RosePopener(object): +class RosePopener: """Wrap Python's subprocess.Popen.""" diff --git a/metomi/rose/reporter.py b/metomi/rose/reporter.py index 633f893f7e..5c494e6eb5 100644 --- a/metomi/rose/reporter.py +++ b/metomi/rose/reporter.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -24,7 +21,7 @@ import time -class Reporter(object): +class Reporter: """Report diagnostic messages. @@ -181,7 +178,7 @@ def report(self, message, kind=None, level=None, prefix=None, clip=None): __call__ = report -class ReporterContext(object): +class ReporterContext: """A context for the reporter object. @@ -260,7 +257,7 @@ def _tty_colour_err(self, str_): return str_ -class Event(object): +class Event: """A base class for events suitable for feeding into a Reporter.""" diff --git a/metomi/rose/resource.py b/metomi/rose/resource.py index 57bf4050e2..7d892f4979 100644 --- a/metomi/rose/resource.py +++ b/metomi/rose/resource.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -22,29 +19,19 @@ """ import os -from metomi.rose.config import ConfigLoader, ConfigNode +from pathlib import Path import inspect import string import sys from importlib.machinery import SourceFileLoader +import metomi.rose +from metomi.rose.config import ConfigLoader, ConfigNode +import metomi.rose.opt_parse +from metomi.rose.reporter import Reporter -ERROR_LOCATE_OBJECT = "Could not locate {0}" - - -def get_util_home(*args): - """Return ROSE_LIB or the dirname of the dirname of sys.argv[0]. - - If args are specified, they are added to the end of returned path. - """ - try: - value = os.environ["ROSE_LIB"] - except KeyError: - value = os.path.abspath(__file__) - for _ in range(3): # assume __file__ under $ROSE_LIB/metomi/rose/ - value = os.path.dirname(value) - return os.path.join(value, *args) +ERROR_LOCATE_OBJECT = "Could not locate {0}" class ResourceError(Exception): @@ -55,12 +42,30 @@ def __init__(self, key): Exception.__init__(self, "%s: resource not found." % key) -class ResourceLocator(object): +ROSE_CONF_PATH = 'ROSE_CONF_PATH' +ROSE_SITE_CONF_PATH = 'ROSE_SITE_CONF_PATH' +ROSE_INSTALL_ROOT = Path(metomi.rose.__file__).parent + + +class ResourceLocator: + """A class for searching resource files. + + Loads files in the following order: + + System: + /etc + Site: + $ROSE_SITE_CONF_PATH + User: + ~/.metomi - """A class for searching resource files.""" + If $ROSE_CONF_PATH is defined these files are skipped and configuration + found in $ROSE_CONF_PATH is loaded instead. - SITE_CONF_PATH = get_util_home("etc") - USER_CONF_PATH = os.path.join(os.path.expanduser("~"), ".metomi") + """ + + SYST_CONF_PATH = Path('/etc') + USER_CONF_PATH = Path('~/.metomi').expanduser() ROSE_CONF = "rose.conf" _DEFAULT_RESOURCE_LOCATOR = None @@ -75,35 +80,53 @@ def __init__(self, namespace=None, util=None, paths=None): self.namespace = namespace self.util = util if paths: - self.paths = list(paths) + self.paths = list(map(Path, paths)) else: - home = self.get_util_home() - name = self.get_util_name("-") - self.paths = [os.path.join(home, "etc", name), - os.path.join(home, "etc")] + self.paths = [ + (ROSE_INSTALL_ROOT / 'etc') / self.get_util_name("-"), + ROSE_INSTALL_ROOT / 'etc' + ] self.conf = None def get_conf(self): """Return the site/user configuration root node.""" if self.conf is None: - paths = [self.SITE_CONF_PATH, self.USER_CONF_PATH] + # base system conf path + paths = [self.SYST_CONF_PATH] + + # add $ROSE_SITE_CONF_PATH if defined + if "ROSE_SITE_CONF_PATH" in os.environ: + path_str = os.environ["ROSE_SITE_CONF_PATH"].strip() + if path_str: + paths.append(Path(path_str)) + + # add user conf path + paths.append(self.USER_CONF_PATH) + + # use $ROSE_CONF_PATH (and ignore all others) if defined if "ROSE_CONF_PATH" in os.environ: paths_str = os.getenv("ROSE_CONF_PATH").strip() if paths_str: - paths = paths_str.split(os.pathsep) + paths = [ + Path(path) + for path in paths_str.split(os.pathsep) + ] else: paths = [] + + # load and cache config self.conf = ConfigNode() config_loader = ConfigLoader() for path in paths: - name = os.path.join(path, self.ROSE_CONF) - if os.path.isfile(name) and os.access(name, os.R_OK): - config_loader.load_with_opts(name, self.conf) + conffile = path / self.ROSE_CONF + if conffile.is_file() and os.access(conffile, os.R_OK): + config_loader.load_with_opts(str(conffile), self.conf) + return self.conf def get_doc_url(self): """Return the URL of Rose documentation.""" - default = "file://%s/doc/" % self.get_util_home() + default = f"file://{ROSE_INSTALL_ROOT}/doc/" return self.get_conf().get_value(["rose-doc"], default=default) def get_synopsis(self): @@ -120,15 +143,6 @@ def get_synopsis(self): except IOError: return None - @classmethod - def get_util_home(cls, *args): - """Return ROSE_HOME or the dirname of the dirname of sys.argv[0]. - - If args are specified, they are added to the end of returned path. - - """ - return get_util_home(*args) - def get_util_name(self, separator=" "): """Return the name of the Rose utility, e.g. "rose app-run". @@ -147,42 +161,16 @@ def get_util_name(self, separator=" "): except KeyError: return os.path.basename(sys.argv[0]) - def get_version(self, ignore_environment=False): - """return the current metomi.rose_version number. - - By default pass through the value of the ``ROSE_VERSION`` environment - variable. - - Args: - ignore_environment (bool): Return the value extracted from the - ``rose-version`` file. - """ - version = None - if not ignore_environment: - version = os.getenv("ROSE_VERSION") - if not version: - for line in open(self.get_util_home("rose-version")): - if line.startswith("ROSE_VERSION="): - value = line.replace("ROSE_VERSION=", "") - version = value.strip(string.whitespace + "\";") - break - return version - def locate(self, key): """Return the location of the resource key.""" key = os.path.expanduser(key) for path in self.paths: - name = os.path.join(path, key) - if os.path.exists(name): + name = path / key + if name.exists(): return name raise ResourceError(key) -def resource_locate(key): - """Return the location of the resource key.""" - return ResourceLocator.default().locate(key) - - def import_object(import_string, from_files, error_handler, module_prefix=None): """Import a Python callable. @@ -239,3 +227,40 @@ def import_object(import_string, from_files, error_handler, if obj_name == class_name and inspect.isclass(obj): return_object = obj return return_object + + +def main(): + """Launcher for the CLI.""" + opt_parser = metomi.rose.opt_parse.RoseOptionParser() + opt_parser.add_my_options() + opts, args = opt_parser.parse_args(sys.argv[1:]) + reporter = Reporter(opts.verbosity - opts.quietness) + is_top_level = False + if len(args) > 1: + reporter.report('Only one argument accepted\n', level=Reporter.FAIL) + sys.exit(1) + if len(args) == 0: + key = ROSE_INSTALL_ROOT / 'etc' + path = ResourceLocator(paths=[ROSE_INSTALL_ROOT]).locate('etc') + is_top_level = True + else: + key = args[0] + try: + path = ResourceLocator().locate(key) + except ResourceError: + reporter.report('Resource not found\n', level=Reporter.FAIL) + sys.exit(1) + if path.is_file(): + print(path) + elif path.is_dir(): + print(f'{key}/') + for item in path.iterdir(): + if is_top_level: + item = item.relative_to(path) + else: + item = item.relative_to(path.parent) + print(f' {item}') + + +if __name__ == "__main__": + main() diff --git a/metomi/rose/run.py b/metomi/rose/run.py index d1be7e3398..0f71074b80 100644 --- a/metomi/rose/run.py +++ b/metomi/rose/run.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -17,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Rose. If not, see . # ----------------------------------------------------------------------------- -"""Shared utilities for app/suite/task run.""" +"""Shared utilities for app/task run.""" import os from metomi.rose.config_processor import ConfigProcessorsManager @@ -83,7 +80,7 @@ def __str__(self): return "%s=%s, --new mode not supported." % self.args -class Dummy(object): +class Dummy: """Convert a dict into an object.""" @@ -92,9 +89,9 @@ def __init__(self, **kwargs): setattr(self, key, value) -class Runner(object): +class Runner: - """Invoke a Rose application or a Rose suite.""" + """Invoke a Rose application.""" CONF_NAME = None NAME = None diff --git a/metomi/rose/run_source_vc.py b/metomi/rose/run_source_vc.py index 315fd7ca50..6eacff572f 100644 --- a/metomi/rose/run_source_vc.py +++ b/metomi/rose/run_source_vc.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -22,7 +19,6 @@ import os from metomi.rose.popen import RosePopener import sys -import _io from metomi.rose.unicode_utils import write_safely diff --git a/metomi/rose/scheme_handler.py b/metomi/rose/scheme_handler.py index ab1a509561..d193164af4 100644 --- a/metomi/rose/scheme_handler.py +++ b/metomi/rose/scheme_handler.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -26,7 +23,7 @@ import sys -class SchemeHandlersManager(object): +class SchemeHandlersManager: """Load and select from a group of related functional classes.""" CAN_HANDLE = "can_handle" diff --git a/metomi/rose/section.py b/metomi/rose/section.py index c2edf09194..746ffe1692 100644 --- a/metomi/rose/section.py +++ b/metomi/rose/section.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -25,7 +22,7 @@ import copy -class Section(object): +class Section: """This class stores the data and metadata of an input section. diff --git a/metomi/rose/stem.py b/metomi/rose/stem.py deleted file mode 100644 index 8c777e8b1c..0000000000 --- a/metomi/rose/stem.py +++ /dev/null @@ -1,497 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Implementation of 'rose stem'""" - -import os -import re -import sys - -import metomi.rose.config -from metomi.rose.fs_util import FileSystemUtil -from metomi.rose.host_select import HostSelector -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopener, RosePopenError -from metomi.rose.reporter import Reporter, Event -from metomi.rose.resource import ResourceLocator -from metomi.rose.suite_run import SuiteRunner - -DEFAULT_TEST_DIR = 'rose-stem' -OPTIONS = ['group', 'source', 'task', ] -ROSE_STEM_VERSION = 1 -SUITE_RC_PREFIX = '[jinja2:suite.rc]' - - -class ConfigVariableSetEvent(Event): - - """Event to report a particular variable has been set.""" - - LEVEL = Event.V - - def __repr__(self): - return "Variable %s set to %s" % (self.args[0], self.args[1]) - - __str__ = __repr__ - - -class ConfigSourceTreeSetEvent(Event): - - """Event to report a source tree for config files.""" - - LEVEL = Event.V - - def __repr__(self): - return "Using config files from source %s" % (self.args[0]) - - __str__ = __repr__ - - -class NameSetEvent(Event): - - """Event to report a name for the suite being set.""" - - LEVEL = Event.V - - def __repr__(self): - return "Suite is named %s" % (self.args[0]) - - __str__ = __repr__ - - -class ProjectNotFoundException(Exception): - - """Exception class when unable to determine project a source belongs to.""" - - def __init__(self, source, error=None): - Exception.__init__(self, source, error) - self.source = source - self.error = error - - def __repr__(self): - if self.error is not None: - return "Cannot ascertain project for source tree %s:\n%s" % ( - self.source, self.error) - else: - return "Cannot ascertain project for source tree %s" % ( - self.source) - - __str__ = __repr__ - - -class RoseStemVersionException(Exception): - - """Exception class when running the wrong metomi.rose-stem version.""" - - def __init__(self, version): - Exception.__init__(self, version) - if version is None: - self.suite_version = "not metomi.rose-stem compatible" - else: - self.suite_version = "at version %s" % (version) - - def __repr__(self): - return "Running metomi.rose-stem version %s but suite is %s" % ( - ROSE_STEM_VERSION, self.suite_version) - - __str__ = __repr__ - - -class RoseSuiteConfNotFoundException(Exception): - - """Exception class when unable to find metomi.rose-suite.conf.""" - - def __init__(self, location): - Exception.__init__(self, location) - self.location = location - - def __repr__(self): - if os.path.isdir(self.location): - return "\nCannot find a suite to run in directory %s" % ( - self.location) - else: - return "\nSuite directory %s is not a valid directory" % ( - self.location) - - __str__ = __repr__ - - -class SourceTreeAddedAsBranchEvent(Event): - - """Event to report a source tree has been added as a branch.""" - - LEVEL = Event.DEFAULT - - def __repr__(self): - return "Source tree %s added as branch" % (self.args[0]) - - __str__ = __repr__ - - -class SourceTreeAddedAsTrunkEvent(Event): - - """Event to report a source tree has been added as a trunk.""" - - LEVEL = Event.DEFAULT - - def __repr__(self): - return "Source tree %s added as trunk" % (self.args[0]) - - __str__ = __repr__ - - -class SuiteSelectionEvent(Event): - - """Event to report a source tree for config files.""" - - LEVEL = Event.DEFAULT - - def __repr__(self): - return "Will run suite from %s" % (self.args[0]) - - __str__ = __repr__ - - -class StemRunner(object): - - """Set up options for running a STEM job through Rose.""" - - def __init__(self, opts, reporter=None, popen=None, fs_util=None): - self.opts = opts - if reporter is None: - self.reporter = Reporter(opts.verbosity - opts.quietness) - else: - self.reporter = reporter - if popen is None: - self.popen = RosePopener(event_handler=self.reporter) - else: - self.popen = popen - if fs_util is None: - self.fs_util = FileSystemUtil(event_handler=self.reporter) - else: - self.fs_util = fs_util - self.host_selector = HostSelector(event_handler=self.reporter, - popen=self.popen) - - def _add_define_option(self, var, val): - """Add a define option passed to the SuiteRunner.""" - - if self.opts.defines: - self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val) - else: - self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val] - self.reporter(ConfigVariableSetEvent(var, val)) - return - - def _get_base_dir(self, item): - """Given a source tree return the following from 'fcm loc-layout': - * url - * sub_tree - * peg_rev - * root - * project - """ - - ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item) - output = output.decode() - if ret_code != 0: - raise ProjectNotFoundException(item, stderr) - - ret = {} - for line in output.splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - if key: - if value: - ret[key] = value.strip() - - return ret - - def _get_project_from_url(self, source_dict): - """Run 'fcm keyword-print' to work out the project name.""" - - repo = source_dict['root'] - if source_dict['project']: - repo += '/' + source_dict['project'] - - kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1] - - project = None - for line in kpoutput.splitlines(): - if line.rstrip().endswith(repo.encode('UTF-8')): - kpresult = re.search(r'^location{primary}\[(.*)\]', - line.decode()) - if kpresult: - project = kpresult.group(1) - break - return project - - def _deduce_mirror(self, source_dict, project): - """Deduce the mirror location of this source tree.""" - - # Root location for project - proj_root = source_dict['root'] + '/' + source_dict['project'] - - # Swap project to mirror - project = re.sub(r'\.x$', r'.xm', project) - mirror_repo = "fcm:" + project - - # Generate mirror location - mirror = re.sub(proj_root, mirror_repo, source_dict['url']) - - # Remove any sub-tree - mirror = re.sub(source_dict['sub_tree'], r'', mirror) - mirror = re.sub(r'/@', r'@', mirror) - - # Add forwards slash after .xm if missing - if '.xm/' not in mirror: - mirror = re.sub(r'\.xm', r'.xm/', mirror) - return mirror - - def _ascertain_project(self, item): - """Set the project name and top-level from 'fcm loc-layout'. - Returns: - * project name - * top-level location of the source tree with revision number - * top-level location of the source tree without revision number - * revision number - """ - - project = None - try: - project, item = item.split("=", 1) - except ValueError: - pass - - if re.search(r'^\.', item): - item = os.path.abspath(os.path.join(os.getcwd(), item)) - - if project is not None: - print("[WARN] Forcing project for '{0}' to be '{1}'".format( - item, project)) - return project, item, item, '', '' - - source_dict = self._get_base_dir(item) - project = self._get_project_from_url(source_dict) - if not project: - raise ProjectNotFoundException(item) - - mirror = self._deduce_mirror(source_dict, project) - - if 'peg_rev' in source_dict and '@' in item: - revision = '@' + source_dict['peg_rev'] - base = re.sub(r'@.*', r'', item) - else: - revision = '' - base = item - - # Remove subtree from base and item - if 'sub_tree' in source_dict: - item = re.sub( - r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', item, count=1) - base = re.sub( - r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', base, count=1) - - # Remove trailing forwards-slash - item = re.sub(r'/$', r'', item) - base = re.sub(r'/$', r'', base) - - # Remove anything after a point - project = re.sub(r'\..*', r'', project) - return project, item, base, revision, mirror - - def _generate_name(self): - """Generate a suite name from the name of the first source tree.""" - try: - basedir = self._ascertain_project(os.getcwd())[1] - except ProjectNotFoundException: - if self.opts.conf_dir: - basedir = os.path.abspath(self.opts.conf_dir) - else: - basedir = os.getcwd() - name = os.path.basename(basedir) - self.reporter(NameSetEvent(name)) - return name - - def _this_suite(self): - """Find the location of the suite in the first source tree.""" - - # Get base of first source - basedir = '' - if self.opts.source: - basedir = self.opts.source[0] - else: - basedir = self._ascertain_project(os.getcwd())[1] - - suitedir = os.path.join(basedir, DEFAULT_TEST_DIR) - suitefile = os.path.join(suitedir, "rose-suite.conf") - - if not os.path.isfile(suitefile): - raise RoseSuiteConfNotFoundException(suitedir) - - self._check_suite_version(suitefile) - - return suitedir - - def _read_auto_opts(self): - """Read the site metomi.rose.conf file.""" - return ResourceLocator.default().get_conf().get_value( - ["rose-stem", "automatic-options"]) - - def _check_suite_version(self, fname): - """Check the suite is compatible with this version of metomi.rose-stem. - """ - if not os.path.isfile(fname): - raise RoseSuiteConfNotFoundException(os.path.dirname(fname)) - config = metomi.rose.config.load(fname) - suite_rose_stem_version = config.get(['ROSE_STEM_VERSION']) - if suite_rose_stem_version: - suite_rose_stem_version = int(suite_rose_stem_version.value) - else: - suite_rose_stem_version = None - if not suite_rose_stem_version == ROSE_STEM_VERSION: - raise RoseStemVersionException(suite_rose_stem_version) - - def _prepend_localhost(self, url): - """Prepend the local hostname to urls which do not point to repository - locations.""" - if ':' not in url or url.split(':', 1)[0] not in ['svn', 'fcm', 'http', - 'https', 'svn+ssh']: - url = self.host_selector.get_local_host() + ':' + url - return url - - def process(self): - """Process STEM options into 'rose suite-run' options.""" - - # Generate options for source trees - repos = {} - repos_with_hosts = {} - if not self.opts.source: - self.opts.source = ['.'] - self.opts.project = list() - - for i, url in enumerate(self.opts.source): - project, url, base, rev, mirror = self._ascertain_project(url) - self.opts.source[i] = url - self.opts.project.append(project) - - # Versions of variables with hostname prepended for working copies - url_host = self._prepend_localhost(url) - base_host = self._prepend_localhost(base) - - if project in repos: - repos[project].append(url) - repos_with_hosts[project].append(url_host) - else: - repos[project] = [url] - repos_with_hosts[project] = [url_host] - self._add_define_option('SOURCE_' + project.upper() + '_REV', - '"' + rev + '"') - self._add_define_option('SOURCE_' + project.upper() + '_BASE', - '"' + base + '"') - self._add_define_option('HOST_SOURCE_' + project.upper() + - '_BASE', '"' + base_host + '"') - self._add_define_option('SOURCE_' + project.upper() + - '_MIRROR', '"' + mirror + '"') - self.reporter(SourceTreeAddedAsBranchEvent(url)) - for project, branches in repos.items(): - var = 'SOURCE_' + project.upper() - branchstring = RosePopener.list_to_shell_str(branches) - self._add_define_option(var, '"' + branchstring + '"') - for project, branches in repos_with_hosts.items(): - var_host = 'HOST_SOURCE_' + project.upper() - branchstring = RosePopener.list_to_shell_str(branches) - self._add_define_option(var_host, '"' + branchstring + '"') - - # Generate the variable containing tasks to run - if self.opts.group: - if not self.opts.defines: - self.opts.defines = [] - expanded_groups = [] - for i in self.opts.group: - expanded_groups.extend(i.split(',')) - self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' + - str(expanded_groups)) - - # Load the config file and return any automatic-options - auto_opts = self._read_auto_opts() - if auto_opts: - automatic_options = auto_opts.split() - for option in automatic_options: - elements = option.split("=") - if len(elements) == 2: - self._add_define_option( - elements[0], '"' + elements[1] + '"') - - # Change into the suite directory - if self.opts.conf_dir: - self.reporter(SuiteSelectionEvent(self.opts.conf_dir)) - self._check_suite_version( - os.path.join(self.opts.conf_dir, 'rose-suite.conf')) - else: - thissuite = self._this_suite() - self.fs_util.chdir(thissuite) - self.reporter(SuiteSelectionEvent(thissuite)) - - # Create a default name for the suite; allow override by user - if not self.opts.name: - self.opts.name = self._generate_name() - - return self.opts - - -def main(): - """Launcher for command line invokation of metomi.rose stem.""" - - # Process options - opt_parser = RoseOptionParser() - - option_keys = SuiteRunner.OPTIONS + OPTIONS - opt_parser.add_my_options(*option_keys) - opts, args = opt_parser.parse_args() - - # Set up a runner instance and process the options - stem = StemRunner(opts) - if opts.debug_mode: - opts = stem.process() - else: - try: - opts = stem.process() - except Exception as exc: - stem.reporter(exc) - sys.exit(1) - - # Get the suiterunner object and execute - runner = SuiteRunner(event_handler=stem.reporter, - popen=stem.popen, - fs_util=stem.fs_util) - if opts.debug_mode: - sys.exit(runner(opts, args)) - try: - sys.exit(runner(opts, args)) - except Exception as exc: - runner.handle_event(exc) - if isinstance(exc, RosePopenError): - sys.exit(exc.ret_code) - else: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/suite_clean.py b/metomi/rose/suite_clean.py deleted file mode 100644 index 491895d495..0000000000 --- a/metomi/rose/suite_clean.py +++ /dev/null @@ -1,206 +0,0 @@ -# -*- coding: utf-8 -*- -# ---------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ---------------------------------------------------------------------------- -"""Implement the "rose suite-clean" command.""" - -import os -from pipes import quote -from metomi.rose.config import ConfigLoader, ConfigNode, ConfigSyntaxError -from metomi.rose.fs_util import FileSystemEvent -from metomi.rose.host_select import HostSelector -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopenError -from metomi.rose.reporter import Reporter -from metomi.rose.suite_engine_proc import ( - SuiteEngineProcessor, SuiteStillRunningError) -import sys -import traceback -from uuid import uuid4 -from functools import cmp_to_key - - -class SuiteRunCleaner(object): - - """Logic to remove items created by the previous runs of suites.""" - - CLEANABLE_PATHS = ["share", "share/cycle", "work"] - - def __init__(self, event_handler=None, host_selector=None, - suite_engine_proc=None): - if event_handler is None: - event_handler = Reporter() - self.event_handler = event_handler - if host_selector is None: - host_selector = HostSelector(event_handler=event_handler) - self.host_selector = host_selector - if suite_engine_proc is None: - suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=event_handler) - self.suite_engine_proc = suite_engine_proc - - def clean(self, suite_name, only_items=None): - """Remove items created by the previous run of a suite. - - Change to user's $HOME for safety. - - """ - os.chdir(os.path.expanduser('~')) - self.suite_engine_proc.check_suite_not_running(suite_name) - self._clean(suite_name, only_items) - - def _clean(self, suite_name, only_items=None): - """Perform the cleaning operations.""" - engine = self.suite_engine_proc - suite_dir_rel = engine.get_suite_dir_rel(suite_name) - locs_file_path = engine.get_suite_dir( - suite_name, "log", "rose-suite-run.locs") - locs_conf = ConfigNode().set(["localhost"], {}) - try: - ConfigLoader().load(locs_file_path, locs_conf) - except IOError: - pass - items = self.CLEANABLE_PATHS + [""] - if only_items: - items = only_items - items.sort() - uuid_str = str(uuid4()) - for auth, node in sorted(locs_conf.value.items(), - key=cmp_to_key(self._auth_node_cmp)): - locs = [] - roots = set([""]) - for item in items: - if item: - locs.append(os.path.join(suite_dir_rel, item)) - else: - locs.append(suite_dir_rel) - if item and os.path.normpath(item) in self.CLEANABLE_PATHS: - item_root = node.get_value(["root-dir{" + item + "}"]) - if item_root is None: # backward compat - item_root = node.get_value(["root-dir-" + item]) - elif item == "": - item_root = node.get_value(["root-dir"]) - else: - continue - if item_root: - loc_rel = suite_dir_rel - if item: - loc_rel = os.path.join(suite_dir_rel, item) - locs.append(os.path.join(item_root, loc_rel)) - roots.add(item_root) - locs.reverse() - # Invoke bash as a login shell. The root location of a path may be - # in $DIR syntax, which can only be expanded correctly in a login - # shell. However, profile scripts invoked on login shell may print - # lots of junks. Hence we use a UUID here as a delimiter. Only - # output after the UUID lines are desirable lines. - command = ["bash", "-l", "-O", "extglob", "-c"] - sh_command = "cd; echo '%s'" % (uuid_str,) - if not self.host_selector.is_local_host(auth): - command = engine.popen.get_cmd("ssh", auth) + command - sh_command += "; ls -d -r %(locs)s; rm -fr %(locs)s" % { - "locs": engine.popen.list_to_shell_str(locs)} - if not only_items: - # Clean empty directories - # Change directory to root level to avoid cleaning them as - # well For cylc suites, e.g. it can clean up to an empty - # "cylc-run/" directory. - for root in roots: - names = [] - # Reverse sort to ensure that e.g. "share/cycle/" is - # cleaned before "share/" - for name in sorted(self.CLEANABLE_PATHS, reverse=True): - names.append(os.path.join(suite_dir_rel, name)) - if os.sep in suite_dir_rel: - names.append(os.path.dirname(suite_dir_rel)) - sh_command += ( - "; " + - "(cd %(root)s; " + - "rmdir -p %(names)s 2>/dev/null || true)" - ) % { - "root": root, - "names": engine.popen.list_to_shell_str(names), - } - if self.host_selector.is_local_host(auth): - command.append(sh_command) - else: - command.append(quote(sh_command)) - is_after_uuid_str = False - for line in engine.popen(*command)[0].splitlines(): - line = line.decode() - if is_after_uuid_str: - engine.handle_event(FileSystemEvent( - FileSystemEvent.DELETE, auth + ":" + line.strip())) - elif line == uuid_str: - is_after_uuid_str = True - - __call__ = clean - - @staticmethod - def _auth_node_cmp(item1, item2): - """Compare (auth1, node1) and (auth2, node2).""" - # This logic replicates output of the deprecated Python2 `cmp` builtin - ret = (item1 > item2) - (item1 < item2) - if ret: - if item1[0] == "localhost": - return -1 - elif item2[0] == "localhost": - return 1 - return ret - - -def main(): - """Implement the "rose suite-clean" command.""" - opt_parser = RoseOptionParser() - opt_parser.add_my_options("name", "non_interactive", "only_items") - opts, args = opt_parser.parse_args() - report = Reporter(opts.verbosity - opts.quietness) - cleaner = SuiteRunCleaner(event_handler=report) - if opts.name: - args.append(opts.name) - if not args: - args = [os.path.basename(os.getcwd())] - os.chdir(os.path.expanduser('~')) - n_done = 0 - for arg in args: - if not opts.non_interactive: - try: - answer = input("Clean %s? y/n (default n) " % arg) - except EOFError: - sys.exit(1) - if answer not in ["Y", "y"]: - continue - try: - cleaner.clean(arg, opts.only_items) - except ( - OSError, - IOError, - ConfigSyntaxError, - RosePopenError, - SuiteStillRunningError, - ) as exc: - report(exc) - if opts.debug_mode: - traceback.print_exc() - else: - n_done += 1 - sys.exit(len(args) - n_done) # Return 0 if everything done - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/suite_control.py b/metomi/rose/suite_control.py deleted file mode 100644 index 7945fc2183..0000000000 --- a/metomi/rose/suite_control.py +++ /dev/null @@ -1,131 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Launch suite engine's control commands.""" - -import os -from metomi.rose.fs_util import FileSystemUtil -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.reporter import Reporter -from metomi.rose.suite_engine_proc import SuiteEngineProcessor -import sys - - -YES = "y" -PROMPT = "Really %s %s? [" + YES + " or n (default)] " - - -class SuiteControl(object): - """Launch suite engine's control commands from the correct suite host.""" - - def __init__(self, event_handler=None): - self.event_handler = event_handler - self.suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=event_handler) - - def shutdown(self, suite_name, confirm=None, stderr=None, - stdout=None, *args): - """Shutdown the suite. - - suite_name: the name of the suite. - confirm: If specified, must be a callable with the interface - b = confirm("shutdown", suite_name, host). This method will - only issue the shutdown command to suite_name at host if b is - True. - stderr: A file handle for stderr, if relevant for suite engine. - stdout: A file handle for stdout, if relevant for suite engine. - args: extra arguments for the suite engine's shutdown command. - - """ - if confirm is None or confirm("shutdown", suite_name): - self.suite_engine_proc.shutdown(suite_name, args, stderr, stdout) - - -class SuiteNotFoundError(Exception): - - """An exception raised when a suite can't be found at or below cwd.""" - def __str__(self): - return ("%s - no suite found for this path." % self.args[0]) - - -def get_suite_name(event_handler=None): - """Find the top level of a suite directory structure""" - fs_util = FileSystemUtil(event_handler) - conf_dir = os.getcwd() - while True: - if os.path.basename(conf_dir) != "rose-stem": - for tail in [ - "rose-suite.conf", - "log/rose-suite-run.conf", - "rose-stem/rose-suite.conf"]: - conf = os.path.join(conf_dir, tail) - if os.path.exists(conf): - return os.path.basename(conf_dir) - up_dir = fs_util.dirname(conf_dir) - if up_dir == conf_dir: - raise SuiteNotFoundError(os.getcwd()) - conf_dir = up_dir - - -def prompt(action, suite_name, host): - """Prompt user to confirm action for suite_name at host.""" - if not host: - host = "localhost" - return input(PROMPT % (action, suite_name, host)).strip() in [YES] - - -def main(): - """Implement "rose suite-shutdown".""" - argv = sys.argv[1:] - method_name = argv.pop(0) - opt_parser = RoseOptionParser() - opt_parser.add_my_options("name", "non_interactive") - opts, args = opt_parser.parse_args(argv) - event_handler = Reporter(opts.verbosity - opts.quietness) - suite_control = SuiteControl(event_handler=event_handler) - method = getattr(suite_control, method_name) - confirm = None - suite_names = [] - if not opts.non_interactive: - confirm = prompt - else: - if opts.name: - suite_names.append(opts.name) - else: - try: - suite_name = get_suite_name(event_handler) - suite_names.append(suite_name) - except SuiteNotFoundError as exc: - event_handler(exc) - sys.exit(1) - - if opts.debug_mode: - for sname in suite_names: - method(sname, confirm, sys.stderr, sys.stdout, *args) - else: - for sname in suite_names: - try: - method(sname, confirm, sys.stderr, sys.stdout, *args) - except Exception as exc: - event_handler(exc) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/suite_engine_proc.py b/metomi/rose/suite_engine_proc.py index 26993f27bf..454aa95636 100644 --- a/metomi/rose/suite_engine_proc.py +++ b/metomi/rose/suite_engine_proc.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -19,21 +16,18 @@ # ----------------------------------------------------------------------------- """Suite engine processor management.""" -from metomi.isodatetime.data import Duration -from metomi.isodatetime.parsers import DurationParser, ISO8601SyntaxError -from glob import glob import os -import pwd import re +import sys + +from metomi.isodatetime.data import Duration +from metomi.isodatetime.parsers import DurationParser, ISO8601SyntaxError from metomi.rose.date import RoseDateTimeOperator, OffsetValueError from metomi.rose.fs_util import FileSystemUtil from metomi.rose.host_select import HostSelector from metomi.rose.popen import RosePopener from metomi.rose.reporter import Event -from metomi.rose.resource import ResourceLocator from metomi.rose.scheme_handler import SchemeHandlersManager -import sys -import webbrowser class NoSuiteLogError(Exception): @@ -62,7 +56,7 @@ def __str__(self): return "%s %s" % self.args -class BaseCycleOffset(object): +class BaseCycleOffset: """Represent a cycle time offset.""" @@ -163,35 +157,6 @@ def to_duration(self): return self.duration -class SuiteEngineGlobalConfCompatError(Exception): - - """An exception raised on incompatible global configuration.""" - - def __str__(self): - engine, key, value = self.args - return ("%s global configuration incompatible to Rose: %s=%s" % - (engine, key, value)) - - -class SuiteNotRunningError(Exception): - - """An exception raised when a suite is not running.""" - - def __str__(self): - return "%s: does not appear to be running" % (self.args) - - -class SuiteStillRunningError(Exception): - - """An exception raised when a suite is still running.""" - - FMT_HEAD = "Suite \"%(suite_name)s\" appears to be running:\n" - - def __str__(self): - suite_name, extras = self.args - return self.FMT_HEAD % {"suite_name": suite_name} + "".join(extras) - - class CycleOffsetError(ValueError): """Unrecognised cycle time offset format.""" @@ -213,7 +178,7 @@ def __str__(self): return self.args[0] + ": unrecognised cycling mode." -class TaskProps(object): +class TaskProps: """Task properties. @@ -291,7 +256,7 @@ def __str__(self): return ret -class SuiteEngineProcessor(object): +class SuiteEngineProcessor: """An abstract suite engine processor.""" TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"} @@ -331,47 +296,6 @@ def __init__(self, event_handler=None, popen=None, fs_util=None, self.host_selector = host_selector self.date_time_oper = RoseDateTimeOperator() - def check_global_conf_compat(self): - """Raise exception on suite engine specific incompatibity. - - Should raise SuiteEngineGlobalConfCompatError. - - """ - raise NotImplementedError() - - def check_suite_not_running(self, suite_name): - """Check that suite is not running. - - This method is not implemented. Sub-class should override. - - Arguments: - suite_name: name of suite to check. - - Raise: - SuiteStillRunningError: - Should raise SuiteStillRunningError if suite is still running. - """ - raise NotImplementedError() - - def cmp_suite_conf( - self, suite_name, run_mode, strict_mode=False, debug_mode=False): - """Compare current suite configuration with that in the previous run. - - An implementation of this method should: - * Raise an exception on failure. - * Return True if suite configuration is unmodified c.f. previous run. - * Return False otherwise. - - """ - raise NotImplementedError() - - def get_suite_contact(self, suite_name): - """Return suite contact information for a user suite. - - Return (dict): suite contact information. - """ - raise NotImplementedError() - def get_suite_dir(self, suite_name, *paths): """Return the path to the suite running directory. @@ -389,48 +313,10 @@ def get_suite_dir_rel(self, suite_name, *paths): """ raise NotImplementedError() - def get_suite_log_url(self, user_name, suite_name): - """Return the "rose bush" URL for a user's suite.""" - prefix = "~" - if user_name: - prefix += user_name - suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name)) - suite_d = os.path.expanduser(suite_d) - if not os.path.isdir(suite_d): - raise NoSuiteLogError(user_name, suite_name) - rose_bush_url = None - for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")): - status = {} - for line in open(f_name): - key, value = line.strip().split("=", 1) - status[key] = value - if status.get("host"): - rose_bush_url = "http://" + status["host"] - if status.get("port"): - rose_bush_url += ":" + status["port"] - rose_bush_url += "/" - break - if not rose_bush_url: - conf = ResourceLocator.default().get_conf() - rose_bush_url = conf.get_value( - ["rose-suite-log", "rose-bush"]) - if not rose_bush_url: - return "file://" + suite_d - if not rose_bush_url.endswith("/"): - rose_bush_url += "/" - if not user_name: - user_name = pwd.getpwuid(os.getuid()).pw_name - return rose_bush_url + "/".join( - ["taskjobs", user_name, suite_name]) - def get_task_auth(self, suite_name, task_name): """Return [user@]host for a remote task in a suite.""" raise NotImplementedError() - def get_tasks_auths(self, suite_name): - """Return a list of [user@]host for remote tasks in a suite.""" - raise NotImplementedError() - def get_task_props(self, *args, **kwargs): """Return a TaskProps object containing suite task's attributes.""" calendar_mode = self.date_time_oper.get_calendar_mode() @@ -531,23 +417,11 @@ def get_task_props_from_env(self): """ raise NotImplementedError() - def get_version(self): - """Return the version string of the suite engine.""" - raise NotImplementedError() - - def get_version_env_name(self): - """Return the name of the suite engine version environment variable.""" - return self.SCHEME.upper() + "_VERSION" - def handle_event(self, *args, **kwargs): """Call self.event_handler if it is callable.""" if callable(self.event_handler): return self.event_handler(*args, **kwargs) - def is_suite_registered(self, suite_name): - """Return whether or not a suite is registered.""" - raise NotImplementedError() - def job_logs_archive(self, suite_name, items): """Archive cycle job logs. @@ -578,30 +452,10 @@ def job_logs_remove_on_server(self, suite_name, items): """ raise NotImplementedError() - def launch_suite_log_browser(self, user_name, suite_name): - """Launch web browser to view suite log. - - Return URL of suite log on success, None otherwise. - - """ - url = self.get_suite_log_url(user_name, suite_name) - browser = webbrowser.get() - browser.open(url, new=True, autoraise=True) - self.handle_event(WebBrowserEvent(browser.name, url)) - return url - def parse_job_log_rel_path(self, f_name): """Return (cycle, task, submit_num, ext) for a job log rel path.""" raise NotImplementedError() - def run(self, suite_name, host=None, run_mode=None, args=None): - """Start a suite (in a specified host).""" - raise NotImplementedError() - - def shutdown(self, suite_name, args=None, stderr=None, stdout=None): - """Shut down the suite.""" - raise NotImplementedError() - def _get_offset_cycle_time(self, cycle, cycle_offset): """Return the actual date time of an BaseCycleOffset against cycle. diff --git a/metomi/rose/suite_engine_procs/__init__.py b/metomi/rose/suite_engine_procs/__init__.py index 8460776f05..f7b71f8284 100644 --- a/metomi/rose/suite_engine_procs/__init__.py +++ b/metomi/rose/suite_engine_procs/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/suite_engine_procs/cylc.py b/metomi/rose/suite_engine_procs/cylc.py index 0ec4fb84e8..0475d9c363 100644 --- a/metomi/rose/suite_engine_procs/cylc.py +++ b/metomi/rose/suite_engine_procs/cylc.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -19,7 +16,6 @@ # ----------------------------------------------------------------------------- """Logic specific to the Cylc suite engine.""" -import filecmp from glob import glob import os import pwd @@ -28,33 +24,26 @@ import socket import sqlite3 import tarfile -from tempfile import mkstemp from time import sleep from uuid import uuid4 from metomi.rose.fs_util import FileSystemEvent from metomi.rose.popen import RosePopenError -from metomi.rose.reporter import Event, Reporter +from metomi.rose.reporter import Reporter from metomi.rose.suite_engine_proc import ( - SuiteEngineProcessor, SuiteEngineGlobalConfCompatError, - SuiteNotRunningError, SuiteStillRunningError, TaskProps) - - -_PORT_SCAN = "port-scan" + SuiteEngineProcessor, + TaskProps +) class CylcProcessor(SuiteEngineProcessor): """Logic specific to the cylc suite engine.""" - CONTACT_KEYS = ( - "CYLC_SUITE_HOST", "CYLC_SUITE_OWNER", "CYLC_SUITE_PORT", - "CYLC_SUITE_PROCESS") - PGREP_CYLC_RUN = r"python.*/bin/cylc-(run|restart)( | .+ )%s( |$)" REC_CYCLE_TIME = re.compile( r"\A[\+\-]?\d+(?:W\d+)?(?:T\d+(?:Z|[+-]\d+)?)?\Z") # Good enough? SCHEME = "cylc" - SUITE_CONF = "suite.rc" + SUITE_CONF = "flow.cylc" SUITE_NAME_ENV = "CYLC_SUITE_NAME" SUITE_DIR_REL_ROOT = "cylc-run" TASK_ID_DELIM = "." @@ -67,108 +56,6 @@ def __init__(self, *args, **kwargs): self.host = None self.user = None - def check_global_conf_compat(self): - """Raise exception on incompatible Cylc global configuration.""" - expected = os.path.join("~", self.SUITE_DIR_REL_ROOT) - expected = os.path.expanduser(expected) - for key in ["[hosts][localhost]run directory", - "[hosts][localhost]work directory"]: - out = self.popen("cylc", "get-global-config", "-i", key)[0] - lines = out.splitlines() - try: - lines[0] = lines[0].decode() - except AttributeError: - pass - if lines and lines[0] != expected: - raise SuiteEngineGlobalConfCompatError( - self.SCHEME, key, lines[0]) - - def check_suite_not_running(self, suite_name): - """Check if a suite is still running. - - Args: - suite_name (str): the name of the suite as known by Cylc. - Raise: - SuiteStillRunningError: if suite is still running. - """ - try: - contact_info = self.get_suite_contact(suite_name) - except SuiteNotRunningError: - return # No contact file, suite not running - else: - fname = self.get_suite_dir(suite_name, ".service", "contact") - extras = ["Contact info from: \"%s\"\n" % fname] - for key, value in sorted(contact_info.items()): - if key in self.CONTACT_KEYS: - extras.append(" %s=%s\n" % (key, value)) - extras.append("Try \"cylc stop '%s'\" first?" % suite_name) - raise SuiteStillRunningError(suite_name, extras) - - def cmp_suite_conf( - self, suite_name, run_mode, strict_mode=False, debug_mode=False): - """Parse and compare current "suite.rc" with that in the previous run. - - (Re-)register and validate the "suite.rc" file. - Raise RosePopenError on failure. - Return True if "suite.rc.processed" is unmodified c.f. previous run. - Return False otherwise. - - """ - suite_dir = self.get_suite_dir(suite_name) - if run_mode == "run": - self.popen.run_simple("cylc", "register", suite_name, suite_dir) - f_desc, new_suite_rc_processed = mkstemp() - os.close(f_desc) - command = ["cylc", "validate", "-o", new_suite_rc_processed] - if debug_mode: - command.append("--debug") - if strict_mode: - command.append("--strict") - command.append(suite_name) - old_suite_rc_processed = os.path.join(suite_dir, "suite.rc.processed") - try: - self.popen.run_simple(*command, stdout_level=Event.V) - return ( - os.path.exists(old_suite_rc_processed) and - filecmp.cmp(old_suite_rc_processed, new_suite_rc_processed)) - finally: - os.unlink(new_suite_rc_processed) - - def get_running_suites(self): - """Return a list containing the names of running suites.""" - rootd = os.path.join(os.path.expanduser('~'), self.SUITE_DIR_REL_ROOT) - sub_names = ['.service', 'log', 'share', 'work', self.SUITE_CONF] - items = [] - for dirpath, dnames, fnames in os.walk(rootd, followlinks=True): - if dirpath != rootd and any( - name in dnames + fnames for name in sub_names): - dnames[:] = [] - else: - continue - if os.path.exists(os.path.join(dirpath, '.service', 'contact')): - items.append(os.path.relpath(dirpath, rootd)) - return items - - def get_suite_contact(self, suite_name): - """Return suite contact information for a user suite. - - Return (dict): suite contact information. - """ - try: - # Note: low level directory open ensures that the file system - # containing the contact file is synchronised, e.g. in an NFS - # environment. - os.close(os.open( - self.get_suite_dir(suite_name, ".service"), os.O_DIRECTORY)) - ret = {} - for line in open( - self.get_suite_dir(suite_name, ".service", "contact")): - key, value = [item.strip() for item in line.split("=", 1)] - ret[key] = value - return ret - except (IOError, OSError, ValueError): - raise SuiteNotRunningError(suite_name) - @classmethod def get_suite_dir_rel(cls, suite_name, *paths): """Return the relative path to the suite running directory. @@ -180,81 +67,33 @@ def get_suite_dir_rel(cls, suite_name, *paths): def get_suite_jobs_auths(self, suite_name, cycle_name_tuples=None): """Return remote ["[user@]host", ...] for submitted jobs.""" - auths = [] - stmt = "SELECT DISTINCT user_at_host FROM task_jobs" - stmt_where_list = [] - stmt_args = [] - if cycle_name_tuples: - for cycle, name in cycle_name_tuples: - stmt_fragments = [] - if cycle is not None: - stmt_fragments.append("cycle==?") - stmt_args.append(cycle) - if name is not None: - stmt_fragments.append("name==?") - stmt_args.append(name) - stmt_where_list.append(" AND ".join(stmt_fragments)) - if stmt_where_list: - stmt += " WHERE (" + ") OR (".join(stmt_where_list) + ")" - for row in self._db_exec(suite_name, stmt, stmt_args): - if row and row[0]: - auth = self._parse_user_host(auth=row[0]) - if auth: - auths.append(auth) - self._db_close(suite_name) - return auths + # TODO: reimplement for Cylc8? + # https://github.com/metomi/rose/issues/2445 + self.handle_event( + Exception( + 'WARNING: Rose cannot currently inspect the platform a Cylc' + ' task has or will run on.\n' + ' https://github.com/metomi/rose/issues/2445' + ) + ) + return [] def get_task_auth(self, suite_name, task_name): - """ - Return [user@]host for a remote task in a suite. + """Return [user@]host for a remote task in a suite. Or None if task does not run remotely. """ - try: - out = self.popen( - "cylc", "get-config", "-o", - "-i", "[runtime][%s][remote]owner" % task_name, - "-i", "[runtime][%s][remote]host" % task_name, - suite_name)[0] - except RosePopenError: - return - user, host = (None, None) - items = out.strip().split(None, 1) - if items: - user = items.pop(0).replace(b"*", b" ") - if items: - host = items.pop(0).replace(b"*", b" ") - return self._parse_user_host(user=user, host=host) - - def get_tasks_auths(self, suite_name): - """Return a list of unique [user@]host for remote tasks in a suite.""" - actual_hosts = {} - auths = [] - out = self.popen("cylc", "get-config", "-ao", - "-i", "[remote]owner", - "-i", "[remote]host", - suite_name)[0] - for line in out.splitlines(): - items = line.split(None, 2) - user, host = (None, None) - items.pop(0) - if items: - user = items.pop(0).decode().replace("*", " ") - if items: - host = items.pop(0).decode().replace("*", " ") - if host in actual_hosts: - host = str(actual_hosts[host]) - auth = self._parse_user_host(user=user, host=host) - else: - auth = self._parse_user_host(user=user, host=host) - if auth and "@" in auth: - actual_hosts[host] = auth.split("@", 1)[1] - else: - actual_hosts[host] = auth - if auth and auth not in auths: - auths.append(auth) - return auths + # TODO: reimplement for Cylc8? + # https://github.com/metomi/rose/issues/2445 + self.handle_event( + Exception( + 'WARNING: Rose cannot currently inspect the platform a Cylc' + ' task has or will run on.\n' + ' https://github.com/metomi/rose/issues/2445' + ) + ) + return None def get_task_props_from_env(self): """Get attributes of a suite task from environment variables. @@ -288,17 +127,6 @@ def get_task_props_from_env(self): task_is_cold_start=task_is_cold_start, cycling_mode=cycling_mode) - def get_version(self): - """Return Cylc's version.""" - return self.popen("cylc", "--version")[0].strip() - - def is_suite_registered(self, suite_name): - """See if a suite is registered - Return True directory for a suite if it is registered - Return False otherwise - """ - return self.popen.run("cylc", "get-directory", suite_name)[0] == 0 - def job_logs_archive(self, suite_name, items): """Archive cycle job logs. @@ -503,62 +331,6 @@ def parse_job_log_rel_path(cls, f_name): """Return (cycle, task, submit_num, ext).""" return f_name.replace("log/job/", "").split("/", 3) - @staticmethod - def process_suite_hook_args(*args, **_): - """Rearrange args for TaskHook.run.""" - task = None - if len(args) == 3: - hook_event, suite, hook_message = args - else: - hook_event, suite, task, hook_message = args - return [suite, task, hook_event, hook_message] - - def run(self, suite_name, host=None, run_mode=None, args=None): - """Invoke "cylc run" (in a specified host). - - The current working directory is assumed to be the suite log directory. - - suite_name: the name of the suite. - host: the host to run the suite. "localhost" if None. - run_mode: call "cylc restart|reload" instead of "cylc run". - args: arguments to pass to "cylc run". - - """ - if run_mode not in ['reload', 'restart', 'run']: - run_mode = 'run' - cmd = ['cylc', run_mode] - if run_mode == 'reload': - cmd.append('--force') - if host and not self.host_selector.is_local_host(host): - cmd.append('--host=%s' % host) - cmd.append(suite_name) - cmd += args - out, err = self.popen(*cmd) - if err: - self.handle_event(err, kind=Event.KIND_ERR) - if out: - self.handle_event(out) - - def shutdown(self, suite_name, args=None, stderr=None, stdout=None): - """Shut down the suite. - - suite_name -- the name of the suite. - stderr -- A file handle for stderr, if relevant for suite engine. - stdout -- A file handle for stdout, if relevant for suite engine. - args -- extra arguments for "cylc shutdown". - - """ - contact_info = self.get_suite_contact(suite_name) - if not contact_info: - raise SuiteNotRunningError(suite_name) - environ = dict(os.environ) - environ.update({'CYLC_VERSION': contact_info['CYLC_VERSION']}) - command = ["cylc", "shutdown", suite_name, "--force"] - if args: - command += args - self.popen.run_simple( - *command, env=environ, stderr=stderr, stdout=stdout) - def _db_close(self, suite_name): """Close a named database connection.""" if self.daos.get(suite_name) is not None: @@ -621,7 +393,7 @@ def _parse_user_host(self, auth=None, user=None, host=None): return auth -class CylcSuiteDAO(object): +class CylcSuiteDAO: """Generic SQLite Data Access Object.""" diff --git a/metomi/rose/suite_hook.py b/metomi/rose/suite_hook.py deleted file mode 100644 index 306db4482b..0000000000 --- a/metomi/rose/suite_hook.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- - -"""Hook functionalities for a suite.""" - -from email.mime.text import MIMEText -import os -import pwd -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopener -from metomi.rose.reporter import Reporter -from metomi.rose.resource import ResourceLocator -from metomi.rose.suite_engine_proc import SuiteEngineProcessor -from smtplib import SMTP, SMTPException -import socket - - -class RoseSuiteHook(object): - - """Hook functionalities for a suite.""" - - def __init__(self, event_handler=None, popen=None, suite_engine_proc=None): - self.event_handler = event_handler - if popen is None: - popen = RosePopener(event_handler) - self.popen = popen - if suite_engine_proc is None: - suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=event_handler, popen=popen) - self.suite_engine_proc = suite_engine_proc - - def handle_event(self, *args, **kwargs): - """Call self.event_handler if it is callabale.""" - if callable(self.event_handler): - return self.event_handler(*args, **kwargs) - - def run(self, suite_name, task_id, hook_event, hook_message=None, - should_mail=False, mail_cc_list=None, should_shutdown=False, - should_retrieve_job_logs=False): - """ - Invoke the hook for a suite. - - 1. For a task hook, if the task runs remotely, retrieve its log from - the remote host. - 2. If "should_mail", send an email notification to the current user, - and those in the "mail_cc_list". - 3. If "should_shutdown", shut down the suite. - - """ - # Retrieve log and populate job logs database - task_ids = [] - if task_id and should_retrieve_job_logs: - task_ids = [task_id] - self.suite_engine_proc.job_logs_pull_remote(suite_name, task_ids) - - # Send email notification if required - email_exc = None - if should_mail: - text = "" - if task_id: - text += "Task: %s\n" % task_id - if hook_message: - text += "Message: %s\n" % hook_message - url = self.suite_engine_proc.get_suite_log_url(None, suite_name) - text += "See: %s\n" % (url) - user = pwd.getpwuid(os.getuid()).pw_name - conf = ResourceLocator.default().get_conf() - host = conf.get_value(["rose-suite-hook", "email-host"], - default="localhost") - msg = MIMEText(text) - msg["From"] = user + "@" + host - msg["To"] = msg["From"] - if mail_cc_list: - mail_cc_addresses = [] - for mail_cc_address in mail_cc_list: - if "@" not in mail_cc_address: - mail_cc_address += "@" + host - mail_cc_addresses.append(mail_cc_address) - msg["Cc"] = ", ".join(mail_cc_addresses) - mail_cc_list = mail_cc_addresses - else: - mail_cc_list = [] - msg["Subject"] = "[%s] %s" % (hook_event, suite_name) - smtp_host = conf.get_value(["rose-suite-hook", "smtp-host"], - default="localhost") - try: - smtp = SMTP(smtp_host) - smtp.sendmail( - msg["From"], [msg["To"]] + mail_cc_list, msg.as_string()) - smtp.quit() - except (socket.error, SMTPException) as email_exc: - pass - - # Shut down if required - if should_shutdown: - self.suite_engine_proc.shutdown(suite_name, args=["--kill"]) - - if email_exc is not None: - raise - - __call__ = run - - -def main(): - """Implement "rose suite-hook" command.""" - opt_parser = RoseOptionParser() - opt_parser.add_my_options( - "mail_cc", "mail", "retrieve_job_logs", "shutdown") - opts, args = opt_parser.parse_args() - for key in ["mail_cc"]: - values = [] - if getattr(opts, key): - for value in getattr(opts, key): - values.extend(value.split(",")) - setattr(opts, key, values) - report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default - popen = RosePopener(event_handler=report) - suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=report, popen=popen) - args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts)) - hook = RoseSuiteHook(event_handler=report, - popen=popen, - suite_engine_proc=suite_engine_proc) - hook(*args, - should_mail=opts.mail, - mail_cc_list=opts.mail_cc, - should_shutdown=opts.shutdown, - should_retrieve_job_logs=opts.retrieve_job_logs) - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/suite_log.py b/metomi/rose/suite_log.py deleted file mode 100644 index 150eeed738..0000000000 --- a/metomi/rose/suite_log.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Implement "rose suite-log" CLI.""" - -import os -import pwd -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.reporter import Event, Reporter -from metomi.rose.suite_engine_proc import SuiteEngineProcessor -from metomi.rose.suite_control import get_suite_name -import sys -from time import sleep -import traceback - - -class RoseBushStartEvent(Event): - - """Event raised on "rose bush start".""" - - def __str__(self): - return ("""Rose bush started:\n%s""" % self.args[0] + - """Run "rose bush stop" when no longer required.""") - - -def main(): - """Implement "rose suite-log" CLI.""" - opt_parser = RoseOptionParser() - opt_parser.add_my_options("archive_mode", "force_mode", "name", - "non_interactive", "prune_remote_mode", - "update_mode", "user", "view_mode") - opts, args = opt_parser.parse_args() - report = Reporter(opts.verbosity - opts.quietness) - - try: - suite_log_view(opts, args, report) - except Exception as exc: - report(exc) - if opts.debug_mode: - traceback.print_exc() - sys.exit(1) - - -def suite_log_view(opts, args, event_handler=None): - """Implement "rose suite-log" CLI functionality.""" - suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=event_handler) - opts.update_mode = ( - opts.update_mode or opts.archive_mode or opts.force_mode) - if opts.force_mode: - args = ["*"] - if not opts.name: - opts.name = get_suite_name(event_handler) - if not opts.update_mode and not opts.user: - opts.user = pwd.getpwuid(os.stat(".").st_uid).pw_name - if opts.archive_mode: - suite_engine_proc.job_logs_archive(opts.name, args) - elif opts.update_mode: - suite_engine_proc.job_logs_pull_remote( - opts.name, args, opts.prune_remote_mode, opts.force_mode) - if opts.view_mode or not opts.update_mode: - n_tries_left = 1 - is_rose_bush_started = False - url = suite_engine_proc.get_suite_log_url(opts.user, opts.name) - if url.startswith("file://"): - if (opts.non_interactive or - input( - "Start rose bush? [y/n] (default=n) ") == "y"): - suite_engine_proc.popen.run_bg( - "rose", "bush", "start", preexec_fn=os.setpgrp) - is_rose_bush_started = True - n_tries_left = 5 # Give the server a chance to start - while n_tries_left: - n_tries_left -= 1 - if n_tries_left: - url = suite_engine_proc.get_suite_log_url(opts.user, opts.name) - if url.startswith("file://"): - sleep(1) - continue - suite_engine_proc.launch_suite_log_browser(opts.user, opts.name) - break - if is_rose_bush_started: - status = suite_engine_proc.popen("rose", "bush")[0] - event_handler(RoseBushStartEvent(status)) - return - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/suite_restart.py b/metomi/rose/suite_restart.py deleted file mode 100644 index e5292721a0..0000000000 --- a/metomi/rose/suite_restart.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Implement "rose suite-restart-only".""" - -import os -import sys -import traceback - -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopenError -from metomi.rose.reporter import Reporter -from metomi.rose.suite_control import get_suite_name, SuiteNotFoundError -from metomi.rose.suite_engine_proc import SuiteEngineProcessor - - -class SuiteRestarter(object): - - """Wrap "cylc restart".""" - - def __init__(self, event_handler=None): - self.event_handler = event_handler - self.suite_engine_proc = SuiteEngineProcessor.get_processor( - event_handler=self.event_handler) - - def handle_event(self, *args, **kwargs): - """Handle event.""" - if callable(self.event_handler): - self.event_handler(*args, **kwargs) - - def restart(self, suite_name=None, host=None, args=None): - """Restart a "cylc" suite.""" - # Check suite engine specific compatibility - self.suite_engine_proc.check_global_conf_compat() - - if not suite_name: - suite_name = get_suite_name(self.event_handler) - - suite_dir = self.suite_engine_proc.get_suite_dir(suite_name) - if not os.path.exists(suite_dir): - raise SuiteNotFoundError(suite_dir) - - # Ensure suite is not running - self.suite_engine_proc.check_suite_not_running(suite_name) - - # Restart the suite - self.suite_engine_proc.run(suite_name, host, "restart", args) - - return - - -def main(): - """Launcher for the CLI.""" - opt_parser = RoseOptionParser() - opt_parser.add_my_options("host", "name") - opts, args = opt_parser.parse_args(sys.argv[1:]) - event_handler = Reporter(opts.verbosity - opts.quietness) - suite_restarter = SuiteRestarter(event_handler) - try: - sys.exit(suite_restarter.restart( - suite_name=opts.name, - host=opts.host, - args=args)) - except Exception as exc: - event_handler(exc) - if opts.debug_mode: - traceback.print_exc() - if isinstance(exc, RosePopenError): - sys.exit(exc.ret_code) - else: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/suite_run.py b/metomi/rose/suite_run.py deleted file mode 100644 index dff53bebaa..0000000000 --- a/metomi/rose/suite_run.py +++ /dev/null @@ -1,694 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Implement "rose suite-run".""" - -import ast -from datetime import datetime -from fnmatch import fnmatchcase -from glob import glob -import os -import pipes -from metomi.rose.config import ConfigDumper, ConfigLoader, ConfigNode -from metomi.rose.env import env_var_process -from metomi.rose.host_select import HostSelector -from metomi.rose.opt_parse import RoseOptionParser -from metomi.rose.popen import RosePopenError -from metomi.rose.reporter import Event, Reporter, ReporterContext -from metomi.rose.resource import ResourceLocator -from metomi.rose.run import ConfigValueError, NewModeError, Runner -from metomi.rose.run_source_vc import write_source_vc_info -from metomi.rose.suite_clean import SuiteRunCleaner -import shutil -import sys -from tempfile import TemporaryFile, mkdtemp -from time import sleep, strftime, time -import traceback - - -class VersionMismatchError(Exception): - - """An exception raised when there is a version mismatch.""" - - def __str__(self): - return "Version expected=%s, actual=%s" % self.args - - -class SkipReloadEvent(Event): - - """An event raised to report that suite configuration reload is skipped.""" - - def __str__(self): - return "%s: reload complete. \"%s\" unchanged" % self.args - - -class SuiteLogArchiveEvent(Event): - - """An event raised to report the archiving of a suite log directory.""" - - def __str__(self): - return "%s <= %s" % self.args - - -class SuiteRunner(Runner): - - """Invoke a Rose suite.""" - - SLEEP_PIPE = 0.05 - NAME = "suite" - OPTIONS = [ - "conf_dir", - "defines", - "defines_suite", - "host", - "install_only_mode", - "local_install_only_mode", - "log_archive_mode", - "log_keep", - "log_name", - "name", - "new_mode", - "no_overwrite_mode", - "opt_conf_keys", - "reload_mode", - "remote", - "restart_mode", - "run_mode", - "strict_mode", - "validate_suite_only"] - - # Lists of rsync (always) exclude globs - SYNC_EXCLUDES = ( - "/.*", - "/cylc-suite.db", - "/log", - "/log.*", - "/state", - "/share", - "/work", - ) - - def __init__(self, *args, **kwargs): - Runner.__init__(self, *args, **kwargs) - self.host_selector = HostSelector(self.event_handler, self.popen) - self.suite_run_cleaner = SuiteRunCleaner( - event_handler=self.event_handler, - host_selector=self.host_selector, - suite_engine_proc=self.suite_engine_proc) - - def run_impl(self, opts, args, uuid, work_files): - # Log file, temporary - if hasattr(self.event_handler, "contexts"): - t_file = TemporaryFile() - log_context = ReporterContext(None, self.event_handler.VV, t_file) - self.event_handler.contexts[uuid] = log_context - - # Check suite engine specific compatibility - self.suite_engine_proc.check_global_conf_compat() - - # Suite name from the current working directory - if opts.conf_dir: - self.fs_util.chdir(opts.conf_dir) - opts.conf_dir = os.getcwd() - - # --remote=KEY=VALUE,... - if opts.remote: - # opts.name always set for remote. - return self._run_remote(opts, opts.name) - - conf_tree = self.config_load(opts) - self.fs_util.chdir(conf_tree.conf_dirs[0]) - - suite_name = opts.name - if not opts.name: - suite_name = os.path.basename(os.getcwd()) - - # Check suite.rc #! line for template scheme - templ_scheme = "jinja2" - if self.suite_engine_proc.SUITE_CONF in conf_tree.files: - suiterc_path = os.path.join( - conf_tree.files[self.suite_engine_proc.SUITE_CONF], - self.suite_engine_proc.SUITE_CONF) - with open(suiterc_path) as fh: - line = fh.readline() - if line.startswith("#!"): - templ_scheme = line[2:].strip().lower() - suite_section = (templ_scheme + ':' + - self.suite_engine_proc.SUITE_CONF) - - extra_defines = [] - if opts.defines_suite: - for define in opts.defines_suite: - extra_defines.append("[" + suite_section + "]" + define) - - # Automatic Rose constants - # ROSE_ORIG_HOST: originating host - # ROSE_VERSION: Rose version (not retained in run_mode=="reload") - # Suite engine version - my_rose_version = ResourceLocator.default().get_version() - suite_engine_key = self.suite_engine_proc.get_version_env_name() - if opts.run_mode in ["reload", "restart"]: - prev_config_path = self.suite_engine_proc.get_suite_dir( - suite_name, "log", "rose-suite-run.conf") - prev_config = ConfigLoader()(prev_config_path) - suite_engine_version = prev_config.get_value( - ["env", suite_engine_key]) - else: - suite_engine_version =\ - self.suite_engine_proc.get_version().decode() - resloc = ResourceLocator.default() - auto_items = [ - (suite_engine_key, suite_engine_version), - ("ROSE_ORIG_HOST", self.host_selector.get_local_host()), - ("ROSE_SITE", resloc.get_conf().get_value(['site'], '')), - ("ROSE_VERSION", resloc.get_version())] - for key, val in auto_items: - requested_value = conf_tree.node.get_value(["env", key]) - if requested_value: - if key == "ROSE_VERSION" and val != requested_value: - exc = VersionMismatchError(requested_value, val) - raise ConfigValueError(["env", key], requested_value, exc) - val = requested_value - else: - conf_tree.node.set(["env", key], val, - state=conf_tree.node.STATE_NORMAL) - extra_defines.append('[%s]%s="%s"' % (suite_section, key, val)) - - # Pass automatic Rose constants as suite defines - self.conf_tree_loader.node_loader.load(extra_defines, conf_tree.node) - - # See if suite is running or not - if opts.run_mode == "reload": - # Check suite is running - self.suite_engine_proc.get_suite_contact(suite_name) - else: - self.suite_engine_proc.check_suite_not_running(suite_name) - - # Install the suite to its run location - suite_dir_rel = self._suite_dir_rel(suite_name) - - # Unfortunately a large try/finally block to ensure a temporary folder - # created in validate only mode is cleaned up. Exceptions are not - # caught here - try: - # Process Environment Variables - environ = self.config_pm(conf_tree, "env") - - if opts.validate_suite_only_mode: - temp_dir = mkdtemp() - suite_dir = os.path.join(temp_dir, suite_dir_rel) - os.makedirs(suite_dir, 0o0700) - else: - suite_dir = os.path.join( - os.path.expanduser("~"), suite_dir_rel) - - suite_conf_dir = os.getcwd() - locs_conf = ConfigNode() - if opts.new_mode: - if os.getcwd() == suite_dir: - raise NewModeError("PWD", os.getcwd()) - elif opts.run_mode in ["reload", "restart"]: - raise NewModeError("--run", opts.run_mode) - self.suite_run_cleaner.clean(suite_name) - if os.getcwd() != suite_dir: - if opts.run_mode == "run": - self._run_init_dir(opts, suite_name, conf_tree, - locs_conf=locs_conf) - os.chdir(suite_dir) - - # Housekeep log files - now_str = None - if not opts.install_only_mode and not opts.local_install_only_mode: - now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - self._run_init_dir_log(opts, now_str) - self.fs_util.makedirs("log/suite") - - # Rose configuration and version logs - self.fs_util.makedirs("log/rose-conf") - run_mode = opts.run_mode - if run_mode not in ["reload", "restart", "run"]: - run_mode = "run" - mode = run_mode - if opts.validate_suite_only_mode: - mode = "validate-suite-only" - elif opts.install_only_mode: - mode = "install-only" - elif opts.local_install_only_mode: - mode = "local-install-only" - prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode) - - # Dump the actual configuration as rose-suite-run.conf - ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf") - - # Install version information file - write_source_vc_info( - suite_conf_dir, "log/" + prefix + ".version", self.popen) - - # If run through rose-stem, install version information - # files for each source tree if they're a working copy - if hasattr(opts, 'source') and hasattr(opts, 'project'): - for i, url in enumerate(opts.source): - if os.path.isdir(url): - write_source_vc_info( - url, "log/" + opts.project[i] + "-" + str(i) + - ".version", self.popen) - - for ext in [".conf", ".version"]: - self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext) - - # Move temporary log to permanent log - if hasattr(self.event_handler, "contexts"): - log_file_path = os.path.abspath( - os.path.join("log", "rose-suite-run.log")) - log_file = open(log_file_path, "ab") - temp_log_file = self.event_handler.contexts[uuid].handle - temp_log_file.seek(0) - log_file.write(temp_log_file.read()) - self.event_handler.contexts[uuid].handle = log_file - temp_log_file.close() - - # Process Files - cwd = os.getcwd() - for rel_path, conf_dir in conf_tree.files.items(): - if (conf_dir == cwd or - any(fnmatchcase(os.sep + rel_path, exclude) - for exclude in self.SYNC_EXCLUDES) or - conf_tree.node.get( - [templ_scheme + ":" + rel_path]) is not None): - continue - # No sub-directories, very slow otherwise - if os.sep in rel_path: - rel_path = rel_path.split(os.sep, 1)[0] - target_key = self.config_pm.get_handler( - "file").PREFIX + rel_path - target_node = conf_tree.node.get([target_key]) - if target_node is None: - conf_tree.node.set([target_key]) - target_node = conf_tree.node.get([target_key]) - elif target_node.is_ignored(): - continue - source_node = target_node.get("source") - if source_node is None: - target_node.set( - ["source"], os.path.join( - conf_dir, rel_path)) - elif source_node.is_ignored(): - continue - self.config_pm(conf_tree, "file", - no_overwrite_mode=opts.no_overwrite_mode) - - # Process suite configuration template header - # (e.g. Jinja2:suite.rc, EmPy:suite.rc) - self.config_pm(conf_tree, templ_scheme, environ=environ) - - # Ask suite engine to parse suite configuration - # and determine if it is up to date (unchanged) - if opts.validate_suite_only_mode: - suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf( - suite_dir, None, opts.strict_mode, - debug_mode=True) - else: - suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf( - suite_name, opts.run_mode, opts.strict_mode, - opts.debug_mode) - finally: - # Ensure the temporary directory created is cleaned up regardless - # of success or failure - if opts.validate_suite_only_mode and os.path.exists(temp_dir): - shutil.rmtree(temp_dir) - - # Only validating so finish now - if opts.validate_suite_only_mode: - return - - # Install share/work directories (local) - for name in ["share", "share/cycle", "work"]: - self._run_init_dir_work( - opts, suite_name, name, conf_tree, locs_conf=locs_conf) - - if opts.local_install_only_mode: - return - - # Install suite files to each remote [user@]host - for name in ["", "log/", "share/", "share/cycle/", "work/"]: - uuid_file = os.path.abspath(name + uuid) - open(uuid_file, "w").close() - work_files.append(uuid_file) - - # Install items to user@host - auths = self.suite_engine_proc.get_tasks_auths(suite_name) - proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...] - for auth in sorted(auths): - host = auth - if "@" in auth: - host = auth.split("@", 1)[1] - # Remote shell - command = self.popen.get_cmd("ssh", "-n", auth) - # Provide ROSE_VERSION and CYLC_VERSION in the environment - shcommand = "env ROSE_VERSION=%s %s=%s" % ( - my_rose_version, suite_engine_key, suite_engine_version) - # Use login shell? - no_login_shell = self._run_conf( - "remote-no-login-shell", host=host, conf_tree=conf_tree) - if not no_login_shell or no_login_shell.lower() != "true": - shcommand += r""" bash -l -c '"$0" "$@"'""" - # Path to "rose" command, if applicable - rose_bin = self._run_conf( - "remote-rose-bin", host=host, conf_tree=conf_tree, - default="rose") - # Build remote "rose suite-run" command - shcommand += " %s suite-run -vv -n %s" % ( - rose_bin, suite_name) - for key in ["new", "debug", "install-only"]: - attr = key.replace("-", "_") + "_mode" - if getattr(opts, attr, None) is not None: - shcommand += " --%s" % key - if opts.log_keep: - shcommand += " --log-keep=%s" % opts.log_keep - if opts.log_name: - shcommand += " --log-name=%s" % opts.log_name - if not opts.log_archive_mode: - shcommand += " --no-log-archive" - shcommand += " --run=%s" % opts.run_mode - # Build --remote= option - shcommand += " --remote=uuid=%s" % uuid - if now_str is not None: - shcommand += ",now-str=%s" % now_str - host_confs = [ - "root-dir", - "root-dir{share}", - "root-dir{share/cycle}", - "root-dir{work}"] - locs_conf.set([auth]) - for key in host_confs: - value = self._run_conf(key, host=host, conf_tree=conf_tree) - if value is not None: - val = self.popen.list_to_shell_str([str(value)]) - shcommand += ",%s=%s" % (key, pipes.quote(val)) - locs_conf.set([auth, key], value) - command.append(shcommand) - proc = self.popen.run_bg(*command) - proc_queue.append([proc, command, "ssh", auth]) - - while proc_queue: - sleep(self.SLEEP_PIPE) - proc, command, command_name, auth = proc_queue.pop(0) - if proc.poll() is None: # put it back in proc_queue - proc_queue.append([proc, command, command_name, auth]) - continue - ret_code = proc.wait() - out, err = proc.communicate() - ret_code, out, err = [ - i.decode() if isinstance(i, bytes) else i for i in [ - ret_code, out, err]] - if ret_code: - raise RosePopenError(command, ret_code, out, err) - if command_name == "rsync": - self.handle_event(out, level=Event.VV) - continue - else: - self.handle_event(out, level=Event.VV, prefix="[%s] " % auth) - for line in out.split("\n"): - if "/" + uuid == line.strip(): - locs_conf.unset([auth]) - break - else: - filters = {"excludes": [], "includes": []} - for name in ["", "log/", "share/", "share/cycle/", "work/"]: - filters["excludes"].append(name + uuid) - target = auth + ":" + suite_dir_rel - cmd = self._get_cmd_rsync(target, **filters) - proc_queue.append( - [self.popen.run_bg(*cmd), cmd, "rsync", auth]) - - # Install ends - ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs")) - if opts.install_only_mode: - return - elif opts.run_mode == "reload" and suite_conf_unchanged: - conf_name = self.suite_engine_proc.SUITE_CONF - self.handle_event(SkipReloadEvent(suite_name, conf_name)) - return - - # Start the suite - self.fs_util.chdir("log") - self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args) - - # Disconnect log file handle, so monitoring tool command will no longer - # be associated with the log file. - self.event_handler.contexts[uuid].handle.close() - self.event_handler.contexts.pop(uuid) - - return 0 - - @classmethod - def _run_conf( - cls, key, default=None, host=None, conf_tree=None, r_opts=None): - """Return the value of a setting given by a key for a given host. If - r_opts is defined, we are already in a remote host, so there is no need - to do a host match. Otherwise, the setting may be found in the run time - configuration, or the default (i.e. site/user configuration). The value - of each setting in the configuration would be in a line delimited list - of PATTERN=VALUE pairs. - """ - if r_opts is not None: - return r_opts.get(key, default) - if host is None: - host = "localhost" - for conf, keys in [ - (conf_tree.node, []), - (ResourceLocator.default().get_conf(), ["rose-suite-run"])]: - if conf is None: - continue - node_value = conf.get_value(keys + [key]) - if node_value is None: - continue - for line in node_value.strip().splitlines(): - pattern, value = line.strip().split("=", 1) - if (pattern.startswith("jinja2:") or - pattern.startswith("empy:")): - section, name = pattern.rsplit(":", 1) - p_node = conf.get([section, name], no_ignore=True) - # Values in "jinja2:*" and "empy:*" sections are quoted. - pattern = ast.literal_eval(p_node.value) - if fnmatchcase(host, pattern): - return value.strip() - return default - - def _run_init_dir(self, opts, suite_name, conf_tree=None, r_opts=None, - locs_conf=None): - """Create the suite's directory.""" - suite_dir_rel = self._suite_dir_rel(suite_name) - home = os.path.expanduser("~") - suite_dir_root = self._run_conf("root-dir", conf_tree=conf_tree, - r_opts=r_opts) - if suite_dir_root: - if locs_conf is not None: - locs_conf.set(["localhost", "root-dir"], suite_dir_root) - suite_dir_root = env_var_process(suite_dir_root) - suite_dir_home = os.path.join(home, suite_dir_rel) - if (suite_dir_root and - os.path.realpath(home) != os.path.realpath(suite_dir_root)): - suite_dir_real = os.path.join(suite_dir_root, suite_dir_rel) - self.fs_util.makedirs(suite_dir_real) - self.fs_util.symlink(suite_dir_real, suite_dir_home, - opts.no_overwrite_mode) - else: - self.fs_util.makedirs(suite_dir_home) - - def _run_init_dir_log(self, opts, now_str=None): - """Create the suite's log/ directory. Housekeep, archive old ones.""" - # Do nothing in log append mode if log directory already exists - if opts.run_mode in ["reload", "restart"] and os.path.isdir("log"): - return - - # Log directory of this run - if now_str is None: - now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - now_log = "log." + now_str - self.fs_util.makedirs(now_log) - self.fs_util.symlink(now_log, "log") - now_log_name = getattr(opts, "log_name", None) - if now_log_name: - self.fs_util.symlink(now_log, "log." + now_log_name) - - # Keep log for this run and named logs - logs = set(glob("log.*") + ["log"]) - for log in list(logs): - if os.path.islink(log): - logs.remove(log) - log_link = os.readlink(log) - if log_link in logs: - logs.remove(log_link) - - # Housekeep old logs, if necessary - log_keep = getattr(opts, "log_keep", None) - if log_keep: - t_threshold = time() - abs(float(log_keep)) * 86400.0 - for log in list(logs): - if os.path.isfile(log): - if t_threshold > os.stat(log).st_mtime: - self.fs_util.delete(log) - logs.remove(log) - else: - for root, _, files in os.walk(log): - keep = False - for file_ in files: - path = os.path.join(root, file_) - if (os.path.exists(path) and - os.stat(path).st_mtime >= t_threshold): - keep = True - break - if keep: - break - else: - self.fs_util.delete(log) - logs.remove(log) - - # Archive old logs, if necessary - if getattr(opts, "log_archive_mode", True): - for log in list(logs): - if os.path.isfile(log): - continue - log_tar_gz = log + ".tar.gz" - try: - self.popen.run_simple("tar", "-czf", log_tar_gz, log) - except RosePopenError: - try: - self.fs_util.delete(log_tar_gz) - except OSError: - pass - raise - else: - self.handle_event(SuiteLogArchiveEvent(log_tar_gz, log)) - self.fs_util.delete(log) - - def _run_init_dir_work(self, opts, suite_name, name, conf_tree=None, - r_opts=None, locs_conf=None): - """Create a named suite's directory.""" - item_path = os.path.realpath(name) - item_path_source = item_path - key = "root-dir{" + name + "}" - item_root = self._run_conf(key, conf_tree=conf_tree, r_opts=r_opts) - if item_root is None: # backward compat - item_root = self._run_conf( - "root-dir-" + name, conf_tree=conf_tree, r_opts=r_opts) - if item_root: - if locs_conf is not None: - locs_conf.set(["localhost", key], item_root) - item_root = env_var_process(item_root) - suite_dir_rel = self._suite_dir_rel(suite_name) - if os.path.isabs(item_root): - item_path_source = os.path.join(item_root, suite_dir_rel, name) - else: - item_path_source = item_root - item_path_source = os.path.realpath(item_path_source) - if item_path == item_path_source: - if opts.new_mode: - self.fs_util.delete(name) - self.fs_util.makedirs(name) - else: - if opts.new_mode: - self.fs_util.delete(item_path_source) - self.fs_util.makedirs(item_path_source) - if os.sep in name: - dirname_of_name = os.path.dirname(name) - self.fs_util.makedirs(dirname_of_name) - item_path_source_rel = os.path.relpath( - item_path_source, os.path.realpath(dirname_of_name)) - else: - item_path_source_rel = os.path.relpath(item_path_source) - if len(item_path_source_rel) < len(item_path_source): - self.fs_util.symlink( - item_path_source_rel, name, opts.no_overwrite_mode) - else: - self.fs_util.symlink( - item_path_source, name, opts.no_overwrite_mode) - - def _run_remote(self, opts, suite_name): - """rose suite-run --remote=KEY=VALUE,...""" - suite_dir_rel = self._suite_dir_rel(suite_name) - r_opts = {} - for item in opts.remote.split(","): - key, val = item.split("=", 1) - r_opts[key] = val - uuid_file = os.path.join(suite_dir_rel, r_opts["uuid"]) - if os.path.exists(uuid_file): - self.handle_event("/" + r_opts["uuid"] + "\n", level=0) - elif opts.new_mode: - self.fs_util.delete(suite_dir_rel) - if opts.run_mode == "run" or not os.path.exists(suite_dir_rel): - self._run_init_dir(opts, suite_name, r_opts=r_opts) - os.chdir(suite_dir_rel) - for name in ["share", "share/cycle", "work"]: - uuid_file = os.path.join(name, r_opts["uuid"]) - if os.path.exists(uuid_file): - self.handle_event(name + "/" + r_opts["uuid"] + "\n", level=0) - else: - self._run_init_dir_work(opts, suite_name, name, r_opts=r_opts) - if not opts.install_only_mode: - uuid_file = os.path.join("log", r_opts["uuid"]) - if os.path.exists(uuid_file): - self.handle_event("log/" + r_opts["uuid"] + "\n", level=0) - else: - self._run_init_dir_log(opts, r_opts.get("now-str")) - self.fs_util.makedirs("log/suite") - - def _get_cmd_rsync(self, target, excludes=None, includes=None): - """rsync relevant suite items to target.""" - if excludes is None: - excludes = [] - if includes is None: - includes = [] - cmd = self.popen.get_cmd("rsync") - for exclude in excludes + list(self.SYNC_EXCLUDES): - cmd.append("--exclude=" + exclude) - for include in includes: - cmd.append("--include=" + include) - cmd.append("./") - cmd.append(target) - return cmd - - def _suite_dir_rel(self, suite_name): - """Return the relative path to the suite running directory.""" - return self.suite_engine_proc.get_suite_dir_rel(suite_name) - - -def main(): - """Launcher for the CLI.""" - opt_parser = RoseOptionParser() - option_keys = SuiteRunner.OPTIONS - opt_parser.add_my_options(*option_keys) - opts, args = opt_parser.parse_args(sys.argv[1:]) - event_handler = Reporter(opts.verbosity - opts.quietness) - runner = SuiteRunner(event_handler) - try: - sys.exit(runner(opts, args)) - except Exception as exc: - runner.handle_event(exc) - if opts.debug_mode: - traceback.print_exc() - if isinstance(exc, RosePopenError): - sys.exit(exc.ret_code) - else: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/metomi/rose/task_env.py b/metomi/rose/task_env.py index 1157f605c5..1de0ee93e9 100644 --- a/metomi/rose/task_env.py +++ b/metomi/rose/task_env.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/task_run.py b/metomi/rose/task_run.py index cfe16f0a10..988df9406a 100644 --- a/metomi/rose/task_run.py +++ b/metomi/rose/task_run.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/.travis/sitecustomize.py b/metomi/rose/tests/conftest.py similarity index 58% rename from .travis/sitecustomize.py rename to metomi/rose/tests/conftest.py index 57efa4a0a7..db9f033850 100644 --- a/.travis/sitecustomize.py +++ b/metomi/rose/tests/conftest.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -16,15 +13,27 @@ # # You should have received a copy of the GNU General Public License # along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""This file is used by Travis-CI to start the coverage process. +from pathlib import Path +from shutil import rmtree +from tempfile import TemporaryDirectory + +import pytest +from _pytest.monkeypatch import MonkeyPatch -In order to make python aware of it, we export PYTHONPATH when running -the tests. -""" +@pytest.fixture(scope='module') +def mod_monkeypatch(): + """A monkeypatch fixture with module-level scope.""" + patch = MonkeyPatch() + yield patch + patch.undo() -import coverage -coverage.process_startup() +@pytest.fixture(scope='module') +def mod_tmp_path(): + """A tmp_path fixture with module-level scope.""" + path = Path(TemporaryDirectory().name) + path.mkdir() + yield path + rmtree(path) diff --git a/metomi/rose/tests/config.py b/metomi/rose/tests/test_config.py similarity index 98% rename from metomi/rose/tests/config.py rename to metomi/rose/tests/test_config.py index 8ea4bd5914..6ac4ad9db9 100644 --- a/metomi/rose/tests/config.py +++ b/metomi/rose/tests/test_config.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/tests/env.py b/metomi/rose/tests/test_env.py similarity index 95% rename from metomi/rose/tests/env.py rename to metomi/rose/tests/test_env.py index 39e8c3956a..8ca4f3d1c5 100644 --- a/metomi/rose/tests/env.py +++ b/metomi/rose/tests/test_env.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/tests/test_host_select_client.py b/metomi/rose/tests/test_host_select_client.py new file mode 100644 index 0000000000..4881875ff6 --- /dev/null +++ b/metomi/rose/tests/test_host_select_client.py @@ -0,0 +1,80 @@ +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +import json +from io import StringIO +from textwrap import dedent + +from metomi.rose.host_select_client import main as host_select + + +def test_empty(monkeypatch, capsys): + """It should not return any results for an empty request.""" + monkeypatch.setattr( + 'sys.stdin', + StringIO(dedent(''' + **start** + [] + **end** + ''')) + ) + host_select() + captured = capsys.readouterr() + assert captured.out == '[]\n' + assert captured.err == '' + + +def test_stdin_pollution(monkeypatch, capsys): + """Any junk before or after the start/end markers should be ignored + + Note this can come from shell profile scripts. + """ + monkeypatch.setattr( + 'sys.stdin', + StringIO(dedent(''' + hello + *&^%$**start** + [] + **end***&^%$E + world + ''')) + ) + host_select() + captured = capsys.readouterr() + assert captured.out == '[]\n' + assert captured.err == '' + + +def test_request(monkeypatch, capsys): + """Test a simple request.""" + monkeypatch.setattr( + 'sys.stdin', + StringIO(dedent(''' + **start** + [["virtual_memory"]] + **end** + ''')) + ) + host_select() + captured = capsys.readouterr() + assert captured.out + assert captured.err == '' + + results = json.loads(captured.out) + assert len(results) == 1 + result = results[0] + for key in ('active', 'available', 'free'): + assert key in result diff --git a/metomi/rose/tests/test_loc_handlers_rsync_remote_check.py b/metomi/rose/tests/test_loc_handlers_rsync_remote_check.py index 79cfc408cb..a71efe8509 100644 --- a/metomi/rose/tests/test_loc_handlers_rsync_remote_check.py +++ b/metomi/rose/tests/test_loc_handlers_rsync_remote_check.py @@ -18,48 +18,48 @@ # along with Rose. If not, see . # ----------------------------------------------------------------------------- -import sys - -from ast import literal_eval +from pathlib import Path from metomi.rose.loc_handlers.rsync_remote_check import main -def test_check_file(monkeypatch, capsys, tmp_path): +def test_check_file(capsys, tmp_path): content = 'blah' - permission_level = '0o100644' + permission_level = 33188 filepath = tmp_path / 'stuff' filepath.write_text(content) - filepath.chmod(int(permission_level, base=8)) - monkeypatch.setattr( - sys, 'argv', ['ignored', str(filepath), 'blob', 'tree'] - ) - main() + filepath.chmod(permission_level) + main(str(filepath), 'blob', 'tree') captured = capsys.readouterr() assert captured.out.splitlines()[0] == 'blob' mode, mtime, size, path = captured.out.splitlines()[1].split() assert path == str(filepath) - assert mode == permission_level + assert int(mode) == permission_level assert size == str(filepath.stat().st_size) -def test_check_folder( - monkeypatch, capsys, tmp_path -): - folder_permission_level = '0o100700' - dirpath = tmp_path / 'stuff' - dirpath.mkdir() - (dirpath / 'more.stuff').write_text('Hi') - (dirpath / 'more.stuff').chmod(int('0o100633', base=8)) - (dirpath / 'even.more.stuff').write_text('Hi') - dirpath.chmod(int(folder_permission_level, base=8)) - monkeypatch.setattr( - sys, 'argv', ['ignored', str(dirpath), 'blob', 'tree'] - ) - main() - captured = capsys.readouterr() - assert captured.out.splitlines()[0] == 'tree' - mode, _, size, path = literal_eval(captured.out.splitlines()[1]) - assert path == str(dirpath / 'more.stuff') - assert mode == '0o100633' - assert size == 2 +def test_check_folder(capsys, tmp_path): + # create a file and chmod it + (tmp_path / 'more.stuff').write_text('Hi') + (tmp_path / 'more.stuff').chmod(int('0o100633', base=8)) + + # create another file + (tmp_path / 'even.more.stuff').write_text('Hi') + + # run the remote check on the dir + main(str(tmp_path), 'blob', 'tree') + lines = capsys.readouterr().out.splitlines() + + # the first line should be the dir + assert lines[0] == 'tree' + + # the following lines should be the files + files = { + # filename: [mode, mod_time, size] + str(Path(line.split()[-1]).relative_to(tmp_path)): line.split()[:-1] + for line in lines[1:] + } + assert list(sorted(files)) == ['even.more.stuff', 'more.stuff'] + mode, _, size = files['more.stuff'] + assert mode == '33179' + assert size == '2' diff --git a/metomi/rose/tests/popen.py b/metomi/rose/tests/test_popen.py similarity index 94% rename from metomi/rose/tests/popen.py rename to metomi/rose/tests/test_popen.py index f3a36e62bc..8bf0a61413 100644 --- a/metomi/rose/tests/popen.py +++ b/metomi/rose/tests/test_popen.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/tests/test_resource_locator.py b/metomi/rose/tests/test_resource_locator.py new file mode 100644 index 0000000000..2b09d72bfe --- /dev/null +++ b/metomi/rose/tests/test_resource_locator.py @@ -0,0 +1,149 @@ +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . + +from textwrap import dedent + +import pytest + +from metomi.rose.resource import ( + ROSE_CONF_PATH, + ROSE_SITE_CONF_PATH, + ResourceLocator +) + + +@pytest.fixture(scope='module') +def sys_site_user_config(mod_monkeypatch, mod_tmp_path): + """Creates system, site and user configurations. + + * Patches ResourceLocator to use the system and user configs. + * Unsets ROSE_*CONF_PATH envvars to prevent them effecting tests. + + """ + # unset ROSE_CONF_PATH env vars + mod_monkeypatch.delenv(ROSE_CONF_PATH, raising=False) + mod_monkeypatch.delenv(ROSE_SITE_CONF_PATH, raising=False) + + # create system, site and user configurations + syst = mod_tmp_path / 'syst' + site = mod_tmp_path / 'site' + user = mod_tmp_path / 'user' + syst.mkdir() + site.mkdir() + user.mkdir() + with open(syst / 'rose.conf', 'w+') as conf: + conf.write(dedent(''' + all=syst + syst=syst + ''')) + with open(site / 'rose.conf', 'w+') as conf: + conf.write(dedent(''' + all=site + site=site + ''')) + with open(user / 'rose.conf', 'w+') as conf: + conf.write(dedent(''' + all=user + user=user + ''')) + + # patch the ResourceLocator + mod_monkeypatch.setattr( + 'metomi.rose.resource.ResourceLocator.SYST_CONF_PATH', syst + ) + mod_monkeypatch.setattr( + 'metomi.rose.resource.ResourceLocator.USER_CONF_PATH', user + ) + + return tuple(map(str, (syst, site, user))) + + +@pytest.fixture +def resource_locator(): + """Return a ResourceLocator instance for testing. + + Wipes the cached instance after each use. + """ + resource_locator = ResourceLocator() + yield resource_locator + # prevent this instance being reused + del resource_locator + ResourceLocator._DEFAULT_RESOURCE_LOCATOR = None + + +def test_default(sys_site_user_config, resource_locator): + """It should pick up the system and user config by default.""" + conf = resource_locator.get_conf() + # both the syst and user config should have been loaded + assert conf.get(['syst']).value == 'syst' + assert conf.get(['user']).value == 'user' + # the syst config should have been loaded before the user config + assert conf.get(['all']).value == 'user' + assert set(conf.value) == {'syst', 'user', 'all'} + + +def test_skip_no_read(sys_site_user_config, resource_locator, monkeypatch): + """It should skip config files it can't read.""" + # make it look like all files are not readable. + monkeypatch.setattr( + 'os.access', + lambda x, y: False + ) + conf = resource_locator.get_conf() + assert conf.value == {} + + +def test_rose_conf_path_blank( + sys_site_user_config, + resource_locator, + monkeypatch +): + """Setting ROSE_CONF_PATH= should prevent any conf files being loaded.""" + monkeypatch.setenv(ROSE_CONF_PATH, '') + conf = resource_locator.get_conf() + assert conf.value == {} + + +def test_rose_conf_path( + sys_site_user_config, + resource_locator, + monkeypatch +): + """If ROSE_CONF_PATH is defined no other files should be loaded.""" + # set ROSE_CONF_PATH to point at the system config + syst, site, *_ = sys_site_user_config + monkeypatch.setenv(ROSE_CONF_PATH, syst) + # set the ROSE_SITE_CONF_PATH just to make sure it is ignored + monkeypatch.setenv(ROSE_SITE_CONF_PATH, site) + conf = resource_locator.get_conf() + assert conf.get(['syst']).value == 'syst' + assert set(conf.value) == {'syst', 'all'} + + +def test_rose_site_conf_path( + sys_site_user_config, + resource_locator, + monkeypatch +): + """If ROSE_SITE_CONF_PATH is defined it should be loaded.""" + # set ROSE_SITE_CONF_PATH to point at the system config + _, site, *_ = sys_site_user_config + monkeypatch.setenv(ROSE_SITE_CONF_PATH, site) + conf = resource_locator.get_conf() + assert conf.get(['site']).value == 'site' + assert conf.get(['user']).value == 'user' + assert conf.get(['all']).value == 'user' + assert set(conf.value) == {'syst', 'site', 'user', 'all'} diff --git a/metomi/rose/tests/trigger_file.py b/metomi/rose/tests/test_trigger_file.py similarity index 93% rename from metomi/rose/tests/trigger_file.py rename to metomi/rose/tests/test_trigger_file.py index 5bceeecb4e..e39899aee3 100644 --- a/metomi/rose/tests/trigger_file.py +++ b/metomi/rose/tests/test_trigger_file.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/tests/unicode_utils.py b/metomi/rose/tests/test_unicode_utils.py similarity index 94% rename from metomi/rose/tests/unicode_utils.py rename to metomi/rose/tests/test_unicode_utils.py index cbe0105fec..371e222a73 100644 --- a/metomi/rose/tests/unicode_utils.py +++ b/metomi/rose/tests/test_unicode_utils.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/unicode_utils.py b/metomi/rose/unicode_utils.py index 7e1b6236b7..0572597014 100644 --- a/metomi/rose/unicode_utils.py +++ b/metomi/rose/unicode_utils.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rose/upgrade.py b/metomi/rose/upgrade.py index a5315646b2..d1bcefb78f 100644 --- a/metomi/rose/upgrade.py +++ b/metomi/rose/upgrade.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -165,22 +162,24 @@ def add_setting(self, config, keys, value=None, forced=False, configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. - value (string - optional): String denoting the new setting value. + value (str): String denoting the new setting value. Required for options but not for settings. - forced (bool - optional) + forced (bool) If True override value if the setting already exists. - state (str - optional): + state (str): The state of the new setting - should be one of the - ``rose.config.ConfigNode`` states e.g. - ``rose.config.ConfigNode.STATE_USER_IGNORED``. Defaults to - ``rose.config.ConfigNode.STATE_NORMAL``. - comments (list - optional): List of comment lines (strings) for + :py:class:`metomi.rose.config.ConfigNode` states e.g. + :py:data:`metomi.rose.config.ConfigNode.STATE_USER_IGNORED`. + Defaults to + :py:data:`metomi.rose.config.ConfigNode.STATE_NORMAL`. + comments (list): List of comment lines (strings) for the new setting or ``None``. - info (string - optional): A short string containing no new lines, + info (str): A short string containing no new lines, describing the addition of the setting. Returns: None + """ section, option = self._get_section_option_from_keys(keys) id_ = self._get_id_from_section_option(section, option) @@ -269,12 +268,12 @@ def change_setting_value(self, config, keys, value, forced=False, configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. - value (string): The new value. Required for options, can be + value (str): The new value. Required for options, can be ``None`` for sections. forced (bool): Create the setting if it is not present in config. - comments (list - optional): List of comment lines (strings) for + comments (list): List of comment lines (strings) for the new setting or ``None``. - info (string - optional): A short string containing no new lines, + info (str): A short string containing no new lines, describing the addition of the setting. Returns: @@ -312,11 +311,12 @@ def get_setting_value(self, config, keys, no_ignore=False): configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. - no_ignore (bool - optional): If ``True`` return ``None`` if the + no_ignore (bool): If ``True`` return ``None`` if the setting is ignored (else return the value). Returns: object - The setting value or ``None`` if not defined. + """ section, option = self._get_section_option_from_keys(keys) if config.get([section, option], no_ignore=no_ignore) is None: @@ -331,11 +331,12 @@ def remove_setting(self, config, keys, info=None): configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. - info (string - optional): A short string containing no new lines, + info (str): A short string containing no new lines, describing the addition of the setting. Returns: None + """ section, option = self._get_section_option_from_keys(keys) if option is None: @@ -356,11 +357,12 @@ def rename_setting(self, config, keys, new_keys, info=None): keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. new_keys (list): The new hierarchy of node.value 'keys'. - info (string - optional): A short string containing no new lines, + info (str): A short string containing no new lines, describing the addition of the setting. Returns: None + """ section, option = self._get_section_option_from_keys(keys) new_section, new_option = self._get_section_option_from_keys(new_keys) @@ -406,11 +408,12 @@ def enable_setting(self, config, keys, info=None): configuration. keys (list): A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. - info (string - optional): A short string containing no new lines, + info (str): A short string containing no new lines, describing the addition of the setting. Returns: False - if the setting's state is not changed else ``None``. + """ return self._ignore_setting( config, @@ -423,17 +426,22 @@ def ignore_setting(self, config, keys, info=None, """User-ignore a setting in the configuration. Args: - config (metomi.rose.config.ConfigNode): The application - configuration. - keys (list): A list defining a hierarchy of node.value 'keys'. + config (metomi.rose.config.ConfigNode): + The application configuration. + keys (list): + A list defining a hierarchy of node.value 'keys'. A section will be a list of one keys, an option will have two. - info (string - optional): A short string containing no new lines, - describing the addition of the setting. - state (string - optional): A ``rose.config.ConfigNode`` state. - ``STATE_USER_IGNORED`` by default. + info (str): + A short string containing no new lines, describing the addition + of the setting. + state (str): + A :py:class:`metomi.rose.config.ConfigNode` state. + :py:data:`metomi.rose.config.ConfigNode.STATE_USER_IGNORED` by + default. Returns: False - if the setting's state is not changed else ``None``. + """ return self._ignore_setting(config, list(keys), info=info, state=state) @@ -474,7 +482,7 @@ def _get_section_option_from_keys(self, keys): return (keys + [None])[:2] -class MacroUpgradeManager(object): +class MacroUpgradeManager: """Manage the upgrades.""" @@ -712,7 +720,6 @@ def parse_upgrade_args(argv=None): opts.conf_dir = os.path.abspath(opts.conf_dir) if opts.output_dir is not None: opts.output_dir = os.path.abspath(opts.output_dir) - sys.path.append(os.getenv("ROSE_LIB")) metomi.rose.macro.add_opt_meta_paths(opts.meta_path) config_name = os.path.basename(opts.conf_dir) config_file_path = os.path.join(opts.conf_dir, diff --git a/metomi/rose/variable.py b/metomi/rose/variable.py index d8107ccffa..4f8fd20a48 100644 --- a/metomi/rose/variable.py +++ b/metomi/rose/variable.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -50,7 +47,7 @@ IGNORED_BY_USER = 'User ignored' -class Variable(object): +class Variable: """This class stores the data and metadata of an input variable. @@ -276,7 +273,7 @@ def get_value_from_metadata(meta_data): return var_value -class RangeSubFunction(object): +class RangeSubFunction: """Holds a checking function.""" @@ -304,7 +301,7 @@ def __repr__(self): self.operator, self.values) -class CombinedRangeSubFunction(object): +class CombinedRangeSubFunction: def __init__(self, *range_insts): self.range_insts = range_insts diff --git a/metomi/rosie/__init__.py b/metomi/rosie/__init__.py index f1b60f1a5d..e7f96eaf6b 100644 --- a/metomi/rosie/__init__.py +++ b/metomi/rosie/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rosie/db.py b/metomi/rosie/db.py index 9ff56b4e39..cc7640cb82 100644 --- a/metomi/rosie/db.py +++ b/metomi/rosie/db.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -59,7 +56,7 @@ def __str__(self): return "Failed to connect to DB '%s'." % self.bad_db_url -class DAO(object): +class DAO: """Retrieves data from the suite database. diff --git a/metomi/rosie/db_create.py b/metomi/rosie/db_create.py index 97bb7ea089..f710a214e9 100644 --- a/metomi/rosie/db_create.py +++ b/metomi/rosie/db_create.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -84,7 +81,7 @@ def __str__(self): return "%s: DB not loaded: %s" % self.args -class RosieDatabaseInitiator(object): +class RosieDatabaseInitiator: """Initiate a database file from the repository information.""" diff --git a/metomi/rosie/graph.py b/metomi/rosie/graph.py index 94aef55216..0912085393 100644 --- a/metomi/rosie/graph.py +++ b/metomi/rosie/graph.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rosie/suite_id.py b/metomi/rosie/suite_id.py index c406b0e245..b98ed3ca57 100644 --- a/metomi/rosie/suite_id.py +++ b/metomi/rosie/suite_id.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -29,7 +26,14 @@ """ import os +from pathlib import Path import re +import shlex +import string +import sys +import traceback +import xml.parsers.expat + import metomi.rose.env from metomi.rose.loc_handlers.svn import SvnInfoXMLParser from metomi.rose.opt_parse import RoseOptionParser @@ -37,11 +41,6 @@ from metomi.rose.reporter import Reporter from metomi.rose.resource import ResourceLocator from metomi.rose.suite_engine_proc import SuiteEngineProcessor, NoSuiteLogError -import shlex -import string -import sys -import traceback -import xml.parsers.expat class SvnCaller(RosePopener): @@ -108,7 +107,7 @@ def __str__(self): return "%s: invalid suite ID" % (self.args[0]) -class SuiteId(object): +class SuiteId: """Represent a suite ID.""" @@ -365,9 +364,32 @@ def _from_id_text(self, id_text): def _from_location(self, location): """Return the ID of a location (origin URL or local copy path).""" - # Is location a "~/cylc-run/$SUITE/" directory? - # Use a hacky way to read the "log/rose-suite-run.version" file suite_engine_proc = SuiteEngineProcessor.get_processor() + suite_dir_rel_root = getattr( + suite_engine_proc, "SUITE_DIR_REL_ROOT", None) + + # Cylc8 run directory + # TODO: extract version control information + loc = Path(location) + sdrr = Path('~', suite_dir_rel_root).expanduser().resolve() + try: + loc.relative_to(sdrr) + except ValueError: + # Not an installed Cylc8 workflow run directory + pass + else: + if (loc / 'rose-suite.info').is_file(): + # This is an installed workflow with a rose-suite.info file + # (most likely a Cylc8 run directory) + + # TODO: extract version control information written by + # Cylc install, see: + # https://github.com/metomi/rose/issues/2432 + # https://github.com/cylc/cylc-flow/issues/3849 + raise SuiteIdLocationError(location) + + # Cylc7 run directory + # Use a hacky way to read the "log/rose-suite-run.version" file suite_dir_rel_root = getattr( suite_engine_proc, "SUITE_DIR_REL_ROOT", None) if suite_dir_rel_root and "/" + suite_dir_rel_root + "/" in location: @@ -536,17 +558,12 @@ def to_web(self): revision = self.REV_HEAD return url + self.FORMAT_VERSION % (branch, revision) - def to_output(self): - """Return the output directory for this suite.""" - suite_engine_proc = SuiteEngineProcessor.get_processor() - return suite_engine_proc.get_suite_log_url(None, str(self)) - def main(): """Implement the "rose suite-id" command.""" opt_parser = RoseOptionParser() opt_parser.add_my_options("latest", "next", "to_local_copy", "to_origin", - "to_output", "to_web") + "to_web") opts, args = opt_parser.parse_args() report = Reporter(opts.verbosity - opts.quietness) SuiteId.svn.event_handler = report # FIXME: ugly? @@ -562,10 +579,6 @@ def main(): for arg in args: report( str(SuiteId(id_text=arg).to_local_copy()) + "\n", level=0) - elif opts.to_output: - for arg in args: - url = SuiteId(id_text=arg).to_output() - report(str(url) + "\n", level=0) elif opts.to_web: for arg in args: report(str(SuiteId(id_text=arg).to_web()) + "\n", level=0) diff --git a/metomi/rosie/svn_post_commit.py b/metomi/rosie/svn_post_commit.py index b0569c9eb5..87c1449782 100644 --- a/metomi/rosie/svn_post_commit.py +++ b/metomi/rosie/svn_post_commit.py @@ -48,7 +48,7 @@ from metomi.rosie.svn_hook import RosieSvnHook, InfoFileError -class RosieWriteDAO(object): +class RosieWriteDAO: """Data Access Object for writing to the Rosie web service database.""" diff --git a/metomi/rosie/usertools/__init__.py b/metomi/rosie/usertools/__init__.py index f9c7376f07..b229a3d772 100644 --- a/metomi/rosie/usertools/__init__.py +++ b/metomi/rosie/usertools/__init__.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/metomi/rosie/usertools/ldaptool.py b/metomi/rosie/usertools/ldaptool.py index c6426d17d4..2452e81ce8 100644 --- a/metomi/rosie/usertools/ldaptool.py +++ b/metomi/rosie/usertools/ldaptool.py @@ -27,7 +27,7 @@ from metomi.rose.resource import ResourceLocator -class LDAPUserTool(object): +class LDAPUserTool: """User information tool via LDAP.""" diff --git a/metomi/rosie/usertools/passwdtool.py b/metomi/rosie/usertools/passwdtool.py index 91cf262218..251c092d1e 100644 --- a/metomi/rosie/usertools/passwdtool.py +++ b/metomi/rosie/usertools/passwdtool.py @@ -22,7 +22,7 @@ from pwd import getpwnam -class PasswdUserTool(object): +class PasswdUserTool: """User information tool via Unix password info.""" diff --git a/metomi/rosie/vc.py b/metomi/rosie/vc.py index 5e67a3dcc2..bfccbb0edf 100644 --- a/metomi/rosie/vc.py +++ b/metomi/rosie/vc.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -163,7 +160,7 @@ def __str__(self): return "delete: %s" % id_.to_origin() -class RosieVCClient(object): +class RosieVCClient: """Client for version control functionalities.""" diff --git a/metomi/rosie/ws.py b/metomi/rosie/ws.py index 8e6bc9eaee..c09af0da4f 100644 --- a/metomi/rosie/ws.py +++ b/metomi/rosie/ws.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -38,15 +35,18 @@ import json import logging import os +from pathlib import Path +import pkg_resources import pwd import signal -import pkg_resources from time import sleep from tornado.ioloop import IOLoop, PeriodicCallback import tornado.log import tornado.web +import wsgiref from metomi.isodatetime.data import get_timepoint_from_seconds_since_unix_epoch +from metomi.rose import __version__ as ROSE_VERSION from metomi.rose.host_select import HostSelector from metomi.rose.opt_parse import RoseOptionParser from metomi.rose.resource import ResourceLocator @@ -82,7 +82,7 @@ def __init__(self, service_root_mode=False, *args, **kwargs): if self.props["host_name"] and "." in self.props["host_name"]: self.props["host_name"] = ( self.props["host_name"].split(".", 1)[0]) - self.props["rose_version"] = ResourceLocator.default().get_version() + self.props["rose_version"] = ROSE_VERSION # Get location of HTML files from package rosie_lib = os.path.join( @@ -139,8 +139,9 @@ def __init__(self, service_root_mode=False, *args, **kwargs): handlers = [root_handler] + prefix_handlers settings = dict( autoreload=True, - static_path=ResourceLocator.default().get_util_home( - "lib", "html", "static"), + static_path=str( + Path(metomi.rosie.__file__).parent / 'lib/html/static' + ) ) super( RosieDiscoServiceApplication, self).__init__(handlers, **settings) diff --git a/metomi/rosie/ws_client.py b/metomi/rosie/ws_client.py index 649cb6ff16..3726aa44de 100644 --- a/metomi/rosie/ws_client.py +++ b/metomi/rosie/ws_client.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -64,7 +61,7 @@ def __str__(self): return "Query syntax error: " + " ".join(self.args[0]) -class RosieWSClient(object): +class RosieWSClient: """A client for the Rosie web service. @@ -75,8 +72,8 @@ class RosieWSClient(object): credentials. Takes and returns the arguments username and password. popen (rose.popen.RosePopener): Use initiated RosePopener instance create a new one if ``None``. - event_handler (object): A callable object for reporting popen output, - see :py:class:`rose.reporter.Reporter`. + event_handler : A callable object for reporting popen output, + see :py:class:`metomi.rose.reporter.Reporter`. """ MAX_LOCAL_QUERIES = 64 diff --git a/metomi/rosie/ws_client_auth.py b/metomi/rosie/ws_client_auth.py index 2ede0ec575..3c5996fdd5 100644 --- a/metomi/rosie/ws_client_auth.py +++ b/metomi/rosie/ws_client_auth.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -18,15 +15,16 @@ # along with Rose. If not, see . # ----------------------------------------------------------------------------- """The authentication manager for the Rosie web service client.""" -GI_FLAG = False import ast from getpass import getpass import os import re +import socket import shlex import sys from urllib.parse import urlparse +import warnings import metomi.rose.config from metomi.rose.env import env_var_process @@ -34,7 +32,23 @@ from metomi.rose.reporter import Reporter from metomi.rose.resource import ResourceLocator -import socket + +try: + from gi import require_version, pygtkcompat + require_version('Gtk', '3.0') # For GTK+ >=v3 use PyGObject; v2 use PyGTK + require_version('Secret', '1') + from gi.repository import Secret + del pygtkcompat + GI_FLAG = True +except (ImportError, ValueError): + GI_FLAG = False +try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + import gtk + import gnomekeyring +except ImportError: + pass class UndefinedRosiePrefixWS(Exception): @@ -63,7 +77,7 @@ def __str__(self): return message -class GnomekeyringStore(object): +class GnomekeyringStore: """Password management with gnomekeyring.""" @@ -121,7 +135,7 @@ def store_password(self, scheme, host, username, password): self.item_ids[(scheme, host, username)] = (None, item_id) -class GPGAgentStore(object): +class GPGAgentStore: """Password management with gpg-agent.""" @@ -232,7 +246,7 @@ def store_password(self, scheme, host, username, password): pass -class LibsecretStore(object): +class LibsecretStore: """Password management with libsecret.""" @@ -275,7 +289,7 @@ def store_password(self, scheme, host, username, password): pass -class RosieWSClientAuthManager(object): +class RosieWSClientAuthManager: """Manage authentication info for a Rosie web service client.""" diff --git a/metomi/rosie/ws_client_cli.py b/metomi/rosie/ws_client_cli.py index 878afe64a4..ab4d97afdd 100644 --- a/metomi/rosie/ws_client_cli.py +++ b/metomi/rosie/ws_client_cli.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/package.json b/package.json new file mode 100644 index 0000000000..f8a8f7fc2d --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "metomi-rose", + "private": true, + "scripts": { + "lint": "eslint -c .eslintrc.js sphinx/" + }, + "dependencies": { + "eslint-plugin-import": "^2.22.1", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^5.0.0" + }, + "devDependencies": { + "eslint": "^7.14.0", + "eslint-config-standard": "^14.1.1" + }, + "bugs": { + "url": "https://github.com/metomi/rose/issues" + } +} diff --git a/pytest.ini b/pytest.ini index 520187d2d8..94c8001f4c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,12 +1,15 @@ [pytest] -addopts = --verbose +addopts = + --verbose --doctest-modules --ignore=metomi/rosie --ignore=metomi/rose/ws.py --ignore=metomi/rose/metadata_graph.py + # these cause import issues + --ignore=metomi/rose/etc/ # these tests do IO, don't run them under sphinx-build rather than pytest: --ignore=metomi/rose/config.py --ignore=metomi/rose/macro.py testpaths = - metomi/rose/tests/* + metomi/ sphinx/ diff --git a/sbin/rosa-rpmbuild b/sbin/rosa-rpmbuild index 4e46c34a8b..87b77a2ac9 100755 --- a/sbin/rosa-rpmbuild +++ b/sbin/rosa-rpmbuild @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -78,8 +78,6 @@ Rose: a framework for managing and running meteorological suites %install rm -fr %{buildroot} -mkdir -p %{buildroot}/etc/bash_completion.d %{buildroot}/opt %{buildroot}/usr/bin -cp -p %_sourcedir/$NAME-$REV_BASE_DOT/etc/rose-bash-completion %{buildroot}/etc/bash_completion.d cp -pr %_sourcedir/$NAME-$REV_BASE_DOT %{buildroot}/opt/$NAME python3 -m compileall %{buildroot}/opt/$NAME/lib cp -p %_sourcedir/$NAME-$REV_BASE_DOT/usr/bin/rose %{buildroot}/usr/bin/rose @@ -89,8 +87,6 @@ cp -p %_sourcedir/$NAME-$REV_BASE_DOT/usr/bin/rose %{buildroot}/usr/bin/rosie rm -fr %{buildroot} %files -/etc/bash_completion.d -/etc/bash_completion.d/rose-bash-completion /opt/$NAME /usr/bin/rose /usr/bin/rosie diff --git a/setup.py b/setup.py index 34b1175aa6..69f7b3486f 100644 --- a/setup.py +++ b/setup.py @@ -49,13 +49,14 @@ def find_version(*file_paths): INSTALL_REQUIRES = [ - "jinja2>=2.10.1", "aiofiles", - "tornado", - "sqlalchemy", + "jinja2>=2.10.1", + "ldap3", "metomi-isodatetime", "requests", - "ldap3", + "sqlalchemy", + "tornado", + 'psutil>=5.6.0', ] EXTRAS_REQUIRE = { 'docs': [ @@ -67,7 +68,13 @@ def find_version(*file_paths): 'cylc-sphinx-extensions[all]>=1.2.0' ] } -EXTRAS_REQUIRE['all'] = list({y for x in EXTRAS_REQUIRE.values() for y in x}) +TESTS_REQUIRE = [ + 'pytest', + 'flake8' +] +EXTRAS_REQUIRE['all'] = list(set( + [y for x in EXTRAS_REQUIRE.values() for y in x] + TESTS_REQUIRE +)) setup( @@ -77,14 +84,13 @@ def find_version(*file_paths): version=find_version("metomi", "rose", "__init__.py"), # Options - scripts=glob(join("bin", "*")) - + glob(join("sbin", "*")) - + glob(join("lib", "bash", "*")), + scripts=( + glob(join("bin", "*")) + + glob(join("sbin", "*")) + ), install_requires=INSTALL_REQUIRES, extras_require=EXTRAS_REQUIRE, - package_data={ - "metomi.rose": ["etc/.*"], - "metomi.rosie": ["lib/*"] - }, + tests_require=TESTS_REQUIRE, + include_package_data=True, packages=find_namespace_packages(include=["metomi.*"]), ) diff --git a/sphinx/Makefile b/sphinx/Makefile old mode 100755 new mode 100644 index 614210ecb5..198bbe5019 --- a/sphinx/Makefile +++ b/sphinx/Makefile @@ -1,186 +1,25 @@ -# Makefile for Sphinx documentation -# +# Minimal makefile for Sphinx documentation -# You can set these variables from the command line. +# You can set these variables from the command line: +ROSE_VERSION = $(shell rose version) SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = -BUILDDIR = ../doc - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext pdf slides +BUILDDIR = ../doc/$(ROSE_VERSION) +SOURCEDIR = . +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -slides: - $(SPHINXBUILD) -b slides $(ALLSPHINXOPTS) $(BUILDDIR)/slides - @echo - @echo "Build finished. The slide-html files are in $(BUILDDIR)/slides." - -pdf: latexpdf - python2 extract-pdf-documents.py "$(BUILDDIR)" - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/pdf." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/rose-api-doc.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rose-api-doc.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/rose-api-doc" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rose-api-doc" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." + @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." +.PHONY: help clean Makefile .EXPORT_ALL_VARIABLES M -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +# NOTE: EXPORT_ALL_VARIABLES exports make vars as env vars +%: Makefile .EXPORT_ALL_VARIABLES + # build documentation + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/sphinx/_static/js/versioning.js b/sphinx/_static/js/versioning.js index 12b2b61d77..672d58fa49 100644 --- a/sphinx/_static/js/versioning.js +++ b/sphinx/_static/js/versioning.js @@ -1,52 +1,59 @@ /* global root_dir current_builder current_page_name current_version */ +/* eslint camelcase: "off" */ // global vars not in camel case -$(document).ready(function() { - $.ajax({ - 'async': false, - 'type': 'GET', - 'url': root_dir + 'versions.json', - dataType: 'json', - success: function (versions) { - // the DOM element to append version and format selectors to - var ele = $('#version-selector'); +$(document).ready(function () { + $.ajax({ + async: false, + type: 'GET', + url: root_dir + 'versions.json', + dataType: 'json', + success: function (versions) { + // the DOM element to append version and format selectors to + var ele = $('#version-selector') - // construct version selector - var ver = ele.append($('
')); - $(ver).append($('
').append('Versions')); - for (let version of Object.keys(versions).sort().reverse()) { - $(ver).append($('
') - .append($('') - .attr({'href': root_dir + version + '/' + - current_builder + '/' + - current_page_name + '.html'}) - .append(version) - ) - ); - } + // construct version selector + var ver = ele.append($('
')) + $(ver).append($('
').append('Versions')) + for (const version of Object.keys(versions).sort().reverse()) { + $(ver).append($('
') + .append($('') + .attr({ + href: root_dir + + version + '/' + + current_builder + + '/' + + current_page_name + '.html' + }) + .append(version) + ) + ) + } - // construct format selector - var bui = ele.append($('
')); - $(bui).append($('
').append('Formats')); - var href; - for (let builder_for_version of versions[current_version].sort()) { - href = root_dir + current_version + '/' + builder_for_version + - '/'; - if (['html', 'slides'].indexOf(builder_for_version) >= 0) { - // format has compatible document structure - href += current_page_name + '.html'; - } else { - // structure different, link to the index.html page - href += 'index.html'; - } - - - $(bui).append($('
') - .append($('') - .attr({'href': href}) - .append(builder_for_version) - ) - ); - } + // construct format selector + var bui = ele.append($('
')) + $(bui).append($('
').append('Formats')) + var href + for (const builderForVersion of versions[current_version].sort()) { + href = root_dir + + current_version + + '/' + + builderForVersion + + '/' + if (['html', 'slides'].indexOf(builderForVersion) >= 0) { + // format has compatible document structure + href += current_page_name + '.html' + } else { + // structure different, link to the index.html page + href += 'index.html' } - }); -}); + + $(bui).append($('
') + .append($('') + .attr({ href: href }) + .append(builderForVersion) + ) + ) + } + } + }) +}) diff --git a/sphinx/api/built-in/rose_ana.rst b/sphinx/api/built-in/rose_ana.rst index d372e99c79..86b8a1d6cc 100644 --- a/sphinx/api/built-in/rose_ana.rst +++ b/sphinx/api/built-in/rose_ana.rst @@ -83,12 +83,12 @@ details. .. rose:conf:: grepper-report-limit Limits the number of lines printed when using the - :py:mod:`rose.apps.ana_builtin.grepper` analysis class. + :py:mod:`metomi.rose.apps.ana_builtin.grepper` analysis class. .. rose:conf:: skip-if-all-files-missing - Causes the :py:mod:`rose.apps.ana_builtin.grepper` class to pass - if all files to be compared are missing. + Causes the :py:mod:`metomi.rose.apps.ana_builtin.grepper` class + to pass if all files to be compared are missing. .. rose:conf:: kgo-database @@ -110,15 +110,5 @@ Analysis Classes There is one built-in module of analysis classes called ``grepper``. -.. To document everything: - - .. automodule:: metomi.rose.apps.ana_builtin.grepper - :members: - -.. autoclass:: metomi.rose.apps.ana_builtin.grepper.FileCommandPattern - -.. autoclass:: metomi.rose.apps.ana_builtin.grepper.FilePattern - -.. autoclass:: metomi.rose.apps.ana_builtin.grepper.SingleCommandPattern - -.. autoclass:: metomi.rose.apps.ana_builtin.grepper.SingleCommandStatus +.. automodule:: metomi.rose.apps.ana_builtin.grepper + :members: FileCommandPattern, FilePattern, SingleCommandPattern, SingleCommandStatus diff --git a/sphinx/api/built-in/rose_arch.rst b/sphinx/api/built-in/rose_arch.rst index 1e8e1a3f93..d82bba7d23 100644 --- a/sphinx/api/built-in/rose_arch.rst +++ b/sphinx/api/built-in/rose_arch.rst @@ -42,7 +42,7 @@ In automatic selection mode, this built-in application will be invoked automatically if a task has a name that starts with ``rose_arch*``. This means that you can use Rose Arch with something like the example below -in your ``suite.rc``: +in your ``flow.cylc``: .. code-block:: cylc diff --git a/sphinx/api/command-reference.rst b/sphinx/api/command-reference.rst index 7ba8f85c0d..70ba7a5ab9 100644 --- a/sphinx/api/command-reference.rst +++ b/sphinx/api/command-reference.rst @@ -11,6 +11,29 @@ Rose Commands ---- +.. _command-rose-config-edit: + +rose config-edit +^^^^^^^^^^^^^^^^ + +TODO: This is here to allow the documentation tests to pass + +.. _command-rose-suite-run: + +rose suite-run +^^^^^^^^^^^^^^ + +TODO: This is here to allow the documentation tests to pass + +.. _command-rose-suite-restart: + +rose suite-restart +^^^^^^^^^^^^^^^^^^ + +TODO: This is here to allow the documentation tests to pass + +---- + .. _command-rose-test-battery: etc/bin/rose-test-battery diff --git a/sphinx/api/configuration/api.rst b/sphinx/api/configuration/api.rst index 1206e66cc7..cda35b76e8 100644 --- a/sphinx/api/configuration/api.rst +++ b/sphinx/api/configuration/api.rst @@ -28,8 +28,8 @@ Python ------ Rose provides a Python API for loading, processing, editing and dumping Rose -configurations via the :py:mod:`rose.config` module located within the Rose -Python library. +configurations via the :py:mod:`metomi.rose.config` module located within the +Rose Python library. .. automodule:: metomi.rose.config :members: diff --git a/sphinx/api/configuration/metadata.rst b/sphinx/api/configuration/metadata.rst index 38eb860c5f..07f3792ed1 100644 --- a/sphinx/api/configuration/metadata.rst +++ b/sphinx/api/configuration/metadata.rst @@ -39,7 +39,7 @@ of precedence: See :ref:`app-meta-loc` for more details. The configuration metadata that controls default behaviour will be located in -``$ROSE_HOME/etc/rose-meta/``. +``etc/rose-meta/`` within the ``metomi.rose`` Python installation. Configuration Metadata File @@ -779,9 +779,9 @@ The metadata options for a configuration fall into four categories: Another useful Rose built-in widget to use is the array element aligning page widget, - ``rose.config_editor.pagewidget.table.PageArrayTable``. You can set - this for a section or namespace to make sure each *n*-th variable value - element lines up horizontally. For example: + ``metomi.rose.config_editor.pagewidget.table.PageArrayTable``. You can + set this for a section or namespace to make sure each *n*-th variable + value element lines up horizontally. For example: .. code-block:: rose diff --git a/sphinx/api/configuration/site-and-user.rst b/sphinx/api/configuration/site-and-user.rst index 22451d075a..fe6715c1af 100644 --- a/sphinx/api/configuration/site-and-user.rst +++ b/sphinx/api/configuration/site-and-user.rst @@ -8,11 +8,7 @@ site configuration file and per user via the user configuration file. Any configuration in the site configuration overrides the default, and any configuration in the user configuration overrides the site configuration and the default. Rose expects these files to be in the modified INI format -described in :ref:`Rose Configuration Format`. Rose utilities search for its -site configuration at ``$ROSE_HOME/etc/rose.conf`` where -``$ROSE_HOME/bin/rose`` is the location of the ``rose`` command, and they -search for the user configuration at ``$HOME/.metomi/rose.conf`` where -``$HOME`` is the home directory of the current user. +described in :ref:`Rose Configuration Format`. You can also override many internal constants of :ref:`command-rose-config-edit` and diff --git a/sphinx/api/configuration/suite.rst b/sphinx/api/configuration/suite.rst index ee7952b130..f2d9a3a153 100644 --- a/sphinx/api/configuration/suite.rst +++ b/sphinx/api/configuration/suite.rst @@ -8,7 +8,7 @@ Suite Configuration The configuration and functionality of a suite will usually be covered by the use of `Cylc`_. In which case, most of the suite configuration will live -in the Cylc ``suite.rc`` file. Otherwise, a suite is just a directory of +in the Cylc ``flow.cylc`` file. Otherwise, a suite is just a directory of files. A suite directory may contain the following: @@ -76,20 +76,20 @@ A suite directory may contain the following: .. rose:conf:: KEY=VALUE Define a `Jinja2`_ variable ``KEY`` with the value ``VALUE`` for use - in the ``suite.rc`` file. + in the ``flow.cylc`` file. The assignment will be inserted after the ``#!jinja2`` line of the - installed ``suite.rc`` file. + installed ``flow.cylc`` file. .. rose:conf:: empy:suite.rc .. rose:conf:: KEY=VALUE Define a `EmPy`_ variable ``KEY`` with the value ``VALUE`` for use - in the ``suite.rc`` file. + in the ``flow.cylc`` file. The assignment will be inserted after the ``#!empy`` line of the - installed ``suite.rc`` file. + installed ``flow.cylc`` file. .. rose:conf:: [file:NAME] @@ -107,62 +107,6 @@ A suite directory may contain the following: used by various Rose utilities, such as the config editor GUI. It can be used to specify the suite type. - .. rose:conf:: root-dir=LIST - - A new line delimited list of ``PATTERN=DIR`` pairs. The ``PATTERN`` - should be a glob-like pattern for matching a host name. The ``DIR`` - should be the root directory to install a suite run directory. E.g.: - - .. code-block:: rose - - root-dir=hpc*=$WORKDIR - =*=$DATADIR - - In this example, :ref:`command-rose-suite-run` of a suite with name - ``$NAME`` will create ``~/cylc-run/$NAME`` as a symbolic link to - ``$DATADIR/cylc-run/$NAME/`` on any machine, except those with their - hostnames matching ``hpc*``. In which case, it will create - ``~/cylc-run/$NAME`` as a symbolic link to ``$WORKDIR/cylc-run/$NAME/``. - - .. warning:: - - If a suite has previously been run changes to any of the ``root-dir`` - settings will take effect on the next clean re-installation i.e:: - - $ rose suite-run --new - - .. rose:conf:: root-dir{share}=LIST - - A new line delimited list of ``PATTERN=DIR`` pairs. The ``PATTERN`` should - be a glob-like pattern for matching a host name. The ``DIR`` should be the - root directory where the suite's ``share/`` directory should be created. - - .. rose:conf:: root-dir{share/cycle}=LIST - - A new line delimited list of ``PATTERN=DIR`` pairs. The ``PATTERN`` should - be a glob-like pattern for matching a host name. The ``DIR`` should be the - root directory where the suite's ``share/cycle/`` directory should be - be created. - - .. rose:conf:: root-dir{work}=LIST - - A new line delimited list of ``PATTERN=DIR`` pairs. The ``PATTERN`` should - be a glob-like pattern for matching a host name. The ``DIR`` should be the - root directory where the suite's ``work/`` directory for tasks should be - created. - - .. rose:conf:: root-dir-share=LIST - - .. deprecated:: 2015.04 - - Equivalent to :rose:conf:`root-dir{share}=LIST`. - - .. rose:conf:: root-dir-work=LIST - - .. deprecated:: 2015.04 - - Equivalent to :rose:conf:`root-dir{work}=LIST`. - .. rose:file:: rose-suite.info The suite information file :rose:file:`rose-suite.info` should contain the diff --git a/sphinx/api/environment-variables.rst b/sphinx/api/environment-variables.rst index 6b53bd0d3e..19b7f7bc31 100644 --- a/sphinx/api/environment-variables.rst +++ b/sphinx/api/environment-variables.rst @@ -56,12 +56,26 @@ Rose Environment Variables .. envvar:: ROSE_CONF_PATH Description - Specify a colon (``:``) separated list of paths for searching and loading - site/user configuration. If this environment variable is not defined, the - normal behaviour is to search for and load :rose:file:`rose.conf` from - ``$ROSE_HOME/etc`` and then ``$HOME/.metomi``. + If defined this will override the default configuration search path. + + Provide a colon (``:``) separated list of paths to search for + ``rose.conf`` files. + + If set to an empty string no config files will be loaded. Used By * :ref:`command-rose-test-battery` + See also + * :envvar:`ROSE_SITE_CONF_PATH` + * :rose:file:`rose.conf` + +.. envvar:: ROSE_SITE_CONF_PATH + + Description + Defines the location of the "site" configuration. Configurations defined + here can be overridden by the "user" configuration. + See also + * :envvar:`ROSE_CONF_PATH` + * :rose:file:`rose.conf` .. envvar:: ROSE_CYCLING_MODE @@ -124,21 +138,6 @@ Rose Environment Variables * :ref:`command-rose-app-run` * :ref:`command-rose-task-run` -.. envvar:: ROSE_HOME - - Description - Specifies the path to the Rose home directory. - Used and Provided By - * ``rose`` - -.. envvar:: ROSE_HOME_BIN - - Description - Specifies the path to the ``bin/`` or ``sbin/`` directory of the current - Rose utility. - Used and Provided By - * ``rose`` - .. envvar:: ROSE_LAUNCHER Description diff --git a/sphinx/api/other.rst b/sphinx/api/other.rst new file mode 100644 index 0000000000..5df6cf5b11 --- /dev/null +++ b/sphinx/api/other.rst @@ -0,0 +1,11 @@ +Other Python API Docs +===================== + + +.. automodule:: metomi.rose.reporter + :members: Reporter + +.. automodule:: metomi.rose.popen + :members: RosePopener + +.. automodule:: metomi.rose.macros diff --git a/sphinx/api/rose-bash-library.rst b/sphinx/api/rose-bash-library.rst index 9f806e2550..17869cbb31 100644 --- a/sphinx/api/rose-bash-library.rst +++ b/sphinx/api/rose-bash-library.rst @@ -1,16 +1,16 @@ Rose Bash Library ================= -The Rose bash library lives in ``lib/bash/``. To import a module, load the file -into your script. E.g. To load ``rose_usage``, you would do:: +Rose includes some bash modules, they can be located by running:: - . $ROSE_HOME/lib/bash/rose_usage + $ rose resource lib/bash + +These modules can be invoked like so:: + + . "$(rose resource lib/bash/rose_log)" The modules are: -``rose_init`` - Called by ``rose`` on initialisation. This is not meant to be for general - use. ``rose_log`` Provide functions to print log messages. ``rose_usage`` diff --git a/sphinx/api/rose-macro.rst b/sphinx/api/rose-macro.rst index 24c0b780cf..54a0468e3c 100644 --- a/sphinx/api/rose-macro.rst +++ b/sphinx/api/rose-macro.rst @@ -46,7 +46,7 @@ A module containing macros should be stored under a directory should be a Python package. When developing macros for Rose internals, macros should be placed in the -:py:mod:`rose.macros` package in the Rose Python library. They should be +:py:mod:`metomi.rose.macros` package in the Rose Python library. They should be referenced by the ``lib/python/rose/macros/__init__.py`` classes and a call to them can be added in the ``lib/python/rose/config_editor/main.py`` module if they need to be run implicitly by the config editor. @@ -60,14 +60,15 @@ Writing Macros For basic usage see the :ref:`macro tutorial `. Validator, transformer and reporter macros are Python classes which subclass -from :py:class:`rose.macro.MacroBase` (:ref:`API `). +from :py:class:`metomi.rose.macro.MacroBase` +(:ref:`API `). These macros implement their behaviours by providing a ``validate``, ``transform`` or ``report`` method. A macro can contain any combination of these methods so, for example, a macro might be both a validator and a transformer. -These methods should accept two :py:class:`rose.config.ConfigNode` +These methods should accept two :py:class:`metomi.rose.config.ConfigNode` instances as arguments - one is the configuration, and one is the metadata configuration that provides information about the configuration items. @@ -92,11 +93,11 @@ A validator macro should look like: # Some check on config appends to self.reports using self.add_report return self.reports -The returned list should be a list of :py:class:`rose.macro.MacroReport` objects -containing the section, option, value, and warning strings (info) for each -setting that is in error. These are initialised behind the scenes by calling the -inherited method :py:meth:`rose.macro.MacroBase.add_report` via -:py:meth:`self.add_report`. This has the form: +The returned list should be a list of :py:class:`metomi.rose.macro.MacroReport` +objects containing the section, option, value, and warning strings (info) for +each setting that is in error. These are initialised behind the scenes by +calling the inherited method :py:meth:`metomi.rose.macro.MacroBase.add_report`. +This has the form: .. code-block:: python @@ -123,7 +124,7 @@ Validator macros have the option to give warnings, which do not count as formal errors in the Rose config editor GUI. These should be used when something *may* be wrong, such as warning when using an advanced-developer-only option. They are invoked by passing a 5th argument -to :py:meth:`self.add_report`, ``is_warning``, like so: +to :py:meth:`metomi.rose.macro.MacroBase.add_report`, ``is_warning``, like so: .. code-block:: python diff --git a/sphinx/api/rose-upgrader-macros.rst b/sphinx/api/rose-upgrader-macros.rst index 638ef99b0d..766cc5f36c 100644 --- a/sphinx/api/rose-upgrader-macros.rst +++ b/sphinx/api/rose-upgrader-macros.rst @@ -10,7 +10,7 @@ metadata versions. They are classes, very similar to * An ``upgrade`` method instead of a ``transform`` method * An optional ``downgrade`` method, identical in API to the ``upgrade`` method, but intended for performing the reverse operation -* A more helpful API via :py:class:`rose.upgrade.MacroUpgrade` methods +* A more helpful API via :py:class:`metomi.rose.upgrade.MacroUpgrade` methods * ``BEFORE_TAG`` and ``AFTER_TAG`` attributes - the version of metadata they apply to (``BEFORE_TAG``) and the version they upgrade to (``AFTER_TAG``) @@ -52,8 +52,9 @@ file - ``rose-meta/CATEGORY/versions.py``. these very modules carefully or use absolute or package level imports like this: ``from .versionXX_YY import FooBar``. -Upgrade macros are subclasses of :py:class:`rose.upgrade.MacroUpgrade`. They -have all the functionality of the :ref:`transformer macros `. +Upgrade macros are subclasses of :py:class:`metomi.rose.upgrade.MacroUpgrade`. +They have all the functionality of the +:ref:`transformer macros `. .. autoclass:: metomi.rose.upgrade.MacroUpgrade :members: diff --git a/sphinx/api/rosie-web.rst b/sphinx/api/rosie-web.rst index 49afc86fb1..8df2e002e2 100644 --- a/sphinx/api/rosie-web.rst +++ b/sphinx/api/rosie-web.rst @@ -29,13 +29,13 @@ supported format is JSON) and use a url that looks like: REST API -------- -.. http:get:: (str:prefix)/get_known_keys +.. http:get:: (prefix)/get_known_keys Return the main property names stored for suites (e.g. ``idx``, ``branch``, ``owner``) plus any additional names specified in the site config. - :arg str prefix: Repository prefix. - :param string format: Desired return format (``json`` or ``None``). + :arg prefix: Repository prefix. + :param format: Desired return format (``json`` or ``None``). Example Request .. code-block:: http @@ -48,13 +48,13 @@ REST API ["access-list", "idx", "branch", "owner", "project", "revision", "status", "title"] -.. http:get:: (str:prefix)/get_optional_keys +.. http:get:: (prefix)/get_optional_keys Return all unique optional or user-defined property names given in suite discovery information. - :arg str prefix: Repository prefix. - :param string format: Desired return format (``json`` or ``None``). + :arg prefix: Repository prefix. + :param format: Desired return format (``json`` or ``None``). Example Request .. code-block:: http @@ -67,13 +67,13 @@ REST API ["access-list", "description", "endgame_status", "operational_flag", "tag-list"] -.. http:get:: (str:prefix)/get_query_operators +.. http:get:: (prefix)/get_query_operators Returns all the SQL-like operators used to compare column values that you - may use in :http:get:`(str:prefix)/query` (e.g. ``eq``, ``ne``, ``contains``, ``like``). + may use in :http:get:`(prefix)/query` (e.g. ``eq``, ``ne``, ``contains``, ``like``). - :arg str prefix: Repository prefix. - :param string format: Desired return format (``json`` or ``None``). + :arg prefix: Repository prefix. + :param format: Desired return format (``json`` or ``None``). Example Request .. code-block:: http @@ -86,23 +86,21 @@ REST API ["eq", "ge", "gt", "le", "lt", "ne", "contains", "endswith", "ilike", "like", "match", "startswith"] -.. http:get:: (str:prefix)/query +.. http:get:: (prefix)/query Return a list of suites matching all search terms. - :arg str prefix: Repository prefix. - :param list q: List of queries. - :param string format: Desired return format (``json`` or ``None``). - :param flag all_revs: Switch on searching older revisions of current suites - and deleted suites. + :arg prefix: Repository prefix. + :param q: List of queries. + :param format: Desired return format (``json`` or ``None``). + :param all_revs: Switch on searching older revisions of current suites and deleted suites. - :queryparameter str CONJUNCTION: ``and`` or ``or``\ . - :queryparameter str OPEN_GROUP: optional, one or more ``(``\ . - :queryparameter str FIELD: e.g. ``idx`` or ``description``\ . - :queryparameter str OPERATOR: e.g. ``contains`` or ``between``, one of - the operators returned by :http:get:`(str:prefix)/get_query_operators`. - :queryparameter str VALUE: e.g. ``euro4m`` or ``200``\ . - :queryparameter str CLOSE_GROUP: optional, one or more ``)``\ . + :queryparameter CONJUNCTION: ``and`` or ``or``\ . + :queryparameter OPEN_GROUP: optional, one or more ``(``\ . + :queryparameter FIELD: e.g. ``idx`` or ``description``\ . + :queryparameter OPERATOR: e.g. ``contains`` or ``between``, one of the operators returned by :http:get:`(prefix)/get_query_operators`. + :queryparameter VALUE: e.g. ``euro4m`` or ``200``\ . + :queryparameter CLOSE_GROUP: optional, one or more ``)``\ . Query Format .. code-block:: none @@ -136,16 +134,14 @@ REST API "access-list": ["*"], "operational": "Y"}] -.. http:get:: (str:prefix)/search +.. http:get:: (prefix)/search Return a list of suites matching one or more search terms. - :arg str prefix: Repository prefix. - :param list s: List of queries in the same format as - :http:get:`(str:prefix)/query` - :param string format: Desired return format (``json`` or ``None``). - :param flag all_revs: Switch on searching older revisions of current suites - and deleted suites. + :arg prefix: Repository prefix. + :param s: List of queries in the same format as :http:get:`(prefix)/query` + :param format: Desired return format (``json`` or ``None``). + :param ll_revs: Switch on searching older revisions of current suites and deleted suites. Example Request Return suites which match ``var``, ``bob`` or ``nowcast``. diff --git a/sphinx/cheat-sheet.rst b/sphinx/cheat-sheet.rst index 73a122ddba..9b041d61b5 100644 --- a/sphinx/cheat-sheet.rst +++ b/sphinx/cheat-sheet.rst @@ -28,7 +28,7 @@ Starting Suites * - :: cylc validate - cylc run + cylc play - :: # run the suite in the current directory: @@ -173,8 +173,8 @@ Visualise A Suite's :term:`graph` cylc graph -View A Suite's ``suite.rc`` Configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +View A Suite's ``flow.cylc`` Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :class: grid-table diff --git a/sphinx/conf.py b/sphinx/conf.py index 35389e5c4f..1660a8430d 100644 --- a/sphinx/conf.py +++ b/sphinx/conf.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -37,6 +34,7 @@ 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.graphviz', + 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', @@ -79,6 +77,18 @@ # vector graphics will be converted to bitmaps in all documents extensions.append('sphinx.ext.imgconverter') +# mapping to other Sphinx projects +# (allows us to reference objects from other projects) +cylc_version = '8.0a2' +intersphinx_mapping = { + 'cylc': ( + f'https://cylc.github.io/cylc-doc/{cylc_version}/html/', None + ), + 'python': ( + 'https://docs.python.org/', None + ) +} + # Slide (hieroglyph) settings. slide_theme = 'single-level' slide_link_to_html = True diff --git a/sphinx/developing/autodoc.rst b/sphinx/developing/autodoc.rst index fa93ed0a4f..09940e2948 100644 --- a/sphinx/developing/autodoc.rst +++ b/sphinx/developing/autodoc.rst @@ -25,7 +25,7 @@ Some quick examples: .. code-block:: python - def Some_Class(object): + def Some_Class: """Some summary. Note __init__ methods are not autodocumented, specify constructor diff --git a/sphinx/developing/building.rst b/sphinx/developing/building.rst index 9548f9d21a..ad1b6368e6 100644 --- a/sphinx/developing/building.rst +++ b/sphinx/developing/building.rst @@ -22,7 +22,8 @@ The following builders are useful for development: ``linkcheck`` Check external links (internal links are checked by a regular build). ``doctest`` - Run any doctests contained within documented code (e.g. see ``rose.config``). + Run any doctests contained within documented code + (e.g. see :py:mod:`metomi.rose.config`). Additionally, if you are not using an editor with a spellchecker you may wish to use aspell/ispell/hunspell to check any changed docs: diff --git a/sphinx/ext/auto_cli_doc.py b/sphinx/ext/auto_cli_doc.py index 88c18e9831..7d805ef98c 100644 --- a/sphinx/ext/auto_cli_doc.py +++ b/sphinx/ext/auto_cli_doc.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -21,7 +18,7 @@ from collections import OrderedDict import re -from subprocess import PIPE, check_output, CalledProcessError +from subprocess import check_output, CalledProcessError import sys from docutils import nodes @@ -128,7 +125,12 @@ def line_strip(lines): ... '' ... ]) ['a'] + >>> line_strip(['']) + [] + """ + if all(not line for line in lines): + return [] lines = list(lines) kill = [] for itt in range(0, len(lines)): diff --git a/sphinx/ext/rose_domain.py b/sphinx/ext/rose_domain.py index 048b76b8d2..fbac87b6cf 100644 --- a/sphinx/ext/rose_domain.py +++ b/sphinx/ext/rose_domain.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -62,7 +59,7 @@ Documentation can be auto-built from RST formatted comments in Rose configuration files using the ``autoconfig`` directive. - Note that due to the implementation of :py:mod:`rose.config` the + Note that due to the implementation of :py:mod:`metomi.rose.config` the autodocumenter will represent empty sections as top level configuration nodes. @@ -76,7 +73,7 @@ .. rose:conf:: jinja2:suite.rc A section for specifying Jinja2 settings for use in the - ``suite.rc`` file. + ``flow.cylc`` file. Note that one ommits the square brackets for config sections. If :rose:conf: contains other :rose:conf:'s then it is implicitly a @@ -925,7 +922,7 @@ class RoseAutoDirective(Directive): """Directive for autodocumenting Rose configuration files. Uses RST formatted comments in Rose configuration files using - :py:mod:`rose.config`. + :py:mod:`metomi.rose.config`. Note the directive only documents config objects not the file itself. diff --git a/sphinx/ext/rose_lang.py b/sphinx/ext/rose_lang.py index 4e85e460c5..8b975d99a5 100644 --- a/sphinx/ext/rose_lang.py +++ b/sphinx/ext/rose_lang.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify @@ -157,4 +154,4 @@ class RoseLexer(RegexLexer): def setup(app): """Sphinx plugin setup function.""" - app.add_lexer('rose', RoseLexer()) + app.add_lexer('rose', RoseLexer) diff --git a/sphinx/ext/script_include.py b/sphinx/ext/script_include.py index 067330fdc7..989869c5be 100644 --- a/sphinx/ext/script_include.py +++ b/sphinx/ext/script_include.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/sphinx/extract-pdf-documents.py b/sphinx/extract-pdf-documents.py deleted file mode 100644 index 1e97d81a5d..0000000000 --- a/sphinx/extract-pdf-documents.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -# ----------------------------------------------------------------------------- -"""Script for extracting PDF documents from a Sphinx LaTeX build. - -* Copies PDF documents from the ``latex`` build directory to a ``pdf`` folder. -* Creates an HTML index of these documents. - -""" - -import errno -import os -import shutil -import sys - -import conf - - -def main(): - try: - build_dir = sys.argv[1] - except IndexError: - sys.exit('usage: extract-pdf-documents build_dir') - latex_dir = os.path.join(build_dir, 'latex') - pdf_dir = os.path.join(build_dir, 'pdf') - os.makedirs(pdf_dir, exist_ok=True) - - # the index html file - html = ( - '' - '' - '' - 'Rose Documentation - PDF Documents' - '' - '' - '

Rose Documentation - PDF Documents

' - '' - '' - '' - ) - - # write index file - with open(os.path.join(pdf_dir, 'index.html'), 'w+') as index: - index.write(html) - - # remove now un-necessary latex directory - shutil.rmtree(latex_dir) - - -if __name__ == '__main__': - main() diff --git a/sphinx/getting-started.rst b/sphinx/getting-started.rst index 9f864ee5c6..0d6f78b87f 100644 --- a/sphinx/getting-started.rst +++ b/sphinx/getting-started.rst @@ -56,37 +56,20 @@ Editor Syntax Highlighting -------------------------- There are ``gedit``, ``kate``, ``vim``, and ``emacs`` plugins for syntax -highlighting of Rose configuration files, located within the Rose installation: +highlighting of Rose configuration files, located within the Rose installation. -* ``etc/rose-conf.lang`` -* ``etc/rose-conf.xml`` -* ``etc/rose-conf.vim`` -* ``etc/rose-conf-mode.el`` +Run the following command to see the available syntax files and their +locations:: -The plugins contain setup instructions within. - -.. _Pygments: https://pygments.org - -Additionally there is a `Pygments`_ lexer located in -``sphinx/ext/rose_lang.py``. - -.. hint:: - - You can locate your Rose installation using:: - - rose version --long + $ rose resource syntax +Each file contains setup instructions within. -Bash Auto-Completion --------------------- - -There is a Rose bash completion script that you can source to enhance the -Rose command line interface within an interactive Bash shell. - -The script allows you to tab-complete Rose commands, options, and arguments. +.. _Pygments: https://pygments.org +.. _Rose Lang: https://github.com/metomi/rose/blob/master/sphinx/ext/rose_lang.py -You can find the script in the Rose installation ``etc/rose-bash-completion``. -The file contains the instructions for using it. +Additionally there is a `Pygments`_ lexer located +`in the source code `_ Configuring Cylc @@ -94,14 +77,3 @@ Configuring Cylc See the "Installation" and "User Config File" sections of the `Cylc User Guide`_. - -.. warning:: - - Do not modify the default values of the following cylc settings: - - * ``[hosts][HOST]run directory`` - * ``[hosts][HOST]work directory`` - - Equivalent functionalities are provided by the - :rose:conf:`rose.conf[rose-suite-run]root-dir` settings in the Rose - site/user configuration. diff --git a/sphinx/glossary.rst b/sphinx/glossary.rst index c5162be67d..87c4832a4a 100644 --- a/sphinx/glossary.rst +++ b/sphinx/glossary.rst @@ -9,7 +9,7 @@ Glossary suite Cylc suite - A Cylc suite is a directory containing a ``suite.rc`` file which contains + A Cylc suite is a directory containing a ``flow.cylc`` file which contains :term:`graphing` representing a workflow. See also: @@ -20,7 +20,7 @@ Glossary suite directory The suite directory contains all of the configuration for a suite (e.g. - the ``suite.rc`` file and for Rose suites the :rose:file:`rose-suite.conf` + the ``flow.cylc`` file and for Rose suites the :rose:file:`rose-suite.conf` file). This is the directory which is registered using ``cylc reg`` or, for Rose @@ -83,7 +83,7 @@ Glossary graph string A graph string is a collection of dependencies which are placed under a - ``graph`` section in the ``suite.rc`` file. E.G: + ``graph`` section in the ``flow.cylc`` file. E.G: .. code-block:: cylc-graph @@ -213,7 +213,7 @@ Glossary been configured to use integer cycling. When a suite uses integer cycling integer :term:`recurrences ` may be used in the :term:`graph`, e.g. ``P3`` means every third cycle. This is configured by setting - ``[scheduling]cycling mode = integer`` in the ``suite.rc`` file. + ``[scheduling]cycling mode = integer`` in the ``flow.cylc`` file. See also: @@ -539,7 +539,7 @@ Glossary what a :term:`job's ` requirements are, e.g. how much memory it requires. - Directives are set in the ``suite.rc`` file in the ``[runtime]`` section + Directives are set in the ``flow.cylc`` file in the ``[runtime]`` section (``[runtime][][directives]``). See also: @@ -566,7 +566,7 @@ Glossary run. This program controls the suite and is what we refer to as "running". - * A :term:`Cylc suite` is started using ``cylc run``. + * A :term:`Cylc suite` is started using ``cylc play``. * A :term:`Rose suite configuration` (or :term:`Rosie Suite`) is started using :ref:`command-rose-suite-run`. @@ -586,7 +586,7 @@ Glossary cold start A cold start is one in which the :term:`suite` :term:`starts ` from the :term:`initial cycle point`. This is the default behaviour of - ``cylc run``. + ``cylc play``. See also: @@ -631,7 +631,7 @@ Glossary * :term:`reload` reload - Any changes made to the ``suite.rc`` file whilst the suite is running + Any changes made to the ``flow.cylc`` file whilst the suite is running will not have any effect until the suite is either: * :term:`shutdown` and :term:`rerun ` @@ -654,7 +654,7 @@ Glossary parameterisation Parameterisation is a way to consolidate configuration in the Cylc - ``suite.rc`` file by implicitly looping over a set of pre-defined + ``flow.cylc`` file by implicitly looping over a set of pre-defined variables e.g: .. code-block:: cylc @@ -823,12 +823,11 @@ Glossary with other optional files and directories which configure the way in which a :term:`Cylc suite` is run. E.g: - * Jinja2 variables to be passed into the ``suite.rc`` file ( + * Jinja2 variables to be passed into the ``flow.cylc`` file ( :rose:conf:`rose-suite.conf[jinja2:suite.rc]`). - * Environment variables to be provided to ``cylc run`` ( + * Environment variables to be provided to ``cylc play`` ( :rose:conf:`rose-suite.conf[env]`). * Installation configuration (e.g. - :rose:conf:`rose-suite.conf|root-dir`, :rose:conf:`rose-suite.conf[file:NAME]`). See also: diff --git a/sphinx/hyperlinks.rst b/sphinx/hyperlinks.rst index f537c2bcd1..7c8c22b6b8 100644 --- a/sphinx/hyperlinks.rst +++ b/sphinx/hyperlinks.rst @@ -10,8 +10,11 @@ :start-line: 1 .. _Cylc: https://cylc.github.io/ +.. _Cylc Flow: https://github.com/cylc/cylc-flow +.. _Cylc Rose: https://github.com/cylc/cylc-rose .. _Cylc User Guide: https://cylc.github.io/documentation/#the-cylc-user-guide -.. _Cylc Suite Design Guide: https://cylc.github.io/doc/built-sphinx/suite-design-guide/suite-design-guide-master.html#suite-design-guide +.. _Cylc Suite Design Guide: https://cylc.github.io/cylc-doc/stable/html/suite-design-guide/suite-design-guide-master.html + .. _EmPy: http://www.alcyone.com/software/empy/ .. _extglob pattern matching: https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html#Pattern-Matching .. _FCM: https://metomi.github.io/fcm/doc/ diff --git a/sphinx/index.rst b/sphinx/index.rst index faf3e58c91..d18adf9d67 100644 --- a/sphinx/index.rst +++ b/sphinx/index.rst @@ -44,7 +44,6 @@ applications. :ref:`What Is Cylc? ` installation getting-started - tutorial/cylc/index tutorial/rose/index cheat-sheet glossary diff --git a/sphinx/installation.rst b/sphinx/installation.rst index 90f50d45a4..3d380ba311 100644 --- a/sphinx/installation.rst +++ b/sphinx/installation.rst @@ -1,122 +1,87 @@ .. include:: hyperlinks.rst :start-line: 1 + Installation ============ -.. _GitHub: https://github.com/metomi/rose -.. _archives: https://github.com/metomi/rose/tags - -The source code for Rose is available via `GitHub`_, code -releases are available in ``zip`` and ``tar.gz`` `archives`_. - -1. If you wish to do so create a new environment using Conda, Venv or some - other environment manager. -2. Install using ``pip install metomi-rose`` -3. Check system compatibility by running :ref:`command-rose-check-software`. +Rose runs on Unix-like systems including Linux and MacOS. +Quick Installation +------------------ -System Requirements -------------------- +With ``conda`` (recommended):: -Rose runs on Unix/Linux systems and is known to run on RHEL6 and a number of -systems including those documented under `metomi-vms`_. + $ conda install metomi-rose -System compatibility can be tested using the :ref:`command-rose-check-software` -command. +With ``pip`` (you will need to ensure `Non Python Dependencies`_ are met):: -.. script-include:: rose check-software --rst + $ pip install metomi-rose +Installing With Cylc +^^^^^^^^^^^^^^^^^^^^ -Host Requirements ------------------ +Rose does not require and is distributed independently of `Cylc`_. -Whilst you can install and use Rose & Cylc on a single host (i.e. machine), -most installations are likely to be more complex. There are five types of -installation: +To use Rose with Cylc you will need to install `Cylc Flow`_ and `Cylc Rose`_ +into the same Python environment as Rose. -Hosts For Running Cylc Suites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +With ``conda`` (recommended):: -Each user can just run their suites on the host where they issue the -:ref:`command-rose-suite-run` command. However, the local host may not meet the -necessary requirements for connectivity (to the hosts running the tasks) -or for availability (the host needs to remain up for the duration of -the suite). Therefore, Rose can be configured to run the suites on -remote hosts. If multiple hosts are available, by default the host with -the lowest system load average will be used. + $ conda install cylc-flow cylc-rose -Installation requirements: - * Rose, Cylc, Bash, Python, jinja2. - * Subversion & FCM *(only if you want* :ref:`command-rose-suite-run` *to - install files from Subversion using FCM keywords).* -Connectivity requirements: - * Must be able to submit tasks to the hosts which run the suite tasks, - either directly or via SSH access to another host. +With ``pip`` (you will need to ensure `Non Python Dependencies`_ are met):: -Hosts For Running Tasks In Suites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + $ pip install cylc-flow cylc-rose -Installation requirements: - * Rose, Cylc, Bash, Python. - * Subversion & FCM *only if you want to use the Rose install utility - to install files from Subversion using FCM keywords*. - -Connectivity requirements: - * Must be able to communicate back to the hosts running the Cylc suites - via HTTPS (preferred), HTTP or SSH. +.. note:: -Hosts For Running User Interactive Tools -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + `Cylc`_ is a distributed system, not all dependencies are required on all + platforms. -Installation requirements: - * Rose, Cylc, Bash, Python, requests, Subversion, FCM, - Pygraphviz (+ graphviz). + See the `Cylc`_ installation instructions for more information. -Connectivity requirements: - * Must have HTTP access to the hosts running the Rosie web service. - * Must have access to the Rosie Subversion repositories via the - appropriate protocol. - * Must have HTTP and SSH access to the hosts running the Cylc suites. - * Must share user accounts and ``$HOME`` directories with the hosts running - the Cylc suites. + .. TODO -Hosts For Running Rose Bush -^^^^^^^^^^^^^^^^^^^^^^^^^^^ + This reference will pass once intersphinx has a more contemporary + version of cylc-doc to point at (see conf.py) -Installation requirements: - * Rose, Bash, Python, jinja2. + See the :ref:`Cylc installation instructions ` for more + information. -Connectivity requirements: - * Must be able to access the home directories of users' Cylc run directories. +Non Python Dependencies +^^^^^^^^^^^^^^^^^^^^^^^ -Hosts For Rosie Subversion Repositories And The Rosie Web Services -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The following packages are installed by ``conda`` but not by ``pip``: -Typically you will only need a single host but you can have multiple -repositories on different hosts if you require this. +* FCM +* Perl +* Python3 +* Subversion -Installation requirements: - * Rose, Bash, Python, jinja2, sqlalchemy, Subversion. +If installing via ``pip`` run :ref:`command-rose-check-software` to ensure +non-Python dependencies are satisfied. .. note:: - This section has assumed that you wish to use Rose & Cylc (including - their respective GUIs) and use Rosie to store your suites. However, - there are many ways of using Rose. For instance: + Subversion & FCM are required for installing files from Subversion using FCM + keywords by: - * You can use Rose without using cylc. - * You can use Rose & Cylc but not use Rosie. - * You can use Rose & Cylc from the command line without using their GUIs. + + ``rose app-run`` + + ``rose task-run`` + + ``cylc install`` Configuring Rose ---------------- -``etc/rose.conf`` - You should add/edit this file to meet the requirement of your site. - Examples can be found at the ``etc/rose.conf.example`` file in your - Rose distribution. See also :rose:file:`rose.conf` in the API reference. +Rose configuration files can be located in the following places: + +* ``/etc/.metomi/rose.conf`` +* ``$ROSE_SITE_CONF_PATH/.metomi/rose.conf`` +* ``$HOME/.metomi/rose.conf`` + +See :rose:file:`rose.conf` in the API reference for more information. Configuring Rosie Client @@ -130,6 +95,7 @@ a web interface for suite discovery and lookup. If users at your site are able to access Rosie services on the Internet or if someone else has already configured Rosie services at your site, all you need to do is configure the client to talk to the servers. + Refer to the `Configuring a Rosie Server`_ section if you need to configure a Rosie server for your site. @@ -236,26 +202,6 @@ under ``/etc/rose-meta/`` at your site. You can do:: See also :ref:`app-meta-loc`. -Configuring Rose Bush ---------------------- - -Rose Bush provides an intranet web service at your site for users to -view their suite logs using a web browser. Depending on settings at -your site, you may or may not be able to set up this service. - -You can start an ad-hoc Rose Bush web server by running:: - - setsid /path/to/rose/bin/rose bush start 0/dev/null 2>&1 & - -You will find the access and error logs under ``~/.metomi/rose-bush*``. - -Alternatively you can run the Rose Bush web service under Apache -``mod_wsgi``. - -Use the Apache log at e.g. ``/var/log/httpd/`` to debug problems. -See also `Configuring a Rosie Server`_. - - Configuring a Rosie Server -------------------------- @@ -300,17 +246,17 @@ Add the following hook scripts to the repository: .. code-block:: sub - #!/bin/bash + #!/usr/bin/env bash exec /sbin/rosa svn-pre-commit "$@" * post-commit: .. code-block:: sub - #!/bin/bash + #!/usr/bin/env bash exec /sbin/rosa svn-post-commit "$@" -You should replace ``/path/to/rose/`` with the location of your Rose +You should replace ```` with the location of your Rose installation. Make sure the hook scripts are executable. @@ -321,9 +267,11 @@ are committed to the repository. Edit the :rose:conf:`rose.conf[rosie-db]` settings to point to your host machine and provide relevant paths such as the location for your repository and database. -Once you have done that, create the Rosie database by running:: +Once you have done that, create the Rosie database by running: + +.. code-block:: sub - /path/to/rose/sbin/rosa db-create + /sbin/rosa db-create Make sure that the account that runs the repository hooks has read/write access to the database and database directory. @@ -331,13 +279,17 @@ access to the database and database directory. You can test that everything is working using the built-in web server. Edit the :rose:conf:`rose.conf[rosie-disco]` settings to configure the web server's log directory and port number. Start the web server -by running:: +by running: + +.. code-block:: sub - setsid /path/to/rose/bin/rosie disco start 0&1 & + setsid /bin/rosie disco start 0&1 & Check that the server is up and running using ``curl`` or a local web browser. E.g. If you have configured the server's port to be 1234, -you can do:: +you can do: + +.. code-block:: sub curl -I http://localhost:1234/ @@ -346,13 +298,14 @@ It should return a HTTP code 200. Alternatively you can run the Rosie web service under Apache ``mod_wsgi``. To do this you will need to set up an Apache module configuration file (typically in ``/etc/httpd/conf.d/rose-wsgi.conf``) containing the -following (with the paths set appropriately):: +following (with the paths set appropriately): + +.. code-block:: sub - WSGIPythonPath /path/to/rose/lib/python - WSGIScriptAlias /rosie /path/to/rose/lib/python/rosie/ws.py + WSGIPythonPath /lib/python + WSGIScriptAlias /rosie /lib/python/rosie/ws.py Use the Apache log at e.g. ``/var/log/httpd/`` to debug problems. -See also `Configuring Rose Bush`_. Hopefully, you should now have a working Rosie service server. Configure the client settings by editing the :rose:conf:`rose.conf[rosie-id]` diff --git a/sphinx/tutorial/cylc/furthertopics/broadcast.rst b/sphinx/tutorial/cylc/furthertopics/broadcast.rst deleted file mode 100644 index 447b49c15b..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/broadcast.rst +++ /dev/null @@ -1,135 +0,0 @@ -Broadcast -========= - -This tutorial walks you through using ``cylc broadcast`` which can be used -to change :ref:`task runtime configuration ` in a -running suite, on-the-fly. - - -Purpose -------- - -``cylc broadcast`` can be used to change any ``[runtime]`` setting whilst the -suite is running. - -The standard use of ``cylc broadcast`` is to update the suite to an -unexpected change in configuration, for example modifying the host a task -runs on. - - -Standalone Example ------------------- - -Create a new suite in the ``cylc-run`` directory called -``tutorial-broadcast``:: - - mkdir ~/cylc-run/tutorial-broadcast - cd ~/cylc-run/tutorial-broadcast - -Copy the following configuration into a ``suite.rc`` file: - -.. code-block:: cylc - - [scheduling] - initial cycle point = 1012 - [[dependencies]] - [[[R1]]] - graph = wipe_log => announce - [[[PT1H]]] - graph = announce[-PT1H] => announce - - [runtime] - [[wipe_log]] - # Delete any files in the suite's "share" directory. - script = rm "${CYLC_SUITE_SHARE_DIR}/knights" || true - - [[announce]] - script = echo "${CYLC_TASK_CYCLE_POINT} - ${MESSAGE}" >> "${FILE}" - [[[environment]]] - WORD = ni - MESSAGE = We are the knights who say \"${WORD}\"! - FILE = "${CYLC_SUITE_SHARE_DIR}/knights" - -We now have a suite with an ``announce`` task which runs every hour, writing a -message to a log file (``share/knights``) when it does so. For the first cycle -the log entry will look like this:: - - 10120101T0000Z - We are the knights who say "ni"! - -The ``cylc broadcast`` command enables us to change runtime configuration -whilst the suite is running. For instance we could change the value of the -``WORD`` environment variable using the command:: - - cylc broadcast tutorial-broadcast -n announce -s "[environment]WORD=it" - -* ``tutorial-broadcast`` is the name of the suite. -* ``-n announce`` tells Cylc we want to change the runtime configuration of the - ``announce`` task. -* ``-s "[environment]WORD=it"`` changes the value of the ``WORD`` environment - variable to ``it``. - -Run the suite then try using the ``cylc broadcast`` command to change the -message:: - - cylc run tutorial-broadcast - cylc broadcast tutorial-broadcast -n announce -s "[environment]WORD=it" - -Inspect the ``share/knights`` file, you should see the message change at -certain points. - -Stop the suite:: - - cylc stop tutorial-broadcast - - -In-Situ Example ---------------- - -We can call ``cylc broadcast`` from within a task's script. This effectively -provides the ability for tasks to communicate between themselves. - -It is almost always better for tasks to communicate using files but there are -some niche situations where communicating via ``cylc broadcast`` is justified. -This tutorial walks you through using ``cylc broadcast`` to communicate between -tasks. - -.. TODO - examples of this? - -Add the following recurrence to the ``dependencies`` section: - -.. code-block:: cylc - - [[[PT3H]]] - graph = announce[-PT1H] => change_word => announce - -The ``change_word`` task runs the ``cylc broadcast`` command to randomly -change the ``WORD`` environment variable used by the ``announce`` task. - -Add the following runtime configuration to the ``runtime`` section: - -.. code-block:: cylc - - [[change_word]] - script = """ - # Select random word. - IFS=',' read -r -a WORDS <<< $WORDS - WORD=${WORDS[$(date +%s) % ${#WORDS[@]}]} - - # Broadcast random word to the announce task. - cylc broadcast $CYLC_SUITE_NAME -n announce -s "[environment]WORD=${WORD}" - """ - [[[environment]]] - WORDS = ni, it, ekke ekke ptang zoo boing - -Run the suite and inspect the log. You should see the message change randomly -after every third entry (because the ``change_word`` task runs every 3 hours) -e.g:: - - 10120101T0000Z - We are the knights who say "ni"! - 10120101T0100Z - We are the knights who say "ni"! - 10120101T0200Z - We are the knights who say "ni"! - 10120101T0300Z - We are the knights who say "ekke ekke ptang zoo boing!" - -Stop the suite:: - - cylc stop tutorial-broadcast diff --git a/sphinx/tutorial/cylc/furthertopics/clock-triggered-tasks.rst b/sphinx/tutorial/cylc/furthertopics/clock-triggered-tasks.rst deleted file mode 100644 index 43181eebf1..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/clock-triggered-tasks.rst +++ /dev/null @@ -1,175 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - -.. _tutorial-cylc-clock-trigger: - -Clock Triggered Tasks -===================== - -.. TODO - - After #2423 has been finalised and merged this tutorial should be - re-factored / re-written to incorporate the usage of ``cylc-graph``. - -In a :term:`datetime cycling` suite the time represented by the -:term:`cycle points ` bear no relation to the real-world time. -Using clock-triggers we can make tasks wait until their cycle point time before -running. - -Clock-triggering effectively enables us to tether the "cycle time" to the -"real world time" which we refer to as the :term:`wall-clock time`. - - -Clock Triggering ----------------- - -When clock-triggering tasks we can use different -:ref:`offsets ` to the cycle time as follows: - -.. code-block:: cylc - - clock-trigger = taskname(CYCLE_OFFSET) - -.. note:: - - Regardless of the offset used, the task still belongs to the cycle from - which the offset has been applied. - - -Example -------- - -Our example suite will simulate a clock chiming on the hour. - -Within your ``~/cylc-run`` directory create a new directory called -``clock-trigger``:: - - mkdir ~/cylc-run/clock-trigger - cd ~/cylc-run/clock-trigger - -Paste the following code into a ``suite.rc`` file: - -.. code-block:: cylc - - [cylc] - UTC mode = True # Ignore DST - - [scheduling] - initial cycle point = TODO - final cycle point = +P1D # Run for one day - [[dependencies]] - [[[PT1H]]] - graph = bell - - [runtime] - [[root]] - [[[events]]] - mail events = failed - [[bell]] - env-script = eval $(rose task-env) - script = printf 'bong%.0s\n' $(seq 1 $(cylc cyclepoint --print-hour)) - -Change the initial cycle point to 00:00 this morning (e.g. if it was -the first of January 2000 we would write ``2000-01-01T00Z``). - -We now have a simple suite with a single task that prints "bong" a number -of times equal to the (cycle point) hour. - -Run your suite using:: - - cylc run clock-trigger - -Stop the suite after a few cycles using the :guilabel:`stop` button in the -``cylc gui``. Notice how the tasks run as soon as possible rather than -waiting for the actual time to be equal to the cycle point. - - -Clock-Triggering Tasks ----------------------- - -We want our clock to only ring in real-time rather than the simulated -cycle time. - -To do this, add the following lines to the ``[scheduling]`` section of -your ``suite.rc``: - -.. code-block:: cylc - - [[special tasks]] - clock-trigger = bell(PT0M) - -This tells the suite to clock trigger the ``bell`` task with a cycle -offset of ``0`` hours. - -Save your changes and run your suite. - -Your suite should now be running the ``bell`` task in real-time. Any cycle times -that have already passed (such as the one defined by ``initial cycle time``) -will be run as soon as possible, while those in the future will wait for that -time to pass. - -At this point you may want to leave your suite running until the next hour -has passed in order to confirm the clock triggering is working correctly. -Once you are satisfied, stop your suite. - -By making the ``bell`` task a clock triggered task we have made it run in -real-time. Thus, when the wall-clock time caught up with the cycle time, the -``bell`` task triggered. - - -Adding More Clock-Triggered Tasks ---------------------------------- - -We will now modify our suite to run tasks at quarter-past, half-past and -quarter-to the hour. - -Open your ``suite.rc`` and modify the ``[runtime]`` section by adding the -following: - -.. code-block:: cylc - - [[quarter_past, half_past, quarter_to]] - script = echo 'chimes' - -Edit the ``[[scheduling]]`` section to read: - -.. code-block:: cylc - - [[special tasks]] - clock-trigger = bell(PT0M), quarter_past(PT15M), half_past(PT30M), quarter_to(PT45M) - [[dependencies]] - [[[PT1H]]] - graph = """ - bell - quarter_past - half_past - quarter_to - """ - -Note the different values used for the cycle offsets of the clock-trigger tasks. - -Save your changes and run your suite using:: - - cylc run clock-trigger now - -.. note:: - - The ``now`` argument will run your suite using the current time for the - initial cycle point. - -Again, notice how the tasks trigger until the current time is reached. - -Leave your suite running for a while to confirm it is working as expected -and then shut it down using the :guilabel:`stop` button in the ``cylc gui``. - - -Summary -------- - -* Clock triggers are a type of :term:`dependency` which cause - :term:`tasks ` to wait for the :term:`wall-clock time` to reach the - :term:`cycle point` time. -* A clock trigger applies only to a single task. -* Clock triggers can only be used in datetime cycling suites. - -For more information see the `Cylc User Guide`_. diff --git a/sphinx/tutorial/cylc/furthertopics/family-triggers.rst b/sphinx/tutorial/cylc/furthertopics/family-triggers.rst deleted file mode 100644 index 29cd03bc6d..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/family-triggers.rst +++ /dev/null @@ -1,208 +0,0 @@ -.. _tutorial-cylc-family-triggers: - -Family Triggers -=============== - -To reduce duplication in the :term:`graph` is is possible to write -:term:`dependencies ` using collections of tasks called -:term:`families `). - -This tutorial walks you through writing such dependencies using family -:term:`triggers `. - - -Explanation ------------ - -Dependencies between tasks can be written using a :term:`qualifier` to describe -the :term:`task state` that the dependency refers to (e.g. ``succeed`` -``fail``, etc). If a dependency does not use a qualifier then it is assumed -that the dependency refers to the ``succeed`` state e.g: - -.. code-block:: cylc-graph - - bake_bread => sell_bread # sell_bread is dependent on bake_bread succeeding. - bake_bread:succeed => sell_bread # sell_bread is dependent on bake_bread succeeding. - sell_bread:fail => through_away # through_away is dependent on sell_bread failing. - -The left-hand side of a :term:`dependency` (e.g. ``sell_bread:fail``) is -referred to as the :term:`trigger `. - -When we write a trigger involving a family, special qualifiers are required -to specify whether the dependency is concerned with *all* or *any* of the tasks -in that family reaching the desired :term:`state ` e.g: - -* ``succeed-all`` -* ``succeed-any`` -* ``fail-all`` - -Such :term:`triggers ` are referred to as -:term:`family triggers ` - -Foo ``cylc gui`` bar - - -Example -------- - -Create a new suite called ``tutorial-family-triggers``:: - - mkdir ~/cylc-run/tutorial-family-triggers - cd ~/cylc-run/tutorial-family-triggers - -Paste the following configuration into the ``suite.rc`` file: - -.. code-block:: cylc - - [cylc] - UTC mode = True # Ignore DST - [scheduling] - [[dependencies]] - graph = visit_mine => MINERS - [runtime] - [[visit_mine]] - script = sleep 5; echo 'off to work we go' - - [[MINERS]] - script = """ - sleep 5; - if (($RANDOM % 2)); then - echo 'Diamonds!'; true; - else - echo 'Nothing...'; false; - fi - """ - [[doc, grumpy, sleepy, happy, bashful, sneezy, dopey]] - inherit = MINERS - -You have now created a suite that: - -* Has a ``visit_mine`` task that sleeps for 5 seconds then outputs a - message. -* Contains a ``MINERS`` family with a command in it that randomly succeeds - or fails. -* Has 7 tasks that inherit from the ``MINERS`` family. - -Open the ``cylc gui`` then run the suite by pressing the "play" button -(top left hand corner) then clicking :guilabel:`Start`:: - - cylc gui tutorial-family-triggers & - -You should see the ``visit_mine`` task run, then trigger the members of the -``MINERS`` family. Note that some of the ``MINERS`` tasks may fail so you -will need to stop your suite using the "stop" button in the ``cylc gui`` in -order to allow it to shutdown. - - -Family Triggering: Success --------------------------- - -As you will have noticed by watching the suite run, some of the tasks in the -``MINERS`` family succeed and some fail. - -We would like to add a task to sell any diamonds we find, but wait for all -the miners to report back first so we only make the one trip. - -We can address this by using *family triggers*. In particular, we are going -to use the ``finish-all`` trigger to check for all members of the ``MINERS`` -family finishing, and the ``succeed-any`` trigger to check for any of the -tasks in the ``MINERS`` family succeeding. - -Open your ``suite.rc`` file and change the ``[[dependencies]]`` to look like -this: - -.. code-block:: cylc - - [[dependencies]] - graph = """visit_mine => MINERS - MINERS:finish-all & MINERS:succeed-any => sell_diamonds""" - -Then, add the following task to the ``[runtime]`` section: - -.. code-block:: cylc - - [[sell_diamonds]] - script = sleep 5 - -These changes add a ``sell_diamonds`` task to the suite which is run once -all the ``MINERS`` tasks have finished and if any of them have succeeded. - -Save your changes and run your suite. You should see the new -``sell_diamonds`` task being run once all the miners have finished and at -least one of them has succeeded. As before, stop your suite using the "stop" -button in the ``cylc gui``. - - -Family Triggering: Failure --------------------------- - -Cylc also allows us to trigger off failure of tasks in a particular family. - -We would like to add another task to close down unproductive mineshafts once -all the miners have reported back and had time to discuss their findings. - -To do this we will make use of family triggers in a similar manner to before. - -Open your ``suite.rc`` file and change the ``[[dependencies]]`` to look like -this: - -.. code-block:: cylc - - [[dependencies]] - graph = """visit_mine => MINERS - MINERS:finish-all & MINERS:succeed-any => sell_diamonds - MINERS:finish-all & MINERS:fail-any => close_shafts - close_shafts => !MINERS - """ - -Alter the ``[[sell_diamonds]]`` section to look like this: - -.. code-block:: cylc - - [[close_shafts, sell_diamonds]] - script = sleep 5 - -These changes add a ``close_shafts`` task which is run once all the -``MINERS`` tasks have finished and any of them have failed. On completion -it applies a *suicide trigger* to the ``MINERS`` family in order to allow -the suite to shutdown. - -Save your changes and run your suite. You should see the new -``close_shafts`` run should any of the ``MINERS`` tasks be in the failed -state once they have all finished. - -.. tip:: - - See the :ref:`tut-cylc-suicide-triggers` tutorial for handling task - failures. - - -Different Triggers ------------------- - -Other family :term:`qualifiers ` beyond those covered in the -example are also available. - -The following types of "all" qualifier are available: - -* ``:start-all`` - all the tasks in the family have started -* ``:succeed-all`` - all the tasks in the family have succeeded -* ``:fail-all`` - all the tasks in the family have failed -* ``:finish-all`` - all the tasks in the family have finished - -The following types of "any" qualifier are available: - -* ``:start-any`` - at least one task in the family has started -* ``:succeed-any`` - at least one task in the family has succeeded -* ``:fail-any`` - at least one task in the family has failed -* ``:finish-any`` - at least one task in the family has finished - - -Summary -------- - -* Family triggers allow you to write dependencies for collections of tasks. -* Like :term:`task triggers `, family triggers can be based on - success, failure, starting and finishing of tasks in a family. -* Family triggers can trigger off either *all* or *any* of the tasks in a - family. diff --git a/sphinx/tutorial/cylc/furthertopics/index.rst b/sphinx/tutorial/cylc/furthertopics/index.rst deleted file mode 100644 index 17e4126286..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -Further Topics -============== - -This section looks at further topics in cylc. - -.. toctree:: - :name: cylc-futher-topics - :maxdepth: 1 - - clock-triggered-tasks - broadcast - family-triggers - inheritance - queues - retries - suicide-triggers diff --git a/sphinx/tutorial/cylc/furthertopics/inheritance.rst b/sphinx/tutorial/cylc/furthertopics/inheritance.rst deleted file mode 100644 index c1bf03af3c..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/inheritance.rst +++ /dev/null @@ -1,610 +0,0 @@ -Inheritance -=========== - -We have seen in the :ref:`runtime tutorial ` how -tasks can be grouped into families. - -In this tutorial we will look at nested families, inheritance order and -multiple inheritance. - - -Inheritance Hierarchy ---------------------- - -Create a new suite by running the command:: - - rose tutorial inheritance-tutorial - cd ~/cylc-run/inheritance-tutorial - -You will now have a ``suite.rc`` file that defines two tasks each representing -a different aircraft, the Airbus A380 jumbo jet and the Robson R44 helicopter: - -.. image:: https://upload.wikimedia.org/wikipedia/commons/0/09/A6-EDY_A380_Emirates_31_jan_2013_jfk_%288442269364%29_%28cropped%29.jpg - :width: 49% - :alt: A380 - -.. image:: https://upload.wikimedia.org/wikipedia/commons/2/2f/Robinson-R44_1.jpg - :width: 49% - :alt: R44 - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = a380 & r44 - - [runtime] - [[VEHICLE]] - init-script = echo 'Boarding' - pre-script = echo 'Departing' - post-script = echo 'Arriving' - - [[AIR_VEHICLE]] - inherit = VEHICLE - [[[meta]]] - description = A vehicle which can fly. - [[AIRPLANE]] - inherit = AIR_VEHICLE - [[[meta]]] - description = An air vehicle with fixed wings. - [[[environment]]] - CAN_TAKE_OFF_VERTICALLY = false - [[HELICOPTER]] - inherit = AIR_VEHICLE - [[[meta]]] - description = An air vehicle with rotors. - [[[environment]]] - CAN_TAKE_OFF_VERTICALLY = true - - [[a380]] - inherit = AIRPLANE - [[[meta]]] - title = Airbus A380 Jumbo-Jet. - [[r44]] - inherit = HELICOPTER - [[[meta]]] - title = Robson R44 Helicopter. - -.. note:: - - The ``[meta]`` section is a freeform section where we can define metadata - to be associated with a task, family or the suite itself. - - This metadata should not be mistaken with Rose :ref:`conf-meta`. - -.. admonition:: Reminder - :class: hint - - By convention we write family names in upper case (with the exception of the - special ``root`` family) and task names in lower case. - -These two tasks sit at the bottom of an inheritance tree. The ``cylc graph`` -command has an option (``-n``) for drawing such inheritance hierarchies:: - - cylc graph -n . & - -Running this command will generate the following output: - -.. digraph:: Example - :align: center - - AIRPLANE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - a380 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - AIRPLANE -> a380 [color=royalblue]; - HELICOPTER [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - r44 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - HELICOPTER -> r44 [color=royalblue]; - root [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - root -> VEHICLE [color=royalblue]; - AIR_VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE -> AIR_VEHICLE [color=royalblue]; - AIR_VEHICLE -> AIRPLANE [color=royalblue]; - AIR_VEHICLE -> HELICOPTER [color=royalblue]; - -.. note:: - - The ``root`` family sits at the top of the inheritance tree as all - tasks/families automatically inherit it: - -Cylc handles inheritance by starting with the root family and working down the -inheritance tree applying each section in turn. - -To see the resulting configuration for the ``a380`` task use the -``cylc get-config`` command:: - - cylc get-config . --sparse -i "[runtime][a380]" - -You should see some settings which have been inherited from the ``VEHICLE`` and -``AIRPLANE`` families as well as a couple defined in the ``a380`` task. - -.. code-block:: cylc - - init-script = echo 'Boarding' # Inherited from VEHICLE - pre-script = echo 'Departing' # Inherited from VEHICLE - post-script = echo 'Arriving' # Inherited from VEHICLE - inherit = AIRPLANE # Defined in a380 - [[[meta]]] - description = An air vehicle with fixed wings. # Inherited from AIR_VEHICLE - overwritten by AIRPLANE - title = Airbus A380 Jumbo-Jet. # Defined in a380 - [[[environment]]] - CAN_TAKE_OFF_VERTICALLY = false # Inherited from AIRPLANE - -Note that the ``description`` setting is defined in the ``AIR_VEHICLE`` -family but is overwritten by the value specified in the ``AIRPLANE`` family. - - -Multiple Inheritance --------------------- - -Next we want to add a vehicle called the V-22 Osprey to the suite. The V-22 -is a cross between a plane and a helicopter - it has wings but can take-off and -land vertically. - -.. image:: https://upload.wikimedia.org/wikipedia/commons/e/e3/MV-22_mcas_Miramar_2014.JPG - :width: 300px - :align: center - -As the V-22 can be thought of as both a plane and a helicopter we want it to -inherit from both the ``AIRPLANE`` and ``HELICOPTER`` families. In Cylc we can -inherit from multiple families by separating their names with commas: - -Add the following task to your ``suite.rc`` file. - -.. code-block:: cylc - - [[v22]] - inherit = AIRPLANE, HELICOPTER - [[[meta]]] - title = V-22 Osprey Military Aircraft. - -Refresh your ``cylc graph`` window or re-run the ``cylc graph`` command. - -The inheritance hierarchy should now look like this: - -.. digraph:: Example - :align: center - - AIRPLANE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - v22 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - AIRPLANE -> v22 [color=royalblue]; - a380 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - AIRPLANE -> a380 [color=royalblue]; - HELICOPTER [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - HELICOPTER -> v22 [color=royalblue]; - r44 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - HELICOPTER -> r44 [color=royalblue]; - root [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - root -> VEHICLE [color=royalblue]; - AIR_VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE -> AIR_VEHICLE [color=royalblue]; - AIR_VEHICLE -> AIRPLANE [color=royalblue]; - AIR_VEHICLE -> HELICOPTER [color=royalblue]; - -Inspect the configuration of the ``v22`` task using the ``cylc get-config`` -command. - -.. spoiler:: Hint warning - - .. code-block:: bash - - cylc get-config . --sparse -i "[runtime][v22]" - -You should see that the ``CAN_TASK_OFF_VERTICALLY`` environment variable has -been set to ``false`` which isn't right. This is because of the order in which -inheritance is applied. - -Cylc handles multiple-inheritance by applying each family from right to left. -For the ``v22`` task we specified ``inherit = AIRPLANE, HELICOPTER`` so the -``HELICOPTER`` family will be applied first and the ``AIRPLANE`` family after. - -The inheritance order would be as follows: - -.. code-block:: bash - - root - VEHICLE - AIR_VEHICLE - HELICOPTER # sets "CAN_TAKE_OFF_VERTICALLY to "true" - AIRPLANE # sets "CAN_TAKE_OFF_VERTICALLY to "false" - v22 - -We could fix this problem by changing the order of inheritance: - -.. code-block:: cylc - - inherit = HELICOPTER, AIRPLANE - -Now the ``HELICOPTER`` family is applied second so its values will override any -in the ``AIRPLANE`` family. - -.. code-block:: bash - - root - VEHICLE - AIR_VEHICLE - AIRPLANE # sets "CAN_TAKE_OFF_VERTICALLY to "false" - HELICOPTER # sets "CAN_TAKE_OFF_VERTICALLY to "true" - v22 - -Inspect the configuration of the ``v22`` task using ``cylc get-config`` to -confirm this. - - -More Inheritance ----------------- - -We will now add some more families and tasks to the suite. - -Engine Type -^^^^^^^^^^^ - -Next we will define four families to represent three different types of engine. - -.. digraph:: Example - :align: center - - size = "5,5" - - ENGINE [color=royalblue, fillcolor=powderblue, shape=box, style=filled, - margin="0.3,0.055"] - TURBINE_ENGINE [color=royalblue, fillcolor=powderblue, shape=box, - style=filled, margin="0.3,0.055"] - INTERNAL_COMBUSTION_ENGINE [color=royalblue, fillcolor=powderblue, - shape=box, style=filled, margin="0.3,0.055"] - HUMAN_ENGINE [color=royalblue, fillcolor=powderblue, shape=box, - style=filled, margin="0.3,0.055"] - - "ENGINE" -> "TURBINE_ENGINE" - "ENGINE" -> "INTERNAL_COMBUSTION_ENGINE" - "ENGINE" -> "HUMAN_ENGINE" - -Each engine type should set an environment variable called ``FUEL`` which we -will assign to the following values: - -* Turbine - kerosene -* Internal Combustion - petrol -* Human - pizza - -Add lines to the ``runtime`` section to represent these four families. - -.. spoiler:: Solution warning - - .. code-block:: cylc - - [[ENGINE]] - [[TURBINE_ENGINE]] - inherit = ENGINE - [[[environment]]] - FUEL = kerosene - [[INTERNAL_COMBUSTION_ENGINE]] - inherit = ENGINE - [[[environment]]] - FUEL = petrol - [[HUMAN_ENGINE]] - inherit = ENGINE - [[[environment]]] - FUEL = pizza - -We now need to make the three aircraft inherit from one of the three engines. -The aircraft use the following types of engine: - -* A380 - turbine -* R44 - internal combustion -* V22 - turbine - -Modify the three tasks so that they inherit from the relevant engine families. - -.. spoiler:: Solution warning - - .. code-block:: cylc - - [[a380]] - inherit = AIRPLANE, TURBINE_ENGINE - [[[meta]]] - title = Airbus A380 Jumbo-Jet. - [[r44]] - inherit = HELICOPTER, INTERNAL_COMBUSTION_ENGINE - [[[meta]]] - title = Robson R44 Helicopter. - [[v22]] - inherit = AIRPLANE, HELICOPTER, TURBINE_ENGINE - [[[meta]]] - title = V-22 Ofsprey Military Aircraft. - -Penny Farthing -^^^^^^^^^^^^^^ - -Next we want to add a new type of vehicle, an old-fashioned bicycle called a -penny farthing. - -.. image:: https://upload.wikimedia.org/wikipedia/commons/a/a7/Ordinary_bicycle01.jpg - :width: 300px - :alt: Penny Farthing Bicycle - :align: center - -To do this we will need to add two new families, ``LAND_VEICHLE`` and -``BICYCLE`` as well as a new task, ``penny_farthing`` related in the -following manner: - -.. digraph:: Example - :align: center - - VEHICLE [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - LAND_VEHICLE [color=royalblue, fillcolor=powderblue, shape=box, - style=filled] - BICYCLE [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - HUMAN_ENGINE [color=royalblue, fillcolor=powderblue, shape=box, - style=filled, margin="0.3,0.055"] - penny_farthing [color=royalblue, fillcolor=powderblue, shape=box, - style=filled, margin="0.3,0.055"] - VEHICLE -> LAND_VEHICLE -> BICYCLE -> penny_farthing - HUMAN_ENGINE -> penny_farthing - -Add lines to the ``runtime`` section to represent the two new families and one -task outlined above. - -Add a description (``[meta]description``) to the ``LAND_VEHICLE`` and -``BICYCLE`` families and a title (``[meta]title``) to the ``penny_farthing`` -task. - -.. spoiler:: Solution warning - - .. code-block:: cylc - - [[LAND_VEHICLE]] - inherit = VEHICLE - [[[meta]]] - description = A vehicle which can travel over the ground. - - [[BICYCLE]] - inherit = LAND_VEHICLE - [[[meta]]] - description = A small two-wheeled vehicle. - - [[penny_farthing]] - inherit = BICYCLE, HUMAN_ENGINE - [[[meta]]] - title = An old-fashioned bicycle. - - -Using ``cylc get-config`` to inspect the configuration of the ``penny_farthing`` -task we can see that it inherits settings from the ``VEHICLE``, -``BICYCLE`` and ``HUMAN_ENGINE`` families. - -.. code-block:: cylc - - inherit = BICYCLE, HUMAN_ENGINE - init-script = echo 'Boarding' # Inherited from VEHICLE - pre-script = echo 'Departing' # Inherited from VEHICLE - post-script = echo 'Arriving' # Inherited from VEHICLE - [[[environment]]] - FUEL = pizza # Inherited from HUMAN_ENGINE - [[[meta]]] - description = A small two-wheeled vehicle. # Inherited from LAND_VEHICLE - overwritten by BICYCLE - title = An old-fashioned bicycle. # Defined in penny_farthing - -.. spoiler:: Hint hint - - .. code-block:: bash - - cylc get-config . --sparse -i "[runtime]penny_farthing" - -Hovercraft -^^^^^^^^^^ - -We will now add a hovercraft called the Hoverwork BHT130, better known to some -as the Isle Of Wight Ferry. - -.. image:: https://upload.wikimedia.org/wikipedia/commons/e/e7/Hovercraft_leaving_Ryde.JPG - :width: 300px - :align: center - :alt: Hoverwork BHT130 Hovercraft - -Hovercraft can move over both land and water and in some respects can be thought -of as flying vehicles. - -.. digraph:: Example - :align: center - - size = "7,5" - - VEHICLE [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - AIR_VEHICLE [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - LAND_VEHICLE [color=royalblue, fillcolor=powderblue, shape=box, - style=filled] - WATER_VEHICLE [color=royalblue, fillcolor=powderblue, shape=box, - style=filled] - HOVERCRAFT [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - bht130 [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - ENGINE [color=royalblue, fillcolor=powderblue, shape=box, style=filled] - INTERNAL_COMBUSTION_ENGINE [color=royalblue, fillcolor=powderblue, - shape=box, style=filled, margin="0.3,0.055"] - VEHICLE -> AIR_VEHICLE -> HOVERCRAFT - VEHICLE -> LAND_VEHICLE -> HOVERCRAFT - VEHICLE -> WATER_VEHICLE -> HOVERCRAFT - HOVERCRAFT -> bht130 - ENGINE -> INTERNAL_COMBUSTION_ENGINE -> bht130 - -Write new families and one new task to represent the above structure. - -Add a description (``[meta]description``) to the ``WATER_VEHICLE`` and -``HOVERCRAFT`` families and a title (``[meta]title``) to the ``bht130`` task. - -.. spoiler:: Solution warning - - .. code-block:: cylc - - [[WATER_VEHICLE]] - inherit = VEHICLE - [[[meta]]] - description = A vehicle which can travel over water. - - [[HOVERCRAFT]] - inherit = LAND_VEHICLE, AIR_VEHICLE, WATER_VEHICLE - [[[meta]]] - description = A vehicle which can travel over ground, water and ice. - - [[bht130]] - inherit = HOVERCRAFT, INTERNAL_COMBUSTION_ENGINE - [[[meta]]] - title = Griffon Hoverwork BHT130 (Isle Of Whight Ferry). - - -Finished Suite --------------- - -You should now have a suite with an inheritance hierarchy which looks like -this: - -.. digraph:: Example - - size = "7, 5" - - root [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - ENGINE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - root -> ENGINE [color=royalblue]; - VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - root -> VEHICLE [color=royalblue]; - INTERNAL_COMBUSTION_ENGINE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled, - margin="0.3,0.055"]; - ENGINE -> INTERNAL_COMBUSTION_ENGINE [color=royalblue]; - TURBINE_ENGINE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled, - margin="0.3,0.055"]; - ENGINE -> TURBINE_ENGINE [color=royalblue]; - HUMAN_ENGINE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled, - margin="0.3,0.055"]; - ENGINE -> HUMAN_ENGINE [color=royalblue]; - LAND_VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE -> LAND_VEHICLE [color=royalblue]; - WATER_VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE -> WATER_VEHICLE [color=royalblue]; - AIR_VEHICLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - VEHICLE -> AIR_VEHICLE [color=royalblue]; - r44 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - INTERNAL_COMBUSTION_ENGINE -> r44 [color=royalblue]; - bht130 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - INTERNAL_COMBUSTION_ENGINE -> bht130 [color=royalblue]; - v22 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - TURBINE_ENGINE -> v22 [color=royalblue]; - a380 [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - TURBINE_ENGINE -> a380 [color=royalblue]; - penny_farthing [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled, - margin="0.3,0.055"]; - HUMAN_ENGINE -> penny_farthing [color=royalblue]; - AIRPLANE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - AIRPLANE -> v22 [color=royalblue]; - AIRPLANE -> a380 [color=royalblue]; - HELICOPTER [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - HELICOPTER -> v22 [color=royalblue]; - HELICOPTER -> r44 [color=royalblue]; - HOVERCRAFT [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - HOVERCRAFT -> bht130 [color=royalblue]; - LAND_VEHICLE -> HOVERCRAFT [color=royalblue]; - BICYCLE [color=royalblue, - fillcolor=powderblue, - shape=box, - style=filled]; - LAND_VEHICLE -> BICYCLE [color=royalblue]; - WATER_VEHICLE -> HOVERCRAFT [color=royalblue]; - AIR_VEHICLE -> AIRPLANE [color=royalblue]; - AIR_VEHICLE -> HELICOPTER [color=royalblue]; - AIR_VEHICLE -> HOVERCRAFT [color=royalblue]; - BICYCLE -> penny_farthing [color=royalblue]; diff --git a/sphinx/tutorial/cylc/furthertopics/queues.rst b/sphinx/tutorial/cylc/furthertopics/queues.rst deleted file mode 100644 index a04dfa48cb..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/queues.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - -Queues -====== - -Queues are used to put a limit on the number of tasks that will be active at -any one time, even if their dependencies are satisfied. This avoids swamping -systems with too many tasks at once. - - -Example -------- - -In this example, our suite manages a particularly understaffed restaurant. - -Create a new suite called ``queues-tutorial``:: - - rose tutorial queues-tutorial - cd ~/cylc-run/queues-tutorial - -You will now have a ``suite.rc`` file that looks like this: - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - open_restaurant => steak1 & steak2 & pasta1 & pasta2 & pasta3 & \ - pizza1 & pizza2 & pizza3 & pizza4 - steak1 => ice_cream1 - steak2 => cheesecake1 - pasta1 => ice_cream2 - pasta2 => sticky_toffee1 - pasta3 => cheesecake2 - pizza1 => ice_cream3 - pizza2 => ice_cream4 - pizza3 => sticky_toffee2 - pizza4 => ice_cream5 - """ - - [runtime] - [[open_restaurant]] - [[MAINS]] - [[DESSERT]] - [[steak1,steak2,pasta1,pasta2,pasta3,pizza1,pizza2,pizza3,pizza4]] - inherit = MAINS - [[ice_cream1,ice_cream2,ice_cream3,ice_cream4,ice_cream5]] - inherit = DESSERT - [[cheesecake1,cheesecake2,sticky_toffee1,sticky_toffee2]] - inherit = DESSERT - -.. note:: - - In graph sections backslash (``\``) is a line continuation character i.e. the - following two examples are equivalent: - - .. code-block:: cylc - - foo => bar & \ - baz - - .. code-block:: cylc - - foo => bar & baz - -Open the ``cylc gui`` then run the suite:: - - cylc gui queues-tutorial & - cylc run queues-tutorial - -You will see that all the ``steak``, ``pasta``, and ``pizza`` tasks are run -at once, swiftly followed by all the ``ice_cream``, ``cheesecake``, -``sticky_toffee`` tasks as the customers order from the dessert menu. - -This will overwhelm our restaurant staff! The chef responsible for ``MAINS`` -can only handle 3 tasks at any given time, and the ``DESSERT`` chef can only -handle 2. - -We need to add some queues. Add a ``[queues]`` section to the ``[scheduling]`` -section like so: - -.. code-block:: cylc - - [scheduling] - [[queues]] - [[[mains_chef_queue]]] - limit = 3 # Only 3 mains dishes at one time. - members = MAINS - [[[dessert_chef_queue]]] - limit = 2 # Only 2 dessert dishes at one time. - members = DESSERT - -Re-open the ``cylc gui`` if you have closed it and re-run the suite. - -You should see that there are now never more than 3 active ``MAINS`` tasks -running and never more than 2 active ``DESSERT`` tasks running. - -The customers will obviously have to wait! - - -Further Reading ---------------- - -For more information, see the `Cylc User Guide`_. diff --git a/sphinx/tutorial/cylc/furthertopics/retries.rst b/sphinx/tutorial/cylc/furthertopics/retries.rst deleted file mode 100644 index d412e0ea5c..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/retries.rst +++ /dev/null @@ -1,156 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - -Retries -======= - -Retries allow us to automatically re-submit tasks which have failed due to -failure in submission or execution. - - -Purpose -------- - -Retries can be useful for tasks that may occasionally fail due to external -events, and are routinely fixable when they do - an example would be a task -that is dependent on a system that experiences temporary outages. - -If a task fails, the Cylc retry mechanism can resubmit it after a -pre-determined delay. An environment variable, ``$CYLC_TASK_TRY_NUMBER`` -is incremented and passed into the task - this means you can write your -task script so that it changes behaviour accordingly. - - -Example -------- - -.. image:: https://upload.wikimedia.org/wikipedia/commons/7/73/Double-six-dice.jpg - :width: 200px - :align: right - :alt: Two dice both showing the number six - -Create a new suite by running the following commands:: - - rose tutorial retries-tutorial - cd retries-tutorial - -You will now have a suite with a ``roll_doubles`` task which simulates -trying to roll doubles using two dice: - -.. code-block:: cylc - - [cylc] - UTC mode = True # Ignore DST - - [scheduling] - [[dependencies]] - graph = start => roll_doubles => win - - [runtime] - [[start]] - [[win]] - [[roll_doubles]] - script = """ - sleep 10 - RANDOM=$$ # Seed $RANDOM - DIE_1=$((RANDOM%6 + 1)) - DIE_2=$((RANDOM%6 + 1)) - echo "Rolled $DIE_1 and $DIE_2..." - if (($DIE_1 == $DIE_2)); then - echo "doubles!" - else - exit 1 - fi - """ - - -Running Without Retries ------------------------ - -Let's see what happens when we run the suite as it is. Open the ``cylc gui``:: - - cylc gui retries-tutorial & - -Then run the suite:: - - cylc run retries-tutorial - -Unless you're lucky, the suite should fail at the roll_doubles task. - -Stop the suite:: - - cylc stop retries-tutorial - - -Configuring Retries -------------------- - -We need to tell Cylc to retry it a few times. To do this, add the following -to the end of the ``[[roll_doubles]]`` task section in the ``suite.rc`` file: - -.. code-block:: cylc - - [[[job]]] - execution retry delays = 5*PT6S - -This means that if the ``roll_doubles`` task fails, Cylc expects to -retry running it 5 times before finally failing. Each retry will have -a delay of 6 seconds. - -We can apply multiple retry periods with the ``execution retry delays`` setting -by separating them with commas, for example the following line would tell Cylc -to retry a task four times, once after 15 seconds, then once after 10 minutes, -then once after one hour then once after three hours. - -.. code-block:: cylc - - execution retry delays = PT15S, PT10M, PT1H, PT3H - - -Running With Retries --------------------- - -If you closed it, re-open the ``cylc gui``:: - - cylc gui retries-tutorial & - -Re-run the suite:: - - cylc run retries-tutorial - -What you should see is Cylc retrying the ``roll_doubles`` task. Hopefully, -it will succeed (there is only about a about a 1 in 3 chance of every task -failing) and the suite will continue. - - -Altering Behaviour ------------------- - -We can alter the behaviour of the task based on the number of retries, using -``$CYLC_TASK_TRY_NUMBER``. - -Change the ``script`` setting for the ``roll_doubles`` task to this:: - - sleep 10 - RANDOM=$$ # Seed $RANDOM - DIE_1=$((RANDOM%6 + 1)) - DIE_2=$((RANDOM%6 + 1)) - echo "Rolled $DIE_1 and $DIE_2..." - if (($DIE_1 == $DIE_2)); then - echo "doubles!" - elif (($CYLC_TASK_TRY_NUMBER >= 2)); then - echo "look over there! ..." - echo "doubles!" # Cheat! - else - exit 1 - fi - -If your suite is still running, stop it, then run it again. - -This time, the task should definitely succeed before the third retry. - - -Further Reading ---------------- - -For more information see the `Cylc User Guide`_. diff --git a/sphinx/tutorial/cylc/furthertopics/suicide-triggers.rst b/sphinx/tutorial/cylc/furthertopics/suicide-triggers.rst deleted file mode 100644 index 067fb974d8..0000000000 --- a/sphinx/tutorial/cylc/furthertopics/suicide-triggers.rst +++ /dev/null @@ -1,676 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - - -.. _tut-cylc-suicide-triggers: - -Suicide Triggers -================ - -Suicide triggers allow us to remove a task from the suite's graph whilst the -suite is running. - -The main use of suicide triggers is for handling failures in the workflow. - - -Stalled Suites --------------- - -Imagine a bakery which has a workflow that involves making cake. - -.. minicylc:: - :snippet: - :theme: none - - make_cake_mixture => bake_cake => sell_cake - -There is a 50% chance that the cake will turn out fine, and a 50% chance that -it will get burnt. In the case that we burn the cake the workflow gets stuck. - -.. digraph:: Example - :align: center - - make_cake_mixture [style="filled" color="#ada5a5"] - bake_cake [style="filled" color="#ff0000" fontcolor="white"] - sell_cake [color="#88c6ff"] - - make_cake_mixture -> bake_cake -> sell_cake - -In this event the ``sell_cake`` task will be unable to run as it depends on -``bake_cake``. We would say that this suite has :term:`stalled `. -When Cylc detects that a suite has stalled it sends you an email to let you -know that the suite has got stuck and requires human intervention to proceed. - - -Handling Failures ------------------ - -In order to prevent the suite from entering a stalled state we need to handle -the failure of the ``bake_cake`` task. - -At the bakery if they burn a cake they eat it and make another. - -The following diagram outlines this workflow with its two possible pathways, -``:succeed`` in the event that ``bake_cake`` is successful and ``:fail`` -otherwise. - -.. digraph:: Example - :align: center - - make_cake_mixture - bake_cake - - subgraph cluster_1 { - label = ":succeed" - labelloc = "b" - color = "green" - fontcolor = "green" - style = "dashed" - sell_cake - } - - subgraph cluster_2 { - label = ":fail" - labelloc = "b" - color = "red" - fontcolor = "red" - style = "dashed" - eat_cake - } - - make_cake_mixture -> bake_cake - bake_cake -> sell_cake - bake_cake -> eat_cake - -We can add this logic to our workflow using the ``fail`` :term:`qualifier`. - -.. code-block:: cylc-graph - - bake_cake => sell_cake - bake_cake:fail => eat_cake - -.. admonition:: Reminder - :class: hint - - If you don't specify a qualifier Cylc assumes you mean ``:succeed`` so the - following two lines are equivalent: - - .. code-block:: cylc-graph - - foo => bar - foo:succeed => bar - - -Why Do We Need To Remove Tasks From The Graph? ----------------------------------------------- - -Create a new suite called ``suicide-triggers``:: - - mkdir -p ~/cylc-run/suicide-triggers - cd ~/cylc-run/suicide-triggers - -Paste the following code into the ``suite.rc`` file: - -.. code-block:: cylc - - [scheduling] - cycling mode = integer - initial cycle point = 1 - [[dependencies]] - [[[P1]]] - graph = """ - make_cake_mixture => bake_cake => sell_cake - bake_cake:fail => eat_cake - """ - [runtime] - [[root]] - script = sleep 2 - [[bake_cake]] - # Random outcome 50% chance of success 50% chance of failure. - script = sleep 2; if (( $RANDOM % 2 )); then true; else false; fi - -Open the ``cylc gui`` and run the suite:: - - cylc gui suicide-triggers & - cylc run suicide-triggers - -The suite will run for three cycles then get stuck. You should see something -similar to the diagram below. As the ``bake_cake`` task fails randomly what -you see might differ slightly. You may receive a "suite stalled" email. - -.. digraph:: Example - :align: center - - size = "7,5" - - subgraph cluster_1 { - label = "1" - style = "dashed" - "make_cake_mixture.1" [ - label="make_cake_mixture\n1", - style="filled", - color="#ada5a5"] - "bake_cake.1" [ - label="bake_cake\n1", - style="filled", - color="#ada5a5"] - "sell_cake.1" [ - label="sell_cake\n1", - style="filled", - color="#ada5a5"] - "eat_cake.1" [ - label="eat_cake\1", - color="#88c6ff"] - } - - subgraph cluster_2 { - label = "2" - style = "dashed" - "make_cake_mixture.2" [ - label="make_cake_mixture\n2", - style="filled", - color="#ada5a5"] - "bake_cake.2" [ - label="bake_cake\n2", - style="filled", - color="#ff0000", - fontcolor="white"] - "sell_cake.2" [ - label="sell_cake\2", - color="#88c6ff"] - "eat_cake.2" [ - label="eat_cake\n2", - color="#888888", - fontcolor="#888888"] - } - - subgraph cluster_3 { - label = "3" - style = "dashed" - "make_cake_mixture.3" [ - label="make_cake_mixture\n3", - style="filled", - color="#ada5a5"] - "bake_cake.3" [ - label="bake_cake\n3", - style="filled", - color="#ff0000", - fontcolor="white"] - "sell_cake.3" [ - label="sell_cake\n3", - color="#888888", - fontcolor="#888888"] - "eat_cake.3" [ - label="eat_cake\3", - color="#888888", - fontcolor="#888888"] - } - - "make_cake_mixture.1" -> "bake_cake.1" -> "sell_cake.1" - "bake_cake.1" -> "eat_cake.1" - - "make_cake_mixture.2" -> "bake_cake.2" -> "sell_cake.2" - "bake_cake.2" -> "eat_cake.2" - - "make_cake_mixture.3" -> "bake_cake.3" -> "sell_cake.3" - "bake_cake.3" -> "eat_cake.3" - -The reason the suite stalls is that, by default, Cylc will run a maximum of -three cycles concurrently. As each cycle has at least one task which hasn't -either succeeded or failed Cylc cannot move onto the next cycle. - -.. tip:: - - For more information search ``max active cycle points`` in the - `Cylc User Guide`_. - -You will also notice that some of the tasks (e.g. ``eat_cake`` in cycle ``2`` -in the above example) are drawn in a faded gray. This is because these tasks -have not yet been run in earlier cycles and as such cannot run. - -.. TODO - Spawn On Demand! - - -Removing Tasks From The Graph ------------------------------ - -In order to get around these problems and prevent the suite from stalling we -must remove the tasks that are no longer needed. We do this using suicide -triggers. - -A suicide trigger is written like a normal dependency but with an exclamation -mark in-front of the task on the right-hand-side of the dependency meaning -*"remove the following task from the graph at the current cycle point."* - -For example the following :term:`graph string` would remove the task ``bar`` -from the graph if the task ``foo`` were to succeed. - -.. code-block:: cylc-graph - - foo => ! bar - -There are three cases where we would need to remove a task in the cake-making -example: - -#. If the ``bake_cake`` task succeeds we don't need the ``eat_cake`` task so - should remove it. - - .. code-block:: cylc-graph - - bake_cake => ! eat_cake - -#. If the ``bake_cake`` task fails we don't need the ``sell_cake`` task so - should remove it. - - .. code-block:: cylc-graph - - bake_cake:fail => ! sell_cake - -#. If the ``bake_cake`` task fails then we will need to remove it else the - suite will stall. We can do this after the ``eat_cake`` task has succeeded. - - .. code-block:: cylc-graph - - eat_cake => ! bake_cake - -Add the following three lines to the suite's graph: - -.. code-block:: cylc-graph - - bake_cake => ! eat_cake - bake_cake:fail => ! sell_cake - eat_cake => ! bake_cake - -We can view suicide triggers in ``cylc graph`` by un-selecting the -:guilabel:`Ignore Suicide Triggers` button in the toolbar. Suicide triggers -will then appear as dashed lines with circular endings. You should see -something like this: - -.. digraph:: Example - :align: center - - make_cake_mixture -> bake_cake - bake_cake -> sell_cake [style="dashed" arrowhead="dot"] - bake_cake -> eat_cake [style="dashed" arrowhead="dot"] - eat_cake -> bake_cake [style="dashed" arrowhead="dot"] - - -Downstream Dependencies ------------------------ - -If we wanted to make the cycles run in order we might write an -:term:`inter-cycle dependency` like this: - -.. code-block:: cylc-graph - - sell_cake[-P1] => make_cake_mixture - -In order to handle the event that the ``sell_cake`` task has been removed from -the graph by a suicide trigger we can write our dependency with an or -symbol ``|`` like so: - -.. code-block:: cylc-graph - - eat_cake[-P1] | sell_cake[-P1] => make_cake_mixture - -Now the ``make_cake_mixture`` task from the next cycle will run after whichever -of the ``sell_cake`` or ``eat_cake`` tasks is run. - -.. digraph:: Example - :align: center - - subgraph cluster_1 { - style="dashed" - label="1" - "make_cake_mixture.1" [label="make_cake_mixture\n1"] - "bake_cake.1" [label="bake_cake\n1"] - "make_cake_mixture.1" -> "bake_cake.1" - "bake_cake.1" -> "sell_cake.1" [style="dashed" arrowhead="dot"] - "bake_cake.1" -> "eat_cake.1" [style="dashed" arrowhead="dot"] - "eat_cake.1" -> "bake_cake.1" [style="dashed" arrowhead="dot"] - subgraph cluster_a { - label = ":fail" - fontcolor = "red" - color = "red" - style = "dashed" - "eat_cake.1" [label="eat_cake\n1" color="red" fontcolor="red"] - } - subgraph cluster_b { - label = ":success" - fontcolor = "green" - color = "green" - style = "dashed" - "sell_cake.1" [label="sell_cake\n1" color="green" fontcolor="green"] - } - } - - subgraph cluster_2 { - style="dashed" - label="2" - "make_cake_mixture.2" [label="make_cake_mixture\n2"] - "bake_cake.2" [label="bake_cake\n2"] - "make_cake_mixture.2" -> "bake_cake.2" - "bake_cake.2" -> "sell_cake.2" [style="dashed" arrowhead="dot"] - "bake_cake.2" -> "eat_cake.2" [style="dashed" arrowhead="dot"] - "eat_cake.2" -> "bake_cake.2" [style="dashed" arrowhead="dot"] - subgraph cluster_c { - label = ":fail" - fontcolor = "red" - color = "red" - style = "dashed" - "eat_cake.2" [label="eat_cake\n2" color="red" fontcolor="red"] - } - subgraph cluster_d { - label = ":success" - fontcolor = "green" - color = "green" - style = "dashed" - "sell_cake.2" [label="sell_cake\n2" color="green" fontcolor="green"] - } - } - - "eat_cake.1" -> "make_cake_mixture.2" [arrowhead="onormal"] - "sell_cake.1" -> "make_cake_mixture.2" [arrowhead="onormal"] - -Add the following :term:`graph string` to your suite. - -.. code-block:: cylc-graph - - eat_cake[-P1] | sell_cake[-P1] => make_cake_mixture - -Open the ``cylc gui`` and run the suite. You should see that if the -``bake_cake`` task fails both it and the ``sell_cake`` task disappear and -are replaced by the ``eat_cake`` task. - - -Comparing "Regular" and "Suicide" Triggers ------------------------------------------- - -In Cylc "regular" and "suicide" triggers both work in the same way. For example -the following graph lines implicitly combine using an ``&`` operator: - -.. highlight:: cylc-graph - -.. list-table:: - :class: grid-table - - * - :: - - foo => pub - bar => pub - - :: - - foo & bar => pub - -Suicide triggers combine in the same way: - -.. list-table:: - :class: grid-table - - * - :: - - foo => !pub - bar => !pub - - :: - - foo & bar => !pub - -.. highlight:: python - -This means that suicide triggers are treated as "invisible tasks" rather than -as "events". Suicide triggers can have pre-requisites just like a normal task. - - -Variations ----------- - -The following sections outline examples of how to use suicide triggers. - -Recovery Task -^^^^^^^^^^^^^ - -A common use case where a ``recover`` task is used to handle a task failure. - -.. digraph:: Example - :align: center - - subgraph cluster_1 { - label = ":fail" - color = "red" - fontcolor = "red" - style = "dashed" - recover - } - - foo -> bar - bar -> recover - recover -> baz [arrowhead="onormal"] - bar -> baz [arrowhead="onormal"] - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - # Regular graph. - foo => bar - - # The fail case. - bar:fail => recover - - # Remove the "recover" task in the success case. - bar => ! recover - - # Remove the "bar" task in the fail case. - recover => ! bar - - # Downstream dependencies. - bar | recover => baz - """ - [runtime] - [[root]] - script = sleep 1 - [[bar]] - script = false - -Branched Workflow -^^^^^^^^^^^^^^^^^ - -A workflow where sub-graphs of tasks are to be run in the success and or fail -cases. - -.. digraph:: Example - :align: center - - foo -> bar - bar -> tar -> par - bar -> jar -> par - bar -> baz -> jaz - - subgraph cluster_1 { - label = ":success" - fontcolor = "green" - color = "green" - style = "dashed" - tar - jar - par - } - - subgraph cluster_2 { - label = ":fail" - fontcolor = "red" - color = "red" - style = "dashed" - baz - jaz - } - - tar -> pub [arrowhead="onormal"] - jaz -> pub [arrowhead="onormal"] - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - # Regular graph. - foo => bar - - # Success case. - bar => tar & jar - - # Fail case. - bar:fail => baz => jaz - - # Remove tasks from the fail branch in the success case. - bar => ! baz & ! jaz - - # Remove tasks from the success branch in the fail case. - bar:fail => ! tar & ! jar & ! par - - # Remove the bar task in the fail case. - baz => ! bar - - # Downstream dependencies. - tar | jaz => pub - """ - [runtime] - [[root]] - script = sleep 1 - [[bar]] - script = true - -Triggering Based On Other States -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In these examples we have been using suicide triggers to handle task failure. -The suicide trigger mechanism works with other qualifiers as well for example: - -.. code-block:: cylc-graph - - foo:start => ! bar - -Suicide triggers can also be used with custom outputs. In the following example -the task ``showdown`` produces one of three possible custom outputs, ``good``, -``bad`` or ``ugly``. - -.. TODO - link to custom task outputs / write an advanced tutorial for them. - -.. digraph:: Example - :align: center - - subgraph cluster_1 { - label = ":good" - color = "green" - fontcolor = "green" - style = "dashed" - good - } - subgraph cluster_2 { - label = ":bad" - color = "red" - fontcolor = "red" - style = "dashed" - bad - } - subgraph cluster_3 { - label = ":ugly" - color = "purple" - fontcolor = "purple" - style = "dashed" - ugly - } - showdown -> good - showdown -> bad - showdown -> ugly - good -> fin [arrowhead="onormal"] - bad -> fin [arrowhead="onormal"] - ugly -> fin [arrowhead="onormal"] - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - # The "regular" dependencies - showdown:good => good - showdown:bad => bad - showdown:ugly => ugly - good | bad | ugly => fin - - # The "suicide" dependencies for each case - showdown:good | showdown:bad => ! ugly - showdown:bad | showdown:ugly => ! good - showdown:ugly | showdown:good => ! bad - """ - [runtime] - [[root]] - script = sleep 1 - [[showdown]] - # Randomly return one of the three custom outputs. - script = """ - SEED=$RANDOM - if ! (( $SEED % 3 )); then - cylc message 'The-Good' - elif ! (( ( $SEED + 1 ) % 3 )); then - cylc message 'The-Bad' - else - cylc message 'The-Ugly' - fi - """ - [[[outputs]]] - # Register the three custom outputs with cylc. - good = 'The-Good' - bad = 'The-Bad' - ugly = 'The-Ugly' - -Self-Suiciding Task -^^^^^^^^^^^^^^^^^^^ - -An example of a workflow where there are no tasks which are dependent on the -task to suicide trigger. - -.. digraph:: Example - :align: center - - subgraph cluster_1 { - label = "Faulty\nTask" - color = "orange" - fontcolor = "orange" - style = "dashed" - labelloc = "b" - pub - } - - foo -> bar -> baz - bar -> pub - - -It is possible for a task to suicide trigger itself e.g: - -.. code-block:: cylc-graph - - foo:fail => ! foo - -.. warning:: - - This is usually not recommended but in the case where there are no tasks - dependent on the one to remove it is an acceptable approach. - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - foo => bar => baz - bar => pub - - # Remove the "pub" task in the event of failure. - pub:fail => ! pub - """ - [runtime] - [[root]] - script = sleep 1 - [[pub]] - script = false diff --git a/sphinx/tutorial/cylc/img/cylc-graph-cluster.png b/sphinx/tutorial/cylc/img/cylc-graph-cluster.png deleted file mode 100644 index 4fb6df22d7..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-graph-cluster.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-graph-refresh.png b/sphinx/tutorial/cylc/img/cylc-graph-refresh.png deleted file mode 100644 index 49158ce972..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-graph-refresh.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-graph-reversible.svg b/sphinx/tutorial/cylc/img/cylc-graph-reversible.svg deleted file mode 100644 index 7ec62d936d..0000000000 --- a/sphinx/tutorial/cylc/img/cylc-graph-reversible.svg +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - -_anonymous_0 -cluster_1 - - -cluster_2 - - - -foo.1 - -foo - - -baz.1 - -baz - - -foo.1->baz.1 - - - - -bar.1 - -bar - - -bar.1->baz.1 - - - - -bar.2 - -bar - - -baz.2 - -baz - - -foo.2->baz.2 - - - - -foo.2 - -foo - - -bar.2->baz.2 - - - - - diff --git a/sphinx/tutorial/cylc/img/cylc-graph.png b/sphinx/tutorial/cylc/img/cylc-graph.png deleted file mode 100644 index 65e1ffa957..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-graph.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui-dot.png b/sphinx/tutorial/cylc/img/cylc-gui-dot.png deleted file mode 100644 index 0cfede9f3d..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui-dot.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui-graph.png b/sphinx/tutorial/cylc/img/cylc-gui-graph.png deleted file mode 100644 index 01b16b9029..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui-graph.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui-suite-start.png b/sphinx/tutorial/cylc/img/cylc-gui-suite-start.png deleted file mode 100644 index 6cbc05681d..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui-suite-start.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui-tree.png b/sphinx/tutorial/cylc/img/cylc-gui-tree.png deleted file mode 100644 index fc537c3fbb..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui-tree.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui-view-log.png b/sphinx/tutorial/cylc/img/cylc-gui-view-log.png deleted file mode 100644 index 0b3338fd0b..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui-view-log.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui-view-selector.png b/sphinx/tutorial/cylc/img/cylc-gui-view-selector.png deleted file mode 100644 index 9b3c8c08bb..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui-view-selector.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/cylc-gui.png b/sphinx/tutorial/cylc/img/cylc-gui.png deleted file mode 100644 index f60097fa2b..0000000000 Binary files a/sphinx/tutorial/cylc/img/cylc-gui.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/gcylc-play.png b/sphinx/tutorial/cylc/img/gcylc-play.png deleted file mode 100644 index 608436508b..0000000000 Binary files a/sphinx/tutorial/cylc/img/gcylc-play.png and /dev/null differ diff --git a/sphinx/tutorial/cylc/img/iso8601-dates.svg b/sphinx/tutorial/cylc/img/iso8601-dates.svg deleted file mode 100644 index 83b561be6a..0000000000 --- a/sphinx/tutorial/cylc/img/iso8601-dates.svg +++ /dev/null @@ -1,265 +0,0 @@ - - - - - - - - - - - - - image/svg+xml - - - - - - - - 1985-04-12 - 23:20:30Z - T - Date Components - Time Components - - - - - - - - - - - Separator - Year - Month - Day - Hour - Minute - Second - Time Zone - - - diff --git a/sphinx/tutorial/cylc/index.rst b/sphinx/tutorial/cylc/index.rst deleted file mode 100644 index c2d34aba8c..0000000000 --- a/sphinx/tutorial/cylc/index.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Cylc Tutorial: - -Cylc Tutorial -============= - -Cylc is a workflow engine for running suites of inter-dependent jobs. - -.. image:: ../../img/cylc-logo.png - :width: 250px - :align: center - -This section will cover the Cylc framework and writing basic Cylc suites. - -.. toctree:: - :name: cylc-tutorial - :maxdepth: 2 - - introduction - scheduling/index - runtime/index - furthertopics/index diff --git a/sphinx/tutorial/cylc/introduction.rst b/sphinx/tutorial/cylc/introduction.rst deleted file mode 100644 index ca9ecc678a..0000000000 --- a/sphinx/tutorial/cylc/introduction.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. _cylc-introduction: - -Introduction -============ - - -What Is A Workflow? -------------------- - -.. epigraph:: - - A workflow consists of an orchestrated and repeatable pattern of business - activity enabled by the systematic organization of resources into processes - that transform materials, provide services, or process information. - - -- Wikipedia - -.. ifnotslides:: - - In research, business and other fields we may have processes that we repeat - in the course of our work. At its simplest a workflow is a set of steps that - must be followed in a particular order to achieve some end goal. - - We can represent each "step" in a workflow as a oval and the order with - arrows. - -.. nextslide:: - -.. digraph:: bakery - :align: center - - "purchase ingredients" -> "make dough" -> "bake bread" -> "sell bread" - "bake bread" -> "clean oven" - "pre-heat oven" -> "bake bread" - - -What Is Cylc? -------------- - -.. ifnotslides:: - - Cylc (pronounced silk) is a workflow engine, a system that automatically - executes tasks according to their schedules and dependencies. - - In a Cylc workflow each step is a computational task, a script to execute. - Cylc runs each task as soon as it is appropriate to do so. - -.. minicylc:: - :align: center - :theme: demo - - a => b => c - b => d => f - e => f - -.. nextslide:: - -Cylc can automatically: - -- Submit tasks across computer systems and resource managers. -- Recover from failures. -- Repeat workflows. - -.. ifnotslides:: - - Cylc was originally developed at NIWA (The National Institute of Water and - Atmospheric Research - New Zealand) for running their weather forecasting - workflows. Cylc is now developed by an international partnership including - members from NIWA and the Met Office (UK). Though initially developed for - meteorological purposes Cylc is a general purpose tool as applicable in - business as in scientific research. - -.. nextslide:: - -.. ifslides:: - - * Originally developed at NIWA (New Zealand) - * Now developed by an international partnership including the - Met Office (UK). - * General purpose tool as applicable in business as in - scientific research. - -.. nextslide:: - -Cylc provides a variety of command line and GUI tools for visualising and -interacting with workflows. - -.. image:: img/cylc-gui.png - -.. nextslide:: - -.. ifslides:: - - :ref:`tutorial-cylc-graphing` diff --git a/sphinx/tutorial/cylc/runtime/configuration-consolidation/families.rst b/sphinx/tutorial/cylc/runtime/configuration-consolidation/families.rst deleted file mode 100644 index 48b8729342..0000000000 --- a/sphinx/tutorial/cylc/runtime/configuration-consolidation/families.rst +++ /dev/null @@ -1,333 +0,0 @@ -.. include:: ../../../../hyperlinks.rst - :start-line: 1 - - -.. _tutorial-cylc-families: - -Families -======== - -:term:`Families ` provide a way of grouping tasks together so they can -be treated as one. - - -Runtime -------- - -.. ifnotslides:: - - :term:`Families ` are groups of tasks which share a common - configuration. In the present example the common configuration is: - - .. code-block:: cylc - - script = get-observations - [[[environment]]] - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - We define a family as a new task consisting of the common configuration. By - convention families are named in upper case: - -.. code-block:: cylc - - [[GET_OBSERVATIONS]] - script = get-observations - [[[environment]]] - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - -.. ifnotslides:: - - We "add" tasks to a family using the ``inherit`` setting: - -.. code-block:: cylc - - [[get_observations_heathrow]] - inherit = GET_OBSERVATIONS - [[[environment]]] - SITE_ID = 3772 - -.. ifnotslides:: - - When we add a task to a family in this way it :term:`inherits ` the configuration from the family, i.e. the above example is - equivalent to: - -.. code-block:: cylc - - [[get_observations_heathrow]] - script = get-observations - [[[environment]]] - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - SITE_ID = 3772 - -.. nextslide:: - -.. ifnotslides:: - - It is possible to override inherited configuration within the task. For - example if we wanted the ``get_observations_heathrow`` task to use a - different API key we could write: - -.. code-block:: cylc - :emphasize-lines: 4 - - [[get_observations_heathrow]] - inherit = GET_OBSERVATIONS - [[[environment]]] - API_KEY = special-api-key - SITE_ID = 3772 - -.. nextslide:: - -.. ifnotslides:: - - Using families the ``get_observations`` tasks could be written like so: - -.. code-block:: cylc - - [runtime] - [[GET_OBSERVATIONS]] - script = get-observations - [[[environment]]] - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - [[get_observations_heathrow]] - inherit = GET_OBSERVATIONS - [[[environment]]] - SITE_ID = 3772 - [[get_observations_camborne]] - inherit = GET_OBSERVATIONS - [[[environment]]] - SITE_ID = 3808 - [[get_observations_shetland]] - inherit = GET_OBSERVATIONS - [[[environment]]] - SITE_ID = 3005 - [[get_observations_belmullet]] - inherit = GET_OBSERVATIONS - [[[environment]]] - SITE_ID = 3976 - - -Graphing --------- - -.. ifnotslides:: - - :term:`Families ` can be used in the suite's :term:`graph`, e.g: - -.. code-block:: cylc-graph - - GET_OBSERVATIONS:succeed-all => consolidate_observations - -.. ifnotslides:: - - The ``:succeed-all`` is a special :term:`qualifier` which in this example - means that the ``consolidate_observations`` task will run once *all* of the - members of the ``GET_OBSERVATIONS`` family have succeeded. This is - equivalent to: - -.. code-block:: cylc-graph - - get_observations_heathrow => consolidate_observations - get_observations_camborne => consolidate_observations - get_observations_shetland => consolidate_observations - get_observations_belmullet => consolidate_observations - -.. ifnotslides:: - - The ``GET_OBSERVATIONS:succeed-all`` part is referred to as a - :term:`family trigger`. Family triggers use special qualifiers which are - non-optional. The most commonly used ones are: - - ``succeed-all`` - Run if all of the members of the family have succeeded. - ``succeed-any`` - Run as soon as any one family member has succeeded. - ``finish-all`` - Run as soon as all of the family members have completed (i.e. have each - either succeeded or failed). - - For more information on family triggers see the `Cylc User Guide`_. - -.. ifslides:: - - * ``succeed-all`` - * ``succeed-any`` - * ``finish-all`` - - -The ``root`` Family -------------------- - -.. ifnotslides:: - - There is a special family called `root` (in lowercase) which is used only - in the runtime to provide configuration which will be inherited by all - tasks. - - In the following example the task ``bar`` will inherit the environment - variable ``FOO`` from the ``[root]`` section: - -.. code-block:: cylc - - [runtime] - [[root]] - [[[environment]]] - FOO = foo - [[bar]] - script = echo $FOO - - -Families and ``cylc graph`` ---------------------------- - -.. ifnotslides:: - - By default, ``cylc graph`` groups together all members of a family - in the :term:`graph`. To un-group a family right click on it and select - :menuselection:`UnGroup`. - - For instance if the tasks ``bar`` and ``baz`` both - inherited from ``BAR`` ``cylc graph`` would produce: - -.. digraph:: Example - :align: center - - subgraph cluster_1 { - label = "Grouped" - "foo.1" [label="foo"] - "BAR.1" [label="BAR", shape="doubleoctagon"] - } - - subgraph cluster_2 { - label = "Un-Grouped" - "foo.2" [label="foo"] - "bar.2" [label="bar"] - "baz.2" [label="baz"] - } - - "foo.1" -> "BAR.1" - "foo.2" -> "bar.2" - "foo.2" -> "baz.2" - -.. nextslide:: - -.. ifslides:: - - .. rubric:: In this practical we will consolidate the configuration of the - :ref:`weather-forecasting suite ` - from the previous section. - - Next section: :ref:`Jinja2 ` - - -.. _cylc-tutorial-families-practical: - -.. practical:: - - .. rubric:: In this practical we will consolidate the configuration of the - :ref:`weather-forecasting suite ` - from the previous section. - - 1. **Create A New Suite.** - - To make a new copy of the forecasting suite run the following commands: - - .. code-block:: bash - - rose tutorial consolidation-tutorial - cd ~/cylc-run/consolidation-tutorial - - 2. **Move Site-Wide Settings Into The** ``root`` **Family.** - - The following two environment variables are used by multiple tasks: - - .. code-block:: none - - RESOLUTION = 0.2 - DOMAIN = -12,48,5,61 # Do not change! - - Rather than manually adding them to each task individually we could put - them in the ``root`` family, making them accessible to all tasks. - - Add a ``root`` section containing these two environment variables. - Remove the variables from any other task's ``environment`` sections: - - .. code-block:: diff - - [runtime] - + [[root]] - + [[[environment]]] - + # The dimensions of each grid cell in degrees. - + RESOLUTION = 0.2 - + # The area to generate forecasts for (lng1, lat1, lng2, lat2). - + DOMAIN = -12,48,5,61 # Do not change! - - .. code-block:: diff - - [[consolidate_observations]] - script = consolidate-observations - - [[[environment]]] - - # The dimensions of each grid cell in degrees. - - RESOLUTION = 0.2 - - # The area to generate forecasts for (lng1, lat1, lng2, lat2). - - DOMAIN = -12,48,5,61 # Do not change! - - [[get_rainfall]] - script = get-rainfall - [[[environment]]] - # The key required to get weather data from the DataPoint service. - # To use archived data comment this line out. - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - # The dimensions of each grid cell in degrees. - - RESOLUTION = 0.2 - - # The area to generate forecasts for (lng1, lat1, lng2, lat2). - - DOMAIN = -12,48,5,61 # Do not change! - - [[forecast]] - script = forecast 60 5 # Generate 5 forecasts at 60 minute intervals. - [[[environment]]] - - # The dimensions of each grid cell in degrees. - - RESOLUTION = 0.2 - - # The area to generate forecasts for (lng1, lat1, lng2, lat2) - - DOMAIN = -12,48,5,61 # Do not change! - # The path to the files containing wind data (the {variables} will - # get substituted in the forecast script). - WIND_FILE_TEMPLATE = $CYLC_SUITE_WORK_DIR/{cycle}/consolidate_observations/wind_{xy}.csv - # List of cycle points to process wind data from. - WIND_CYCLES = 0, -3, -6 - - # The path to the rainfall file. - RAINFALL_FILE = $CYLC_SUITE_WORK_DIR/$CYLC_TASK_CYCLE_POINT/get_rainfall/rainfall.csv - # Create the html map file in the task's log directory. - MAP_FILE = "${CYLC_TASK_LOG_ROOT}-map.html" - # The path to the template file used to generate the html map. - MAP_TEMPLATE = "$CYLC_SUITE_RUN_DIR/lib/template/map.html" - - [[post_process_exeter]] - # Generate a forecast for Exeter 60 minutes into the future. - script = post-process exeter 60 - - [[[environment]]] - - # The dimensions of each grid cell in degrees. - - RESOLUTION = 0.2 - - # The area to generate forecasts for (lng1, lat1, lng2, lat2). - - DOMAIN = -12,48,5,61 # Do not change! - - To ensure that the environment variables are being inherited correctly - by the tasks, inspect the ``[runtime]`` section using ``cylc get-config`` - by running the following command: - - .. code-block:: bash - - cylc get-config . --sparse -i "[runtime]" - - You should see the environment variables from the ``[root]`` section - in the ``[environment]`` section for all tasks. - - .. tip:: - - You may find it easier to open the output of this command in a text - editor, e.g:: - - cylc get-config . --sparse -i "[runtime]" | gvim - diff --git a/sphinx/tutorial/cylc/runtime/configuration-consolidation/index.rst b/sphinx/tutorial/cylc/runtime/configuration-consolidation/index.rst deleted file mode 100644 index 8cb159e1f0..0000000000 --- a/sphinx/tutorial/cylc/runtime/configuration-consolidation/index.rst +++ /dev/null @@ -1,191 +0,0 @@ -.. include:: ../../../../hyperlinks.rst - :start-line: 1 - - -.. _tutorial-cylc-consolidating-configuration: - -Consolidating Configuration -=========================== - -.. ifnotslides:: - - In the last section we wrote out the following code in the ``suite.rc`` file: - -.. slide:: Weather Forecasting Suite - :level: 2 - :inline-contents: True - - .. code-block:: cylc - - [runtime] - [[get_observations_heathrow]] - script = get-observations - [[[environment]]] - SITE_ID = 3772 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - [[get_observations_camborne]] - script = get-observations - [[[environment]]] - SITE_ID = 3808 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - [[get_observations_shetland]] - script = get-observations - [[[environment]]] - SITE_ID = 3005 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - [[get_observations_belmullet]] - script = get-observations - [[[environment]]] - SITE_ID = 3976 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - -.. ifnotslides:: - - In this code the ``script`` item and the ``API_KEY`` environment variable have - been repeated for each task. This is bad practice as it makes the - configuration lengthy and making changes can become difficult. - - Likewise the graphing relating to the ``get_observations`` tasks is highly - repetitive: - -.. ifslides:: - - .. slide:: Weather Forecasting Suite - :level: 2 - - Repetition - - * ``script`` - * ``API_KEY`` - -.. slide:: Weather Forecasting Suite - :level: 2 - :inline-contents: True - - .. code-block:: cylc - - [scheduling] - [[dependencies]] - [[[T00/PT3H]]] - graph = """ - get_observations_belmullet => consolidate_observations - get_observations_camborne => consolidate_observations - get_observations_heathrow => consolidate_observations - get_observations_shetland => consolidate_observations - """ - -.. nextslide:: - -Cylc offers three ways of consolidating configurations to help improve the -structure of a suite and avoid duplication. - -.. toctree:: - :maxdepth: 1 - - families - jinja2 - parameters - - -The ``cylc get-config`` Command -------------------------------- - -.. ifnotslides:: - - The ``cylc get-config`` command reads in then prints out the ``suite.rc`` file - to the terminal. - - Throughout this section we will be introducing methods for consolidating - the ``suite.rc`` file, the ``cylc get-config`` command can be used to - "expand" the ``suite.rc`` file back to its full form. - - .. note:: - - The main use of ``cylc get-config`` is inspecting the - ``[runtime]`` section of a suite. The ``cylc get-config`` command does not - expand :term:`parameterisations ` and - :term:`families ` in the suite's :term:`graph`. To inspect the - graphing use the ``cylc graph`` command. - - Call ``cylc get-config`` with the path of the suite (``.`` if you are already - in the :term:`suite directory`) and the ``--sparse`` option which hides - default values. - -.. code-block:: sub - - cylc get-config --sparse - -.. ifnotslides:: - - To view the configuration of a particular section or setting refer to it by - name using the ``-i`` option (see :ref:`Cylc file format` for details), e.g: - -.. code-block:: sub - - # Print the contents of the [scheduling] section. - cylc get-config --sparse -i '[scheduling]' - # Print the contents of the get_observations_heathrow task. - cylc get-config --sparse -i '[runtime][get_observations_heathrow]' - # Print the value of the script setting in the get_observations_heathrow task - cylc get-config --sparse -i '[runtime][get_observations_heathrow]script' - -.. nextslide:: - -.. ifslides:: - - Note that ``cylc get-config`` doesn't expand families or parameterisations - in the :term:`graph`. Use ``cylc graph`` to visualise these. - - .. TODO - Raise and issue for this, note cylc get-config and cylc view. - - -The Three Approaches --------------------- - -.. ifnotslides:: - - The next three sections cover the three consolidation approaches and how we - could use them to simplify the suite from the previous tutorial. *Work - through them in order!* - -* :ref:`families ` -* :ref:`jinja2 ` -* :ref:`parameters ` - - -.. _cylc-tutorial-consolidation-conclusion: - -Which Approach To Use ---------------------- - -.. ifnotslides:: - - Each approach has its uses. Cylc permits mixing approaches, allowing us to - use what works best for us. As a rule of thumb: - - * :term:`Families ` work best consolidating runtime configuration by - collecting tasks into broad groups, e.g. groups of tasks which run on a - particular machine or groups of tasks belonging to a particular system. - * `Jinja2`_ is good at configuring settings which apply to the entire suite - rather than just a single task, as we can define variables then use them - throughout the suite. - * :term:`Parameterisation ` works best for describing tasks - which are very similar but which have subtly different configurations - (e.g. different arguments or environment variables). - -.. ifslides:: - - As a rule of thumb each method works best for: - - Families - Collecting tasks into broad groups. - Jinja2 - Configuration settings which apply to the entire suite. - Parameterisation - Tasks which are similar. - -.. nextslide:: - -.. ifslides:: - - Next section: :ref:`Rose Tutorial ` diff --git a/sphinx/tutorial/cylc/runtime/configuration-consolidation/jinja2.rst b/sphinx/tutorial/cylc/runtime/configuration-consolidation/jinja2.rst deleted file mode 100644 index 6dc5963bb1..0000000000 --- a/sphinx/tutorial/cylc/runtime/configuration-consolidation/jinja2.rst +++ /dev/null @@ -1,236 +0,0 @@ -.. _shebang: https://en.wikipedia.org/wiki/Shebang_(Unix) - - -.. _tutorial-cylc-jinja2: - -Jinja2 -====== - -`Jinja2 `_ -is a templating language often used in web design with some -similarities to python. It can be used to make a suite definition more -dynamic. - - -The Jinja2 Language -------------------- - -In Jinja2 statements are wrapped with ``{%`` characters, i.e: - -.. code-block:: none - - {% ... %} - -Variables are initiated using the ``set`` statement, e.g: - -.. code-block:: css+jinja - - {% set foo = 3 %} - -.. nextslide:: - -Expressions wrapped with ``{{`` characters will be replaced with the value of -the evaluation of the expression, e.g: - -.. code-block:: css+jinja - - There are {{ foo }} methods for consolidating the suite.rc file - -Would result in:: - - There are 3 methods for consolidating the suite.rc file - -.. nextslide:: - -Loops are written with ``for`` statements, e.g: - -.. code-block:: css+jinja - - {% for x in range(foo) %} - {{ x }} - {% endfor %} - -Would result in: - -.. code-block:: none - - 0 - 1 - 2 - -.. nextslide:: - -To enable Jinja2 in the ``suite.rc`` file, add the following `shebang`_ to the -top of the file: - -.. code-block:: cylc - - #!Jinja2 - -For more information see the -`Jinja2 Tutorial `_. - - -Example -------- - -To consolidate the configuration for the ``get_observations`` tasks we could -define a dictionary of station and ID pairs: - -.. code-block:: css+jinja - - {% set stations = {'belmullet': 3976, - 'camborne': 3808, - 'heathrow': 3772, - 'shetland': 3005} %} - -.. nextslide:: - -We could then loop over the stations like so: - -.. code-block:: css+jinja - - {% for station in stations %} - {{ station }} - {% endfor %} - -After processing, this would result in: - -.. code-block:: none - - belmullet - camborne - heathrow - shetland - -.. nextslide:: - -We could also loop over both the stations and corresponding IDs like so: - -.. code-block:: css+jinja - - {% for station, id in stations.items() %} - {{ station }} - {{ id }} - {% endfor %} - -This would result in: - -.. code-block:: none - - belmullet - 3976 - camborne - 3808 - heathrow - 3772 - shetland - 3005 - -.. nextslide:: - -.. ifnotslides:: - - Putting this all together, the ``get_observations`` configuration could be - written as follows: - -.. code-block:: cylc - - #!Jinja2 - - {% set stations = {'belmullet': 3976, - 'camborne': 3808, - 'heathrow': 3772, - 'shetland': 3005} %} - - [scheduling] - [[dependencies]] - [[[T00/PT3H]]] - graph = """ - {% for station in stations %} - get_observations_{{station}} => consolidate_observations - {% endfor %} - """ - -.. nextslide:: - -.. code-block:: cylc - - [runtime] - {% for station, id in stations.items() %} - [[get_observations_{{station}}]] - script = get-observations - [[[environment]]] - SITE_ID = {{ id }} - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - {% endfor %} - -.. nextslide:: - -.. ifslides:: - - .. rubric:: This practical continues on from the - :ref:`families practical `. - - Next section: :ref:`tutorial-cylc-parameterisation` - - -.. _cylc-tutorial-jinja2-practical: - -.. practical:: - - .. rubric:: This practical continues on from the - :ref:`families practical `. - - 3. **Use Jinja2 To Avoid Duplication.** - - The ``API_KEY`` environment variable is used by both the - ``get_observations`` and ``get_rainfall`` tasks. Rather than writing it - out multiple times we will use Jinja2 to centralise this configuration. - - At the top of the ``suite.rc`` file add the Jinja2 shebang line. Then - copy the value of the ``API_KEY`` environment variable and use it to - define an ``API_KEY`` Jinja2 variable: - - .. code-block:: cylc - - #!Jinja2 - - {% set API_KEY = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' %} - - Next replace the key, where it appears in the suite, with - ``{{ API_KEY }}``: - - .. code-block:: diff - - [runtime] - [[get_observations_heathrow]] - script = get-observations - [[[environment]]] - SITE_ID = 3772 - - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - + API_KEY = {{ API_KEY }} - [[get_observations_camborne]] - script = get-observations - [[[environment]]] - SITE_ID = 3808 - - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - + API_KEY = {{ API_KEY }} - [[get_observations_shetland]] - script = get-observations - [[[environment]]] - SITE_ID = 3005 - - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - + API_KEY = {{ API_KEY }} - [[get_observations_belmullet]] - script = get-observations - [[[environment]]] - SITE_ID = 3976 - - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - + API_KEY = {{ API_KEY }} - [[get_rainfall]] - script = get-rainfall - [[[environment]]] - # The key required to get weather data from the DataPoint service. - # To use archived data comment this line out. - - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - + API_KEY = {{ API_KEY }} - - Check the result with ``cylc get-config``. The Jinja2 will be processed - so you should not see any difference after making these changes. diff --git a/sphinx/tutorial/cylc/runtime/configuration-consolidation/parameters.rst b/sphinx/tutorial/cylc/runtime/configuration-consolidation/parameters.rst deleted file mode 100644 index c0329fdd15..0000000000 --- a/sphinx/tutorial/cylc/runtime/configuration-consolidation/parameters.rst +++ /dev/null @@ -1,349 +0,0 @@ -.. include:: ../../../../hyperlinks.rst - :start-line: 1 - -.. _tutorial-cylc-parameterisation: - - -Parameterised Tasks -=================== - -Parameterised tasks (see :term:`parameterisation`) provide a way of implicitly -looping over tasks without the need for Jinja2. - - -Cylc Parameters ---------------- - -.. ifnotslides:: - - Parameters are defined in their own section, e.g: - -.. code-block:: cylc - - [cylc] - [[parameters]] - world = Mercury, Venus, Earth - - -.. ifnotslides:: - - They can then be referenced by writing the name of the parameter in angle - brackets, e.g: - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = start => hello => end - [runtime] - [[hello]] - script = echo 'Hello World!' - -.. nextslide:: - -.. ifnotslides:: - - When the ``suite.rc`` file is read by Cylc, the parameters will be expanded. - For example the code above is equivalent to: - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - start => hello_Mercury => end - start => hello_Venus => end - start => hello_Earth => end - """ - [runtime] - [[hello_Mercury]] - script = echo 'Hello World!' - [[hello_Venus]] - script = echo 'Hello World!' - [[hello_Earth]] - script = echo 'Hello World!' - -.. nextslide:: - -.. ifnotslides:: - - We can refer to a specific parameter by writing it after an ``=`` sign: - -.. code-block:: cylc - - [runtime] - [[hello]] - script = echo 'Greetings Earth!' - - -Environment Variables ---------------------- - -.. ifnotslides:: - - The name of the parameter is provided to the job as an environment variable - called ``CYLC_TASK_PARAM_`` where ```` is the name of - the parameter (in the present case ``world``): - -.. code-block:: cylc - - [runtime] - [[hello]] - script = echo "Hello ${CYLC_TASK_PARAM_world}!" - - -Parameter Types ---------------- - -Parameters can be either words or integers: - -.. code-block:: cylc - - [cylc] - [[parameters]] - foo = 1..5 - bar = 1..5..2 - baz = pub, qux, bol - -.. nextslide:: - -.. hint:: - - Remember that Cylc automatically inserts an underscore between the task and - the parameter, e.g. the following lines are equivalent: - - .. code-block:: cylc-graph - - task - task_pub - -.. nextslide:: - -.. hint:: - - .. ifnotslides:: - - When using integer parameters, to prevent confusion, Cylc prefixes the - parameter value with the parameter name. For example: - - .. ifslides:: - - Cylc prefixes integer parameters with the parameter name: - - .. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - # task would result in: - task_bar1 - task_bar3 - task_bar5 - - # task would result in: - task_pub - task_qux - task_bol - """ - -.. nextslide:: - -.. ifnotslides:: - - Using parameters the ``get_observations`` configuration could be written like - so: - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - [[[T00/PT3H]]] - graph = """ - get_observations => consolidate_observations - """ - [runtime] - [[get_observations]] - script = get-observations - [[[environment]]] - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - [[get_observations]] - [[[environment]]] - SITE_ID = 3976 - [[get_observations]] - [[[environment]]] - SITE_ID = 3808 - [[get_observations]] - [[[environment]]] - SITE_ID = 3772 - [[get_observations]] - [[[environment]]] - SITE_ID = 3005 - -.. nextslide:: - -.. ifnotslides:: - - For more information see the `Cylc User Guide`_. - -.. ifslides:: - - .. rubric:: This practical continues on from the - :ref:`Jinja2 practical `. - - Next section: :ref:`Which approach to use - ` - - -.. _cylc-tutorial-parameters-practical: - -.. practical:: - - .. rubric:: This practical continues on from the - :ref:`Jinja2 practical `. - - 4. **Use Parameterisation To Consolidate The** ``get_observations`` - **Tasks**. - - Next we will parameterise the ``get_observations`` tasks. - - Add a parameter called ``station``: - - .. code-block:: diff - - [cylc] - UTC mode = True - + [[parameters]] - + station = belmullet, camborne, heathrow, shetland - - Remove the four ``get_observations`` tasks and insert the following code - in their place: - - .. code-block:: cylc - - [[get_observations]] - script = get-observations - [[[environment]]] - API_KEY = {{ API_KEY }} - - Using ``cylc get-config`` you should see that Cylc replaces the - ```` with each of the stations in turn, creating a new task for - each: - - .. code-block:: bash - - cylc get-config . --sparse -i "[runtime]" - - The ``get_observations`` tasks are now missing the ``SITE_ID`` - environment variable. Add a new section for each station with a - ``SITE_ID``: - - .. code-block:: cylc - - [[get_observations]] - [[[environment]]] - SITE_ID = 3772 - - .. hint:: - - The relevant IDs are: - - * Belmullet - ``3976`` - * Camborne - ``3808`` - * Heathrow - ``3772`` - * Shetland - ``3005`` - - .. spoiler:: Solution warning - - .. code-block:: cylc - - [[get_observations]] - [[[environment]]] - SITE_ID = 3976 - [[get_observations]] - [[[environment]]] - SITE_ID = 3808 - [[get_observations]] - [[[environment]]] - SITE_ID = 3772 - [[get_observations]] - [[[environment]]] - SITE_ID = 3005 - - Using ``cylc get-config`` you should now see four ``get_observations`` - tasks, each with a ``script``, an ``API_KEY`` and a ``SITE_ID``: - - .. code-block:: bash - - cylc get-config . --sparse -i "[runtime]" - - Finally we can use this parameterisation to simplify the suite's - graphing. Replace the ``get_observations`` lines in the graph with - ``get_observations``: - - .. code-block:: diff - - [[[PT3H]]] - # Repeat every three hours starting at the initial cycle point. - graph = """ - - get_observations_belmullet => consolidate_observations - - get_observations_camborne => consolidate_observations - - get_observations_heathrow => consolidate_observations - - get_observations_shetland => consolidate_observations - + get_observations => consolidate_observations - """ - - .. hint:: - - The ``cylc get-config`` command does not expand parameters or families - in the graph so you must use ``cylc graph`` to inspect changes to the - graphing. - - #. **Use Parameterisation To Consolidate The** ``post_process`` **Tasks**. - - At the moment we only have one ``post_process`` task - (``post_process_exeter``), but suppose we wanted to add a second task for - Edinburgh. - - Create a new parameter called ``site`` and set it to contain ``exeter`` - and ``edinburgh``. Parameterise the ``post_process`` task using this - parameter. - - .. hint:: - - The first argument to the ``post-process`` task is the name of the - site. We can use the ``CYLC_TASK_PARAM_site`` environment variable to - avoid having to write out this section twice. - - .. TODO - use parameter environment templates instead of - CYLC_TASK_PARAM. - - .. spoiler:: Solution warning - - First we must create the ``site`` parameter: - - .. code-block:: diff - - [cylc] - UTC mode = True - [[parameters]] - station = belmullet, camborne, heathrow, shetland - + site = exeter, edinburgh - - Next we parameterise the task in the graph: - - .. code-block:: diff - - - -get_rainfall => forecast => post_process_exeter - +get_rainfall => forecast => post_process - - And also the runtime: - - .. code-block:: diff - - -[[post_process_exeter]] - +[[post_process]] - # Generate a forecast for Exeter 60 minutes in the future. - - script = post-process exeter 60 - + script = post-process $CYLC_TASK_PARAM_site 60 diff --git a/sphinx/tutorial/cylc/runtime/index.rst b/sphinx/tutorial/cylc/runtime/index.rst deleted file mode 100644 index 2bca48173a..0000000000 --- a/sphinx/tutorial/cylc/runtime/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _tutorial-runtime: - -Runtime -======= - -This section covers: - -* Associating tasks with scripts and executables. -* Providing executables with runtime configurations. -* Running Cylc suites. - -.. toctree:: - :name: rug-runtime-toc - :maxdepth: 2 - - introduction - runtime-configuration - configuration-consolidation/index diff --git a/sphinx/tutorial/cylc/runtime/introduction.rst b/sphinx/tutorial/cylc/runtime/introduction.rst deleted file mode 100644 index 36a1222d9c..0000000000 --- a/sphinx/tutorial/cylc/runtime/introduction.rst +++ /dev/null @@ -1,518 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - - -.. _tutorial-cylc-runtime-introduction: - -Introduction -============ - -.. ifnotslides:: - - So far we have been working with the ``[scheduling]`` section. This is where - the workflow is defined in terms of :term:`tasks ` and - :term:`dependencies `. - - In order to make the workflow runnable we must associate tasks with scripts - or binaries to be executed when the task runs. This means working with the - ``[runtime]`` section which determines what runs, as well as where and how - it runs. - -.. ifslides:: - - ``[scheduling]`` - Defines the workflow in terms of :term:`tasks ` and - :term:`dependencies `. - ``[runtime]`` - Defines what runs, where and how it runs. - - -The Task Section ----------------- - -.. ifnotslides:: - - The runtime settings for each task are stored in a sub-section of the - ``[runtime]`` section. E.g. for a task called ``hello_world`` we would write - settings inside the following section: - -.. code-block:: cylc - - [runtime] - [[hello_world]] - - -The ``script`` Setting ----------------------- - -.. ifnotslides:: - - We tell Cylc *what* to execute when a task is run using the ``script`` - setting. - - This setting is interpreted as a bash script. The following example defines a - task called ``hello_world`` which writes ``Hello World!`` to stdout upon - execution. - -.. code-block:: cylc - - [runtime] - [[hello_world]] - script = echo 'Hello World!' - -.. note:: - - If you do not set the ``script`` for a task then nothing will be run. - -We can also call other scripts or executables in this way, e.g: - -.. code-block:: cylc - - [runtime] - [[hello_world]] - script = ~/foo/bar/baz/hello_world - - -:envvar:`PATH` and :envvar:`PYTHONPATH` ---------------------------------------- - -.. ifnotslides:: - - It is often a good idea to keep our scripts with the Cylc suite rather than - leaving them somewhere else on the system. - - If you create a ``bin/`` sub-directory within the :term:`suite directory` - Cylc will automatically prepend it to the :envvar:`PATH` environment - variable when the task runs. - -.. code-block:: bash - :caption: bin/hello_world - - #!/usr/bin/bash - echo 'Hello World!' - -.. code-block:: cylc - :caption: suite.rc - - [runtime] - [[hello_world]] - script = hello_world - -.. nextslide:: - -.. ifnotslides:: - - Similarly the ``lib/python/`` directory gets prepended to the - :envvar:`PYTHONPATH` variable. - -.. code-block:: python - :caption: lib/python/hello.py - - def world(): - print('Hello World!') - -.. code-block:: cylc - :caption: suite.rc - - [runtime] - [[hello_world]] - script = python -c 'import hello; hello.world()' - - -.. _tutorial-tasks-and-jobs: - -Tasks And Jobs --------------- - -.. ifnotslides:: - - When a :term:`task` is "Run" it creates a :term:`job`. The job is a bash - file containing the script you have told the task to run along with - configuration specifications and a system for trapping errors. It is the - :term:`job` which actually gets executed and not the task itself. This - "job file" is called the :term:`job script`. - - During its life a typical :term:`task` goes through the following states: - - Waiting - :term:`Tasks ` wait for their dependencies to be satisfied before - running. In the meantime they are in the "Waiting" state. - Submitted - When a :term:`task's ` dependencies have been met it is ready for - submission. During this phase the :term:`job script` is created. - The :term:`job` is then submitted to the specified batch system. - There is more about this in the :ref:`next section - `. - Running - A :term:`task` is in the "Running" state as soon as the :term:`job` is - executed. - Succeeded - If the :term:`job` submitted by a :term:`task` has successfully - completed (i.e. there is zero return code) then it is said to have - succeeded. - - These descriptions, and a few more (e.g. failed), are called the - :term:`task states `. - -.. ifslides:: - - When a :term:`task` is "Run" it creates a :term:`job`. - - The life-cycle of a job: - - * Waiting - * Submitted - * Running - * Succeeded / Failed - - -The Cylc GUI ------------- - -.. ifnotslides:: - - To help you to keep track of a running suite Cylc has a graphical user - interface (the Cylc GUI) which can be used for monitoring and - interaction. - - The Cylc GUI looks quite like ``cylc graph`` but the tasks are colour-coded - to represent their state, as in the following diagram. - -.. digraph:: example - :align: center - - Waiting [color="#88c6ff"] - Running [style="filled" color="#00c410"] - Succeeded [style="filled" color="#ada5a5"] - -.. minicylc:: - :align: center - - a => b => c - b => d => f - e => f - -.. nextslide:: - -.. ifnotslides:: - - This is the "graph view". The Cylc GUI has two other views called "tree" and - "dot". - -.. figure:: ../img/cylc-gui-graph.png - :figwidth: 75% - :align: center - - Screenshot of the Cylc GUI in "Graph View" mode. - -.. nextslide:: - -.. figure:: ../img/cylc-gui-tree.png - :figwidth: 75% - :align: center - - Screenshot of the Cylc GUI in "Tree View" mode. - -.. nextslide:: - -.. figure:: ../img/cylc-gui-dot.png - :figwidth: 75% - :align: center - - Screenshot of the Cylc GUI in "Dot View" mode. - - -Where Do All The Files Go? --------------------------- - -.. ifnotslides:: - - The Work Directory - ^^^^^^^^^^^^^^^^^^ - - When a :term:`task` is run Cylc creates a directory for the :term:`job` to - run in. This is called the :term:`work directory`. - - By default the work directory is located in a directory structure - under the relevant :term:`cycle point` and :term:`task` name: - - .. code-block:: sub - - ~/cylc-run//work// - - The Job Log Directory - ^^^^^^^^^^^^^^^^^^^^^ - - When a task is run Cylc generates a :term:`job script` which is stored in the - :term:`job log directory` as the file ``job``. - - When the :term:`job script` is executed the stdout and stderr are redirected - into the ``job.out`` and ``job.err`` files which are also stored in the - :term:`job log directory`. - - The :term:`job log directory` lives in a directory structure under the - :term:`cycle point`, :term:`task` name and :term:`job submission number`: - - .. code-block:: sub - - ~/cylc-run//log/job//// - - The :term:`job submission number` starts at 1 and increments by 1 each time - a task is re-run. - - .. tip:: - - If a task has run and is still visible in the Cylc GUI you can view its - :term:`job log files ` by right-clicking on the task and - selecting "View". - - .. image:: ../img/cylc-gui-view-log.png - :align: center - :scale: 75% - -.. ifslides:: - - The Work Directory - .. code-block:: sub - - ~/cylc-run//work// - The Job Log Directory - .. code-block:: sub - - ~/cylc-run//log/job//// - - .. image:: ../img/cylc-gui-view-log.png - :align: center - :scale: 75% - - -Running A Suite ---------------- - -.. ifnotslides:: - - It is a good idea to check a suite for errors before running it. - Cylc provides a command which automatically checks for any obvious - configuration issues called ``cylc validate``, run via: - -.. code-block:: sub - - cylc validate - -.. ifnotslides:: - - Here ```` is the path to the suite's location within the - filesystem (so if we create a suite in ``~/cylc-run/foo`` we would put - ``~/cylc-run/foo/suite.rc``). - - Next we can run the suite using the ``cylc run`` command. - -.. code-block:: sub - - cylc run - -.. ifnotslides:: - - The ``name`` is the name of the :term:`suite directory` (i.e. ```` - would be ``foo`` in the above example). - -.. note:: - - In this tutorial we are writing our suites in the ``cylc-run`` directory. - - It is possible to write them elsewhere on the system. If we do so we - must register the suite with Cylc before use. - - We do this using the ``cylc reg`` command which we supply with a name which - will be used to refer to the suite in place of the path i.e: - - .. code-block:: sub - - cylc reg - cylc validate - cylc run - - The ``cylc reg`` command will create a directory for the suite in the - ``cylc-run`` directory meaning that we will have separate - :term:`suite directories ` and - :term:`run directories `. - - -Suite Files ------------ - -.. ifnotslides:: - - Cylc generates files and directories when it runs a suite, namely: - - ``log/`` - Directory containing log files, including: - - ``log/db`` - The database which Cylc uses to record the state of the suite; - ``log/job`` - The directory where the :term:`job log files ` live; - ``log/suite`` - The directory where the :term:`suite log files ` live. - These files are written by Cylc as the suite is run and are useful for - debugging purposes in the event of error. - - ``suite.rc.processed`` - A copy of the ``suite.rc`` file made after any `Jinja2`_ has been - processed - we will cover this in the - :ref:`tutorial-cylc-consolidating-configuration` section. - ``share/`` - The :term:`share directory` is a place where :term:`tasks ` can - write files which are intended to be shared within that cycle. - ``work/`` - A directory hierarchy containing task's :term:`work directories - `. - -.. ifslides:: - - * ``log/`` - * ``log/db`` - * ``log/job`` - * ``log/suite`` - * ``suite.rc.processed`` - * ``share/`` - * ``work/`` - - .. nextslide:: - - .. rubric:: In this practical we will add some scripts to, and run, the - :ref:`weather forecasting suite ` - from the :ref:`scheduling tutorial `. - - Next section: :ref:`tutorial-cylc-runtime-configuration` - - -.. practical:: - - .. rubric:: In this practical we will add some scripts to, and run, the - :ref:`weather forecasting suite ` - from the :ref:`scheduling tutorial `. - - #. **Create A New Suite.** - - The following command will copy some files for us to work with into - a new suite called ``runtime-introduction``: - - .. code-block:: bash - - rose tutorial runtime-introduction - cd ~/cylc-run/runtime-introduction - - In this directory we have the ``suite.rc`` file from the - :ref:`weather forecasting suite ` - with some runtime configuration added to it. - - There is also a script called ``get-observations`` located in the bin - directory. - - Take a look at the ``[runtime]`` section in the ``suite.rc`` file. - - #. **Run The Suite.** - - First validate the suite by running: - - .. code-block:: bash - - cylc validate . - - Open the Cylc GUI (in the background) by running the following command: - - .. code-block:: bash - - cylc gui runtime-introduction & - - Finally run the suite by executing: - - .. code-block:: bash - - cylc run runtime-introduction - - The tasks will start to run - you should see them going through the - "Waiting", "Running" and "Succeeded" states. - - When the suite reaches the final cycle point and all tasks have succeeded - it will shutdown automatically and the GUI will go blank. - - .. tip:: - - You can also run a suite from the Cylc GUI by pressing the "play" - button. - - .. image:: ../img/gcylc-play.png - :align: center - - A box will appear. Ensure that "Cold Start" is selected then press - "Start". - - .. image:: ../img/cylc-gui-suite-start.png - :align: center - - #. **Inspect A Job Log.** - - Try opening the file ``job.out`` for one of the - ``get_observations`` jobs in a text editor. The file will be - located within the :term:`job log directory`: - - .. code-block:: sub - - log/job//get_observations_heathrow/01/job.out - - You should see something like this: - - .. code-block:: none - - Suite : runtime-introduction - Task Job : 20000101T0000Z/get_observations_heathrow/01 (try 1) - User@Host: username@hostname - - Guessing Weather Conditions - Writing Out Wind Data - 1970-01-01T00:00:00Z NORMAL - started - 2038-01-19T03:14:08Z NORMAL - succeeded - - * The first three lines are information which Cylc has written to the file - to provide information about the job. - * The last two lines were also written by cylc. They provide timestamps - marking the stages in the job's life. - * The lines in the middle are the stdout of the job itself. - - #. **Inspect A Work Directory.** - - The ``get_rainfall`` task should create a file called ``rainfall`` in its - :term:`work directory`. Try opening this file, recalling that the - format of the relevant path from within the work directory will be: - - .. code-block:: sub - - work//get_rainfall/rainfall - - .. hint:: - - The ``get_rainfall`` task only runs every third cycle. - - #. **Extension: Explore The Cylc GUI** - - * Try re-running the suite. - - * Try changing the current view(s). - - .. tip:: - - You can do this from the "View" menu or from the toolbar: - - .. image:: ../img/cylc-gui-view-selector.png - :align: center - :scale: 75% - - * Try pressing the "Pause" button which is found in the top left-hand - corner of the GUI. - - * Try right-clicking on a task. From the right-click menu you could try: - - * "Trigger (run now)" - * "Reset State" diff --git a/sphinx/tutorial/cylc/runtime/runtime-configuration.rst b/sphinx/tutorial/cylc/runtime/runtime-configuration.rst deleted file mode 100644 index 90f1dc5d75..0000000000 --- a/sphinx/tutorial/cylc/runtime/runtime-configuration.rst +++ /dev/null @@ -1,501 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - -.. _DataPoint: https://www.metoffice.gov.uk/services/data/datapoint - - -.. _tutorial-cylc-runtime-configuration: - -Runtime Configuration -===================== - -In the last section we associated tasks with scripts and ran a simple suite. In -this section we will look at how we can configure these tasks. - - -Environment Variables ---------------------- - -.. ifnotslides:: - - We can specify environment variables in a task's ``[environment]`` section. - These environment variables are then provided to :term:`jobs ` when they - run. - -.. code-block:: cylc - - [runtime] - [[countdown]] - script = seq $START_NUMBER - [[[environment]]] - START_NUMBER = 5 - -.. ifnotslides:: - - Each job is also provided with some standard environment variables e.g: - - ``CYLC_SUITE_RUN_DIR`` - The path to the suite's :term:`run directory` - *(e.g. ~/cylc-run/suite)*. - ``CYLC_TASK_WORK_DIR`` - The path to the associated task's :term:`work directory` - *(e.g. run-directory/work/cycle/task)*. - ``CYLC_TASK_CYCLE_POINT`` - The :term:`cycle point` for the associated task - *(e.g. 20171009T0950)*. - - There are many more environment variables - see the `Cylc User Guide`_ for more - information. - -.. ifslides:: - - * ``CYLC_SUITE_RUN_DIR`` - * ``CYLC_TASK_WORK_DIR`` - * ``CYLC_TASK_CYCLE_POINT`` - - -.. _tutorial-batch-system: - -Job Submission --------------- - -.. ifnotslides:: - - By default Cylc runs :term:`jobs ` on the machine where the suite is - running. We can tell Cylc to run jobs on other machines by setting the - ``[remote]host`` setting to the name of the host, e.g. to run a task on the - host ``computehost`` you might write: - -.. code-block:: cylc - - [runtime] - [[hello_computehost]] - script = echo "Hello Compute Host" - [[[remote]]] - host = computehost - -.. _background processes: https://en.wikipedia.org/wiki/Background_process -.. _job scheduler: https://en.wikipedia.org/wiki/Job_scheduler - -.. nextslide:: - -.. ifnotslides:: - - Cylc also executes jobs as `background processes`_ by default. - When we are running jobs on other compute hosts we will often want to - use a :term:`batch system` (`job scheduler`_) to submit our job. - Cylc supports the following :term:`batch systems `: - -* at -* loadleveler -* lsf -* pbs -* sge -* slurm -* moab - -.. nextslide:: - -.. ifnotslides:: - - :term:`Batch systems ` typically require - :term:`directives ` in some form. :term:`Directives ` - inform the :term:`batch system` of the requirements of a :term:`job`, for - example how much memory a given job requires or how many CPUs the job will - run on. For example: - -.. code-block:: cylc - - [runtime] - [[big_task]] - script = big-executable - - # Submit to the host "big-computer". - [[[remote]]] - host = big-computer - - # Submit the job using the "slurm" batch system. - [[[job]]] - batch system = slurm - - # Inform "slurm" that this job requires 500MB of RAM and 4 CPUs. - [[[directives]]] - --mem = 500 - --ntasks = 4 - - -Timeouts --------- - -.. ifnotslides:: - - We can specify a time limit after which a job will be terminated using the - ``[job]execution time limit`` setting. The value of the setting is an - :term:`ISO8601 duration`. Cylc automatically inserts this into a job's - directives as appropriate. - -.. code-block:: cylc - - [runtime] - [[some_task]] - script = some-executable - [[[job]]] - execution time limit = PT15M # 15 minutes. - - -Retries -------- - -Sometimes jobs fail. This can be caused by two factors: - -* Something going wrong with the job's execution e.g: - - * A bug; - * A system error; - * The job hitting the ``execution time limit``. - -* Something going wrong with the job submission e.g: - - * A network problem; - * The :term:`job host` becoming unavailable or overloaded; - * An issue with the directives. - -.. nextslide:: - -.. ifnotslides:: - - In the event of failure Cylc can automatically re-submit (retry) jobs. We - configure retries using the ``[job]execution retry delays`` and - ``[job]submission retry delays`` settings. These settings are both set to an - :term:`ISO8601 duration`, e.g. setting ``execution retry delays`` to ``PT10M`` - would cause the job to retry every 10 minutes in the event of execution - failure. - - We can limit the number of retries by writing a multiple in front of the - duration, e.g: - -.. code-block:: cylc - - [runtime] - [[some-task]] - script = some-script - [[[job]]] - # In the event of execution failure, retry a maximum - # of three times every 15 minutes. - execution retry delays = 3*PT15M - # In the event of submission failure, retry a maximum - # of two times every ten minutes and then every 30 - # minutes thereafter. - submission retry delays = 2*PT10M, PT30M - - -Start, Stop, Restart --------------------- - -.. ifnotslides:: - - We have seen how to start and stop Cylc suites with ``cylc run`` and - ``cylc stop`` respectively. The ``cylc stop`` command causes Cylc to wait - for all running jobs to finish before it stops the suite. There are two - options which change this behaviour: - - ``cylc stop --kill`` - When the ``--kill`` option is used Cylc will kill all running jobs - before stopping. *Cylc can kill jobs on remote hosts and uses the - appropriate command when a* :term:`batch system` *is used.* - ``cylc stop --now --now`` - When the ``--now`` option is used twice Cylc stops straight away, leaving - any jobs running. - - Once a suite has stopped it is possible to restart it using the - ``cylc restart`` command. When the suite restarts it picks up where it left - off and carries on as normal. - - .. code-block:: bash - - # Run the suite "name". - cylc run - # Stop the suite "name", killing any running tasks. - cylc stop --kill - # Restart the suite "name", picking up where it left off. - cylc restart - -.. ifslides:: - - .. code-block:: sub - - cylc run - cylc stop - cylc restart - - cylc stop --kill - cylc stop --now --now - - .. nextslide:: - - .. rubric:: In this practical we will add runtime configuration to the - :ref:`weather-forecasting suite ` - from the :ref:`scheduling tutorial `. - - Next section: :ref:`tutorial-cylc-consolidating-configuration` - - -.. _tutorial-cylc-runtime-forecasting-suite: - -.. practical:: - - .. rubric:: In this practical we will add runtime configuration to the - :ref:`weather-forecasting suite ` - from the :ref:`scheduling tutorial `. - - #. **Create A New Suite.** - - Create a new suite by running the command: - - .. code-block:: bash - - rose tutorial runtime-tutorial - cd ~/cylc-run/runtime-tutorial - - You will now have a copy of the weather-forecasting suite along with some - executables and python modules. - - #. **Set The Initial And Final Cycle Points.** - - We want the suite to run for 6 hours, starting at least 7 hours ago, on - the hour. - - We could work out the dates and times manually, or we could let Cylc do - the maths for us. - - Set the :term:`initial cycle point`: - - .. code-block:: cylc - - initial cycle point = previous(T-00) - PT7H - - * ``previous(T-00)`` returns the current time ignoring minutes and - seconds. - - *e.g. if the current time is 12:34 this will return 12:00* - - * ``-PT7H`` subtracts 7 hours from this value. - - Set the :term:`final cycle point`: - - .. code-block:: cylc - - final cycle point = +PT6H - - This sets the :term:`final cycle point` six hours after the - :term:`initial cycle point`. - - Run `cylc validate` to check for any errors:: - - cylc validate . - - #. **Add Runtime Configuration For The** ``get_observations`` **Tasks.** - - In the ``bin`` directory is a script called ``get-observations``. This - script gets weather data from the MetOffice `DataPoint`_ service. - It requires two environment variables: - - ``SITE_ID``: - A four digit numerical code which is used to identify a - weather station, e.g. ``3772`` is Heathrow Airport. - ``API_KEY``: - An authentication key required for access to the service. - - .. TODO: Add instructions for offline configuration - - Generate a Datapoint API key:: - - rose tutorial api-key - - Add the following lines to the bottom of the ``suite.rc`` file replacing - ``xxx...`` with your API key: - - .. code-block:: cylc - - [runtime] - [[get_observations_heathrow]] - script = get-observations - [[[environment]]] - SITE_ID = 3772 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - - Add three more ``get_observations`` tasks for each of the remaining - weather stations. - - You will need the codes for the other three weather stations, which are: - - * Camborne - ``3808`` - * Shetland - ``3005`` - * Belmullet - ``3976`` - - .. spoiler:: Solution warning - - .. code-block:: cylc - - [runtime] - [[get_observations_heathrow]] - script = get-observations - [[[environment]]] - SITE_ID = 3772 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - [[get_observations_camborne]] - script = get-observations - [[[environment]]] - SITE_ID = 3808 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - [[get_observations_shetland]] - script = get-observations - [[[environment]]] - SITE_ID = 3005 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - [[get_observations_belmullet]] - script = get-observations - [[[environment]]] - SITE_ID = 3976 - API_KEY = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - - Check the ``suite.rc`` file is valid by running the command: - - .. code-block:: bash - - cylc validate . - - .. TODO: Add advice on what to do if the command fails. - - #. **Test The** ``get_observations`` **Tasks.** - - Next we will test the ``get_observations`` tasks. - - Open the Cylc GUI by running the following command: - - .. code-block:: bash - - cylc gui runtime-tutorial & - - Run the suite either by pressing the play button in the Cylc GUI or by - running the command: - - .. code-block:: bash - - cylc run runtime-tutorial - - If all goes well the suite will startup and the tasks will run and - succeed. Note that the tasks which do not have a ``[runtime]`` section - will still run though they will not do anything as they do not call any - scripts. - - Once the suite has reached the final cycle point and all tasks have - succeeded the suite will automatically shutdown. - - .. TODO: Advise on what to do if all does not go well. - - The ``get-observations`` script produces a file called ``wind.csv`` which - specifies the wind speed and direction. This file is written in the task's - :term:`work directory`. - - Try and open one of the ``wind.csv`` files. Note that the path to the - :term:`work directory` is: - - .. code-block:: sub - - work// - - You should find a file containing four numbers: - - * The longitude of the weather station; - * The latitude of the weather station; - * The wind direction (*the direction the wind is blowing towards*) - in degrees; - * The wind speed in miles per hour. - - .. spoiler:: Hint hint - - If you run ``ls work`` you should see a - list of cycles. Pick one of them and open the file:: - - work//get_observations_heathrow/wind.csv - - #. **Add runtime configuration for the other tasks.** - - The runtime configuration for the remaining tasks has been written out - for you in the ``runtime`` file which you will find in the - :term:`suite directory`. Copy the code in the ``runtime`` file to the - bottom of the ``suite.rc`` file. - - Check the ``suite.rc`` file is valid by running the command: - - .. code-block:: bash - - cylc validate . - - .. TODO: Add advice on what to do if the command fails. - - #. **Run The Suite.** - - Open the Cylc GUI (if not already open) and run the suite. - - .. spoiler:: Hint hint - - .. code-block:: bash - - cylc gui runtime-tutorial & - - Run the suite either by: - - * Pressing the play button in the Cylc GUI. Then, ensuring that - "Cold Start" is selected within the dialogue window, pressing the - "Start" button. - * Running the command ``cylc run runtime-tutorial``. - - #. **View The Forecast Summary.** - - The ``post_process_exeter`` task will produce a one-line summary of the - weather in Exeter, as forecast two hours ahead of time. This summary can - be found in the ``summary.txt`` file in the :term:`work directory`. - - Try opening the summary file - it will be in the last cycle. The path to - the :term:`work directory` is: - - .. code-block:: sub - - work// - - .. spoiler:: Hint hint - - * ``cycle-point`` - this will be the last cycle of the suite, - i.e. the final cycle point. - * ``task-name`` - set this to "post_process_exeter". - - #. **View The Rainfall Data.** - - .. TODO: Skip this if you don't have internet connection. - - The ``forecast`` task will produce a html page where the rainfall - data is rendered on a map. This html file is called ``job-map.html`` and - is saved alongside the :term:`job log`. - - Try opening this file in a web browser, e.g via: - - .. code-block:: sub - - firefox & - - The path to the :term:`job log directory` is: - - .. code-block:: sub - - log/job/// - - .. spoiler:: Hint hint - - * ``cycle-point`` - this will be the last cycle of the suite, - i.e. the final cycle point. - * ``task-name`` - set this to "forecast". - * ``submission-number`` - set this to "01". - diff --git a/sphinx/tutorial/cylc/scheduling/datetime-cycling.rst b/sphinx/tutorial/cylc/scheduling/datetime-cycling.rst deleted file mode 100644 index 7eb9e47470..0000000000 --- a/sphinx/tutorial/cylc/scheduling/datetime-cycling.rst +++ /dev/null @@ -1,632 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - -.. _nowcasting: https://www.metoffice.gov.uk/services/data/met-office-data-for-reuse/model - -.. _tutorial-datetime-cycling: - -Date-Time Cycling -================= - - -In the last section we looked at writing an :term:`integer cycling` workflow, -one where the :term:`cycle points ` are numbered. - -.. ifnotslides:: - - Typically workflows are repeated at a regular time interval, say every day - or every few hours. To make this easier Cylc has a date-time cycling mode - where the :term:`cycle points ` use date and time specifications - rather than numbers. - -.. admonition:: Reminder - :class: tip - - :term:`Cycle points ` are labels. Cylc runs tasks as soon as - their dependencies are met so cycles do not necessarily run in order. - - -.. _tutorial-iso8601: - -ISO8601 -------- - -In Cylc, dates, times and durations are written using the :term:`ISO8601` format -- an international standard for representing dates and times. - -.. _tutorial-iso8601-datetimes: - -ISO8601 Date-Times -^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - In ISO8601, datetimes are written from the largest unit to the smallest - (i.e: year, month, day, hour, minute, second in succession) with the ``T`` - character separating the date and time components. For example, midnight - on the 1st of January 2000 is written ``20000101T000000``. - - For brevity we may omit seconds (and minutes) from the time i.e: - ``20000101T0000`` (``20000101T00``). - - For readability we may add hyphen (``-``) characters between the date - components and colon (``:``) characters between the time components, i.e: - ``2000-01-01T00:00``. This is the "extended" format. - - Time-zone information can be added onto the end. UTC is written ``Z``, - UTC+1 is written ``+01``, etc. E.G: ``2000-01-01T00:00Z``. - -.. Diagram of an iso8601 datetime's components. - -.. image:: ../img/iso8601-dates.svg - :width: 75% - :align: center - -.. nextslide:: - -.. warning:: - - The "basic" (purely numeric except for ``T``) and "extended" (written with - hyphens and colons) formats cannot be mixed. For example the following - date-times are invalid: - - .. code-block:: none - - 2000-01-01T0000 - 20000101T00:00 - -.. _tutorial-iso8601-durations: - -ISO8601 Durations -^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - In ISO8601, durations are prefixed with a ``P`` and are written with a - character following each unit: - -* ``Y`` for year. -* ``M`` for month. -* ``D`` for day. -* ``W`` for week. -* ``H`` for hour. -* ``M`` for minute. -* ``S`` for second. - -.. nextslide:: - -.. ifnotslides:: - - As with datetimes the components are written in order from largest to - smallest and the date and time components are separated by the ``T`` - character. E.G: - -* ``P1D``: one day. -* ``PT1H``: one hour. -* ``P1DT1H``: one day and one hour. -* ``PT1H30M``: one and a half hours. -* ``P1Y1M1DT1H1M1S``: a year and a month and a day and an hour and a - minute and a second. - - -Date-Time Recurrences ---------------------- - -In :term:`integer cycling`, suites' recurrences are written ``P1``, ``P2``, -etc. - -In :term:`date-time cycling ` there are two ways to write -recurrences: - -1. Using ISO8601 durations (e.g. ``P1D``, ``PT1H``). -2. Using ISO8601 date-times with inferred recurrence. - -Inferred Recurrence -^^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - A recurrence can be inferred from a date-time by omitting digits from the - front. For example, if the year is omitted then the recurrence can be - inferred to be annual. E.G: - -.. code-block:: sub - - 2000-01-01T00 # Datetime - midnight on the 1st of January 2000. - - 01-01T00 # Every year on the 1st of January. - 01T00 # Every month on the first of the month. - T00 # Every day at midnight. - T-00 # Every hour at zero minutes past (every hour on the hour). - -.. note:: - - To omit hours from a date time we must place a ``-`` after the - ``T`` character. - -Recurrence Formats -^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - As with integer cycling, recurrences start, by default, at the - :term:`initial cycle point`. We can override this in one of two ways: - -1. By defining an arbitrary cycle point (``datetime/recurrence``): - - * ``2000/P1Y``: every year starting with the year 2000. - * ``2000-01-01T00/T00``: every day at midnight starting on the 1st of January - 2000 - * ``2000-01-01T12/T00``: every day at midnight starting on the first midnight - after the 1st of January at 12:00 (i.e. ``2000-01-02T00``). - -.. nextslide:: - -.. _tutorial-cylc-datetime-offset-icp: - -2. By defining an offset from the initial cycle point (``offset/recurrence``). - This offset is an ISO8601 duration preceded by a plus character: - - * ``+P1Y/P1Y``: every year starting one year after the initial cycle point. - * ``+PT1H/T00``: every day starting on the first midnight after the point one - hour after the initial cycle point. - -Durations And The Initial Cycle Point -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When using durations, beware that a change in the initial cycle point -might produce different results for the recurrences. - -.. nextslide:: - -.. list-table:: - :class: grid-table - - * - .. code-block:: cylc - :emphasize-lines: 2 - - [scheduling] - initial cycle point = 2000-01-01T00 - [[dependencies]] - [[[P1D]]] - graph = foo[-P1D] => foo - - - .. code-block:: cylc - :emphasize-lines: 2 - - [scheduling] - initial cycle point = 2000-01-01T12 - [[dependencies]] - [[[P1D]]] - graph = foo[-P1D] => foo - - * - .. digraph:: Example - :align: center - - size = "3,3" - - "foo.1" [label="foo\n2000-01-01T00"] - "foo.2" [label="foo\n2000-01-02T00"] - "foo.3" [label="foo\n2000-01-03T00"] - - "foo.1" -> "foo.2" -> "foo.3" - - - .. digraph:: Example - :align: center - - size = "3,3" - - "foo.1" [label="foo\n2000-01-01T12"] - "foo.2" [label="foo\n2000-01-02T12"] - "foo.3" [label="foo\n2000-01-03T12"] - - "foo.1" -> "foo.2" -> "foo.3" - -.. nextslide:: - -We could write the recurrence "every midnight" independent from the initial -cycle point by: - -* Use an `inferred recurrence`_ instead (i.e. ``T00``). -* Overriding the recurrence start point (i.e. ``T00/P1D``) -* Using the ``[scheduling]initial cycle point constraints`` setting to - constrain the initial cycle point (e.g. to a particular time of day). See - the `Cylc User Guide`_ for details. - -The Initial & Final Cycle Points -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - There are two special recurrences for the initial and final cycle points: - -* ``R1``: repeat once at the initial cycle point. -* ``R1/P0Y``: repeat once at the final cycle point. - -.. TODO - change terminology as done in the cylc user guide, "repeat" can be - confusing. Use occur? - -Inter-Cycle Dependencies -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - Inter-cycle dependencies are written as ISO8601 durations, e.g: - -* ``foo[-P1D]``: the task ``foo`` from the cycle one day before. -* ``bar[-PT1H30M]``: the task ``bar`` from the cycle 1 hour 30 minutes before. - -.. ifnotslides:: - - The initial cycle point can be referenced using a caret character ``^``, e.g: - -* ``baz[^]``: the task ``baz`` from the initial cycle point. - - -.. _tutorial-cylc-datetime-utc: - -UTC Mode --------- - -.. ifnotslides:: - - Due to all of the difficulties caused by time zones, particularly with - respect to daylight savings, we typically use UTC (that's the ``+00`` time - zone) in Cylc suites. - - When a suite uses UTC all of the cycle points will be written in the - ``+00`` time zone. - - To make your suite use UTC set the ``[cylc]UTC mode`` setting to ``True``, - i.e: - -.. code-block:: cylc - - [cylc] - UTC mode = True - - -.. _tutorial-datetime-cycling-practical: - -Putting It All Together ------------------------ - -.. ifslides:: - - We will now develop a simple weather forecasting suite. - -.. ifnotslides:: - - Cylc was originally developed for running operational weather forecasting. In - this section we will outline a basic (dummy) weather-forecasting suite and - explain how to implement it in cylc. - - .. note:: - - Technically the suite outlined in this section is a `nowcasting`_ suite. - We will refer to it as forecasting for simplicity. - - A basic weather-forecasting workflow consists of three main steps: - -1. Gathering Observations -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - We gather observations from different weather stations and use them to - build a picture of the current weather. Our dummy weather forecast - will get wind observations from four weather stations: - - * Belmullet - * Camborne - * Heathrow - * Shetland - - The tasks which retrieve observation data will be called - ``get_observations_`` where ``site`` is the name of the weather - station in question. - - Next we need to consolidate these observations so that our forecasting - system can work with them. To do this we have a - ``consolidate_observations`` task. - - We will fetch wind observations **every three hours starting from the initial - cycle point**. - - The ``consolidate_observations`` task must run after the - ``get_observations`` tasks. - -.. digraph:: example - :align: center - - size = "7,4" - - get_observations_belmullet -> consolidate_observations - get_observations_camborne -> consolidate_observations - get_observations_heathrow -> consolidate_observations - get_observations_shetland -> consolidate_observations - - hidden [style="invis"] - get_observations_belmullet -> hidden [style="invis"] - get_observations_camborne -> hidden [style="invis"] - hidden -> consolidate_observations [style="invis"] - -.. ifnotslides:: - - We will also use the UK radar network to get rainfall data with a task - called ``get_rainfall``. - - We will fetch rainfall data **every six hours starting six hours after the - initial cycle point**. - -2. Running computer models to generate forecast data -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - We will do this with a task called ``forecast`` which will run - **every six hours starting six hours after the initial cycle point**. - The ``forecast`` task will be dependent on: - - * The ``consolidate_observations`` task from the previous two cycles as well - as from the present cycle. - * The ``get_rainfall`` task from the present cycle. - -.. digraph:: example - :align: center - - size = "7,4" - - subgraph cluster_T00 { - label="+PT0H" - style="dashed" - "observations.t00" [label="consolidate observations\n+PT0H"] - } - - subgraph cluster_T03 { - label="+PT3H" - style="dashed" - "observations.t03" [label="consolidate observations\n+PT3H"] - } - - subgraph cluster_T06 { - label="+PT6H" - style="dashed" - "forecast.t06" [label="forecast\n+PT6H"] - "get_rainfall.t06" [label="get_rainfall\n+PT6H"] - "observations.t06" [label="consolidate observations\n+PT6H"] - } - - "observations.t00" -> "forecast.t06" - "observations.t03" -> "forecast.t06" - "observations.t06" -> "forecast.t06" - "get_rainfall.t06" -> "forecast.t06" - -3. Processing the data output to produce user-friendly forecasts -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. ifnotslides:: - - This will be done with a task called ``post_process_`` where - ``location`` is the place we want to generate the forecast for. For - the moment we will use Exeter. - - The ``post_process_exeter`` task will run **every six hours starting six - hours after the initial cycle point** and will be dependent on the - ``forecast`` task. - -.. digraph:: example - :align: center - - size = "2.5,2" - - "forecast" -> "post_process_exeter" - -.. nextslide:: - -.. ifslides:: - - .. rubric:: Next Steps - - 1. Read through the "Putting It All Together" section. - 2. Complete the practical. - - Next section: :ref:`tutorial-cylc-further-scheduling` - - -.. _datetime cycling practical: - -.. practical:: - - .. rubric:: In this practical we will create a dummy forecasting suite - using date-time cycling. - - #. **Create A New Suite.** - - Within your ``~/cylc-run`` directory create a new directory called - ``datetime-cycling`` and move into it: - - .. code-block:: bash - - mkdir ~/cylc-run/datetime-cycling - cd ~/cylc-run/datetime-cycling - - Create a ``suite.rc`` file and paste the following code into it: - - .. code-block:: cylc - - [cylc] - UTC mode = True - [scheduling] - initial cycle point = 20000101T00Z - [[dependencies]] - - #. **Add The Recurrences.** - - The weather-forecasting suite will require two - recurrences. Add sections under the dependencies section for these, - based on the information given above. - - .. hint:: - - See :ref:`Date-Time Recurrences`. - - .. spoiler:: Solution warning - - The two recurrences you need are - - * ``PT3H``: repeat every three hours starting from the initial cycle - point. - * ``+PT6H/PT6H``: repeat every six hours starting six hours after the - initial cycle point. - - .. code-block:: diff - - [cylc] - UTC mode = True - [scheduling] - initial cycle point = 20000101T00Z - [[dependencies]] - + [[[PT3H]]] - + [[[+PT6H/PT6H]]] - - #. **Write The Graphing.** - - With the help of the graphs and the information above add dependencies to - your suite to implement the weather-forecasting workflow. - - You will need to consider the inter-cycle dependencies between tasks. - - Use ``cylc graph`` to inspect your work. - - .. spoiler:: Hint hint - - The dependencies you will need to formulate are as follows: - - * The ``consolidate_observations`` task is dependent on the - ``get_observations_`` tasks. - * The ``forecast`` task is dependent on: - - * the ``get_rainfall`` task; - * the ``consolidate_observations`` tasks from: - - * the same cycle; - * the cycle 3 hours before (``-PT3H``); - * the cycle 6 hours before (``-PT6H``). - - * The ``post_process_exeter`` task is dependent on the ``forecast`` - task. - - To launch ``cylc graph`` run the command: - - .. code-block:: sub - - cylc graph - - .. spoiler:: Solution warning - - .. code-block:: cylc - - [cylc] - UTC mode = True - [scheduling] - initial cycle point = 20000101T00Z - [[dependencies]] - [[[PT3H]]] - graph = """ - get_observations_belmullet => consolidate_observations - get_observations_camborne => consolidate_observations - get_observations_heathrow => consolidate_observations - get_observations_shetland => consolidate_observations - """ - [[[+PT6H/PT6H]]] - graph = """ - consolidate_observations => forecast - consolidate_observations[-PT3H] => forecast - consolidate_observations[-PT6H] => forecast - get_rainfall => forecast => post_process_exeter - """ - - #. **Inter-Cycle Offsets.** - - To ensure the ``forecast`` tasks for different cycles run in order the - ``forecast`` task will also need to be dependent on the previous run - of ``forecast``. - - .. digraph:: example - :align: center - - size = "4,1.5" - rankdir=LR - - subgraph cluster_T06 { - label="T06" - style="dashed" - "forecast.t06" [label="forecast\nT06"] - } - - subgraph cluster_T12 { - label="T12" - style="dashed" - "forecast.t12" [label="forecast\nT12"] - } - - subgraph cluster_T18 { - label="T18" - style="dashed" - "forecast.t18" [label="forecast\nT18"] - } - - "forecast.t06" -> "forecast.t12" -> "forecast.t18" - - We can express this dependency as ``forecast[-PT6H] => forecast``. - - Try adding this line to your suite then visualising it with ``cylc - graph``. - - .. hint:: - - Try adjusting the number of cycles displayed by ``cylc graph``: - - .. code-block:: console - - $ cylc graph . 2000 20000101T12Z & - - You will notice that there is a dependency which looks like this: - - .. digraph:: example - :align: center - - size = "4,1" - rankdir=LR - - "forecast.t00" [label="forecast\n20000101T0000Z" - color="#888888" - fontcolor="#888888"] - "forecast.t06" [label="forecast\n20000101T0600Z"] - - - "forecast.t00" -> "forecast.t06" - - Note in particular that the ``forecast`` task in the 00:00 cycle is - grey. The reason for this is that this task does not exist. Remember - the forecast task runs every six hours - **starting 6 hours after the initial cycle point**, so the - dependency is only valid from 12:00 onwards. To fix the problem we - must add a new dependency section which repeats every six hours - **starting 12 hours after the initial cycle point**. - - Make the following changes to your suite and the grey task should - disappear: - - .. code-block:: diff - - [[[+PT6H/PT6H]]] - graph = """ - ... - - forecast[-PT6H] => forecast - """ - + [[[+PT12H/PT6H]]] - + graph = """ - + forecast[-PT6H] => forecast - + """ diff --git a/sphinx/tutorial/cylc/scheduling/further-scheduling.rst b/sphinx/tutorial/cylc/scheduling/further-scheduling.rst deleted file mode 100644 index b728cd3cda..0000000000 --- a/sphinx/tutorial/cylc/scheduling/further-scheduling.rst +++ /dev/null @@ -1,107 +0,0 @@ -.. include:: ../../../hyperlinks.rst - :start-line: 1 - -.. _tutorial-cylc-further-scheduling: - -Further Scheduling -================== - -In this section we will quickly run through some of the more advanced features -of Cylc's scheduling logic. - - -.. _tutorial-qualifiers: - -Qualifiers ----------- - -.. ifnotslides:: - - So far we have written dependencies like ``foo => bar``. This is, in fact, - shorthand for ``foo:succeed => bar``. It means that the task ``bar`` will run - once ``foo`` has finished successfully. If ``foo`` were to fail then ``bar`` - would not run. We will talk more about these :term:`task states ` - in the :ref:`Runtime Section `: - - We refer to the ``:succeed`` descriptor as a :term:`qualifier`. - There are qualifiers for different :term:`task states ` e.g: - -.. ifslides:: - - .. code-block:: cylc-graph - - foo => bar - foo:succeed => bar - foo:fail => bar - -``:start`` - When the task has started running. -``:fail`` - When the task finishes if it fails (produces non-zero return code). -``:finish`` - When the task has completed (either succeeded or failed). - -.. nextslide:: - -It is also possible to create your own custom :term:`qualifiers ` -to handle events within your code (custom outputs). - -.. ifnotslides:: - - *For more information see the* `Cylc User Guide`_. - - -Clock Triggers --------------- - -.. ifnotslides:: - - In Cylc, :term:`cycle points ` are just labels. Tasks are - triggered when their dependencies are met irrespective of the cycle they are - in, but we can force cycles to wait for a particular time before running - using clock triggers. This is necessary for certain operational and - monitoring systems. - - For example in the following suite the cycle ``2000-01-01T12Z`` will wait - until 11:00 on the 1st of January 2000 before running: - -.. code-block:: cylc - - [scheduling] - initial cycle point = 2000-01-01T00Z - [[special tasks]] - clock-trigger = daily(-PT1H) - [[dependencies]] - [[[T12]]] - graph = daily # "daily" will run, at the earliest, one hour - # before midday. - -.. tip:: - - See the :ref:`tutorial-cylc-clock-trigger` tutorial for more information. - - -Alternative Calendars ---------------------- - -.. ifnotslides:: - - By default Cylc uses the Gregorian calendar for :term:`datetime cycling`, - but Cylc also supports the 360-day calendar (12 months of 30 days each in - a year). - -.. code-block:: cylc - - [scheduling] - cycling mode = 360day - -.. ifnotslides:: - - *For more information see the* `Cylc User Guide`_. - -.. nextslide:: - -.. ifslides:: - - Next section: :ref:`Runtime Introduction - ` diff --git a/sphinx/tutorial/cylc/scheduling/graphing.rst b/sphinx/tutorial/cylc/scheduling/graphing.rst deleted file mode 100644 index 87816438a6..0000000000 --- a/sphinx/tutorial/cylc/scheduling/graphing.rst +++ /dev/null @@ -1,408 +0,0 @@ -.. include:: ../../../hyperlinks.rst - - -.. _tutorial-cylc-graphing: - -Graphing -======== - -In this section we will cover writing basic workflows in cylc. - - -.. _Cylc file format: - -The ``suite.rc`` File Format ----------------------------- - -.. ifnotslides:: - - We refer to a Cylc workflow as a :term:`Cylc suite`. A Cylc suite is a - directory containing a ``suite.rc`` file. This configuration file is where - we define our workflow. The ``suite.rc`` file uses a nested `INI`_-based - format: - -.. ifslides:: - - * Cylc workflow == Cylc suite - * Cylc suite is a directory containing a ``suite.rc`` file - * The ``suite.rc`` file is written in a nested `INI`_-based format - -.. ifnotslides:: - - * Comments start with a ``#`` character. - * Settings are written as ``key = value`` pairs. - * Settings can be contained within sections. - * Sections are written inside square brackets i.e. ``[section-name]``. - * Sections can be nested, by adding an extra square bracket with each level, - so a sub-section would be written ``[[sub-section]]``, a sub-sub-section - ``[[[sub-sub-section]]]``, and so on. - -Example -^^^^^^^ - -.. code-block:: cylc - - # Comment - [section] - key = value - [[sub-section]] - another-key = another-value # Inline comment - yet-another-key = """ - A - Multi-line - String - """ - -Shorthand -^^^^^^^^^ - -Throughout this tutorial we will refer to settings in the following format: - -``[section]`` - Refers to the entire section. -``[section]key`` - Refers to a setting within the section. -``[section]key=value`` - Expresses the value of the setting. -``[section][sub-section]another-key`` - Note we only use one set of square brackets with nested sections. - -Duplicate Items -^^^^^^^^^^^^^^^ - -Duplicate sections get merged: - -.. list-table:: - :class: grid-table - - * - - .. code-block:: cylc - :caption: input - - [a] - c = C - [b] - d = D - [a] # duplicate - e = E - - - - .. code-block:: cylc - :caption: result - - [a] - c = C - e = E - [b] - d = D - -.. nextslide:: - -Duplicate settings get overwritten: - -.. list-table:: - :class: grid-table - - * - - .. code-block:: cylc - :caption: input - - a = foo - a = bar # duplicate - - - - .. code-block:: cylc - :caption: result - - a = bar - -Indentation -^^^^^^^^^^^ - -It is advisable to indent ``suite.rc`` files. - -However, Cylc ignores this indentation meaning the following two examples -are equivalent: - -.. list-table:: - :class: grid-table - - * - - .. code-block:: cylc - :caption: input - - [section] - a = A - [[sub-section]] - b = B - b = C - # this setting is still - # in [[sub-section]] - - - - - .. code-block:: cylc - :caption: result - - [section] - a = A - [[sub-section]] - b = C - - -Graph Strings -------------- - -In Cylc we consider workflows in terms of :term:`tasks ` and -:term:`dependencies `. - -.. ifnotslides:: - - Task are represented as words and dependencies as arrows (``=>``), so the - following text defines two tasks where ``make_dough`` is dependent on - ``purchase_ingredients``: - -.. minicylc:: - :align: center - :snippet: - :theme: demo - - purchase_ingredients => make_dough - -.. nextslide:: - -.. ifnotslides:: - - In a Cylc workflow this would mean that ``make_dough`` would only run when - ``purchase_ingredients`` has succeeded. These :term:`dependencies - ` can be chained together: - -.. minicylc:: - :align: center - :snippet: - :theme: demo - - purchase_ingredients => make_dough => bake_bread => sell_bread - -.. nextslide:: - -.. ifnotslides:: - - This line of text is referred to as a :term:`graph string`. These graph - strings can be combined to form more complex workflows: - -.. minicylc:: - :align: center - :snippet: - :theme: demo - - purchase_ingredients => make_dough => bake_bread => sell_bread - pre_heat_oven => bake_bread - bake_bread => clean_oven - -.. nextslide:: - -.. ifnotslides:: - - Graph strings can also contain "and" (``&``) and "or" (``|``) operators, for - instance the following lines are equivalent to the ones just above: - -.. code-block:: cylc-graph - - purchase_ingredients => make_dough - pre_heat_oven & make_dough => bake_bread => sell_bread & clean_oven - -.. nextslide:: - -Collectively these :term:`graph strings` are referred to as a -:term:`graph`. - -.. admonition:: Note - :class: tip - - .. ifnotslides:: - - The order in which lines appear in the graph section doesn't matter, for - instance the following examples are the same as each other: - - .. code-block:: cylc-graph - - foo => bar - bar => baz - - .. code-block:: cylc-graph - - bar => baz - foo => bar - - -Cylc Graphs ------------ - -.. ifnotslides:: - - In a :term:`Cylc suite` the :term:`graph` is stored under the - ``[scheduling][dependencies]graph`` setting, i.e: - -.. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - purchase_ingredients => make_dough - pre_heat_oven & make_dough => bake_bread => sell_bread & clean_oven - """ - -.. nextslide:: - -.. ifnotslides:: - - This is a minimal :term:`Cylc suite`, in which we have defined a - :term:`graph` representing a workflow for Cylc to run. - We have not yet provided Cylc with the scripts or binaries to run for - each task. This will be covered later in the - :ref:`runtime tutorial `. - - Cylc provides a GUI for visualising :term:`graphs `. It is run on the - command line using the ``cylc graph `` command where the path ``path`` - is to the ``suite.rc`` file you wish to visualise. - - When run, ``cylc graph`` will display a diagram similar to the ones you have - seen so far. The number ``1`` which appears below each task is the - :term:`cycle point`. We will explain what this means in the next section. - -.. image:: ../img/cylc-graph.png - :align: center - -.. nextslide:: - -.. hint:: - - .. ifnotslides:: - - A graph can be drawn in multiple ways, for instance the following two - examples are equivalent: - - .. ifslides:: - - A graph can be drawn in multiple ways: - - .. image:: ../img/cylc-graph-reversible.svg - :align: center - - .. ifnotslides:: - - The graph drawn by ``cylc graph`` may vary slightly from one run to - another but the tasks and dependencies will always be the same. - -.. nextslide:: - -.. ifslides:: - - .. rubric:: In this practical we will create a new Cylc suite and write a - graph for it to use. - - Next session: :ref:`tutorial-integer-cycling` - -.. practical:: - - .. rubric:: In this practical we will create a new Cylc suite and write a - graph for it to use. - - #. **Create a Cylc suite.** - - A Cylc suite is just a directory containing a ``suite.rc`` file. - - If you don't have one already, create a ``cylc-run`` directory in your - user space i.e:: - - ~/cylc-run - - Within this directory create a new folder called ``graph-introduction``, - which is to be our :term:`suite directory`. Move into it: - - .. code-block:: bash - - mkdir ~/cylc-run/graph-introduction - cd ~/cylc-run/graph-introduction - - Inside this directory create a ``suite.rc`` file and paste in the - following text: - - .. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - # Write graph strings here! - """ - - #. **Write a graph.** - - We now have a blank Cylc suite, next we need to define a workflow. - - Edit your ``suite.rc`` file to add graph strings representing the - following graph: - - .. digraph:: graph_tutorial - :align: center - - foo -> bar -> baz -> qux - pub -> bar -> wop - - #. **Use** ``cylc graph`` **to visualise the workflow.** - - Once you have written some graph strings try using ``cylc graph`` to - display the workflow. Run the following command: - - .. code-block:: bash - - cylc graph . - - .. admonition:: Note - :class: hint - - ``cylc graph`` takes the path to the suite as an argument. As we are - inside the :term:`suite directory` we can run ``cylc graph .``. - - If the results don't match the diagram above try going back to the - suite.rc file and making changes. - - .. tip:: - - In the top right-hand corner of the ``cylc graph`` window there is a - refresh button which will reload the GUI with any changes you have - made. - - .. image:: ../img/cylc-graph-refresh.png - :align: center - - - .. spoiler:: Solution warning - - There are multiple correct ways to write this graph. So long as what - you see in ``cylc graph`` matches the above diagram then you have a - correct solution. - - Two valid examples: - - .. code-block:: cylc-graph - - foo & pub => bar => baz & wop - baz => qux - - .. code-block:: cylc-graph - - foo => bar => baz => qux - pub => bar => wop - - The whole suite should look something like this: - - .. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - foo & pub => bar => baz & wop - baz => qux - """ diff --git a/sphinx/tutorial/cylc/scheduling/index.rst b/sphinx/tutorial/cylc/scheduling/index.rst deleted file mode 100644 index f52df39fa2..0000000000 --- a/sphinx/tutorial/cylc/scheduling/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _tutorial-scheduling: - -Scheduling -========== - -This section looks at how to write workflows in cylc. - -.. toctree:: - :name: rug-scheduling-toc - :maxdepth: 2 - - graphing - integer-cycling - datetime-cycling - further-scheduling diff --git a/sphinx/tutorial/cylc/scheduling/integer-cycling.rst b/sphinx/tutorial/cylc/scheduling/integer-cycling.rst deleted file mode 100644 index db15986453..0000000000 --- a/sphinx/tutorial/cylc/scheduling/integer-cycling.rst +++ /dev/null @@ -1,612 +0,0 @@ -.. _tutorial-integer-cycling: - -Basic Cycling -============= - - -In this section we will look at how to write :term:`cycling` (repeating) -workflows. - - -Repeating Workflows -------------------- - -.. ifnotslides:: - - Often, we will want to repeat the same workflow multiple times. In Cylc this - "repetition" is called :term:`cycling` and each repetition of the workflow is - referred to as a :term:`cycle`. - - Each :term:`cycle` is given a unique label. This is called a - :term:`cycle point`. For now these :term:`cycle points` will be - integers *(they can also be dates as we will see in the next section)*. - -To make a workflow repeat we must tell Cylc three things: - -.. ifslides:: - - * :term:`recurrence` - * :term:`initial cycle point` - * :term:`final cycle point` - -.. ifnotslides:: - - The :term:`recurrence` - How often we want the workflow to repeat. - The :term:`initial cycle point` - At what cycle point we want to start the workflow. - The :term:`final cycle point` - *Optionally* we can also tell Cylc what cycle point we want to stop the - workflow. - -.. nextslide:: - -.. ifnotslides:: - - Let's take the bakery example from the previous section. Bread is - produced in batches so the bakery will repeat this workflow for each - batch of bread they bake. We can make this workflow repeat with the addition - of three lines: - -.. code-block:: diff - - [scheduling] - + cycling mode = integer - + initial cycle point = 1 - [[dependencies]] - + [[[P1]]] - graph = """ - purchase_ingredients => make_dough - pre_heat_oven & make_dough => bake_bread => sell_bread & clean_oven - """ - -.. nextslide:: - -.. ifnotslides:: - - * The ``cycling mode = integer`` setting tells Cylc that we want our - :term:`cycle points ` to be numbered. - * The ``initial cycle point = 1`` setting tells Cylc to start counting - from 1. - * ``P1`` is the :term:`recurrence`. The :term:`graph` within the ``[[[P1]]]`` - section will be repeated at each :term:`cycle point`. - - The first three :term:`cycles` would look like this, with the entire - workflow repeated at each cycle point: - -.. digraph:: example - :align: center - - size = "7,15" - - subgraph cluster_1 { - label = 1 - style = dashed - "pur.1" [label="purchase_ingredients\n1"] - "mak.1" [label="make_dough\n1"] - "bak.1" [label="bake_bread\n1"] - "sel.1" [label="sell_bread\n1"] - "cle.1" [label="clean_oven\n1"] - "pre.1" [label="pre_heat_oven\n1"] - } - - subgraph cluster_2 { - label = 2 - style = dashed - "pur.2" [label="purchase_ingredients\n2"] - "mak.2" [label="make_dough\n2"] - "bak.2" [label="bake_bread\n2"] - "sel.2" [label="sell_bread\n2"] - "cle.2" [label="clean_oven\n2"] - "pre.2" [label="pre_heat_oven\n2"] - } - - subgraph cluster_3 { - label = 3 - style = dashed - "pur.3" [label="purchase_ingredients\n3"] - "mak.3" [label="make_dough\n3"] - "bak.3" [label="bake_bread\n3"] - "sel.3" [label="sell_bread\n3"] - "cle.3" [label="clean_oven\n3"] - "pre.3" [label="pre_heat_oven\n3"] - } - - "pur.1" -> "mak.1" -> "bak.1" -> "sel.1" - "pre.1" -> "bak.1" -> "cle.1" - "pur.2" -> "mak.2" -> "bak.2" -> "sel.2" - "pre.2" -> "bak.2" -> "cle.2" - "pur.3" -> "mak.3" -> "bak.3" -> "sel.3" - "pre.3" -> "bak.3" -> "cle.3" - -.. ifnotslides:: - - Note the numbers under each task which represent the :term:`cycle point` each - task is in. - - -Inter-Cycle Dependencies ------------------------- - -.. ifnotslides:: - - We've just seen how to write a workflow that repeats every :term:`cycle`. - - Cylc runs tasks as soon as their dependencies are met so cycles are not - necessarily run in order. This could cause problems, for instance we could - find ourselves pre-heating the oven in one cycle whist we are still - cleaning it in another. - - To resolve this we must add :term:`dependencies` *between* the - cycles. We do this by adding lines to the :term:`graph`. Tasks in the - previous cycle can be referred to by suffixing their name with ``[-P1]``, - for example. So to ensure the ``clean_oven`` task has been completed before - the start of the ``pre_heat_oven`` task in the next cycle, we would write - the following dependency: - - .. code-block:: cylc-graph - - clean_oven[-P1] => pre_heat_oven - - This dependency can be added to the suite by adding it to the other graph - lines: - -.. code-block:: diff - - [scheduling] - cycling mode = integer - initial cycle point = 1 - [[dependencies]] - [[[P1]]] - graph = """ - purchase_ingredients => make_dough - pre_heat_oven & make_dough => bake_bread => sell_bread & clean_oven - + clean_oven[-P1] => pre_heat_oven - """ - -.. nextslide:: - -.. ifnotslides:: - - The resulting suite would look like this: - -.. digraph:: example - :align: center - - size = "7,15" - - subgraph cluster_1 { - label = 1 - style = dashed - "pur.1" [label="purchase_ingredients\n1"] - "mak.1" [label="make_dough\n1"] - "bak.1" [label="bake_bread\n1"] - "sel.1" [label="sell_bread\n1"] - "cle.1" [label="clean_oven\n1"] - "pre.1" [label="pre_heat_oven\n1"] - } - - subgraph cluster_2 { - label = 2 - style = dashed - "pur.2" [label="purchase_ingredients\n2"] - "mak.2" [label="make_dough\n2"] - "bak.2" [label="bake_bread\n2"] - "sel.2" [label="sell_bread\n2"] - "cle.2" [label="clean_oven\n2"] - "pre.2" [label="pre_heat_oven\n2"] - } - - subgraph cluster_3 { - label = 3 - style = dashed - "pur.3" [label="purchase_ingredients\n3"] - "mak.3" [label="make_dough\n3"] - "bak.3" [label="bake_bread\n3"] - "sel.3" [label="sell_bread\n3"] - "cle.3" [label="clean_oven\n3"] - "pre.3" [label="pre_heat_oven\n3"] - } - - "pur.1" -> "mak.1" -> "bak.1" -> "sel.1" - "pre.1" -> "bak.1" -> "cle.1" - "cle.1" -> "pre.2" - "pur.2" -> "mak.2" -> "bak.2" -> "sel.2" - "pre.2" -> "bak.2" -> "cle.2" - "cle.2" -> "pre.3" - "pur.3" -> "mak.3" -> "bak.3" -> "sel.3" - "pre.3" -> "bak.3" -> "cle.3" - -.. nextslide:: - -.. ifnotslides:: - - Adding this dependency "strings together" the cycles, forcing them to run in - order. We refer to dependencies between cycles as - :term:`inter-cycle dependencies`. - - In the dependency the ``[-P1]`` suffix tells Cylc that we are referring to a - task in the previous cycle. Equally ``[-P2]`` would refer to a task two - cycles ago. - - Note that the ``purchase_ingredients`` task has no arrows pointing at it - meaning that it has no dependencies. Consequently the ``purchase_ingredients`` - tasks will all run straight away. This could cause our bakery to run into - cash-flow problems as they would be purchasing ingredients well in advance - of using them. - - To solve this, but still make sure that they never run out of - ingredients, the bakery wants to purchase ingredients two batches ahead. - This can be achieved by adding the following dependency: - -.. ifslides:: - - We need ``purchase_ingredients`` to be dependent on ``sell_bread`` from - two cycles before. - -.. nextslide:: - -.. code-block:: diff - - [scheduling] - cycling mode = integer - initial cycle point = 1 - [[dependencies]] - [[[P1]]] - graph = """ - purchase_ingredients => make_dough - pre_heat_oven & make_dough => bake_bread => sell_bread & clean_oven - clean_oven[-P1] => pre_heat_oven - + sell_bread[-P2] => purchase_ingredients - """ - -.. nextslide:: - -.. ifnotslides:: - - This dependency means that the ``purchase_ingredients`` task will run after - the ``sell_bread`` task two cycles before. - -.. note:: - - The ``[-P2]`` suffix is used to reference a task two cycles before. For the - first two cycles this doesn't make sense as there was no cycle two cycles - before, so this dependency will be ignored. - - Any inter-cycle dependencies stretching back to before the - :term:`initial cycle point` will be ignored. - -.. digraph:: example - :align: center - - size = "4.5,15" - - subgraph cluster_1 { - label = 1 - style = dashed - "pur.1" [label="purchase_ingredients\n1"] - "mak.1" [label="make_dough\n1"] - "bak.1" [label="bake_bread\n1"] - "sel.1" [label="sell_bread\n1"] - "cle.1" [label="clean_oven\n1"] - "pre.1" [label="pre_heat_oven\n1"] - } - - subgraph cluster_2 { - label = 2 - style = dashed - "pur.2" [label="purchase_ingredients\n2"] - "mak.2" [label="make_dough\n2"] - "bak.2" [label="bake_bread\n2"] - "sel.2" [label="sell_bread\n2"] - "cle.2" [label="clean_oven\n2"] - "pre.2" [label="pre_heat_oven\n2"] - } - - subgraph cluster_3 { - label = 3 - style = dashed - "pur.3" [label="purchase_ingredients\n3"] - "mak.3" [label="make_dough\n3"] - "bak.3" [label="bake_bread\n3"] - "sel.3" [label="sell_bread\n3"] - "cle.3" [label="clean_oven\n3"] - "pre.3" [label="pre_heat_oven\n3"] - } - - subgraph cluster_4 { - label = 4 - style = dashed - "pur.4" [label="purchase_ingredients\n4"] - "mak.4" [label="make_dough\n4"] - "bak.4" [label="bake_bread\n4"] - "sel.4" [label="sell_bread\n4"] - "cle.4" [label="clean_oven\n4"] - "pre.4" [label="pre_heat_oven\n4"] - } - - "pur.1" -> "mak.1" -> "bak.1" -> "sel.1" - "pre.1" -> "bak.1" -> "cle.1" - "cle.1" -> "pre.2" - "sel.1" -> "pur.3" - "pur.2" -> "mak.2" -> "bak.2" -> "sel.2" - "pre.2" -> "bak.2" -> "cle.2" - "cle.2" -> "pre.3" - "sel.2" -> "pur.4" - "pur.3" -> "mak.3" -> "bak.3" -> "sel.3" - "pre.3" -> "bak.3" -> "cle.3" - "cle.3" -> "pre.4" - "pur.4" -> "mak.4" -> "bak.4" -> "sel.4" - "pre.4" -> "bak.4" -> "cle.4" - - -Recurrence Sections -------------------- - -.. ifnotslides:: - - In the previous examples we made the workflow repeat by placing the graph - within the ``[[[P1]]]`` section. Here ``P1`` is a :term:`recurrence` meaning - repeat every cycle, where ``P1`` means every cycle, ``P2`` means every - *other* cycle, and so on. To build more complex workflows we can use multiple - recurrences: - -.. code-block:: cylc - - [scheduling] - cycling mode = integer - initial cycle point = 1 - [[dependencies]] - [[[P1]]] # Repeat every cycle. - graph = foo - [[[P2]]] # Repeat every second cycle. - graph = bar - [[[P3]]] # Repeat every third cycle. - graph = baz - -.. digraph:: example - :align: center - - subgraph cluster_1 { - label = 1 - style = dashed - "foo.1" [label="foo\n1"] - "bar.1" [label="bar\n1"] - "baz.1" [label="baz\n1"] - } - - subgraph cluster_2 { - label = 2 - style = dashed - "foo.2" [label="foo\n2"] - } - - subgraph cluster_3 { - label = 3 - style = dashed - "foo.3" [label="foo\n3"] - "bar.3" [label="bar\n3"] - } - -.. nextslide:: - -.. ifnotslides:: - - By default recurrences start at the :term:`initial cycle point`, however it - is possible to make them start at an arbitrary cycle point. This is done by - writing the cycle point and the recurrence separated by a forward slash - (``/``), e.g. ``5/P3`` means repeat every third cycle starting *from* cycle - number 5. - - The start point of a recurrence can also be defined as an offset from the - :term:`initial cycle point`, e.g. ``+P5/P3`` means repeat every third cycle - starting 5 cycles *after* the initial cycle point. - -.. ifslides:: - - ``5/P3`` - Repeat every third cycle starting *from* cycle number 5. - ``+P5/P3`` - Repeat every third cycle starting 5 cycles *after* the initial cycle - point. - - .. nextslide:: - - .. rubric:: In this practical we will take the :term:`suite ` - we wrote in the previous section and turn it into a - :term:`cycling suite `. - - Next section: :ref:`tutorial-datetime-cycling` - -.. _basic cycling practical: - -.. practical:: - - .. rubric:: In this practical we will take the :term:`suite ` - we wrote in the previous section and turn it into a - :term:`cycling suite `. - - If you have not completed the previous practical use the following code for - your ``suite.rc`` file. - - .. code-block:: cylc - - [scheduling] - [[dependencies]] - graph = """ - foo & pub => bar => baz & wop - baz => qux - """ - - #. **Create a new suite.** - - Within your ``~/cylc-run/`` directory create a new (sub-)directory called - ``integer-cycling`` and move into it: - - .. code-block:: bash - - mkdir -p ~/cylc-run/integer-cycling - cd ~/cylc-run/integer-cycling - - Copy the above code into a ``suite.rc`` file in that directory. - - #. **Make the suite cycle.** - - Add in the following lines. - - .. code-block:: diff - - [scheduling] - + cycling mode = integer - + initial cycle point = 1 - [[dependencies]] - + [[[P1]]] - graph = """ - foo & pub => bar => baz & wop - baz => qux - """ - - #. **Visualise the suite.** - - Try visualising the suite using ``cylc graph``. - - .. code-block:: none - - cylc graph . - - .. tip:: - - You can get Cylc graph to draw dotted boxes around the cycles by - clicking the "Organise by cycle point" button on the toolbar: - - .. image:: ../img/cylc-graph-cluster.png - :align: center - - .. tip:: - - By default ``cylc graph`` displays the first three cycles of a suite. - You can tell ``cylc graph`` to visualise the cycles between two points - by providing them as arguments, for instance the following example - would show all cycles between ``1`` and ``5`` (inclusive):: - - cylc graph . 1 5 & - - #. **Add another recurrence.** - - Suppose we wanted the ``qux`` task to run every *other* cycle as opposed - to every cycle. We can do this by adding another recurrence. - - Make the following changes to your ``suite.rc`` file. - - .. code-block:: diff - - [scheduling] - cycling mode = integer - initial cycle point = 1 - [[dependencies]] - [[[P1]]] - graph = """ - foo & pub => bar => baz & wop - - baz => qux - """ - + [[[P2]]] - + graph = """ - + baz => qux - + """ - - Use ``cylc graph`` to see the effect this has on the workflow. - - #. **Inter-cycle dependencies.** - - Next we need to add some inter-cycle dependencies. We are going to add - three inter-cycle dependencies: - - #. Between ``wop`` from the previous cycle and ``pub``. - #. Between ``baz`` from the previous cycle and ``foo`` - *every odd cycle* (e.g. baz.2 => foo.3). - #. Between ``qux`` from the previous cycle and ``foo`` - *every even cycle* (e.g. qux.1 => foo.2). - - Have a go at adding inter-cycle dependencies to your ``suite.rc`` file to - make your workflow match the diagram below. - - .. hint:: - - * ``P2`` means every odd cycle. - * ``2/P2`` means every even cycle. - - .. digraph:: example - :align: center - - size = "4.5,7" - - subgraph cluster_1 { - label = 1 - style = dashed - "foo.1" [label="foo\n1"] - "bar.1" [label="bar\n1"] - "baz.1" [label="baz\n1"] - "wop.1" [label="wop\n1"] - "pub.1" [label="pub\n1"] - "qux.1" [label="qux\n1"] - } - - subgraph cluster_2 { - label = 2 - style = dashed - "foo.2" [label="foo\n2"] - "bar.2" [label="bar\n2"] - "baz.2" [label="baz\n2"] - "wop.2" [label="wop\n2"] - "pub.2" [label="pub\n2"] - } - - subgraph cluster_3 { - label = 3 - style = dashed - "foo.3" [label="foo\n3"] - "bar.3" [label="bar\n3"] - "baz.3" [label="baz\n3"] - "wop.3" [label="wop\n3"] - "pub.3" [label="pub\n3"] - "qux.3" [label="qux\n3"] - } - - "foo.1" -> "bar.1" -> "wop.1" - "bar.1" -> "baz.1" - "pub.1" -> "bar.1" - "foo.2" -> "bar.2" -> "wop.2" - "bar.2" -> "baz.2" - "pub.2" -> "bar.2" - "foo.3" -> "bar.3" -> "wop.3" - "bar.3" -> "baz.3" - "pub.3" -> "bar.3" - "baz.1" -> "qux.1" -> "foo.2" - "baz.3" -> "qux.3" - "baz.2" -> "foo.3" - "wop.1" -> "pub.2" - "wop.2" -> "pub.3" - - .. spoiler:: Solution warning - - .. code-block:: cylc - - - [scheduling] - cycling mode = integer - initial cycle point = 1 - [[dependencies]] - [[[P1]]] - graph = """ - foo & pub => bar => baz & wop - wop[-P1] => pub # (1) - """ - [[[P2]]] - graph = """ - baz => qux - baz[-P1] => foo # (2) - """ - [[[2/P2]]] - graph = """ - qux[-P1] => foo # (3) - """ diff --git a/sphinx/tutorial/rose/applications.rst b/sphinx/tutorial/rose/applications.rst index f23df168b7..94109aedf1 100644 --- a/sphinx/tutorial/rose/applications.rst +++ b/sphinx/tutorial/rose/applications.rst @@ -9,7 +9,7 @@ Rose Applications .. ifnotslides:: - The Cylc ``suite.rc`` file allows us to define environment variables for + The Cylc ``flow.cylc`` file allows us to define environment variables for use by :term:`tasks ` e.g: .. slide:: Cylc Task Environment @@ -222,9 +222,9 @@ An application can be run using the :ref:`command-rose-app-run` command: rose tutorial test-data file/test-data - #. **Move environment variables defined in the** ``suite.rc`` **file.** + #. **Move environment variables defined in the** ``flow.cylc`` **file.** - In the ``[runtime][forecast][environment]`` section of the ``suite.rc`` + In the ``[runtime][forecast][environment]`` section of the ``flow.cylc`` file in the :ref:`weather-forecasting suite ` we set a few environment variables: @@ -263,7 +263,7 @@ An application can be run using the :ref:`command-rose-app-run` command: To make this application work outside of the weather forecasting suite we will also need to provide the ``DOMAIN`` and ``RESOLUTION`` environment variables defined - in the ``[runtime][root][environment]`` section of the ``suite.rc`` + in the ``[runtime][root][environment]`` section of the ``flow.cylc`` file as well as the ``CYLC_TASK_CYCLE_POINT`` environment variable provided by Cylc when it runs a task. diff --git a/sphinx/tutorial/rose/configurations.rst b/sphinx/tutorial/rose/configurations.rst index 94888a70c3..bfd8f8f100 100644 --- a/sphinx/tutorial/rose/configurations.rst +++ b/sphinx/tutorial/rose/configurations.rst @@ -28,7 +28,7 @@ behaviours such as: :term:`Rose suite configuration` A Rose configuration designed to run :term:`Cylc suites `. For instance it may be used to define Jinja2 variables for use in the - ``suite.rc`` file. + ``flow.cylc`` file. .. ifslides:: @@ -52,7 +52,7 @@ Rose Configuration Format .. ifslides:: - .. rubric:: Like the ``suite.rc`` format: + .. rubric:: Like the ``flow.cylc`` format: * Comments start with a ``#`` character. * Settings are written as ``key=value`` pairs. @@ -67,7 +67,7 @@ Rose Configuration Format .. ifslides:: - .. rubric:: Unlike the ``suite.rc`` format: + .. rubric:: Unlike the ``flow.cylc`` format: * Sections cannot be nested. * Settings should not be indented. diff --git a/sphinx/tutorial/rose/furthertopics/command-keys.rst b/sphinx/tutorial/rose/furthertopics/command-keys.rst index 25137711bd..3991e48b8c 100644 --- a/sphinx/tutorial/rose/furthertopics/command-keys.rst +++ b/sphinx/tutorial/rose/furthertopics/command-keys.rst @@ -20,7 +20,7 @@ Create a new Rose suite configuration called ``command-keys``:: mkdir -p ~/rose-tutorial/command-keys cd ~/rose-tutorial/command-keys -Create a blank :rose:file:`rose-suite.conf` and a ``suite.rc`` file that +Create a blank :rose:file:`rose-suite.conf` and a ``flow.cylc`` file that looks like this: .. code-block:: cylc @@ -57,7 +57,7 @@ This sets up a simple suite that contains the following: Save your changes then run the suite using :ref:`command-rose-suite-run`. -Once it has finished use :ref:`command-rose-suite-log` to view the suite log. +Once it has finished use ``cylc cat-log`` to view the suite log. In the page that appears, click the "out" link for the breadmaker task. In the page you are taken to you should see a line saying "fresh bread". @@ -74,7 +74,7 @@ Open the :rose:file:`rose-app.conf` file and edit to look like this: make_dough=sleep 8; echo 'dough for later' timed_bread=sleep 15; echo 'fresh bread when you want it' -Save your changes and open up your ``suite.rc`` file. Alter the +Save your changes and open up your ``flow.cylc`` file. Alter the ``[[breadmaker]]`` task to look like this: .. code-block:: cylc diff --git a/sphinx/tutorial/rose/furthertopics/macro-development.rst b/sphinx/tutorial/rose/furthertopics/macro-development.rst index 745240f531..2cac189c08 100644 --- a/sphinx/tutorial/rose/furthertopics/macro-development.rst +++ b/sphinx/tutorial/rose/furthertopics/macro-development.rst @@ -115,7 +115,7 @@ Open ``planet.py`` in a text editor and paste in the following code: return self.reports This is the bare bones of a Rose macro - a bit of Python that is a -subclass of :py:class:`rose.macro.MacroBase`. At the moment, it doesn't +subclass of :py:class:`metomi.rose.macro.MacroBase`. At the moment, it doesn't do anything. We need to check the value of the option (``env=WORLD``) in our diff --git a/sphinx/tutorial/rose/furthertopics/polling.rst b/sphinx/tutorial/rose/furthertopics/polling.rst index 2c52274ee4..5e38b2dbd4 100644 --- a/sphinx/tutorial/rose/furthertopics/polling.rst +++ b/sphinx/tutorial/rose/furthertopics/polling.rst @@ -18,7 +18,7 @@ Create a new Rose suite configuration:: mkdir -p ~/rose-tutorial/polling cd ~/rose-tutorial/polling -Create a blank :rose:file:`rose-suite.conf` and a ``suite.rc`` +Create a blank :rose:file:`rose-suite.conf` and a ``flow.cylc`` file that looks like this: .. code-block:: cylc @@ -37,7 +37,7 @@ This is a simple suite which consists of the following: * A ``bob`` task which we will be using to poll with. * A ``read_letter`` task which will run once the polling task is complete. -It will need some runtime. Add the following to your ``suite.rc`` file: +It will need some runtime. Add the following to your ``flow.cylc`` file: .. code-block:: cylc diff --git a/sphinx/tutorial/rose/furthertopics/rose-arch.rst b/sphinx/tutorial/rose/furthertopics/rose-arch.rst index 516c753c2e..1bc60d625e 100644 --- a/sphinx/tutorial/rose/furthertopics/rose-arch.rst +++ b/sphinx/tutorial/rose/furthertopics/rose-arch.rst @@ -18,7 +18,7 @@ Create a new Rose suite configuration:: mkdir -p ~/rose-tutorial/rose-arch-tutorial -Create a blank :rose:file:`rose-suite.conf` and a ``suite.rc`` +Create a blank :rose:file:`rose-suite.conf` and a ``flow.cylc`` file that looks like this: .. code-block:: cylc @@ -155,7 +155,7 @@ Save your changes and run the suite:: rose suite-run -View the suite output using :ref:`command-rose-suite-log` and inspect the +View the suite output using ``cylc cat-log`` and inspect the output of the ``make_files``, ``archive_files_rsync`` and ``archive_files_scp`` tasks. diff --git a/sphinx/tutorial/rose/furthertopics/rose-bunch.rst b/sphinx/tutorial/rose/furthertopics/rose-bunch.rst index adf3e91b25..69a024ca20 100644 --- a/sphinx/tutorial/rose/furthertopics/rose-bunch.rst +++ b/sphinx/tutorial/rose/furthertopics/rose-bunch.rst @@ -49,7 +49,7 @@ Create a new Rose suite configuration:: mkdir -p ~/rose-tutorial/rose-bunch cd ~/rose-tutorial/rose-bunch -Create a blank :rose:file:`rose-suite.conf` and a ``suite.rc`` +Create a blank :rose:file:`rose-suite.conf` and a ``flow.cylc`` file that looks like this: .. code-block:: cylc @@ -99,7 +99,7 @@ In the ``app/lander/`` directory create a ``bin/`` directory:: Using your editor of choice, create a file named ``land`` under the ``bin`` directory and paste in these lines:: - #!/bin/bash + #!/usr/bin/env bash CLASS=$1 PASSENGERS=$2 @@ -134,7 +134,7 @@ directory of the lander app and running:: chmod +x land -Navigate to the top directory of your suite (where the ``suite.rc`` and +Navigate to the top directory of your suite (where the ``flow.cylc`` and :rose:file:`rose-suite.conf` files can be found) and run :ref:`command-rose-suite-run`. @@ -149,7 +149,7 @@ its output (note that you can close the Cylc GUI at this point):: .. note:: You can quickly get to the relevant page by running - :ref:`command-rose-suite-log` from within the :term:`suite directory`. + ``cylc cat-log`` from within the :term:`suite directory`. In the Rose Bush jobs page for your suite you should be presented with a page containing a single row for the ``lander`` task, from which you can diff --git a/sphinx/tutorial/rose/furthertopics/rose-stem-tutorial.rst b/sphinx/tutorial/rose/furthertopics/rose-stem-tutorial.rst index c2a607a5cd..f97afeb35c 100644 --- a/sphinx/tutorial/rose/furthertopics/rose-stem-tutorial.rst +++ b/sphinx/tutorial/rose/furthertopics/rose-stem-tutorial.rst @@ -129,12 +129,12 @@ The ``$SOURCE_SPACESHIP`` environment variable will be set using the Jinja2 variable of the same name which is provided by Rose Stem. -The ``suite.rc`` file ---------------------- +The ``flow.cylc`` file +---------------------- -Next we will look at the ``rose-stem/suite.rc`` file. +Next we will look at the ``rose-stem/flow.cylc`` file. -The ``suite.rc`` file starts off with ``UTC mode = True``, which you +The ``flow.cylc`` file starts off with ``UTC mode = True``, which you should already be :ref:`familiar with `. The next part is a Jinja2 block which links the group names the user can specify with the :term:`graph ` for that group. In this @@ -285,7 +285,7 @@ Further Exercises ----------------- If you wish, you can try extending the suite to include the ``fire_lasers`` -group of tasks which was in the list of groups in the ``suite.rc`` file. +group of tasks which was in the list of groups in the ``flow.cylc`` file. Using the same technique as we've just demonstrated for piloting the spaceship, you should be able to aim and fire the ship's weapons. @@ -306,5 +306,5 @@ option on the ``rose stem`` command line. For example: automatic-options=GRAVITY=newtonian PLANET=jupiter sets the variable ``GRAVITY`` to have the value ``newtonian``, and -``PLANET`` to be ``jupiter``. These can then be used in the ``suite.rc`` +``PLANET`` to be ``jupiter``. These can then be used in the ``flow.cylc`` file as Jinja2 variables. diff --git a/sphinx/tutorial/rose/furthertopics/rose-stem.rst b/sphinx/tutorial/rose/furthertopics/rose-stem.rst index 970a88b0cb..494d34cb95 100644 --- a/sphinx/tutorial/rose/furthertopics/rose-stem.rst +++ b/sphinx/tutorial/rose/furthertopics/rose-stem.rst @@ -239,8 +239,8 @@ of precedence): variable. The only analysis module provided with Rose is -:py:mod:`rose.apps.ana_builtin.grepper`, it provides the following analysis -tasks and options: +:py:mod:`metomi.rose.apps.ana_builtin.grepper`, it provides the following +analysis tasks and options: .. autoclass:: metomi.rose.apps.ana_builtin.grepper.SingleCommandStatus :noindex: @@ -262,7 +262,7 @@ route to understanding how they should be arranged is likely to look at the built-in ``grepper`` module. But the key concepts are as follows. To be recognised as a valid analysis module, the Python file must contain at least one class which inherits and extends -:py:mod:`rose.apps.rose_ana.AnalysisTask`: +:py:mod:`metomi.rose.apps.rose_ana.AnalysisTask`: .. autoclass:: metomi.rose.apps.rose_ana.AnalysisTask diff --git a/sphinx/tutorial/rose/furthertopics/upgrading-macro-development.rst b/sphinx/tutorial/rose/furthertopics/upgrading-macro-development.rst index ed3863c800..72feab8b02 100644 --- a/sphinx/tutorial/rose/furthertopics/upgrading-macro-development.rst +++ b/sphinx/tutorial/rose/furthertopics/upgrading-macro-development.rst @@ -221,7 +221,7 @@ Upgrade macros are bits of Python code that essentially look like this: return config, self.reports They are sub-classes of a particular class, -:py:class:`rose.upgrade.MacroUpgrade`, +:py:class:`metomi.rose.upgrade.MacroUpgrade`, which means that some of the Python functionality is done 'under the hood' to make things easier. diff --git a/sphinx/tutorial/rose/suites.rst b/sphinx/tutorial/rose/suites.rst index 3a6b6e9095..9adb0e1c1a 100644 --- a/sphinx/tutorial/rose/suites.rst +++ b/sphinx/tutorial/rose/suites.rst @@ -33,9 +33,9 @@ A Rose suite configuration is a Cylc :term:`suite directory` containing a :rose:conf:`rose-suite.conf[env]` Environment variables for use by the whole suite. :rose:conf:`rose-suite.conf[jinja2:suite.rc]` - `Jinja2`_ variables for use in the ``suite.rc`` file. + `Jinja2`_ variables for use in the ``flow.cylc`` file. :rose:conf:`rose-suite.conf[empy:suite.rc]` - `EmPy`_ variables for use in the ``suite.rc`` file. + `EmPy`_ variables for use in the ``flow.cylc`` file. :rose:conf:`rose-suite.conf[file:NAME]` Files and resources to be installed in the :term:`run directory` when the suite is run. @@ -52,7 +52,7 @@ A Rose suite configuration is a Cylc :term:`suite directory` containing a In the following example the environment variable ``GREETING`` and the Jinja2 variable ``WORLD`` are both set in the :rose:file:`rose-suite.conf` - file. These variables can then be used in the ``suite.rc`` file: + file. These variables can then be used in the ``flow.cylc`` file: .. code-block:: rose :caption: rose-suite.conf @@ -64,7 +64,7 @@ A Rose suite configuration is a Cylc :term:`suite directory` containing a WORLD=Earth .. code-block:: cylc - :caption: suite.rc + :caption: flow.cylc [scheduling] [[dependencies]] @@ -81,7 +81,7 @@ Suite Directory Vs Run Directory -------------------------------- :term:`suite directory` - The directory in which the suite is written. The ``suite.rc`` and + The directory in which the suite is written. The ``flow.cylc`` and :rose:file:`rose-suite.conf` files live here. :term:`run directory` The directory in which the suite runs. The ``work``, ``share`` and ``log`` @@ -137,7 +137,7 @@ Running Rose Suite Configurations it becomes the :term:`run directory`. #. Any files defined in the :rose:file:`rose-suite.conf` file are installed. #. Jinja2 variables defined in the :rose:file:`rose-suite.conf` file are added - to the top of the ``suite.rc`` file. + to the top of the ``flow.cylc`` file. #. The Cylc suite is validated. #. The Cylc suite is run. #. The Cylc GUI is launched. @@ -239,7 +239,7 @@ Start, Stop, Restart :ref:`Starting Suites` Suites must be run using the :ref:`command-rose-suite-run` command which - in turn calls the ``cylc run`` command. + in turn calls the ``cylc play`` command. :ref:`Stopping Suites` Suites can be stopped using the ``cylc stop `` command, as for regular Cylc suites. @@ -257,7 +257,7 @@ Start, Stop, Restart .. ifslides:: Starting Suites - ``rose suite-run`` which in turn calls ``cylc run`` + ``rose suite-run`` which in turn calls ``cylc play`` Stopping Suites ``cylc stop `` Restarting Suites @@ -315,7 +315,7 @@ See the :ref:`Cheat Sheet` for more information. file does not need to have anything in it but it is required to run :ref:`command-rose-suite-run`. - There are three things defined in the ``suite.rc`` file which it might be + There are three things defined in the ``flow.cylc`` file which it might be useful to be able to configure: ``station`` @@ -377,9 +377,9 @@ See the :ref:`Cheat Sheet` for more information. This will contain the environment and Jinja2 variables we have just defined. - #. **Use Suite Variables In The** ``suite.rc`` **File.** + #. **Use Suite Variables In The** ``flow.cylc`` **File.** - Next we need to make use of these settings in the ``suite.rc`` file. + Next we need to make use of these settings in the ``flow.cylc`` file. We can delete the ``RESOLUTION`` and ``DOMAIN`` settings in the ``[runtime][root][environment]`` section which would otherwise override @@ -470,7 +470,7 @@ Rose Applications In Rose Suite Configurations .. nextslide:: .. code-block:: cylc - :caption: suite.rc + :caption: flow.cylc [runtime] [[hello]] @@ -492,7 +492,7 @@ Rose Applications In Rose Suite Configurations :term:`app ` in the task defined below. .. code-block:: cylc - :caption: suite.rc + :caption: flow.cylc [runtime] [[greetings]] @@ -600,12 +600,12 @@ Rose Applications In Rose Suite Configurations We have moved the map template file (``map-template.html``) into the ``forecast`` application so we can delete the ``MAP_TEMPLATE`` environment variable from the ``[runtime]forecast`` section of the - ``suite.rc`` file. + ``flow.cylc`` file. Copy the remaining environment variables defined in the ``forecast`` - task within the ``suite.rc`` file into the :rose:file:`rose-app.conf` + task within the ``flow.cylc`` file into the :rose:file:`rose-app.conf` file of the ``forecast`` application, replacing any values already - specified if necessary. Remove the lines from the ``suite.rc`` file + specified if necessary. Remove the lines from the ``flow.cylc`` file when you are done. Remember, in Rose configuration files: @@ -631,7 +631,7 @@ Rose Applications In Rose Suite Configurations Finally we need to change the ``forecast`` task to run :ref:`command-rose-task-run`. The ``[runtime]forecast`` section of the - ``suite.rc`` file should now look like this: + ``flow.cylc`` file should now look like this: .. code-block:: cylc diff --git a/sphinx/tutorial/rose/summary.rst b/sphinx/tutorial/rose/summary.rst index 8ec5e8abaf..7632c863a8 100644 --- a/sphinx/tutorial/rose/summary.rst +++ b/sphinx/tutorial/rose/summary.rst @@ -37,18 +37,18 @@ So far we have covered: fontsize = "20" fontcolor = "#5050aa" labelloc = "r" - "suite.rc" [fontsize="18", + "flow.cylc" [fontsize="18", fontname="mono", fontcolor="black"] "rcinfo" [label="Defines the workflow\nin terms of tasks\nand dependencies"] - "suite.rc" -- "rcinfo" + "flow.cylc" -- "rcinfo" subgraph cluster_2 { label = "Rose Suite Configuration" "rose-suite.conf" [fontsize="18", fontname="mono", fontcolor="black"] - "confinfo" [label="Defines Jinja2 variables for\nthe suite.rc and environment\nvariables for use throughout\nthe suite"] + "confinfo" [label="Defines Jinja2 variables for\nthe flow.cylc and environment\nvariables for use throughout\nthe suite"] "rose-suite.conf" -- "confinfo" subgraph cluster_3 { @@ -88,10 +88,10 @@ Suite Commands ``cylc graph`` Draws the suite's :term:`graph`. ``cylc get-config`` - Processes the ``suite.rc`` file and prints it back out. + Processes the ``flow.cylc`` file and prints it back out. ``cylc validate`` - Validates the Cylc ``suite.rc`` file to check for any obvious errors. - ``cylc run`` + Validates the Cylc ``flow.cylc`` file to check for any obvious errors. + ``cylc play`` Runs a suite. ``cylc stop`` Stops a suite, in a way that: @@ -109,7 +109,7 @@ Suite Commands * ``cylc graph`` * ``cylc get-config`` * ``cylc validate`` - * ``cylc run`` + * ``cylc play`` * ``cylc stop`` * ``--kill`` * ``--now --now`` diff --git a/t/docs/01-doctest.t b/t/docs/01-doctest.t deleted file mode 100644 index 50240744b2..0000000000 --- a/t/docs/01-doctest.t +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Run doctests in sphinx extensions. -#------------------------------------------------------------------------------- -THIS_TEST_HOME=$(pwd) -. "$(dirname "$0")/test_header" -#------------------------------------------------------------------------------- -if ! rose check-software --docs 2>'/dev/null'; then - skip_all "Software dependencies for documentation not met." -fi -#------------------------------------------------------------------------------- -FILES=($(find "${THIS_TEST_HOME}/sphinx/ext" -name "*.py")) -tests $(( ${#FILES[@]} * 2 )) -#------------------------------------------------------------------------------- -TERM= # Nasty solution to prevent control chars being printed in python output. -for file in "${FILES[@]}"; do - TEST_KEY="${TEST_KEY_BASE}-$(basename ${file})" - run_pass "${TEST_KEY}" python3 -m doctest "${file}" - file_cmp "${TEST_KEY}-err" "${TEST_KEY}.out" '/dev/null' -done -#------------------------------------------------------------------------------- -exit diff --git a/t/docs/02-tutorial-suites.t b/t/docs/02-tutorial-suites.t index 4b524d3dba..62112c5990 100644 --- a/t/docs/02-tutorial-suites.t +++ b/t/docs/02-tutorial-suites.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -53,7 +53,7 @@ for tutorial in $(ls -1 "${TUTORIALS_PATH}"); do done else # Tutorial has no validate file - run cylc validate. - TESTS+=('cylc validate "'"${tutorial_path}/suite.rc"'"') + TESTS+=('cylc validate "'"${tutorial_path}/flow.cylc"'"') TEST_KEYS+=("${tutorial}-0") TUT_DIRS+=( "$tutorial_path" ) continue diff --git a/t/lib/bash/test_header b/t/lib/bash/test_header index 2ec6db19bb..4e256b03ed 100644 --- a/t/lib/bash/test_header +++ b/t/lib/bash/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,15 +21,12 @@ # test_header # # SYNOPSIS -# . $ROSE_HOME/t/lib/bash/test_header +# . t/lib/bash/test_header # # DESCRIPTION # Provide bash shell functions for writing tests for "rose" commands to # output in Perl's TAP format. Add "set -eu". Create a temporary working # directory $TEST_DIR and change to it. Automatically increment test number. -# If $ROSE_HOME is not specified, set it to point to the "rose" source tree -# containing this script. Add $ROSE_HOME/bin to the front of $PATH. -# # FUNCTIONS # tests N # echo "1..$N". @@ -51,6 +48,7 @@ # Compare contents in $FILE_ACTUAL and $FILE_EXPECT. pass/fail # $TEST_KEY if contents are identical/different. If $FILE_EXPECT is "-" # or not defined, compare $FILE_ACTUAL with STDIN to this function. +# Uses `diff -i` if DIFF_CASE_INSENSITIVE is set. # file_cmp_any TEST_KEY FILE_ACTUAL [FILE_EXPECT] # As file_cmp, but FILE_EXPECT should consist of more than one # contents set to compare against, separated by a line matching @@ -86,7 +84,6 @@ # * TEST_ROSE_WS_PID - PID of the service server # * TEST_ROSE_WS_PORT - Port of the service server on localhost. # * TEST_ROSE_WS_URL - URL of service server. -# E.g. rose_ws_init rose bush # rose_ws_kill # Kill a web service server started by "rose_ws_init" and remove # generated log and status files. @@ -99,6 +96,13 @@ # array index. It can also be a dict {attr_key: attr_value, ...}. In # which case, the expected data item is under a list of dicts, where a # unique dict in the list contains all elements attr_key: attr_value. +# get_reg +# Generates a unique Cylc registration name. +# exports FLOW, FLOW_RUN_DIR +# purge [FLOW] +# Runs `cylc clean` on the provided reg (else uses the $FLOW env var) +# only if none of the subtests failed. Otherwise it passes so that +# the test files are left behind for debugging. # # VARIABLES # TEST_DIR @@ -131,6 +135,7 @@ trap 'test_finally EXIT' 'EXIT' trap 'test_finally INT' 'INT' TEST_NUMBER=0 +FAILURES=0 tests() { echo "1..$1" @@ -156,6 +161,11 @@ pass() { fail() { echo "not ok $((++TEST_NUMBER)) - $*" + ((++FAILURES)) + if [[ -f "${TEST_KEY}.err" ]]; then + # output the last 10 lines of stderr for debug + tail -n 10 "${TEST_KEY}.err" >&2 + fi } run_pass() { @@ -182,7 +192,11 @@ file_cmp() { local TEST_KEY=$1 local FILE_ACTUAL=$2 local FILE_EXPECT=${3:--} - if diff -u "${FILE_EXPECT}" "${FILE_ACTUAL}" >&2; then + local diff_opts=(-u) + if [[ -n ${DIFF_CASE_INSENSITIVE:-} ]]; then + diff_opts+=(-i) + fi + if diff "${diff_opts[@]}" "${FILE_EXPECT}" "${FILE_ACTUAL}" >&2; then pass "$TEST_KEY" return fi @@ -352,14 +366,15 @@ poll() { } port_is_busy() { - local PORT="${1}" - if type -P netcat 1>/dev/null; then - HOSTNAME="${HOSTNAME:-"$(hostname)"}" - HOSTNAME="${HOSTNAME:-'localhost'}" - netcat -z "${HOSTNAME}" "${PORT}" - else - netstat -atun | grep -q "0.0.0.0:${PORT}" - fi + # use Python rather than netcat/netstat which aren't as portable + python3 -c ' +import socket +import sys + +print(sys.argv[1]) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + assert sock.connect_ex(("localhost", int(sys.argv[1]))) == 0 + ' "$1" 2>/dev/null } TEST_ROSE_WS_PID= @@ -440,13 +455,34 @@ __PYTHON__ fi } +get_reg () { + local FLOW_UID + FLOW_UID="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" + FLOW="rtb.$ROSE_TEST_TIME_INIT/${TEST_KEY_BASE}/${FLOW_UID}" + FLOW_RUN_DIR="$HOME/cylc-run/$FLOW" + echo "${FLOW}" + export FLOW FLOW_RUN_DIR +} + +purge () { + local FLOW="${1:-$FLOW}" + if [[ -z "$FLOW" ]]; then + echo 'no flow to purge' >&2 + return 1 + elif ((FAILURES == 0)); then + cylc clean "${FLOW}" + fi +} + +mkdir -p "$HOME/cylc-run" + ROSE_TEST_HOME=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) export ROSE_TEST_HOME DIFFTOOL="$(rose config '--default=diff -u' t difftool)" TEST_KEY_BASE="$(basename "$0" .t)" # shellcheck disable=SC2034 TEST_SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)" -TEST_DIR="$(mktemp -d)" +TEST_DIR="$(realpath "$(mktemp -d)")" cd "$TEST_DIR" set +e diff --git a/t/rosa-db-create/00-basic.t b/t/rosa-db-create/00-basic.t index 1f601c28b0..fe5d49afb1 100755 --- a/t/rosa-db-create/00-basic.t +++ b/t/rosa-db-create/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -45,7 +45,7 @@ prefix-location.foo=$SVN_URL __ROSE_CONF__ export ROSE_CONF_PATH=$PWD/conf cat >repos/foo/hooks/post-commit <<__POST_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$ROSE_CONF_PATH rosa svn-post-commit --debug "\$@" \\ 1>$PWD/rosa-svn-post-commit.out 2>$PWD/rosa-svn-post-commit.err diff --git a/t/rosa-svn-post-commit/00-basic.t b/t/rosa-svn-post-commit/00-basic.t index b929cd9801..a67afb03a8 100755 --- a/t/rosa-svn-post-commit/00-basic.t +++ b/t/rosa-svn-post-commit/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -44,7 +44,7 @@ __ROSE_CONF__ export ROSE_CONF_PATH=$PWD/conf ROSE_BIN_HOME=$(dirname $(command -v rose)) cat >repos/foo/hooks/post-commit <<__POST_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=${ROSE_CONF_PATH} export PATH=$PATH:${ROSE_BIN_HOME} rosa svn-post-commit --debug "\$@" \\ @@ -73,6 +73,8 @@ file_cmp "$TEST_KEY-hook.err" $PWD/rosa-svn-post-commit.err "$TEST_KEY-main.out" file_cmp "$TEST_KEY-main.out" "$TEST_KEY-main.out" <<__OUT__ foo-aa000|trunk|1|ivy|hook|test post commit hook: create|$LOGNAME|A | diff --git a/t/rosa-svn-post-commit/01-mail-passwd.conf b/t/rosa-svn-post-commit/01-mail-passwd.conf index 3dbf160f1a..c09243b499 100644 --- a/t/rosa-svn-post-commit/01-mail-passwd.conf +++ b/t/rosa-svn-post-commit/01-mail-passwd.conf @@ -3,5 +3,5 @@ recips.new=$USER recips.new-access-list=$USER root recips.mod-owner=$USER root recips.mod-access-list=$USER root -recips.mod-access-list-2=$USER bin root -recips.del=$USER bin root +recips.mod-access-list-2=$USER $USER2 root +recips.del=$USER $USER2 root diff --git a/t/rosa-svn-post-commit/01-mail-passwd.t b/t/rosa-svn-post-commit/01-mail-passwd.t index de2fc409dc..3d350d6131 100755 --- a/t/rosa-svn-post-commit/01-mail-passwd.t +++ b/t/rosa-svn-post-commit/01-mail-passwd.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -30,6 +30,17 @@ if [[ -z ${TEST_SMTPD_HOST:-} ]]; then skip_all "cannot start mock SMTP server" fi +# pick a second user account for testing +# (must exist else tests fail) +USERS=(bin nobody) +for user in "${USERS[@]}"; do + if grep "^$user" /etc/passwd; then + USER2="$user" + break + fi +done +export USER2 + TEST_CONF="${TEST_SOURCE_DIR}/${TEST_KEY_BASE}.conf" # Get recipients from configuration get_recips() { @@ -73,7 +84,7 @@ export ROSE_CONF_PATH=$PWD/conf ROSE_BIN_HOME=$(dirname $(command -v rose)) cat >repos/foo/hooks/post-commit <<__POST_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$ROSE_CONF_PATH export PATH=$PATH:${ROSE_BIN_HOME} rosa svn-post-commit --debug "\$@" \\ @@ -182,7 +193,7 @@ file_cmp "$TEST_KEY-rc" "$PWD/rosa-svn-post-commit.rc" <<<'0' TEST_KEY="$TEST_KEY_BASE-mod-access-list-2" rosie checkout -q foo-aa001 cat >$PWD/roses/foo-aa001/rose-suite.info <<__ROSE_SUITE_INFO -access-list=root bin +access-list=root $USER2 owner=$USER project=hook sub-project=post-commit @@ -198,7 +209,7 @@ file_grep "$TEST_KEY-smtpd.log.recips" "^recips: \[$RECIPS\]" "$TEST_SMTPD_LOG" file_grep "$TEST_KEY-smtpd.log.subject" \ "^Data: b'.*Subject: foo-aa001/trunk@6" "$TEST_SMTPD_LOG" file_grep "$TEST_KEY-smtpd.log.text" \ - "^Data: b'.*-access-list=\\*.*+access-list=root bin" "$TEST_SMTPD_LOG" + "^Data: b'.*-access-list=\\*.*+access-list=root $USER2" "$TEST_SMTPD_LOG" file_cmp "$TEST_KEY-rc" "$PWD/rosa-svn-post-commit.rc" <<<'0' #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-del" diff --git a/t/rosa-svn-post-commit/02-mail-owner-passwd.conf b/t/rosa-svn-post-commit/02-mail-owner-passwd.conf index 3c9ab7afeb..4f02b2915d 100644 --- a/t/rosa-svn-post-commit/02-mail-owner-passwd.conf +++ b/t/rosa-svn-post-commit/02-mail-owner-passwd.conf @@ -3,5 +3,5 @@ recips.new= recips.new-access-list=root recips.mod-owner=root recips.mod-access-list=root -recips.mod-access-list-2=bin root -recips.del=bin root +recips.mod-access-list-2=$USER2 root +recips.del=$USER2 root diff --git a/t/rosa-svn-post-commit/03-unicode.t b/t/rosa-svn-post-commit/03-unicode.t index 653be518c6..4a97223e08 100755 --- a/t/rosa-svn-post-commit/03-unicode.t +++ b/t/rosa-svn-post-commit/03-unicode.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -45,7 +45,7 @@ __ROSE_CONF__ export ROSE_CONF_PATH="${PWD}/conf" ROSE_BIN_HOME=$(dirname $(command -v rose)) cat >'repos/foo/hooks/post-commit' <<__POST_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH="${ROSE_CONF_PATH}" export PATH=$PATH:${ROSE_BIN_HOME} rosa svn-post-commit --debug "\$@" \\ diff --git a/t/rosa-svn-pre-commit/00-basic.t b/t/rosa-svn-pre-commit/00-basic.t index e1672cc8ee..3172038ca0 100755 --- a/t/rosa-svn-pre-commit/00-basic.t +++ b/t/rosa-svn-pre-commit/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -34,10 +34,10 @@ mkdir repos svnadmin create repos/foo SVN_URL=file://$PWD/repos/foo ROSE_BIN=$(dirname $(command -v rose)) -ROSE_LIB=$(dirname $(python -c "import metomi.rose; print(metomi.rose.__file__)")) +ROSE_LIB=$(dirname $(python3 -c "import metomi.rose; print(metomi.rose.__file__)")) export ROSE_LIB ROSE_BIN cat >repos/foo/hooks/pre-commit <<__PRE_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$PWD/conf export PATH=$PATH:${ROSE_BIN} export ROSE_LIB=${ROSE_LIB} diff --git a/t/rosa-svn-pre-commit/01-rosie-create.t b/t/rosa-svn-pre-commit/01-rosie-create.t index 38e8822705..8680a6fbcd 100755 --- a/t/rosa-svn-pre-commit/01-rosie-create.t +++ b/t/rosa-svn-pre-commit/01-rosie-create.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -36,10 +36,10 @@ prefix-owner-default.foo=fred prefix-location.foo=$SVN_URL __ROSE_CONF__ ROSE_BIN_HOME=$(dirname $(command -v rose)) -ROSE_LIB=$(dirname $(python -c "import metomi.rose; print(metomi.rose.__file__)")) +ROSE_LIB=$(dirname $(python3 -c "import metomi.rose; print(metomi.rose.__file__)")) export ROSE_CONF_PATH=$PWD/conf cat >repos/foo/hooks/pre-commit <<__PRE_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$ROSE_CONF_PATH export PATH=$PATH:${ROSE_BIN_HOME} export ROSE_LIB="${ROSE_LIB}" diff --git a/t/rosa-svn-pre-commit/02-passwd.t b/t/rosa-svn-pre-commit/02-passwd.t index a55ec7ba23..0423c225bf 100755 --- a/t/rosa-svn-pre-commit/02-passwd.t +++ b/t/rosa-svn-pre-commit/02-passwd.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -35,9 +35,9 @@ mkdir repos svnadmin create repos/foo SVN_URL=file://$PWD/repos/foo ROSE_BIN_HOME=$(dirname $(command -v rose)) -ROSE_LIB=$(dirname $(python -c "import metomi.rose; print(metomi.rose.__file__)")) +ROSE_LIB=$(dirname $(python3 -c "import metomi.rose; print(metomi.rose.__file__)")) cat >repos/foo/hooks/pre-commit <<__PRE_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$PWD/conf export PATH=$PATH:${ROSE_BIN_HOME} export ROSE_LIB="${ROSE_LIB}" diff --git a/t/rosa-svn-pre-commit/03-meta.t b/t/rosa-svn-pre-commit/03-meta.t index 368701f5a5..6c7b68edcc 100755 --- a/t/rosa-svn-pre-commit/03-meta.t +++ b/t/rosa-svn-pre-commit/03-meta.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -41,9 +41,9 @@ mkdir 'repos' svnadmin create 'repos/foo' SVN_URL="file://${PWD}/repos/foo" ROSE_BIN_HOME=$(dirname $(command -v rose)) -ROSE_LIB=$(dirname $(python -c "import metomi.rose; print(metomi.rose.__file__)")) +ROSE_LIB=$(dirname $(python3 -c "import metomi.rose; print(metomi.rose.__file__)")) cat >'repos/foo/hooks/pre-commit' <<__PRE_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=${PWD}/conf export PATH=$PATH:${ROSE_BIN_HOME} export ROSE_LIB="${ROSE_LIB}" diff --git a/t/rosa-svn-pre-commit/04-unicode.t b/t/rosa-svn-pre-commit/04-unicode.t index 30c7b31a67..bb9eb1a2fd 100644 --- a/t/rosa-svn-pre-commit/04-unicode.t +++ b/t/rosa-svn-pre-commit/04-unicode.t @@ -47,16 +47,13 @@ chmod +x repos/foo/hooks/pre-commit export LANG=C TEST_KEY="${TEST_KEY_BASE}-western" -cat > rose-suite.info << '__INFO__' -owner=ivy -project=euro -title=We should not éñçödê config files in latin-1/western -__INFO__ -iconv -f UTF-8 -t LATIN1 rose-suite.info -o rose-suite.info -# ----------------------------------------------------------------------------- run_fail "$TEST_KEY" \ - svn import rose-suite.info -q -m 't' --non-interactive \ - "${SVN_URL}/a/a/0/0/0/trunk/rose-suite.info" + svn import \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}/rose-suite.info" \ + -q \ + -m 't' \ + --non-interactive \ + "${SVN_URL}/a/a/0/0/0/trunk/rose-suite.info" file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" < /dev/null sed -i '/^\[FAIL\]/!d' "$TEST_KEY.err" file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" << '__ERR__' diff --git a/t/rosa-svn-pre-commit/04-unicode/rose-suite.info b/t/rosa-svn-pre-commit/04-unicode/rose-suite.info new file mode 100644 index 0000000000..008d0f3859 --- /dev/null +++ b/t/rosa-svn-pre-commit/04-unicode/rose-suite.info @@ -0,0 +1,3 @@ +owner=ivy +project=euro +title=We should not éñçödê config files in latin-1/western diff --git a/t/rose-ana/00-run-basic.t b/t/rose-ana/00-run-basic.t index dadd37c4b0..7c3148888d 100644 --- a/t/rose-ana/00-run-basic.t +++ b/t/rose-ana/00-run-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,8 +21,7 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header #------------------------------------------------------------------------------- -N_TESTS=81 -tests $N_TESTS +tests 82 #------------------------------------------------------------------------------- # The database test is only valid if the user has set things up to use it, so @@ -34,22 +33,32 @@ kgo-database=.true. __CONF__ # Run the suite. +get_reg export CYLC_CONF_PATH= export ROSE_CONF_PATH=$PWD/conf -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) + +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="${FLOW}" \ + --no-run-name + +TEST_KEY="${TEST_KEY_BASE}-run" run_fail "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc play \ + "${FLOW}" \ + --host=localhost \ + --no-detach \ + --debug + #------------------------------------------------------------------------------- # Test the output # # Note that t1 and t5 are identical except t5 uses threading, so use a loop here for t in t1 t5 ; do - OUTPUT="${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_$t/01/job.out" - file_pcregrep "${TEST_KEY_BASE}-$t-exact_list_fail}" \ + OUTPUT="${FLOW_RUN_DIR}/log/job/1/rose_ana_$t/01/job.out" + file_pcregrep "${TEST_KEY_BASE}-$t-exact_list_fail" \ 'Running task #([0-9]+).*\n.*Exact List Match Fail.*\n.*Task #\1 did not pass' \ "${OUTPUT}" file_pcregrep "${TEST_KEY_BASE}-$t-exact_list_success" \ @@ -133,12 +142,12 @@ for t in t1 t5 ; do done # Now check that the threading option is reflected in the output -OUTPUT="${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t1/01/job.out" +OUTPUT="${FLOW_RUN_DIR}/log/job/1/rose_ana_t1/01/job.out" TEST_KEY=$TEST_KEY_BASE-t1-serial_statement REGEXP="Running in SERIAL mode" file_grep $TEST_KEY "$REGEXP" $OUTPUT -OUTPUT="${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t5/01/job.out" +OUTPUT="${FLOW_RUN_DIR}/log/job/1/rose_ana_t5/01/job.out" TEST_KEY=$TEST_KEY_BASE-t5-threading_statement REGEXP="Running in THREADED mode, with 4 threads" file_grep $TEST_KEY "$REGEXP" $OUTPUT @@ -146,7 +155,7 @@ file_grep $TEST_KEY "$REGEXP" $OUTPUT #------------------------------------------------------------------------------- # Test of ignoring a task # First, test that the basic task ran ok -OUTPUT="${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t2_activated/01/job.out" +OUTPUT="${FLOW_RUN_DIR}/log/job/1/rose_ana_t2_activated/01/job.out" file_pcregrep "${TEST_KEY_BASE}-ignore-basic-1" \ 'Running task #([0-9]+).*\n.*First Test.*\n.*Task #\1 did not pass' \ "${OUTPUT}" @@ -155,16 +164,16 @@ file_pcregrep "${TEST_KEY_BASE}-ignore-basic-2" \ # Then test that ignoring a test means the output is not present file_pcregrep "${TEST_KEY_BASE}-ignore-notpresent" \ 'Running task #([0-9]+).*\n.*Second Test.*\n.*Task #\1 passed' \ - "${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t2_deactivated/01/job.out" + "${FLOW_RUN_DIR}/log/job/1/rose_ana_t2_deactivated/01/job.out" #------------------------------------------------------------------------------- # Test tolerance as an environment variable # First, test that the basic task ran ok file_pcregrep "${TEST_KEY_BASE}-tolerance-env-var-pass" \ 'Running task #([0-9]+).*\n.*First Test.*\n.*Task #\1 passed' \ - "${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t3_within_tolerance/01/job.out" + "${FLOW_RUN_DIR}/log/job/1/rose_ana_t3_within_tolerance/01/job.out" file_pcregrep "${TEST_KEY_BASE}-tolerance-env-var-fail" \ 'Running task #([0-9]+).*\n.*First Test.*\n.*Task #\1 did not pass' \ - "${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t3_outside_tolerance/01/job.out" + "${FLOW_RUN_DIR}/log/job/1/rose_ana_t3_outside_tolerance/01/job.out" #------------------------------------------------------------------------------- # Test of comparison database @@ -172,7 +181,7 @@ file_pcregrep "${TEST_KEY_BASE}-tolerance-env-var-fail" \ # Regexp for any number of digits (re-used a lot below) COMP_NUMBER="[0-9][0-9]*" -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/db_check/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/db_check/01/job.out" # For each of the 3 tasks check that a task entry exists with a status of 0 for TASK_NAME in "rose_ana_t1" "rose_ana_t2_activated" "rose_ana_t2_deactivated" ; do TEST_KEY=$TEST_KEY_BASE-db_check_${TASK_NAME}_success @@ -272,7 +281,7 @@ file_grep $TEST_KEY "$REGEXP" $OUTPUT #------------------------------------------------------------------------------- # Test of a few config options -OUTPUT="${HOME}/cylc-run/${NAME}/log/job/1/rose_ana_t4/01/job.out" +OUTPUT="${FLOW_RUN_DIR}/log/job/1/rose_ana_t4/01/job.out" file_pcregrep "${TEST_KEY_BASE}-report_limit_working" \ 'Running task #([0-9]+).*Check report limit is active.*Some output omitted due to limit.*Task #\1 passed' \ "${OUTPUT}" @@ -281,7 +290,5 @@ file_pcregrep "${TEST_KEY_BASE}-missing_skip_working" \ "${OUTPUT}" #------------------------------------------------------------------------------- -#Clean suite -rose suite-clean -q -y $NAME -#------------------------------------------------------------------------------- +purge exit 0 diff --git a/t/rose-ana/00-run-basic/suite.rc b/t/rose-ana/00-run-basic/flow.cylc similarity index 100% rename from t/rose-ana/00-run-basic/suite.rc rename to t/rose-ana/00-run-basic/flow.cylc diff --git a/t/rose-ana/01-run-basic-v1.t b/t/rose-ana/01-run-basic-v1.t index a46e189035..10c7e13e0e 100644 --- a/t/rose-ana/01-run-basic-v1.t +++ b/t/rose-ana/01-run-basic-v1.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,21 +21,29 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header #------------------------------------------------------------------------------- -N_TESTS=36 +N_TESTS=37 tests $N_TESTS #------------------------------------------------------------------------------- # Run the suite. export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -run_fail "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "$TEST_KEY" \ + cylc play \ + "${FLOW}" \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- # Test the output -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/rose_ana_t1/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/rose_ana_t1/01/job.out" TEST_KEY=$TEST_KEY_BASE-exact_numeric_success file_grep $TEST_KEY "[ OK ].*Semi-major Axis.*all: 0%:" $OUTPUT TEST_KEY=$TEST_KEY_BASE-exact_numeric_fail @@ -63,22 +71,22 @@ file_grep $TEST_KEY "[FAIL].*Inclination/Axial Tilt.*285.74423480[0-9]*% > 5%:.* #------------------------------------------------------------------------------- # Test of ignoring a task # First, test that the basic task ran ok -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/rose_ana_t2_activated/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/rose_ana_t2_activated/01/job.out" TEST_KEY=$TEST_KEY_BASE-ignore-basic-1 file_grep $TEST_KEY "[FAIL].*Species" $OUTPUT TEST_KEY=$TEST_KEY_BASE-ignore-basic-2 file_grep $TEST_KEY "[ OK ].*Class" $OUTPUT # Then test that ignoring a test means the output is not present -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/rose_ana_t2_deactivated/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/rose_ana_t2_deactivated/01/job.out" TEST_KEY=$TEST_KEY_BASE-ignore-notpresent file_grep_fail $TEST_KEY "[FAIL].*Species" $OUTPUT #------------------------------------------------------------------------------- # Test tolerance as an environment variable # First, test that the basic task ran ok -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/rose_ana_t3_within_tolerance/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/rose_ana_t3_within_tolerance/01/job.out" TEST_KEY=$TEST_KEY_BASE-tolerance-env-var-pass file_grep $TEST_KEY "[PASS].*data" $OUTPUT -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/rose_ana_t3_outside_tolerance/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/rose_ana_t3_outside_tolerance/01/job.out" TEST_KEY=$TEST_KEY_BASE-tolerance-env-var-fail file_grep $TEST_KEY "[FAIL].*data" $OUTPUT #------------------------------------------------------------------------------- @@ -88,7 +96,7 @@ file_grep $TEST_KEY "[FAIL].*data" $OUTPUT # Regexp for any number of digits (re-used a lot below) COMP_NUMBER="[0-9][0-9]*" -OUTPUT=$HOME/cylc-run/$NAME/log/job/1/db_check/01/job.out +OUTPUT="$FLOW_RUN_DIR/log/job/1/db_check/01/job.out" # For each of the 3 tasks check that a task entry exists with a status of 0 for TASK_NAME in "rose_ana_t1" "rose_ana_t2_activated" "rose_ana_t2_deactivated" ; do TEST_KEY=$TEST_KEY_BASE-db_check_${TASK_NAME}_success @@ -206,9 +214,6 @@ TASK_METHOD=".*Species.*" TEST_KEY=$TEST_KEY_BASE-db_check_t2_ignore_notpresent REGEXP="$COMP_NUMBER | $TASK_NAME | $COMP_FILES | $TASK_STATUS | $TASK_METHOD" file_grep_fail $TEST_KEY "$REGEXP" $OUTPUT - -#------------------------------------------------------------------------------- -#Clean suite -rose suite-clean -q -y $NAME #------------------------------------------------------------------------------- +purge exit 0 diff --git a/t/rose-ana/01-run-basic-v1/suite.rc b/t/rose-ana/01-run-basic-v1/flow.cylc similarity index 100% rename from t/rose-ana/01-run-basic-v1/suite.rc rename to t/rose-ana/01-run-basic-v1/flow.cylc diff --git a/t/rose-app-run/00-null.t b/t/rose-app-run/00-null.t index 2fbcb5595f..ccd5f95a79 100755 --- a/t/rose-app-run/00-null.t +++ b/t/rose-app-run/00-null.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/01-basic.t b/t/rose-app-run/01-basic.t index 8c2fb4d9cf..6010bb36e6 100755 --- a/t/rose-app-run/01-basic.t +++ b/t/rose-app-run/01-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/02-command.t b/t/rose-app-run/02-command.t index b1b68fbf7d..1e6050666a 100755 --- a/t/rose-app-run/02-command.t +++ b/t/rose-app-run/02-command.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -31,7 +31,7 @@ hello-from-env = hello run by env __CONFIG__ mkdir $PWD/config/bin cat > $PWD/config/bin/hello <<'__SCRIPT__' -#!/bin/bash +#!/usr/bin/env bash echo "Hello ${@:-world}!" __SCRIPT__ chmod +x $PWD/config/bin/hello diff --git a/t/rose-app-run/03-env.t b/t/rose-app-run/03-env.t index 1375159a1e..234af9aadf 100755 --- a/t/rose-app-run/03-env.t +++ b/t/rose-app-run/03-env.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/04-stdin.t b/t/rose-app-run/04-stdin.t index 8819c84c73..9ecdb6a0f4 100755 --- a/t/rose-app-run/04-stdin.t +++ b/t/rose-app-run/04-stdin.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/05-file.t b/t/rose-app-run/05-file.t index 56f744cf7d..f1fa0d51af 100755 --- a/t/rose-app-run/05-file.t +++ b/t/rose-app-run/05-file.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/06-namelist.t b/t/rose-app-run/06-namelist.t index 59d9ab081c..d98c53cc54 100755 --- a/t/rose-app-run/06-namelist.t +++ b/t/rose-app-run/06-namelist.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/07-opt.t b/t/rose-app-run/07-opt.t index 641e0dd5f2..29faaf3028 100755 --- a/t/rose-app-run/07-opt.t +++ b/t/rose-app-run/07-opt.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/08-poll.t b/t/rose-app-run/08-poll.t index 921b76d7bc..14e53abe43 100755 --- a/t/rose-app-run/08-poll.t +++ b/t/rose-app-run/08-poll.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/09-file-incr-0.t b/t/rose-app-run/09-file-incr-0.t index c915409800..c0f4164e65 100755 --- a/t/rose-app-run/09-file-incr-0.t +++ b/t/rose-app-run/09-file-incr-0.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/10-file-incr-1.t b/t/rose-app-run/10-file-incr-1.t index 21164e49cb..74f83df4d8 100755 --- a/t/rose-app-run/10-file-incr-1.t +++ b/t/rose-app-run/10-file-incr-1.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/11-file-incr-2.t b/t/rose-app-run/11-file-incr-2.t index bd040726b3..3c83efff4f 100755 --- a/t/rose-app-run/11-file-incr-2.t +++ b/t/rose-app-run/11-file-incr-2.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/12-file-incr-3.t b/t/rose-app-run/12-file-incr-3.t index bdad378f34..3edf08eda2 100755 --- a/t/rose-app-run/12-file-incr-3.t +++ b/t/rose-app-run/12-file-incr-3.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/13-file-source-tilde.t b/t/rose-app-run/13-file-source-tilde.t index f14f9634d1..b328e5ed74 100755 --- a/t/rose-app-run/13-file-source-tilde.t +++ b/t/rose-app-run/13-file-source-tilde.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/14-file-source-optional.t b/t/rose-app-run/14-file-source-optional.t index 27b9ae8fd1..de8ae2857e 100755 --- a/t/rose-app-run/14-file-source-optional.t +++ b/t/rose-app-run/14-file-source-optional.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/15-file-permission.t b/t/rose-app-run/15-file-permission.t index 87588397d7..4eec25df91 100755 --- a/t/rose-app-run/15-file-permission.t +++ b/t/rose-app-run/15-file-permission.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/16-file-mode-bad.t b/t/rose-app-run/16-file-mode-bad.t index 60d3f960b8..8412e27fde 100755 --- a/t/rose-app-run/16-file-mode-bad.t +++ b/t/rose-app-run/16-file-mode-bad.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -27,8 +27,8 @@ test_init <<'__CONFIG__' [command] default=true -[file:COPYING] -source=$ROSE_HOME/COPYING +[file:beef] +source=wellington # Oops, typos mode=5ym1ink __CONFIG__ @@ -38,7 +38,7 @@ test_setup run_fail "$TEST_KEY" rose app-run --config=../config -q file_cmp "$TEST_KEY.out" "$TEST_KEY.out" "$TEST_DIR/bin/egg.sh" <<'__BASH__' -#!/bin/bash +#!/usr/bin/env bash echo 'Bash it, crack it, bin it.' __BASH__ diff --git a/t/rose-app-run/19-file-unchanged.t b/t/rose-app-run/19-file-unchanged.t index 8aff0a61db..201909a921 100755 --- a/t/rose-app-run/19-file-unchanged.t +++ b/t/rose-app-run/19-file-unchanged.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/20-file-symlink-no-source.t b/t/rose-app-run/20-file-symlink-no-source.t index 835fb39714..f14658aaa0 100755 --- a/t/rose-app-run/20-file-symlink-no-source.t +++ b/t/rose-app-run/20-file-symlink-no-source.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/21-run-config-load-event.t b/t/rose-app-run/21-run-config-load-event.t index 6e4ff6483a..a9790a85b8 100755 --- a/t/rose-app-run/21-run-config-load-event.t +++ b/t/rose-app-run/21-run-config-load-event.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/22-opt-conf-key-missing.t b/t/rose-app-run/22-opt-conf-key-missing.t index d7da502edf..c8b6f839cc 100755 --- a/t/rose-app-run/22-opt-conf-key-missing.t +++ b/t/rose-app-run/22-opt-conf-key-missing.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/23-file-both.t b/t/rose-app-run/23-file-both.t index 742d7c6c99..04aff3999b 100755 --- a/t/rose-app-run/23-file-both.t +++ b/t/rose-app-run/23-file-both.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/24-app-unicode.t b/t/rose-app-run/24-app-unicode.t index f31a33c280..03134d018d 100755 --- a/t/rose-app-run/24-app-unicode.t +++ b/t/rose-app-run/24-app-unicode.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/25-file-symlink-plus.t b/t/rose-app-run/25-file-symlink-plus.t index 76370adc3a..09fb288d76 100755 --- a/t/rose-app-run/25-file-symlink-plus.t +++ b/t/rose-app-run/25-file-symlink-plus.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/26-define-overrides.t b/t/rose-app-run/26-define-overrides.t index 03e7cb3a26..e6b7be424e 100644 --- a/t/rose-app-run/26-define-overrides.t +++ b/t/rose-app-run/26-define-overrides.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-run/test_header b/t/rose-app-run/test_header index 4ed5fab271..a6dc7f1efc 100644 --- a/t/rose-app-run/test_header +++ b/t/rose-app-run/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/00-null.t b/t/rose-app-upgrade/00-null.t index 6a230b3b73..7c0c97db53 100755 --- a/t/rose-app-upgrade/00-null.t +++ b/t/rose-app-upgrade/00-null.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/01-upgrade-basic.t b/t/rose-app-upgrade/01-upgrade-basic.t index b91d1999f3..1968035b1a 100755 --- a/t/rose-app-upgrade/01-upgrade-basic.t +++ b/t/rose-app-upgrade/01-upgrade-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/02-downgrade-basic.t b/t/rose-app-upgrade/02-downgrade-basic.t index 54bb91794e..de1b794c22 100755 --- a/t/rose-app-upgrade/02-downgrade-basic.t +++ b/t/rose-app-upgrade/02-downgrade-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/03-complex.t b/t/rose-app-upgrade/03-complex.t index f8baabd7ab..955593fad8 100755 --- a/t/rose-app-upgrade/03-complex.t +++ b/t/rose-app-upgrade/03-complex.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/04-upgrade-trigger.t b/t/rose-app-upgrade/04-upgrade-trigger.t index 83e28f68f0..b3cab24c24 100755 --- a/t/rose-app-upgrade/04-upgrade-trigger.t +++ b/t/rose-app-upgrade/04-upgrade-trigger.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/05-cwd.t b/t/rose-app-upgrade/05-cwd.t index 4f0d61a13f..aecca5af16 100755 --- a/t/rose-app-upgrade/05-cwd.t +++ b/t/rose-app-upgrade/05-cwd.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/06-broken.t b/t/rose-app-upgrade/06-broken.t index 6f17e58d90..4ce1dd7fad 100755 --- a/t/rose-app-upgrade/06-broken.t +++ b/t/rose-app-upgrade/06-broken.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/07-add-existing.t b/t/rose-app-upgrade/07-add-existing.t index 3d7f58d5a2..eca3ffe5ba 100755 --- a/t/rose-app-upgrade/07-add-existing.t +++ b/t/rose-app-upgrade/07-add-existing.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/08-category-is-package.t b/t/rose-app-upgrade/08-category-is-package.t index 457c8c0f5b..9980cf1468 100755 --- a/t/rose-app-upgrade/08-category-is-package.t +++ b/t/rose-app-upgrade/08-category-is-package.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/09-opt.t b/t/rose-app-upgrade/09-opt.t index f2edae6be2..032ec94e4c 100755 --- a/t/rose-app-upgrade/09-opt.t +++ b/t/rose-app-upgrade/09-opt.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/10-prettify-trigger-bug.t b/t/rose-app-upgrade/10-prettify-trigger-bug.t index 5582146e0a..c530dbec13 100755 --- a/t/rose-app-upgrade/10-prettify-trigger-bug.t +++ b/t/rose-app-upgrade/10-prettify-trigger-bug.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/11-upgrade-basic-tree.t b/t/rose-app-upgrade/11-upgrade-basic-tree.t index 623a91f962..266bd5fdf8 100755 --- a/t/rose-app-upgrade/11-upgrade-basic-tree.t +++ b/t/rose-app-upgrade/11-upgrade-basic-tree.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/12-downgrade-basic-tree.t b/t/rose-app-upgrade/12-downgrade-basic-tree.t index 923caa674c..03431792cd 100755 --- a/t/rose-app-upgrade/12-downgrade-basic-tree.t +++ b/t/rose-app-upgrade/12-downgrade-basic-tree.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/13-bad-upgrade-macro.t b/t/rose-app-upgrade/13-bad-upgrade-macro.t index 502f921c53..4017b4bbf7 100755 --- a/t/rose-app-upgrade/13-bad-upgrade-macro.t +++ b/t/rose-app-upgrade/13-bad-upgrade-macro.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-app-upgrade/lib/versions_cwd.py b/t/rose-app-upgrade/lib/versions_cwd.py index 1d87ff94f7..c59974399d 100644 --- a/t/rose-app-upgrade/lib/versions_cwd.py +++ b/t/rose-app-upgrade/lib/versions_cwd.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import os diff --git a/t/rose-app-upgrade/test_header b/t/rose-app-upgrade/test_header index 23bd11cf5a..bafb1f8185 100644 --- a/t/rose-app-upgrade/test_header +++ b/t/rose-app-upgrade/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-cli-bash-completion/00-static.t b/t/rose-cli-bash-completion/00-static.t deleted file mode 100755 index 14b20e72fc..0000000000 --- a/t/rose-cli-bash-completion/00-static.t +++ /dev/null @@ -1,861 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test the rose CLI bash completion script. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -tests 193 -setup -#------------------------------------------------------------------------------- -# Source the script. -. $ROSE_TEST_HOME/etc/rose-bash-completion || exit 1 -#------------------------------------------------------------------------------- -# List Rose subcommands. -TEST_KEY=$TEST_KEY_BASE-rose-subcommands -COMP_WORDS=( rose "") -COMP_CWORD=1 -COMPREPLY= -run_pass "$TEST_KEY" _rose -compreply_test "$TEST_KEY.reply" "rose help" -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" ok_users -compreply_cmp "$TEST_KEY.reply" < ok_users -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" ok_users -compreply_cmp "$TEST_KEY.reply" < ok_users -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" . -#------------------------------------------------------------------------------- -# Test the rose CLI bash completion script for running suites. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -export ROSE_CONF_PATH= -tests 16 -#------------------------------------------------------------------------------- -# Source the script. -. $ROSE_TEST_HOME/etc/rose-bash-completion || exit 1 -#------------------------------------------------------------------------------- -# Run the suite. -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -#------------------------------------------------------------------------------- -# rose suite-log -n -TEST_KEY=$TEST_KEY_BASE -TEST_KEY=$TEST_KEY_BASE-log-n -COMP_WORDS=( rose suite-log -n "" ) -COMP_CWORD=3 -COMPREPLY= -run_pass "$TEST_KEY" _rose -compreply_grep "$TEST_KEY.reply" '^'"$NAME"'$' -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" . -#------------------------------------------------------------------------------- -# Provides init, setup and teardown functions for rose macro tests. -#------------------------------------------------------------------------------- -. $(dirname $0)/../lib/bash/test_header - -function init() { - mkdir -p $TEST_DIR/config - cat >$TEST_DIR/config/rose-app.conf -} - -function init_macro() { - mkdir -p $TEST_DIR/config/etc/rose-meta/$1/HEAD/lib/python/macros/ - touch $TEST_DIR/config/etc/rose-meta/$1/HEAD/rose-meta.conf - cat >$TEST_DIR/config/etc/rose-meta/$1/HEAD/lib/python/macros/$2 -} - -function init_meta() { - mkdir -p $TEST_DIR/config/meta - cat >$TEST_DIR/config/meta/rose-meta.conf -} - -function init_opt_app() { - mkdir -p $TEST_DIR/config/opt - touch $TEST_DIR/config/opt/rose-app-"$1".conf -} - -function init_opt_suite() { - mkdir -p $TEST_DIR/config/opt - touch $TEST_DIR/config/opt/rose-suite-"$1".conf -} - -function init_rose_stem_meta() { - svnadmin create --fs-type fsfs $TEST_DIR/test_repos 1>/dev/null 2>&1 - REPOS_URL="file://$TEST_DIR/test_repos" - svn mkdir -q $REPOS_URL/trunk -m "make trunk" - svn co -q $REPOS_URL/trunk $TEST_DIR/config - mkdir -p $TEST_DIR/config/rose-stem/meta - touch $TEST_DIR/config/rose-stem/suite.rc - cat >$TEST_DIR/config/rose-stem/meta/rose-meta.conf - svn add $TEST_DIR/config/rose-stem 1>/dev/null 2>&1 -} - -function init_suite() { - mkdir -p $TEST_DIR/config - cat >$TEST_DIR/config/rose-suite.conf -} - -function init_upgrade_macro() { - category=$1 - cat >$TEST_DIR/rose-meta/$category/versions.py -} - -function init_upgrade_meta() { - category=$1 - mkdir -p $TEST_DIR/rose-meta/$category/ - shift - for version; do - mkdir -p $TEST_DIR/rose-meta/$category/$version/ - touch $TEST_DIR/rose-meta/$category/$version/rose-meta.conf - done -} - -function setup() { - mkdir $TEST_DIR/run - cd $TEST_DIR/run -} - -function teardown() { - cd $TEST_DIR - rm -rf $TEST_DIR/run - rm -rf $TEST_DIR/config/meta - rm -rf $TEST_DIR/config/rose-stem - rm -rf $TEST_DIR/config/rose*conf - rm -rf $TEST_DIR/config/.svn - rm -rf $TEST_DIR/test_repos -} - -function compreply_cmp() { - local TEST_KEY=$1 - local ENV_ACTUAL_TEMP_FILE=$(mktemp) - if [[ -n ${COMPREPLY:-} ]]; then - printf "%s\n" "${COMPREPLY[@]}" > $ENV_ACTUAL_TEMP_FILE - fi - if cmp $ENV_ACTUAL_TEMP_FILE -; then - pass $TEST_KEY - return - fi - fail $TEST_KEY -} - -function compreply_grep() { - local TEST_KEY=$1 - local TEST_PATTERN=$2 - local ENV_ACTUAL_TEMP_FILE=$(mktemp) - if [[ -n ${COMPREPLY:-} ]]; then - printf "%s\n" "${COMPREPLY[@]}" > $ENV_ACTUAL_TEMP_FILE - fi - if grep -q "$TEST_PATTERN" "$ENV_ACTUAL_TEMP_FILE"; then - pass $TEST_KEY - return - fi - fail $TEST_KEY -} - -function compreply_test() { - local TEST_KEY=$1 - local TEST_CMD=$2 - if [[ -z ${COMPREPLY:-} ]]; then - fail $TEST_KEY - fi - for reply in "${COMPREPLY[@]}"; do - $TEST_CMD $reply 1>/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - fail $TEST_KEY - return - fi - done - pass $TEST_KEY -} diff --git a/t/rose-config-diff/00-basic.t b/t/rose-config-diff/00-basic.t index f8dc251168..40c19d022a 100755 --- a/t/rose-config-diff/00-basic.t +++ b/t/rose-config-diff/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config-diff/test_header b/t/rose-config-diff/test_header index 7b7c101ba6..21ab8e0f0f 100644 --- a/t/rose-config-diff/test_header +++ b/t/rose-config-diff/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config-dump/00-basic.t b/t/rose-config-dump/00-basic.t index 2f38d062ab..efc093aba3 100755 --- a/t/rose-config-dump/00-basic.t +++ b/t/rose-config-dump/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config-dump/01-indices.t b/t/rose-config-dump/01-indices.t index 91c7cac1ae..d4181ab135 100755 --- a/t/rose-config-dump/01-indices.t +++ b/t/rose-config-dump/01-indices.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config-dump/02-files.t b/t/rose-config-dump/02-files.t index 5953342e4d..3493efb28f 100755 --- a/t/rose-config-dump/02-files.t +++ b/t/rose-config-dump/02-files.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config-dump/03-prettyprint.t b/t/rose-config-dump/03-prettyprint.t index 0764c9c568..af48987ac4 100644 --- a/t/rose-config-dump/03-prettyprint.t +++ b/t/rose-config-dump/03-prettyprint.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config-dump/test_header b/t/rose-config-dump/test_header index fa2e8f0a67..9cff165e94 100644 --- a/t/rose-config-dump/test_header +++ b/t/rose-config-dump/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config/00-basic.t b/t/rose-config/00-basic.t index 5fa7514ab6..4b837fc158 100755 --- a/t/rose-config/00-basic.t +++ b/t/rose-config/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config/01-opt.t b/t/rose-config/01-opt.t index ad68a5f30d..3ab4aea474 100755 --- a/t/rose-config/01-opt.t +++ b/t/rose-config/01-opt.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-config/03-syntax-error.t b/t/rose-config/03-syntax-error.t index 66803e7b5e..595f8742b4 100755 --- a/t/rose-config/03-syntax-error.t +++ b/t/rose-config/03-syntax-error.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -27,7 +27,7 @@ echo ' foo=bar' >rose-bad.conf run_fail "$TEST_KEY" rose config -f rose-bad.conf file_cmp "$TEST_KEY.out" "$TEST_KEY.out" rose-bad.conf run_fail "$TEST_KEY" rose config -f rose-bad.conf file_cmp "$TEST_KEY.out" "$TEST_KEY.out" rose-bad.conf <<'__CONF__' -[flower:daisy((2)] +[flower:daisy((line 2)] ivy=poison __CONF__ run_fail "$TEST_KEY" rose config -f rose-bad.conf file_cmp "$TEST_KEY.out" "$TEST_KEY.out" rose-bad.conf <<'__CONF__' -[flower:daisy(2))] +[flower:daisy(line 2))] ivy=poison __CONF__ run_fail "$TEST_KEY" rose config -f rose-bad.conf file_cmp "$TEST_KEY.out" "$TEST_KEY.out" rose-bad.conf <<'__CONF__' -[flower:daisy(2){white}] +[flower:daisy(line 2){white}] ivy=poison __CONF__ run_fail "$TEST_KEY" rose config -f rose-bad.conf file_cmp "$TEST_KEY.out" "$TEST_KEY.out" rose-bad.conf <<'__CONF__' -[flower:daisy{white(2)}] +[flower:daisy{white(line 2)}] ivy=poison __CONF__ run_fail "$TEST_KEY" rose config -f rose-bad.conf file_cmp "$TEST_KEY.out" "$TEST_KEY.out" . -#------------------------------------------------------------------------------- -# Test "rose date". -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -tests 104 -#------------------------------------------------------------------------------- -# Produce the correct format for the current date/time. -TEST_KEY=$TEST_KEY_BASE-current-format -run_pass "$TEST_KEY" rose date -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" = $T_START )) && (( $T_TEST <= $T_END )); then - pass "$TEST_KEY.out" -else - fail "$TEST_KEY.out" -fi -#------------------------------------------------------------------------------- -# Parse its own current date/time output. -TEST_KEY=$TEST_KEY_BASE-current-parse -run_pass "$TEST_KEY" rose date -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" 3540 )) && (( T_OFFSET - T_START < 3660 )); then - pass "$TEST_KEY.out" -else - fail "$TEST_KEY.out" -fi -#------------------------------------------------------------------------------- -# Produce a print format from the current date/time. -TEST_KEY=$TEST_KEY_BASE-current-print-format -run_pass "$TEST_KEY" rose date --print-format="%D %R" now -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" $TEST_KEY.out.truncated -echo "" >> $TEST_KEY.out.truncated -file_cmp "$TEST_KEY.out.truncated" "$TEST_KEY.out.truncated" <<'__OUT__' -0.01666666 -__OUT__ -#------------------------------------------------------------------------------- -# Test rose date --as-total=h P832DT23H12M45S -TEST_KEY=$TEST_KEY_BASE-as-total-P832DT23H12M45S-h -run_pass "$TEST_KEY" rose date --as-total=h P832DT23H12M45S -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<'__OUT__' -19991.2125 -__OUT__ -#------------------------------------------------------------------------------- -# Test rose date --as-total=FORMAT fails for invalid format -TEST_KEY=$TEST_KEY_BASE-as-total-invalid-format -run_fail "$TEST_KEY" rose date --as-total=y PT1M -#------------------------------------------------------------------------------- -# Test rose date --as-total=FORMAT for negative durations -TEST_KEY=$TEST_KEY_BASE-as-total-negative -run_pass "$TEST_KEY" rose date --as-total=S \\-PT1M1S -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<'__OUT__' --61.0 -__OUT__ -#------------------------------------------------------------------------------- -# Test rose date --as-total=FORMAT for use case 2 -TEST_KEY=$TEST_KEY_BASE-as-total-between-dates -run_pass "$TEST_KEY" rose date 2000-01-01T00:00:00 2000-01-01T01:00:00 --as-total=s -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<'__OUT__' -3600.0 -__OUT__ -#------------------------------------------------------------------------------- -# Test rose date --as-total=FORMAT for use case 2 with an offset -if python3 -c "import argparse; argparse.ArgumentParser.parse_intermixed_args"; then - TEST_KEY=$TEST_KEY_BASE-as-total-between-dates-with-offset - run_pass "$TEST_KEY" rose date 2000-01-01T00:00:00 --offset=PT1H 2000-01-01T01:00:00 --as-total=s -else - run_pass "$TEST_KEY" rose date 2000-01-01T00:00:00 2000-01-01T01:00:00 --as-total=s --offset=PT1H -fi -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<'__OUT__' -0.0 -__OUT__ -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-date/00-reference-time.t b/t/rose-date/00-reference-time.t deleted file mode 100755 index 9f3791b170..0000000000 --- a/t/rose-date/00-reference-time.t +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose date". -# Test logic triggered by `if "-c" in sys.args`. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -tests 15 -#------------------------------------------------------------------------------- -# Test -c option where ROSE_TASK_CYCLE_TIME is set -TEST_KEY=$TEST_KEY_BASE-ROSE_TASK_CYCLE_TIME-set -ROSE_TASK_CYCLE_TIME=20121225T0000Z \ - run_pass "$TEST_KEY" rose date -c -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<'__OUT__' -20121225T0000Z -__OUT__ -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" . -#------------------------------------------------------------------------------- -# Test "rose date calendar swapping". -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -tests 28 -#------------------------------------------------------------------------------- -# Check correct setting of 360 day calendar via environment. -TEST_KEY=$TEST_KEY_BASE-360-env -ROSE_CYCLING_MODE=360day \ - run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130230 -__OUT__ -ROSE_CYCLING_MODE=360day \ - run_pass "$TEST_KEY-fwd" rose date 20130230 --offset=P1D -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -# Check that ROSE_CYCLING_MODE over-rides ISODATETIMECALENDAR -TEST_KEY=$TEST_KEY_BASE-360-env -ROSE_CYCLING_MODE=360day \ - ISODATETIMECALENDAR=366day\ - run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130230 -__OUT__ -ROSE_CYCLING_MODE=360day \ - ISODATETIMECALENDAR=366day\ - run_pass "$TEST_KEY-fwd" rose date 20130230 --offset=P1D -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -# Check that ROSE_CYCLING_MODE over-rides ISODATETIMECALENDAR -TEST_KEY=$TEST_KEY_BASE-360-env -ISODATETIMECALENDAR=360day\ - run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130230 -__OUT__ -ROSE_CYCLING_MODE=360day \ - ISODATETIMECALENDAR=366day\ - run_pass "$TEST_KEY-fwd" rose date 20130230 --offset=P1D -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -# Check correct setting of 360 day calendar via args. -TEST_KEY=$TEST_KEY_BASE-360-switch -run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D --calendar=360day -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130230 -__OUT__ -run_pass "$TEST_KEY-fwd" rose date 20130230 --offset=P1D --calendar=360day -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -# Check correct setting of 365 day calendar via args. -TEST_KEY=$TEST_KEY_BASE-365-switch -run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D --calendar=365day -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130228 -__OUT__ -run_pass "$TEST_KEY-fwd" rose date 20130228 --offset=P1D --calendar=365day -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -# Check correct setting of 366 day calendar via args. -TEST_KEY=$TEST_KEY_BASE-366-switch -run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D --calendar=366day -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130229 -__OUT__ -run_pass "$TEST_KEY-fwd" rose date 20130229 --offset=P1D --calendar=366day -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -# Check args correctly override environment. -TEST_KEY=$TEST_KEY_BASE-gregorian-override -ROSE_CYCLING_MODE=gregorian \ - run_pass "$TEST_KEY-back" rose date 20130301 --offset=-P1D --calendar=360day -file_cmp "$TEST_KEY-back.out" "$TEST_KEY-back.out" <<__OUT__ -20130230 -__OUT__ -ROSE_CYCLING_MODE=gregorian \ - run_pass "$TEST_KEY-fwd" rose date 20130230 --offset=P1D --calendar=360day -file_cmp "$TEST_KEY-fwd.out" "$TEST_KEY-fwd.out" <<__OUT__ -20130301 -__OUT__ -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-date/test_header b/t/rose-date/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-date/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-env-cat/00-null.t b/t/rose-env-cat/00-null.t index d5c3c25fb4..2965ddfc76 100755 --- a/t/rose-env-cat/00-null.t +++ b/t/rose-env-cat/00-null.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-env-cat/01-basic.t b/t/rose-env-cat/01-basic.t index 90ece23d68..8d0539528c 100755 --- a/t/rose-env-cat/01-basic.t +++ b/t/rose-env-cat/01-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-env-cat/02-braced.t b/t/rose-env-cat/02-braced.t index adfaefb00c..0ad0824078 100644 --- a/t/rose-env-cat/02-braced.t +++ b/t/rose-env-cat/02-braced.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-env-cat/test_header b/t/rose-env-cat/test_header index fa2e8f0a67..9cff165e94 100644 --- a/t/rose-env-cat/test_header +++ b/t/rose-env-cat/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-help/00-alias.t b/t/rose-help/00-alias.t index 97ab167df8..3908781484 100755 --- a/t/rose-help/00-alias.t +++ b/t/rose-help/00-alias.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,21 +21,16 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header #------------------------------------------------------------------------------- -tests 12 +tests 4 #------------------------------------------------------------------------------- export PAGER=cat -cat >'rose-aliases.txt' <<'__TXT__' -config-edit edit -suite-hook task-hook -suite-log slv suite-log-view -__TXT__ cat >'rosie-aliases.txt' <<'__TXT__' checkout co create copy __TXT__ #------------------------------------------------------------------------------- -for PREFIX in 'rose' 'rosie'; do +for PREFIX in 'rosie'; do while read; do COMMAND=$(cut -d' ' -f 1 <<<"${REPLY}") "${PREFIX}" help "${COMMAND}" >'help.txt' diff --git a/t/rose-host-select/00-basic.t b/t/rose-host-select/00-basic.t index 07df8a469b..849bbe700c 100755 --- a/t/rose-host-select/00-basic.t +++ b/t/rose-host-select/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -30,7 +30,15 @@ if [[ -n $HOST_GROUPS ]]; then else N_HOST_GROUPS=1 fi -tests $((N_HOST_GROUPS * 2 + 4)) +tests $((N_HOST_GROUPS * 2 + 7)) +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-ssh-fail" +run_fail "${TEST_KEY}" rose 'host-select' 'electric-monkey-eggs' +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" mock-ssh.out.sorted # N.B. Tab between 1 and sleepy? file_cmp "$TEST_KEY.mock-ssh.out" mock-ssh.out.sorted <<'__OUT__' -sleepy1 bash -sleepy2 bash +sleepy1 rose host-select-client +sleepy2 rose host-select-client __OUT__ # Make sure there is no lingering processes run_fail "$TEST_KEY.ps" ps $(cut -f1 mock-ssh.out) diff --git a/t/rose-host-select/01-timeout/bin/mock-ssh b/t/rose-host-select/01-timeout/bin/mock-ssh index 04858b4ad5..c919a20558 100755 --- a/t/rose-host-select/01-timeout/bin/mock-ssh +++ b/t/rose-host-select/01-timeout/bin/mock-ssh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ME=$(basename $0) echo -e "$$\t$*" >>$ME.out sleep 10 # A bit longer, so lingering process can be tested diff --git a/t/rose-host-select/02-localhost.t b/t/rose-host-select/02-localhost.t index 3637b3ff15..0c5f95caee 100755 --- a/t/rose-host-select/02-localhost.t +++ b/t/rose-host-select/02-localhost.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -27,7 +27,7 @@ MORE_HOST=$(rose config --default= 't' 'job-host-with-share') export ROSE_CONF_PATH= LOCAL_HOSTS='localhost 127.0.0.1' -for CMD in 'hostname -s' 'hostname' 'hostname --fqdn' 'hostname -I'; do +for CMD in 'hostname -s' 'hostname' 'hostname -f' 'hostname -I'; do if LOCAL_HOST=$(eval "$CMD"); then if ssh -n -q -oBatchMode=yes "${LOCAL_HOST}" true 1>'/dev/null' 2>&1 then @@ -47,9 +47,9 @@ run_pass "${TEST_KEY_BASE}" rose 'host-select' -v -v ${HOSTS} # 1 bash command sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0-9]*/YYYY-MM-DDTHHMM/g'\ "${TEST_KEY_BASE}.out" > stamp-removed.log -grep -F "[INFO] YYYY-MM-DDTHHMM bash" "stamp-removed.log" >"${TEST_KEY_BASE}.out.1" +grep -F "[INFO] YYYY-MM-DDTHHMM rose" "stamp-removed.log" >"${TEST_KEY_BASE}.out.1" file_cmp "${TEST_KEY_BASE}.out.1" "${TEST_KEY_BASE}.out.1" <<'__OUT__' -[INFO] YYYY-MM-DDTHHMM bash <<'__STDIN__' +[INFO] YYYY-MM-DDTHHMM rose host-select-client <<'__STDIN__' __OUT__ # 0 ssh LOCAL_HOST command @@ -67,7 +67,7 @@ if [[ -n ${MORE_HOST} ]]; then "stamp-removed.log" >"${TEST_KEY_BASE}.out.${MORE_HOST}" file_cmp "${TEST_KEY_BASE}.out.${MORE_HOST}" \ "${TEST_KEY_BASE}.out.${MORE_HOST}" <<__OUT__ -[INFO] YYYY-MM-DDTHHMM ssh -oBatchMode=yes -oConnectTimeout=10 ${MORE_HOST} bash <<'__STDIN__' +[INFO] YYYY-MM-DDTHHMM ssh -oBatchMode=yes -oConnectTimeout=10 ${MORE_HOST} rose host-select-client <<'__STDIN__' __OUT__ fi diff --git a/t/rose-macro/00-null.t b/t/rose-macro/00-null.t index 125673d14a..5c28f7f692 100755 --- a/t/rose-macro/00-null.t +++ b/t/rose-macro/00-null.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -31,7 +31,7 @@ setup run_fail "$TEST_KEY" rose macro file_cmp "$TEST_KEY.out" "$TEST_KEY.out" . -#------------------------------------------------------------------------------- -# Test "rose metadata-graph" in the absence of a rose configuration. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -python3 -c "import pygraphviz" 2>/dev/null || \ - skip_all '"pygraphviz" not installed' - -init . -#------------------------------------------------------------------------------- -# Test "rose metadata-graph" against configuration and configuration metadata. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header_extra -. $(dirname $0)/test_header - -python3 -c "import pygraphviz" 2>/dev/null || \ - skip_all '"pygraphviz" not installed' -#------------------------------------------------------------------------------- -tests 12 -#------------------------------------------------------------------------------- -# Check full graphing of our example configuration & configuration metadata. -TEST_KEY=$TEST_KEY_BASE-ok-full -setup -init < $TEST_SOURCE_DIR/lib/rose-app.conf -init_meta < $TEST_SOURCE_DIR/lib/rose-meta.conf -CONFIG_PATH=$(cd ../config && pwd -P) -run_pass "$TEST_KEY" rose metadata-graph --debug --config=../config -filter_graphviz <"$TEST_KEY.out" >"$TEST_KEY.filtered.out" -sort "$TEST_KEY.filtered.out" >"$TEST_KEY.filtered.out.sorted" -sort >"$TEST_KEY.filtered.out.expected" <<__OUTPUT__ -"env=CONTROL" -> "env=CONTROL=None" [color=green, label=foo -"env=CONTROL" -> "env=CONTROL=bar" [color=red, label=foo -"env=CONTROL" -> "env=CONTROL=baz" [color=red, label=foo -"env=CONTROL" -> "env=CONTROL=foo" [color=green, label=foo -"env=CONTROL" [color=green -"env=CONTROL=None" -> "env=DEPENDENT3" [color=green -"env=CONTROL=None" [color=green, label=None, shape=box, style=filled -"env=CONTROL=bar" -> "env=DEPENDENT1" [color=red -"env=CONTROL=bar" -> "env=DEPENDENT2" [color=red -"env=CONTROL=bar" -> "env=DEPENDENT_MISSING1" [arrowhead=empty, color=red -"env=CONTROL=bar" [color=red, label=bar, shape=box, style=filled -"env=CONTROL=baz" -> "env=DEPENDENT1" [color=red -"env=CONTROL=baz" [color=red, label=baz, shape=box, style=filled -"env=CONTROL=foo" -> "env=DEPENDENT_MISSING1" [color=green -"env=CONTROL=foo" [color=green, label=foo, shape=box, style=filled -"env=CONTROL_NAMELIST_QUX" -> "env=CONTROL_NAMELIST_QUX=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_QUX" [color=green -"env=CONTROL_NAMELIST_QUX=bar" -> "namelist:qux" [color=red -"env=CONTROL_NAMELIST_QUX=bar" [color=red, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE" -> "env=CONTROL_NAMELIST_WIBBLE=bar" [color=green, label=bar -"env=CONTROL_NAMELIST_WIBBLE" [color=green -"env=CONTROL_NAMELIST_WIBBLE=bar" -> "namelist:wibble" [color=green -"env=CONTROL_NAMELIST_WIBBLE=bar" [color=green, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" -> "env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" [color=green -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" -> "namelist:wibble=wubble" [color=red -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=red, label=bar, shape=box, style=filled -"env=DEPENDENT1" [color=red, label="!!env=DEPENDENT1" -"env=DEPENDENT2" [color=red, label="!!env=DEPENDENT2" -"env=DEPENDENT3" [color=green -"env=DEPENDENT_DEPENDENT1" [color=red, label="!!env=DEPENDENT_DEPENDENT1" -"env=DEPENDENT_MISSING1" -> "env=DEPENDENT_MISSING1=None" [color=grey -"env=DEPENDENT_MISSING1" [color=grey -"env=DEPENDENT_MISSING1=None" -> "env=DEPENDENT_DEPENDENT1" [color=grey -"env=DEPENDENT_MISSING1=None" [color=grey, label=None, shape=box, style=filled -"env=MISSING_VARIABLE" [color=grey -"env=USER_IGNORED" [color=orange, label="!env=USER_IGNORED" -"namelist:qux" [color=red, label="!!namelist:qux", shape=octagon -"namelist:qux=wobble" -> "namelist:qux=wobble=.true." [color=red, label=".false." -"namelist:qux=wobble" [color=red, label="^namelist:qux=wobble" -"namelist:qux=wobble=.true." -> "namelist:qux=wubble" [color=red -"namelist:qux=wobble=.true." [color=red, label=".true.", shape=box, style=filled -"namelist:qux=wubble" [color=red, label="^!!namelist:qux=wubble" -"namelist:wibble" [color=green, shape=octagon -"namelist:wibble=wobble" -> "namelist:wibble=wobble=.true." [color=green, label=".true." -"namelist:wibble=wobble" [color=green -"namelist:wibble=wobble=.true." -> "namelist:wibble=wubble" [color=green -"namelist:wibble=wobble=.true." [color=green, label=".true.", shape=box, style=filled -"namelist:wibble=wubble" [color=red, label="!!namelist:wibble=wubble" -env [color=green, shape=octagon -graph [label="$CONFIG_PATH", rankdir=LR -node [label="\N" -__OUTPUT__ -file_cmp "$TEST_KEY.out" \ - "$TEST_KEY.filtered.out.sorted" "$TEST_KEY.filtered.out.expected" -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" "$TEST_KEY.filtered.out" -sort "$TEST_KEY.filtered.out" >"$TEST_KEY.filtered.out.sorted" -sort >"$TEST_KEY.filtered.out.expected" <<__OUTPUT__ -"env=CONTROL_NAMELIST_QUX" -> "env=CONTROL_NAMELIST_QUX=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_QUX" [color=green, shape=rectangle -"env=CONTROL_NAMELIST_QUX=bar" -> "namelist:qux" [color=red -"env=CONTROL_NAMELIST_QUX=bar" [color=red, label=bar, shape=box, style=filled -"namelist:qux" [color=red, label="!!namelist:qux", shape=octagon -"namelist:qux=wobble" -> "namelist:qux=wobble=.true." [color=red, label=".false." -"namelist:qux=wobble" [color=red, label="^namelist:qux=wobble" -"namelist:qux=wobble=.true." -> "namelist:qux=wubble" [color=red -"namelist:qux=wobble=.true." [color=red, label=".true.", shape=box, style=filled -"namelist:qux=wubble" [color=red, label="^!!namelist:qux=wubble" -graph [label="$CONFIG_PATH: namelist:qux", rankdir=LR -node [label="\N" -__OUTPUT__ -file_cmp "$TEST_KEY.out" \ - "$TEST_KEY.filtered.out.sorted" "$TEST_KEY.filtered.out.expected" -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" "$TEST_KEY.filtered.out" -sort "$TEST_KEY.filtered.out" >"$TEST_KEY.filtered.out.sorted" -sort >"$TEST_KEY.filtered.out.expected" <<__OUTPUT__ -"env=CONTROL" -> "env=CONTROL=None" [color=green, label=foo -"env=CONTROL" -> "env=CONTROL=bar" [color=red, label=foo -"env=CONTROL" -> "env=CONTROL=baz" [color=red, label=foo -"env=CONTROL" -> "env=CONTROL=foo" [color=green, label=foo -"env=CONTROL" [color=green -"env=CONTROL=None" -> "env=DEPENDENT3" [color=green -"env=CONTROL=None" [color=green, label=None, shape=box, style=filled -"env=CONTROL=bar" -> "env=DEPENDENT1" [color=red -"env=CONTROL=bar" -> "env=DEPENDENT2" [color=red -"env=CONTROL=bar" -> "env=DEPENDENT_MISSING1" [arrowhead=empty, color=red -"env=CONTROL=bar" [color=red, label=bar, shape=box, style=filled -"env=CONTROL=baz" -> "env=DEPENDENT1" [color=red -"env=CONTROL=baz" [color=red, label=baz, shape=box, style=filled -"env=CONTROL=foo" -> "env=DEPENDENT_MISSING1" [color=green -"env=CONTROL=foo" [color=green, label=foo, shape=box, style=filled -"env=CONTROL_NAMELIST_QUX" -> "env=CONTROL_NAMELIST_QUX=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_QUX" [color=green -"env=CONTROL_NAMELIST_QUX=bar" -> "namelist:qux" [color=red -"env=CONTROL_NAMELIST_QUX=bar" [color=red, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE" -> "env=CONTROL_NAMELIST_WIBBLE=bar" [color=green, label=bar -"env=CONTROL_NAMELIST_WIBBLE" [color=green -"env=CONTROL_NAMELIST_WIBBLE=bar" -> "namelist:wibble" [color=green -"env=CONTROL_NAMELIST_WIBBLE=bar" [color=green, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" -> "env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" [color=green -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" -> "namelist:wibble=wubble" [color=red -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=red, label=bar, shape=box, style=filled -"env=DEPENDENT1" [color=red, label="!!env=DEPENDENT1" -"env=DEPENDENT2" [color=red, label="!!env=DEPENDENT2" -"env=DEPENDENT3" [color=green -"env=DEPENDENT_DEPENDENT1" [color=red, label="!!env=DEPENDENT_DEPENDENT1" -"env=DEPENDENT_MISSING1" -> "env=DEPENDENT_MISSING1=None" [color=grey -"env=DEPENDENT_MISSING1" [color=grey -"env=DEPENDENT_MISSING1=None" -> "env=DEPENDENT_DEPENDENT1" [color=grey -"env=DEPENDENT_MISSING1=None" [color=grey, label=None, shape=box, style=filled -"env=MISSING_VARIABLE" [color=grey -"env=USER_IGNORED" [color=orange, label="!env=USER_IGNORED" -"namelist:qux" [color=red, label="!!namelist:qux", shape=octagon -"namelist:qux=wobble" -> "namelist:qux=wobble=.true." [color=red, label=".false." -"namelist:qux=wobble" [color=red, label="^namelist:qux=wobble" -"namelist:qux=wobble=.true." -> "namelist:qux=wubble" [color=red -"namelist:qux=wobble=.true." [color=red, label=".true.", shape=box, style=filled -"namelist:qux=wubble" [color=red, label="^!!namelist:qux=wubble" -"namelist:wibble" [color=green, shape=octagon -"namelist:wibble=wobble" -> "namelist:wibble=wobble=.true." [color=green, label=".true." -"namelist:wibble=wobble" [color=green -"namelist:wibble=wobble=.true." -> "namelist:wibble=wubble" [color=green -"namelist:wibble=wobble=.true." [color=green, label=".true.", shape=box, style=filled -"namelist:wibble=wubble" [color=red, label="!!namelist:wibble=wubble" -env [color=green, shape=octagon -graph [label="$CONFIG_PATH (trigger)", rankdir=LR -node [label="\N" -__OUTPUT__ -file_cmp "$TEST_KEY.out" \ - "$TEST_KEY.filtered.out.sorted" "$TEST_KEY.filtered.out.expected" -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" "$TEST_KEY.filtered.out" -sort "$TEST_KEY.filtered.out" >"$TEST_KEY.filtered.out.sorted" -sort >"$TEST_KEY.filtered.out.expected" <<__OUTPUT__ -"env=CONTROL" -> "env=CONTROL=None" [color=green, label=foo -"env=CONTROL" -> "env=CONTROL=bar" [color=red, label=foo -"env=CONTROL" -> "env=CONTROL=baz" [color=red, label=foo -"env=CONTROL" -> "env=CONTROL=foo" [color=green, label=foo -"env=CONTROL" [color=green -"env=CONTROL=None" -> "env=DEPENDENT3" [color=green -"env=CONTROL=None" [color=green, label=None, shape=box, style=filled -"env=CONTROL=bar" -> "env=DEPENDENT1" [color=red -"env=CONTROL=bar" -> "env=DEPENDENT2" [color=red -"env=CONTROL=bar" -> "env=DEPENDENT_MISSING1" [arrowhead=empty, color=red -"env=CONTROL=bar" [color=red, label=bar, shape=box, style=filled -"env=CONTROL=baz" -> "env=DEPENDENT1" [color=red -"env=CONTROL=baz" [color=red, label=baz, shape=box, style=filled -"env=CONTROL=foo" -> "env=DEPENDENT_MISSING1" [color=green -"env=CONTROL=foo" [color=green, label=foo, shape=box, style=filled -"env=CONTROL_NAMELIST_QUX" -> "env=CONTROL_NAMELIST_QUX=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_QUX" [color=green -"env=CONTROL_NAMELIST_QUX=bar" -> "namelist:qux" [color=red -"env=CONTROL_NAMELIST_QUX=bar" [color=red, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE" -> "env=CONTROL_NAMELIST_WIBBLE=bar" [color=green, label=bar -"env=CONTROL_NAMELIST_WIBBLE" [color=green -"env=CONTROL_NAMELIST_WIBBLE=bar" -> "namelist:wibble" [color=green -"env=CONTROL_NAMELIST_WIBBLE=bar" [color=green, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" -> "env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=red, label=foo -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" [color=green -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" -> "namelist:wibble=wubble" [color=red -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=red, label=bar, shape=box, style=filled -"env=DEPENDENT1" [color=red, label="!!env=DEPENDENT1" -"env=DEPENDENT2" [color=red, label="!!env=DEPENDENT2" -"env=DEPENDENT3" [color=green -"env=DEPENDENT_DEPENDENT1" [color=red, label="!!env=DEPENDENT_DEPENDENT1" -"env=DEPENDENT_MISSING1" -> "env=DEPENDENT_MISSING1=None" [color=grey -"env=DEPENDENT_MISSING1" [color=grey -"env=DEPENDENT_MISSING1=None" -> "env=DEPENDENT_DEPENDENT1" [color=grey -"env=DEPENDENT_MISSING1=None" [color=grey, label=None, shape=box, style=filled -"env=MISSING_VARIABLE" [color=grey -"env=USER_IGNORED" [color=orange, label="!env=USER_IGNORED" -"namelist:qux" [color=red, label="!!namelist:qux", shape=rectangle -"namelist:wibble" [color=green, shape=rectangle -"namelist:wibble=wubble" [color=red, label="!!namelist:wibble=wubble", shape=rectangle -env [color=green, shape=octagon -graph [label="$CONFIG_PATH: env (trigger)", rankdir=LR -node [label="\N" -__OUTPUT__ -file_cmp "$TEST_KEY.out" \ - "$TEST_KEY.filtered.out.sorted" "$TEST_KEY.filtered.out.expected" -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" . -#------------------------------------------------------------------------------- -# Test "rose metadata-graph" against just configuration metadata. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header_extra -. $(dirname $0)/test_header - -python3 -c "import pygraphviz" 2>/dev/null || \ - skip_all '"pygraphviz" not installed' - -#------------------------------------------------------------------------------- -tests 3 -#------------------------------------------------------------------------------- -# Check full graphing. -TEST_KEY=$TEST_KEY_BASE-ok-full -setup -init < $TEST_SOURCE_DIR/lib/rose-app.conf -init_meta < $TEST_SOURCE_DIR/lib/rose-meta.conf -META_CONFIG_PATH=$(cd ../config/meta && pwd -P) -run_pass "$TEST_KEY" rose metadata-graph --debug --config=../config/meta -filter_graphviz <"$TEST_KEY.out" >"$TEST_KEY.filtered.out" -sort "$TEST_KEY.filtered.out" >"$TEST_KEY.filtered.out.sorted" -sort >"$TEST_KEY.filtered.out.expected" <<__OUTPUT__ -"env=CONTROL" -> "env=CONTROL=None" [color=grey -"env=CONTROL" -> "env=CONTROL=bar" [color=grey -"env=CONTROL" -> "env=CONTROL=baz" [color=grey -"env=CONTROL" -> "env=CONTROL=foo" [color=grey -"env=CONTROL" [ -"env=CONTROL=None" -> "env=DEPENDENT3" [color=grey -"env=CONTROL=None" [color=grey, label=None, shape=box, style=filled -"env=CONTROL=bar" -> "env=DEPENDENT1" [color=grey -"env=CONTROL=bar" -> "env=DEPENDENT2" [color=grey -"env=CONTROL=bar" -> "env=DEPENDENT_MISSING1" [color=grey -"env=CONTROL=bar" [color=grey, label=bar, shape=box, style=filled -"env=CONTROL=baz" -> "env=DEPENDENT1" [color=grey -"env=CONTROL=baz" [color=grey, label=baz, shape=box, style=filled -"env=CONTROL=foo" -> "env=DEPENDENT_MISSING1" [color=grey -"env=CONTROL=foo" [color=grey, label=foo, shape=box, style=filled -"env=CONTROL_NAMELIST_QUX" -> "env=CONTROL_NAMELIST_QUX=bar" [color=grey -"env=CONTROL_NAMELIST_QUX" [ -"env=CONTROL_NAMELIST_QUX=bar" -> "namelist:qux" [color=grey -"env=CONTROL_NAMELIST_QUX=bar" [color=grey, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE" -> "env=CONTROL_NAMELIST_WIBBLE=bar" [color=grey -"env=CONTROL_NAMELIST_WIBBLE" [ -"env=CONTROL_NAMELIST_WIBBLE=bar" -> "namelist:wibble" [color=grey -"env=CONTROL_NAMELIST_WIBBLE=bar" [color=grey, label=bar, shape=box, style=filled -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" -> "env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=grey -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE" [ -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" -> "namelist:wibble=wubble" [color=grey -"env=CONTROL_NAMELIST_WIBBLE_WUBBLE=bar" [color=grey, label=bar, shape=box, style=filled -"env=DEPENDENT1" [ -"env=DEPENDENT2" [ -"env=DEPENDENT3" [ -"env=DEPENDENT_DEPENDENT1" [ -"env=DEPENDENT_MISSING1" -> "env=DEPENDENT_MISSING1=None" [color=grey -"env=DEPENDENT_MISSING1" [ -"env=DEPENDENT_MISSING1=None" -> "env=DEPENDENT_DEPENDENT1" [color=grey -"env=DEPENDENT_MISSING1=None" [color=grey, label=None, shape=box, style=filled -"env=MISSING_VARIABLE" [ -"env=USER_IGNORED" [ -"namelist:qux" [shape=octagon -"namelist:qux=wobble" -> "namelist:qux=wobble=.true." [color=grey -"namelist:qux=wobble" [ -"namelist:qux=wobble=.true." -> "namelist:qux=wubble" [color=grey -"namelist:qux=wobble=.true." [color=grey, label=".true.", shape=box, style=filled -"namelist:qux=wubble" [ -"namelist:wibble" [shape=octagon -"namelist:wibble=wobble" -> "namelist:wibble=wobble=.true." [color=grey -"namelist:wibble=wobble" [ -"namelist:wibble=wobble=.true." -> "namelist:wibble=wubble" [color=grey -"namelist:wibble=wobble=.true." [color=grey, label=".true.", shape=box, style=filled -"namelist:wibble=wubble" [ -env [shape=octagon -graph [label="$META_CONFIG_PATH", rankdir=LR -node [label="\N" -__OUTPUT__ -file_cmp "$TEST_KEY.out" \ - "$TEST_KEY.filtered.out.sorted" "$TEST_KEY.filtered.out.expected" -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" . -#------------------------------------------------------------------------------- -# Provides init, setup and teardown functions for rose macro tests. -#------------------------------------------------------------------------------- -. $(dirname $0)/../lib/bash/test_header - -function init() { - mkdir -p $TEST_DIR/config - cat >$TEST_DIR/config/rose-app.conf -} - -function init_meta() { - mkdir -p $TEST_DIR/config/meta - cat >$TEST_DIR/config/meta/rose-meta.conf -} - -function init_macro() { - mkdir -p $TEST_DIR/config/meta/lib/python/macros/ - cat >$TEST_DIR/config/meta/lib/python/macros/$1 -} - -function setup() { - mkdir $TEST_DIR/run - cd $TEST_DIR/run -} - -function teardown() { - cd $TEST_DIR - rm -rf $TEST_DIR/run - rm -rf $TEST_DIR/config/meta -} diff --git a/t/rose-metadata-graph/test_header_extra b/t/rose-metadata-graph/test_header_extra deleted file mode 100644 index 52820e516d..0000000000 --- a/t/rose-metadata-graph/test_header_extra +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Provides extra functions for rose metadata-graph testing. -#------------------------------------------------------------------------------- - -function filter_graphviz() { - FILTER_TMP_FILE=$(mktemp) - cat >"$FILTER_TMP_FILE" - # Sort and filter out non-essential properties from the output file. - # Get rid of line-broken newer graphviz output (replace ",\n"). - # Sort the file. - python3 << __PYTHON__ -import re -filename = '$FILTER_TMP_FILE' -f = open(filename, 'r') -text = f.read() -f.close() -text = text.replace(",\n", ", ") -text = re.sub("\s+\[", " [", text) -lines = text.splitlines() -f = open(filename, 'w') -for line in lines: - if '[' not in line: - if line.startswith("\t") and line.strip() != "];": - f.write(line.lstrip() + '\n') - continue - props = dict([_.strip().split('=', 1) for _ in - re.split(', ', - line.split('[', 1)[1].replace('];', ''))]) - new_prop_string = '' - for key in ['arrowhead', 'color', 'label', 'rankdir', 'shape', 'style']: - if key in props: - new_prop_string += key + '=' + props[key] + ', ' - new_prop_string = new_prop_string.rstrip().rstrip(',') - new_line = line.lstrip().split('[')[0] + '[' + new_prop_string + '\n' - if new_line.strip() != 'graph [': - f.write(new_line) -__PYTHON__ - LANG=C sort "$FILTER_TMP_FILE" - rm "$FILTER_TMP_FILE" -} diff --git a/t/rose-mpi-launch/00-command.t b/t/rose-mpi-launch/00-command.t index b0e41fb55a..57978ea5ba 100755 --- a/t/rose-mpi-launch/00-command.t +++ b/t/rose-mpi-launch/00-command.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-mpi-launch/01-command-inner.t b/t/rose-mpi-launch/01-command-inner.t index 162891278c..1b3a4582bf 100755 --- a/t/rose-mpi-launch/01-command-inner.t +++ b/t/rose-mpi-launch/01-command-inner.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,10 +21,14 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header #------------------------------------------------------------------------------- +if [[ "$OSTYPE" == darwin* ]]; then + skip_all 'Does not work on MacOS' +fi tests 12 #------------------------------------------------------------------------------- # Basic. TEST_KEY=$TEST_KEY_BASE +ROSE_HOME_BIN="$(rose version --long | sed 's/.*(\(.*\))/\1/')" ROSE_LAUNCHER_ULIMIT_OPTS='-a' \ run_pass "$TEST_KEY" rose mpi-launch echo hello world file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ diff --git a/t/rose-mpi-launch/02-file.t b/t/rose-mpi-launch/02-file.t index 2daebf1f71..fe55c0cdb8 100755 --- a/t/rose-mpi-launch/02-file.t +++ b/t/rose-mpi-launch/02-file.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-mpi-launch/test_header b/t/rose-mpi-launch/test_header index 7382d101f7..4aedabcd75 100644 --- a/t/rose-mpi-launch/test_header +++ b/t/rose-mpi-launch/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-namelist-dump/00-null.t b/t/rose-namelist-dump/00-null.t index 7c34d2cb70..7c0e4625ad 100755 --- a/t/rose-namelist-dump/00-null.t +++ b/t/rose-namelist-dump/00-null.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-namelist-dump/01-empty.t b/t/rose-namelist-dump/01-empty.t index 485065b9b7..f2222c57a7 100755 --- a/t/rose-namelist-dump/01-empty.t +++ b/t/rose-namelist-dump/01-empty.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-namelist-dump/02-type.t b/t/rose-namelist-dump/02-type.t index f155eb49fd..f82b7e036e 100755 --- a/t/rose-namelist-dump/02-type.t +++ b/t/rose-namelist-dump/02-type.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-namelist-dump/test_header b/t/rose-namelist-dump/test_header index 646fb420a7..ba812932c2 100644 --- a/t/rose-namelist-dump/test_header +++ b/t/rose-namelist-dump/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-stem/00-run-basic.t b/t/rose-stem/00-run-basic.t deleted file mode 100755 index a2427349d1..0000000000 --- a/t/rose-stem/00-run-basic.t +++ /dev/null @@ -1,250 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose stem" without site/user configuration -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -if ! fcm --version 1>/dev/null 2>&1; then - skip_all '"FCM" not installed' -fi -#------------------------------------------------------------------------------- -#Create repository to run on -REPO=$PWD/rose-test-battery-stemtest-repo -mkdir -p $REPO -svnadmin create $REPO/foo -URL=file://$REPO/foo -BASEINSTALL=$(mktemp -d --tmpdir=$PWD) -(cd $BASEINSTALL; mkdir -p trunk/rose-stem; svn import -q -m "" $URL) -#Keywords for the foo repository -mkdir -p conf -echo "location{primary}[foo.x]=$URL" >conf/keyword.cfg -export FCM_CONF_PATH=$PWD/conf -cd $TEST_DIR -#------------------------------------------------------------------------------- -#Check out a copy of the repository -WORKINGCOPY=$(mktemp -d --tmpdir=$PWD) -SUITENAME=$(basename $WORKINGCOPY) -fcm checkout -q fcm:foo.x_tr $WORKINGCOPY -#------------------------------------------------------------------------------- -#Copy suite into working copy -cp $TEST_SOURCE_DIR/00-run-basic/suite.rc $WORKINGCOPY/rose-stem -cp $TEST_SOURCE_DIR/00-run-basic/rose-suite.conf $WORKINGCOPY/rose-stem -touch $WORKINGCOPY/rose-stem/rose-suite.conf -#We should now have a valid rose-stem suite. -#------------------------------------------------------------------------------- -N_TESTS=56 -tests $N_TESTS -#------------------------------------------------------------------------------- -#Test for successful execution -TEST_KEY=$TEST_KEY_BASE-basic-check -run_pass "$TEST_KEY" \ - rose stem --group=earl_grey --task=milk,sugar --group=spoon,cup,milk \ - --source=$WORKINGCOPY --source=fcm:foo.x_tr@head \ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-basic-groups-to-run -file_grep $TEST_KEY "RUN_NAMES=\[earl_grey, milk, sugar, spoon, cup, milk\]" \ - $OUTPUT -TEST_KEY=$TEST_KEY_BASE-basic-source -file_grep $TEST_KEY "SOURCE_FOO=$WORKINGCOPY fcm:foo.x_tr@head" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-basic-host-source -file_grep $TEST_KEY "HOST_SOURCE_FOO=$HOSTNAME:$WORKINGCOPY fcm:foo.x_tr@head" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-basic-source-base -file_grep $TEST_KEY "SOURCE_FOO_BASE=$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-basic-host-source-base -file_grep $TEST_KEY "SOURCE_FOO_BASE=$HOSTNAME:$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-basic-source-rev -file_grep $TEST_KEY "SOURCE_FOO_REV=\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-basic-source-mirror -file_grep $TEST_KEY "SOURCE_FOO_MIRROR=fcm:foo.xm/trunk@1\$" $OUTPUT -#------------------------------------------------------------------------------- -# Test using manual project override -TEST_KEY=$TEST_KEY_BASE-project-override -run_pass "$TEST_KEY" \ - rose stem --group=earl_grey --task=milk,sugar --group=spoon,cup,milk \ - --source=bar=$WORKINGCOPY --source=fcm:foo.x_tr@head \ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-basic-groups-to-run -file_grep $TEST_KEY "RUN_NAMES=\[earl_grey, milk, sugar, spoon, cup, milk\]" \ - $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-foo -file_grep $TEST_KEY "SOURCE_FOO=fcm:foo.x_tr@head" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-host-source-foo -file_grep $TEST_KEY "HOST_SOURCE_FOO=fcm:foo.x_tr@head" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-bar -file_grep $TEST_KEY "SOURCE_BAR=$WORKINGCOPY" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-host-source-bar -file_grep $TEST_KEY "HOST_SOURCE_BAR=$HOSTNAME:$WORKINGCOPY" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-base-foo -file_grep $TEST_KEY "SOURCE_FOO_BASE=fcm:foo.x_tr" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-host-source-base-foo -file_grep $TEST_KEY "HOST_SOURCE_FOO_BASE=fcm:foo.x_tr" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-base-bar -file_grep $TEST_KEY "SOURCE_BAR_BASE=$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-base-bar -file_grep $TEST_KEY "HOST_SOURCE_BAR_BASE=$HOSTNAME:$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-rev-foo -file_grep $TEST_KEY "SOURCE_FOO_REV=@1" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-rev-bar -file_grep $TEST_KEY "SOURCE_BAR_REV=\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-project-override-source-mirror -file_grep $TEST_KEY "SOURCE_FOO_MIRROR=fcm:foo.xm/trunk@1\$" $OUTPUT -#------------------------------------------------------------------------------- -# Second test, using suite redirection -TEST_KEY=$TEST_KEY_BASE-suite-redirection -run_pass "$TEST_KEY" \ - rose stem --group=lapsang -C $WORKINGCOPY/rose-stem --source=fcm:foo.x_tr@head\ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-suite-redirection-groups-to-run -file_grep $TEST_KEY "RUN_NAMES=\[lapsang\]" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-suite-redirection-source -file_grep $TEST_KEY "SOURCE_FOO=fcm:foo.x_tr@head\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-suite-redirection-source-base -file_grep $TEST_KEY "SOURCE_FOO_BASE=fcm:foo.x_tr\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-suite-redirection-source-rev -file_grep $TEST_KEY "SOURCE_FOO_REV=@1\$" $OUTPUT -#------------------------------------------------------------------------------- -# Third test, checking subdirectory is working -TEST_KEY=$TEST_KEY_BASE-subdirectory -run_pass "$TEST_KEY" \ - rose stem --group=assam --source=$WORKINGCOPY/rose-stem \ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-subdirectory-groups-to-run -file_grep $TEST_KEY "RUN_NAMES=\[assam\]" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-subdirectory-source -file_grep $TEST_KEY "SOURCE_FOO=$WORKINGCOPY" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-subdirectory-host-source -file_grep $TEST_KEY "HOST_SOURCE_FOO=$HOSTNAME:$WORKINGCOPY" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-subdirectory-source-base -file_grep $TEST_KEY "SOURCE_FOO_BASE=$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-subdirectory-host-source-base -file_grep $TEST_KEY "HOST_SOURCE_FOO_BASE=$HOSTNAME:$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-subdirectory-source-rev -file_grep $TEST_KEY "SOURCE_FOO_REV=\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-subdirectory-source-mirror -file_grep $TEST_KEY "SOURCE_FOO_MIRROR=fcm:foo.xm/trunk@1\$" $OUTPUT -#------------------------------------------------------------------------------- -# Fourth test, checking relative path with -C is working -TEST_KEY=$TEST_KEY_BASE-relative-path -cd $WORKINGCOPY -run_pass "$TEST_KEY" \ - rose stem --group=ceylon -C rose-stem \ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-relative-path-groups-to-run -file_grep $TEST_KEY "RUN_NAMES=\[ceylon\]" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-relative-path-source -file_grep $TEST_KEY "SOURCE_FOO=$WORKINGCOPY" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-relative-path-host-source -file_grep $TEST_KEY "HOST_SOURCE_FOO=$HOSTNAME:$WORKINGCOPY" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-relative-path-source-base -file_grep $TEST_KEY "SOURCE_FOO_BASE=$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-relative-path-host-source-base -file_grep $TEST_KEY "HOST_SOURCE_FOO_BASE=$HOSTNAME:$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-relative-path-source-rev -file_grep $TEST_KEY "SOURCE_FOO_REV=\$" $OUTPUT -#------------------------------------------------------------------------------- -cd $TEST_DIR -#------------------------------------------------------------------------------- -# Test "rose stem" with site/user configuration -export ROSE_CONF_PATH=$TEST_DIR -cat > rose.conf << EOF -[rose-stem] -automatic-options=MILK=true -EOF -#------------------------------------------------------------------------------- -# Fifth test - for successful execution with site/user configuration -TEST_KEY=$TEST_KEY_BASE-check-with-config -run_pass "$TEST_KEY" \ - rose stem --group=earl_grey --task=milk,sugar --group=spoon,cup,milk \ - --source=$WORKINGCOPY --source=fcm:foo.x_tr@head \ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-check-with-config-groups-to-run -file_grep $TEST_KEY "RUN_NAMES=\[earl_grey, milk, sugar, spoon, cup, milk\]" \ - $OUTPUT -TEST_KEY=$TEST_KEY_BASE-check-with-config-source -file_grep $TEST_KEY "SOURCE_FOO=$WORKINGCOPY fcm:foo.x_tr@head" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-check-with-config-host-source -file_grep $TEST_KEY "HOST_SOURCE_FOO=$HOSTNAME:$WORKINGCOPY fcm:foo.x_tr@head" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-check-with-config-source-base -file_grep $TEST_KEY "SOURCE_FOO_BASE=$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-check-with-config-host-source-base -file_grep $TEST_KEY "HOST_SOURCE_FOO_BASE=$HOSTNAME:$WORKINGCOPY\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-check-with-config-source-rev -file_grep $TEST_KEY "SOURCE_FOO_REV=\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-check-with-config-single-auto-option -file_grep $TEST_KEY "MILK=true\$" $OUTPUT -#------------------------------------------------------------------------------- -# Sixth test - multiple automatic-options in the site/user configuration -cat > rose.conf << EOF -[rose-stem] -automatic-options=MILK=true TEA=darjeeling -EOF -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE-check-with-config -run_pass "$TEST_KEY" \ - rose stem --group=assam --source=$WORKINGCOPY/rose-stem \ - --name $SUITENAME -- --no-detach --debug -#Test output -OUTPUT=$HOME/cylc-run/$SUITENAME/log/job/1/my_task_1/01/job.out -TEST_KEY=$TEST_KEY_BASE-multi-auto-config-first -file_grep $TEST_KEY "MILK=true\$" $OUTPUT -TEST_KEY=$TEST_KEY_BASE-multi-auto-config-second -file_grep $TEST_KEY "TEA=darjeeling\$" $OUTPUT -#------------------------------------------------------------------------------- -# Seventh test - check incompatible rose-stem versions -cp $TEST_SOURCE_DIR/00-run-basic/rose-suite2.conf $WORKINGCOPY/rose-stem/rose-suite.conf -TEST_KEY=$TEST_KEY_BASE-incompatible_versions -run_fail "$TEST_KEY" \ - rose stem --group=earl_grey --task=milk,sugar --group=spoon,cup,milk \ - --source=$WORKINGCOPY --source=fcm:foo.x_tr@head \ - --name $SUITENAME -- --no-detach --debug -OUTPUT=$TEST_DIR/${TEST_KEY}.err -TEST_KEY=$TEST_KEY_BASE-incompatible-versions-correct_error -file_grep $TEST_KEY "Running metomi.rose-stem version 1 but suite is at version 0" $OUTPUT -#------------------------------------------------------------------------------- -# Eighth test - test error message when project not in keywords -# Remove the keywords file and re-copy the original rose-suite.conf file -unset FCM_CONF_PATH -cp $TEST_SOURCE_DIR/00-run-basic/rose-suite.conf $WORKINGCOPY/rose-stem -TEST_KEY=$TEST_KEY_BASE-project-not-in-keywords -run_fail "$TEST_KEY" \ - rose stem --group=earl_grey --task=milk,sugar --group=spoon,cup,milk \ - --source=$WORKINGCOPY --source=fcm:foo.x_tr@head \ - --name $SUITENAME -- --no-detach --debug -OUTPUT=$TEST_DIR/${TEST_KEY}.err -TEST_KEY=$TEST_KEY_BASE-project-not-in-keywords-correct_error -file_grep $TEST_KEY "Cannot ascertain project for source tree" $OUTPUT -#------------------------------------------------------------------------------- -#Clean suite -rose suite-clean -q -y $SUITENAME -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-stem/00-run-basic/rose-suite.conf b/t/rose-stem/00-run-basic/rose-suite.conf deleted file mode 100644 index e7818b8ecb..0000000000 --- a/t/rose-stem/00-run-basic/rose-suite.conf +++ /dev/null @@ -1 +0,0 @@ -ROSE_STEM_VERSION=1 diff --git a/t/rose-stem/00-run-basic/rose-suite2.conf b/t/rose-stem/00-run-basic/rose-suite2.conf deleted file mode 100644 index 787d9323aa..0000000000 --- a/t/rose-stem/00-run-basic/rose-suite2.conf +++ /dev/null @@ -1 +0,0 @@ -ROSE_STEM_VERSION=0 diff --git a/t/rose-stem/00-run-basic/suite.rc b/t/rose-stem/00-run-basic/suite.rc deleted file mode 100644 index 3efa350f6b..0000000000 --- a/t/rose-stem/00-run-basic/suite.rc +++ /dev/null @@ -1,38 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - abort if any task fails=True - [[events]] - abort on timeout=True - timeout=PT2M -[scheduling] - [[dependencies]] - graph=my_task_1 - -[runtime] - [[my_task_1]] - script=""" -echo RUN_NAMES={{RUN_NAMES}} -echo SOURCE_FOO={{SOURCE_FOO}} -echo HOST_SOURCE_FOO={{HOST_SOURCE_FOO}} -echo SOURCE_FOO_BASE={{SOURCE_FOO_BASE}} -echo HOST_SOURCE_FOO_BASE={{HOST_SOURCE_FOO_BASE}} -echo SOURCE_FOO_REV={{SOURCE_FOO_REV}} -echo SOURCE_FOO_MIRROR={{SOURCE_FOO_MIRROR}} -{%- if TEA is defined %} -echo TEA={{TEA}} -{%- endif %} -{%- if MILK is defined %} -echo MILK={{MILK}} -{%- endif %} -{%- if SOURCE_BAR is defined%} -echo SOURCE_BAR={{SOURCE_BAR}} -echo HOST_SOURCE_BAR={{HOST_SOURCE_BAR}} -echo SOURCE_BAR_BASE={{SOURCE_BAR_BASE}} -echo HOST_SOURCE_BAR_BASE={{HOST_SOURCE_BAR_BASE}} -echo SOURCE_BAR_REV={{SOURCE_BAR_REV}} -{%- endif %} - -""" - [[[job]]] - execution time limit=PT1M diff --git a/t/rose-stem/test_header b/t/rose-stem/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-stem/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-suite-clean/00-normal b/t/rose-suite-clean/00-normal deleted file mode 120000 index af1978f616..0000000000 --- a/t/rose-suite-clean/00-normal +++ /dev/null @@ -1 +0,0 @@ -../rose-suite-run/02-install \ No newline at end of file diff --git a/t/rose-suite-clean/00-normal.t b/t/rose-suite-clean/00-normal.t deleted file mode 100755 index fc6c2c6800..0000000000 --- a/t/rose-suite-clean/00-normal.t +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", normal mode. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -install_suite() { - set -e - if [[ -n "$JOB_HOST" ]]; then - rose suite-run --new -q \ - -C $TEST_SOURCE_DIR/$TEST_KEY_BASE -i --name=$NAME \ - -S "HOST=\"$JOB_HOST\"" - ssh "$JOB_HOST" "ls -d cylc-run/$NAME 1>/dev/null" - else - rose suite-run --new -q \ - -C $TEST_SOURCE_DIR/$TEST_KEY_BASE -i --name=$NAME - fi - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} -#------------------------------------------------------------------------------- -JOB_HOST=$(rose config --default= 't' 'job-host') -if [[ -n $JOB_HOST ]]; then - JOB_HOST=$(rose host-select -q $JOB_HOST) - tests 15 -else - tests 10 -fi -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -install_suite -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE-ans-empty -run_fail "$TEST_KEY" rose suite-clean $NAME <<<'' -run_pass "$TEST_KEY.locahost.ls" ls -d $HOME/cylc-run/$NAME -if [[ -n "$JOB_HOST" ]]; then - run_pass "$TEST_KEY.job-host.ls" ssh "$JOB_HOST" "ls -d cylc-run/$NAME" -fi -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE-ans-n -run_fail "$TEST_KEY" rose suite-clean $NAME <<<'n' -run_pass "$TEST_KEY.locahost.ls" ls -d $HOME/cylc-run/$NAME -if [[ -n "$JOB_HOST" ]]; then - run_pass "$TEST_KEY.job-host.ls" ssh "$JOB_HOST" "ls -d cylc-run/$NAME" -fi -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE--name-ans-n -run_fail "$TEST_KEY" rose suite-clean --name=$NAME <<<'n' -run_pass "$TEST_KEY.locahost.ls" ls -d $HOME/cylc-run/$NAME -if [[ -n "$JOB_HOST" ]]; then - run_pass "$TEST_KEY.job-host.ls" ssh "$JOB_HOST" "ls -d cylc-run/$NAME" -fi -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE-ans-y -run_pass "$TEST_KEY" rose suite-clean -v -v $NAME <<<'y' -run_fail "$TEST_KEY.locahost.ls" ls -d $HOME/cylc-run/$NAME -if [[ -n "$JOB_HOST" ]]; then - run_fail "$TEST_KEY.job-host.ls" ssh "$JOB_HOST" "ls -d cylc-run/$NAME" -fi -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE--name-ans-y -install_suite -run_pass "$TEST_KEY" rose suite-clean -v -v -n $NAME <<<'y' -run_fail "$TEST_KEY.locahost.ls" ls -d $HOME/cylc-run/$NAME -if [[ -n "$JOB_HOST" ]]; then - run_fail "$TEST_KEY.job-host.ls" ssh "$JOB_HOST" "ls -d cylc-run/$NAME" -fi -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-suite-clean/01-running b/t/rose-suite-clean/01-running deleted file mode 120000 index c6307c3aee..0000000000 --- a/t/rose-suite-clean/01-running +++ /dev/null @@ -1 +0,0 @@ -../rose-suite-run/00-run-basic \ No newline at end of file diff --git a/t/rose-suite-clean/01-running.t b/t/rose-suite-clean/01-running.t deleted file mode 100755 index a7b9794108..0000000000 --- a/t/rose-suite-clean/01-running.t +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", while the suite is running. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -#------------------------------------------------------------------------------- -tests 7 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -TEST_KEY="${TEST_KEY_BASE}" -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -SUITE_RUN_DIR="$(readlink -f "${SUITE_RUN_DIR}")" -NAME="$(basename "${SUITE_RUN_DIR}")" -# Install suite, and prove that directories are created -rose suite-run --debug -q \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - -- --no-detach --debug & -ROSE_SUITE_RUN_PID=$! -poll ! test -e "${SUITE_RUN_DIR}/log/job/2013010100/my_task_1/01/job" -CONTACT="${HOME}/cylc-run/${NAME}/.service/contact" -SUITE_HOST="$(sed -n 's/CYLC_SUITE_HOST=//p' "${CONTACT}")" -SUITE_OWNER="$(sed -n 's/CYLC_SUITE_OWNER=//p' "${CONTACT}")" -SUITE_PORT="$(sed -n 's/CYLC_SUITE_PORT=//p' "${CONTACT}")" -SUITE_PROCESS="$(sed -n 's/CYLC_SUITE_PROCESS=//p' "${CONTACT}")" -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-running" -run_fail "${TEST_KEY}" rose suite-clean -y "${NAME}" -# Note: Error file CYLC_SUITE_* lines contain \t characters -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] Suite "${NAME}" appears to be running: -[FAIL] Contact info from: "${CONTACT}" -[FAIL] CYLC_SUITE_HOST=${SUITE_HOST} -[FAIL] CYLC_SUITE_OWNER=${SUITE_OWNER} -[FAIL] CYLC_SUITE_PORT=${SUITE_PORT} -[FAIL] CYLC_SUITE_PROCESS=${SUITE_PROCESS} -[FAIL] Try "cylc stop '${NAME}'" first? -__ERR__ -if [[ ! -d "${HOME}/cylc-run/${NAME}" ]]; then - exit 1 -fi -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-running-name" -run_fail "${TEST_KEY}" rose suite-clean -y -n "${NAME}" -# Note: Error file CYLC_SUITE_* lines contain \t characters -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] Suite "${NAME}" appears to be running: -[FAIL] Contact info from: "${CONTACT}" -[FAIL] CYLC_SUITE_HOST=${SUITE_HOST} -[FAIL] CYLC_SUITE_OWNER=${SUITE_OWNER} -[FAIL] CYLC_SUITE_PORT=${SUITE_PORT} -[FAIL] CYLC_SUITE_PROCESS=${SUITE_PROCESS} -[FAIL] Try "cylc stop '${NAME}'" first? -__ERR__ -if [[ ! -d "${HOME}/cylc-run/${NAME}" ]]; then - exit 1 -fi -#------------------------------------------------------------------------------- -touch $SUITE_RUN_DIR/flag # let the suite stop -# Wait for the suite to complete -TIMEOUT=$(($(date +%s) + 120)) # wait 2 minutes -while [[ -e "${CONTACT}" ]] && (($(date +%s) < TIMEOUT)) -do - sleep 1 -done -if [[ -e "${CONTACT}" ]]; then - exit 1 -fi -wait "${ROSE_SUITE_RUN_PID}" -TEST_KEY="${TEST_KEY_BASE}-stopped" -run_pass "${TEST_KEY}" rose suite-clean -y "${NAME}" -sed -i '/\/\.cylc\//d' "$TEST_KEY.out" -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle -[INFO] delete: localhost:cylc-run/${NAME}/share -[INFO] delete: localhost:cylc-run/${NAME} -__OUT__ -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", simple --only= modes. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - if [[ -n "$JOB_HOST" ]]; then - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" -i --name="$NAME" \ - -S "HOST=\"$JOB_HOST\"" -- --no-detach --debug - ssh "$JOB_HOST" "ls -d cylc-run/$NAME 1>/dev/null" - else - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" -i --name=$NAME \ - -- --no-detach --debug - fi - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} - -#------------------------------------------------------------------------------- -N_TESTS=6 -tests "$N_TESTS" -#------------------------------------------------------------------------------- -JOB_HOST=$(rose config --default= 't' 'job-host') -if [[ -n $JOB_HOST ]]; then - JOB_HOST=$(rose host-select -q $JOB_HOST) -fi -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-share" -run_suite -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=share -if [[ -n "$JOB_HOST" ]]; then - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/share -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share -__OUT__ -else - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/share -__OUT__ -fi -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-work" -run_suite -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work -if [[ -n "$JOB_HOST" ]]; then - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work -[INFO] delete: $JOB_HOST:cylc-run/$NAME/work -__OUT__ -else - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work -__OUT__ -fi -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-both" -run_suite -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=share --only=work -if [[ -n "$JOB_HOST" ]]; then - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work -[INFO] delete: localhost:cylc-run/${NAME}/share -[INFO] delete: $JOB_HOST:cylc-run/$NAME/work -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share -__OUT__ -else - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work -[INFO] delete: localhost:cylc-run/${NAME}/share -__OUT__ -fi -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/02-only-1/rose-suite.conf b/t/rose-suite-clean/02-only-1/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-clean/02-only-1/suite.rc b/t/rose-suite-clean/02-only-1/suite.rc deleted file mode 100644 index 2fab1e5d5a..0000000000 --- a/t/rose-suite-clean/02-only-1/suite.rc +++ /dev/null @@ -1,23 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on stalled = True -[scheduling] - [[dependencies]] - graph = """ -my_task_1 -{% if HOST is defined %} -my_task_2 -{% endif %} -""" - -[runtime] - [[root]] - script = true - [[my_task_1]] -{% if HOST is defined %} - [[my_task_2]] - [[[remote]]] - host = {{HOST}} -{% endif %} diff --git a/t/rose-suite-clean/03-only-2.t b/t/rose-suite-clean/03-only-2.t deleted file mode 100755 index 29a3edcab7..0000000000 --- a/t/rose-suite-clean/03-only-2.t +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", --only= mode with globs. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - if [[ -n "$JOB_HOST" ]]; then - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -S "HOST=\"$JOB_HOST\"" -- --no-detach --debug - ssh "$JOB_HOST" "ls -d cylc-run/$NAME 1>/dev/null" - else - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -- --no-detach --debug - fi - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} - -#------------------------------------------------------------------------------- -N_TESTS=4 -tests "$N_TESTS" -#------------------------------------------------------------------------------- -JOB_HOST=$(rose config --default= 't' 'job-host') -if [[ -n $JOB_HOST ]]; then - JOB_HOST=$(rose host-select -q $JOB_HOST) -fi -#------------------------------------------------------------------------------- -export CYLC_CONF_PATH= -export ROSE_CONF_PATH= -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -run_suite -TEST_KEY="$TEST_KEY_BASE-share-tens" -run_pass "$TEST_KEY" \ - rose suite-clean -y -n "$NAME" --only=share/cycle/20?00101T0000Z -LANG=C sort "$TEST_KEY.out" >"$TEST_KEY.sorted.out" -if [[ -n "$JOB_HOST" ]]; then - # We do not know the relative sort order of $SUITE_RUN_DIR and $JOB_HOST. - LANG=C sort >"$TEST_KEY.expected.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20200101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20100101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20000101T0000Z -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20200101T0000Z -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20100101T0000Z -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20000101T0000Z -__OUT__ -else - cat >"$TEST_KEY.expected.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20000101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20100101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20200101T0000Z -__OUT__ -fi -file_cmp "$TEST_KEY.sorted.out" "$TEST_KEY.sorted.out" "$TEST_KEY.expected.out" -TEST_KEY="$TEST_KEY_BASE-multiples" -run_pass "$TEST_KEY" \ - rose suite-clean -y -n "$NAME" \ - --only=share/cycle/*/hello-world.out \ - --only=etc/*/greet-earth.out -if [[ -n "$JOB_HOST" ]]; then - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20150101T0000Z/hello-world.out -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20050101T0000Z/hello-world.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20200101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20150101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20100101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20050101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20000101T0000Z/greet-earth.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20150101T0000Z/hello-world.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20050101T0000Z/hello-world.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20200101T0000Z/greet-earth.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20150101T0000Z/greet-earth.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20100101T0000Z/greet-earth.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20050101T0000Z/greet-earth.out -[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20000101T0000Z/greet-earth.out -__OUT__ -else - file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20150101T0000Z/hello-world.out -[INFO] delete: localhost:cylc-run/${NAME}/share/cycle/20050101T0000Z/hello-world.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20200101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20150101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20100101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20050101T0000Z/greet-earth.out -[INFO] delete: localhost:cylc-run/${NAME}/etc/20000101T0000Z/greet-earth.out -__OUT__ -fi -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/03-only-2/rose-suite.conf b/t/rose-suite-clean/03-only-2/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-clean/03-only-2/suite.rc b/t/rose-suite-clean/03-only-2/suite.rc deleted file mode 100644 index 29b105cfbf..0000000000 --- a/t/rose-suite-clean/03-only-2/suite.rc +++ /dev/null @@ -1,39 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on stalled = True -[scheduling] - initial cycle point = 2000 - final cycle point = 2020 - [[dependencies]] - [[[P5Y]]] - graph = """ -my_task_1[-P5Y] => my_task_1 -{% if HOST is defined %} -my_task_2 -{% endif %} -""" - -[runtime] - [[root]] - env-script = eval $(rose task-env) - script = """ -mkdir -p "$ROSE_SUITE_DIR/etc" || true -mkdir -p "$ROSE_SUITE_DIR/etc/$ROSE_TASK_CYCLE_TIME" || true -for ITEM in 'hello-world.out:Hello World' 'greet-earth.out:Greet Earth'; do - FILE=${ITEM%:*} - CONTENT=${ITEM#*:} - tee \ - "$FILE" \ - "$ROSE_DATAC/$FILE" \ - "$ROSE_SUITE_DIR/etc/$ROSE_TASK_CYCLE_TIME/$FILE" \ - <<<"$CONTENT" -done -""" - [[my_task_1]] -{% if HOST is defined %} - [[my_task_2]] - [[[remote]]] - host = {{HOST}} -{% endif %} diff --git a/t/rose-suite-clean/04-only-3 b/t/rose-suite-clean/04-only-3 deleted file mode 120000 index 2b5e2dadf4..0000000000 --- a/t/rose-suite-clean/04-only-3 +++ /dev/null @@ -1 +0,0 @@ -02-only-1 \ No newline at end of file diff --git a/t/rose-suite-clean/04-only-3.t b/t/rose-suite-clean/04-only-3.t deleted file mode 100755 index f04c3a113a..0000000000 --- a/t/rose-suite-clean/04-only-3.t +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", --only= mode and a localhost root-dir{*} setting. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -- --no-detach --debug - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} - -#------------------------------------------------------------------------------- -tests 2 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH=$PWD/conf -export CYLC_CONF_PATH= -export ROOT_DIR_WORK=$PWD/work - -mkdir 'conf' 'work' -cat >'conf/rose.conf' <<'__CONF__' -[rose-suite-run] -root-dir{work}=*=$ROOT_DIR_WORK -__CONF__ - -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -run_suite -TEST_KEY="$TEST_KEY_BASE-work" -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work -{ - LANG=C sort <<__OUT__ -[INFO] delete: localhost:${ROOT_DIR_WORK}/cylc-run/${NAME}/work -[INFO] delete: localhost:cylc-run/${NAME}/work -__OUT__ -} >"${TEST_KEY}.out.expected" -LANG=C sort "${TEST_KEY}.out" >"${TEST_KEY}.out.sorted" -file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out.sorted" "${TEST_KEY}.out.expected" -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/05-only-4 b/t/rose-suite-clean/05-only-4 deleted file mode 120000 index f8e79ae6d5..0000000000 --- a/t/rose-suite-clean/05-only-4 +++ /dev/null @@ -1 +0,0 @@ -03-only-2 \ No newline at end of file diff --git a/t/rose-suite-clean/05-only-4.t b/t/rose-suite-clean/05-only-4.t deleted file mode 100755 index c8227d4b25..0000000000 --- a/t/rose-suite-clean/05-only-4.t +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", --only= glob and a localhost root-dir{*} setting. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -- --no-detach --debug - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} - -#------------------------------------------------------------------------------- -tests 2 -#------------------------------------------------------------------------------- -export CYLC_CONF_PATH= -export ROSE_CONF_PATH=$PWD/conf -export ROOT_DIR_WORK=$PWD/work - -mkdir 'conf' 'work' -cat >'conf/rose.conf' <<'__CONF__' -[rose-suite-run] -root-dir{work}=*=$ROOT_DIR_WORK -__CONF__ - -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -run_suite -TEST_KEY="$TEST_KEY_BASE-work" -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work/20?00101T0000Z -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work/20200101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20100101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20000101T0000Z -__OUT__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/06-only-5 b/t/rose-suite-clean/06-only-5 deleted file mode 120000 index 2b5e2dadf4..0000000000 --- a/t/rose-suite-clean/06-only-5 +++ /dev/null @@ -1 +0,0 @@ -02-only-1 \ No newline at end of file diff --git a/t/rose-suite-clean/06-only-5.t b/t/rose-suite-clean/06-only-5.t deleted file mode 100755 index 2ae187d263..0000000000 --- a/t/rose-suite-clean/06-only-5.t +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", --only= mode and a remote host root-dir{*} setting. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -S "HOST=\"$JOB_HOST\"" -- --no-detach --debug - ssh "$JOB_HOST" "ls -d cylc-run/$NAME 1>/dev/null" - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} -JOB_HOST=$(rose config --default= 't' 'job-host') -JOB_HOST_WORK=$(rose config --default= 't' 'job-host-run-root') -if [[ -z "$JOB_HOST" || -z "$JOB_HOST_WORK" ]]; then - skip_all '"[t]job-host" or "[t]job-host-run-root" not defined' -fi -JOB_HOST=$(rose host-select -q $JOB_HOST) -#------------------------------------------------------------------------------- -tests 2 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH=$PWD/conf - -mkdir 'conf' -cat >'conf/rose.conf' <<__CONF__ -[rose-suite-run] -root-dir{work}=$JOB_HOST=$JOB_HOST_WORK -__CONF__ -JOB_HOST_WORK=$(ssh -oBatchMode=yes "$JOB_HOST" \ - "bash -l -c \"echo \\$JOB_HOST_WORK\"" | tail -1) - -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -run_suite -TEST_KEY="$TEST_KEY_BASE-work" -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work -{ - echo "[INFO] delete: localhost:cylc-run/${NAME}/work" - LANG=C sort -r <<__OUT__ -[INFO] delete: $JOB_HOST:cylc-run/$NAME/work -[INFO] delete: $JOB_HOST:$JOB_HOST_WORK/cylc-run/$NAME/work -__OUT__ -} >"$TEST_KEY.out.expected" -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" "$TEST_KEY.out.expected" -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/07-only-6 b/t/rose-suite-clean/07-only-6 deleted file mode 120000 index f8e79ae6d5..0000000000 --- a/t/rose-suite-clean/07-only-6 +++ /dev/null @@ -1 +0,0 @@ -03-only-2 \ No newline at end of file diff --git a/t/rose-suite-clean/07-only-6.t b/t/rose-suite-clean/07-only-6.t deleted file mode 100755 index 9b392330b0..0000000000 --- a/t/rose-suite-clean/07-only-6.t +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", --only= glob and a remote host root-dir{*} setting. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -S "HOST=\"$JOB_HOST\"" -- --no-detach --debug - ssh "$JOB_HOST" "ls -d cylc-run/$NAME 1>/dev/null" - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} -JOB_HOST=$(rose config --default= 't' 'job-host') -JOB_HOST_WORK=$(rose config --default= 't' 'job-host-run-root') -if [[ -z "$JOB_HOST" || -z "$JOB_HOST_WORK" ]]; then - skip_all '"[t]job-host" or "[t]job-host-run-root" not defined' -fi -JOB_HOST=$(rose host-select -q $JOB_HOST) -#------------------------------------------------------------------------------- -tests 2 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH=$PWD/conf - -mkdir 'conf' -cat >'conf/rose.conf' <<__CONF__ -[rose-suite-run] -root-dir{work}=$JOB_HOST=$JOB_HOST_WORK -__CONF__ - -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -run_suite -TEST_KEY="$TEST_KEY_BASE-work" -run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work/20?00101T0000Z -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work/20200101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20100101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20000101T0000Z -[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20200101T0000Z -[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20100101T0000Z -[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20000101T0000Z -__OUT__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/08-rmdir.t b/t/rose-suite-clean/08-rmdir.t deleted file mode 100755 index adb36363c8..0000000000 --- a/t/rose-suite-clean/08-rmdir.t +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", normal mode. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" - -cp -pr "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" "${NAME}" - -JOB_HOST="$(rose config 't' 'job-host')" -JOB_HOST_RUN_ROOT="$(rose config 't' 'job-host-run-root')" -JOB_HOST_OPT= -if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then - export JOB_HOST="$(rose host-select -q "${JOB_HOST}")" - export JOB_HOST_RUN_ROOT - cat >>"${NAME}/rose-suite.conf" <<__CONF__ -root-dir{share/cycle}=${JOB_HOST}=${JOB_HOST_RUN_ROOT} - =*=\${ROSE_TEST_ROOT_DIR} -root-dir{work}=${JOB_HOST}=${JOB_HOST_RUN_ROOT} - =*=\${ROSE_TEST_ROOT_DIR} - -[jinja2:suite.rc] -JOB_HOST="${JOB_HOST}" -__CONF__ - tests 5 -else - tests 3 -fi - -# Run suite, create lots of directories -export CYLC_CONF_PATH= -export ROSE_CONF_PATH= -export ROSE_TEST_ROOT_DIR="${PWD}/root.d" -set -e -rose suite-run -q -C "${PWD}/${NAME}" --host='localhost' \ - -- --no-detach --debug -# Prove that the directories exist before clean -test -d "${HOME}/cylc-run/${NAME}" -test -d "${PWD}/root.d/cylc-run/${NAME}" -if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then - SSH='ssh -n -oBatchMode=yes' - ${SSH} "${JOB_HOST}" \ - "bash -l -c 'ls -d cylc-run/${NAME} ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME}'" -fi -set +e - -TEST_KEY="${TEST_KEY_BASE}" -run_pass "${TEST_KEY}" rose suite-clean -y -v -v --debug "${NAME}" -run_fail "${TEST_KEY}-test-d-cylc-run" test -d "${HOME}/cylc-run/${NAME}" -run_fail "${TEST_KEY}-test-d-root-x" test -d "${PWD}/root.d/cylc-run/${NAME}" -if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then - run_pass "${TEST_KEY}-test-d-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \ - "bash -l -c '! test -d cylc-run/${NAME}'" - run_pass "${TEST_KEY}-test-d-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \ - "bash -l -c '! test -d ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME}'" -fi -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-suite-clean/08-rmdir/app/t-1/rose-app.conf b/t/rose-suite-clean/08-rmdir/app/t-1/rose-app.conf deleted file mode 100644 index 8e971429ed..0000000000 --- a/t/rose-suite-clean/08-rmdir/app/t-1/rose-app.conf +++ /dev/null @@ -1,2 +0,0 @@ -[command] -default=echo 'Hello World' | tee 'hello.txt' "${ROSE_DATA}/hello.txt" "${ROSE_DATAC}/hello.txt" diff --git a/t/rose-suite-clean/08-rmdir/rose-suite.conf b/t/rose-suite-clean/08-rmdir/rose-suite.conf deleted file mode 100644 index b8d6a621dd..0000000000 --- a/t/rose-suite-clean/08-rmdir/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -root-dir{share/cycle}=*=$ROSE_TEST_ROOT_DIR -root-dir{work}=*=$ROSE_TEST_ROOT_DIR diff --git a/t/rose-suite-clean/08-rmdir/suite.rc b/t/rose-suite-clean/08-rmdir/suite.rc deleted file mode 100644 index 1f4b7ca2fb..0000000000 --- a/t/rose-suite-clean/08-rmdir/suite.rc +++ /dev/null @@ -1,30 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout = PT2M -[scheduling] - initial cycle point = 2020 - final cycle point = 2020 - [[dependencies]] - [[[R1]]] - graph = """ -t-1 -{% if JOB_HOST is defined %} -t-2 -{% endif %} -""" - -[runtime] - [[root]] - script = rose task-run --app-key=t-1 - [[[job]]] - execution time limit = PT1M - [[t-1]] -{% if JOB_HOST is defined %} - [[t-2]] - [[[remote]]] - host = {{JOB_HOST}} -{% endif %} diff --git a/t/rose-suite-clean/09-rmdir-2.t b/t/rose-suite-clean/09-rmdir-2.t deleted file mode 100755 index 51b31ab279..0000000000 --- a/t/rose-suite-clean/09-rmdir-2.t +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", clean up parent directories. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR0="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME0="$(basename "${SUITE_RUN_DIR0}")" -SUITE_RUN_DIR="${SUITE_RUN_DIR0}/foo/bar" -mkdir -p "${SUITE_RUN_DIR}" -NAME="${NAME0}/foo/bar" - -cp -pr "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" 'source' - -JOB_HOST="$(rose config 't' 'job-host')" -JOB_HOST_RUN_ROOT="$(rose config 't' 'job-host-run-root')" -JOB_HOST_OPT= -if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then - export JOB_HOST="$(rose host-select -q "${JOB_HOST}")" - export JOB_HOST_RUN_ROOT - cat >>'source/rose-suite.conf' <<__CONF__ -root-dir{share/cycle}=${JOB_HOST}=${JOB_HOST_RUN_ROOT} - =*=\${ROSE_TEST_ROOT_DIR} -root-dir{work}=${JOB_HOST}=${JOB_HOST_RUN_ROOT} - =*=\${ROSE_TEST_ROOT_DIR} - -[jinja2:suite.rc] -JOB_HOST="${JOB_HOST}" -__CONF__ - tests 9 -else - tests 5 -fi - -# Run suite, create lots of directories -export CYLC_CONF_PATH= -export ROSE_CONF_PATH= -export ROSE_TEST_ROOT_DIR="${PWD}/root.d" -set -e -rose suite-run --name="${NAME}" -q -C "${PWD}/source" \ - --host='localhost' -- --no-detach --debug -# Prove that the directories exist before clean -test -d "${HOME}/cylc-run/${NAME}" -test -d "${PWD}/root.d/cylc-run/${NAME}" -if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then - SSH='ssh -n -oBatchMode=yes' - ${SSH} "${JOB_HOST}" \ - "bash -l -c 'ls -d cylc-run/${NAME} ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME}'" -fi -set +e - -TEST_KEY="${TEST_KEY_BASE}" -run_pass "${TEST_KEY}" rose suite-clean -y -v -v --debug "${NAME}" -run_fail "${TEST_KEY}-test-d-cylc-run" test -d "${HOME}/cylc-run/${NAME}" -run_fail "${TEST_KEY}-test-d-cylc-run-0" test -d "${HOME}/cylc-run/${NAME0}" -run_fail "${TEST_KEY}-test-d-root-x" test -d "${PWD}/root.d/cylc-run/${NAME}" -run_fail "${TEST_KEY}-test-d-root-x-0" test -d "${PWD}/root.d/cylc-run/${NAME0}" -if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then - run_pass "${TEST_KEY}-test-d-cylc-run-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \ - "bash -l -c '! test -d cylc-run/${NAME}'" - run_pass "${TEST_KEY}-test-d-cylc-run0-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \ - "bash -l -c '! test -d cylc-run/${NAME0}'" - run_pass "${TEST_KEY}-test-d-root-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \ - "bash -l -c '! test -d ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME}'" - run_pass "${TEST_KEY}-test-d-root0-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \ - "bash -l -c '! test -d ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME0}'" -fi -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-suite-clean/09-rmdir-2/app/t-1/rose-app.conf b/t/rose-suite-clean/09-rmdir-2/app/t-1/rose-app.conf deleted file mode 100644 index 8e971429ed..0000000000 --- a/t/rose-suite-clean/09-rmdir-2/app/t-1/rose-app.conf +++ /dev/null @@ -1,2 +0,0 @@ -[command] -default=echo 'Hello World' | tee 'hello.txt' "${ROSE_DATA}/hello.txt" "${ROSE_DATAC}/hello.txt" diff --git a/t/rose-suite-clean/09-rmdir-2/rose-suite.conf b/t/rose-suite-clean/09-rmdir-2/rose-suite.conf deleted file mode 100644 index b8d6a621dd..0000000000 --- a/t/rose-suite-clean/09-rmdir-2/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -root-dir{share/cycle}=*=$ROSE_TEST_ROOT_DIR -root-dir{work}=*=$ROSE_TEST_ROOT_DIR diff --git a/t/rose-suite-clean/09-rmdir-2/suite.rc b/t/rose-suite-clean/09-rmdir-2/suite.rc deleted file mode 100644 index 1f4b7ca2fb..0000000000 --- a/t/rose-suite-clean/09-rmdir-2/suite.rc +++ /dev/null @@ -1,30 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout = PT2M -[scheduling] - initial cycle point = 2020 - final cycle point = 2020 - [[dependencies]] - [[[R1]]] - graph = """ -t-1 -{% if JOB_HOST is defined %} -t-2 -{% endif %} -""" - -[runtime] - [[root]] - script = rose task-run --app-key=t-1 - [[[job]]] - execution time limit = PT1M - [[t-1]] -{% if JOB_HOST is defined %} - [[t-2]] - [[[remote]]] - host = {{JOB_HOST}} -{% endif %} diff --git a/t/rose-suite-clean/10-only-7 b/t/rose-suite-clean/10-only-7 deleted file mode 120000 index f8e79ae6d5..0000000000 --- a/t/rose-suite-clean/10-only-7 +++ /dev/null @@ -1 +0,0 @@ -03-only-2 \ No newline at end of file diff --git a/t/rose-suite-clean/10-only-7.t b/t/rose-suite-clean/10-only-7.t deleted file mode 100755 index 56a8798b8e..0000000000 --- a/t/rose-suite-clean/10-only-7.t +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-clean", --only=extglob and a localhost root-dir{*} setting. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -run_suite() { - set -e - rose suite-run --new -q \ - -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" --name="$NAME" \ - -- --no-detach --debug - ls -d $HOME/cylc-run/$NAME 1>/dev/null - set +e -} - -#------------------------------------------------------------------------------- -tests 2 -#------------------------------------------------------------------------------- -export CYLC_CONF_PATH= -export ROSE_CONF_PATH=$PWD/conf -export ROOT_DIR_WORK=$PWD/work - -mkdir 'conf' 'work' -cat >'conf/rose.conf' <<'__CONF__' -[rose-suite-run] -root-dir{work}=*=$ROOT_DIR_WORK -__CONF__ - -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -SUITE_RUN_DIR=$(readlink -f "$SUITE_RUN_DIR") -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -run_suite -TEST_KEY="$TEST_KEY_BASE-work" -run_pass "$TEST_KEY" \ - rose suite-clean -y -n "${NAME}" --only='work/!(20100101T0000Z)' -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -[INFO] delete: localhost:cylc-run/${NAME}/work/20200101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20150101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20050101T0000Z -[INFO] delete: localhost:cylc-run/${NAME}/work/20000101T0000Z -__OUT__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name="$NAME" -exit 0 diff --git a/t/rose-suite-clean/test_header b/t/rose-suite-clean/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-suite-clean/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-suite-cmp-vc/00-basic.t b/t/rose-suite-cmp-vc/00-basic.t deleted file mode 100755 index c55897c19d..0000000000 --- a/t/rose-suite-cmp-vc/00-basic.t +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Basic tests for "rose suite-cmp-vc" with "svn". -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -if ! which svn 1>'/dev/null' 2>&1; then - skip_all '"svn" unavailable' -fi -tests 2 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -svnadmin create 'repos' -svn import -q -m'who cares' --non-interactive --no-auth-cache \ - "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}/" "file://${PWD}/repos" -svn checkout -q --non-interactive --no-auth-cache "file://${PWD}/repos" 'source' -mkdir -p "${HOME}/cylc-run" -RUND="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${RUND}")" -rose suite-run -C './source' --debug -q --name="${NAME}" -l - -TEST_KEY="${TEST_KEY_BASE}-run-pass" -run_pass "${TEST_KEY}" rose suite-cmp-vc "${NAME}" - -TEST_KEY="${TEST_KEY_BASE}-run-fail" -sed -i 's/meow/miaow/' './source/suite.rc' -run_fail "${TEST_KEY}" rose suite-cmp-vc "${NAME}" -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-cmp-vc/00-basic/rose-suite.conf b/t/rose-suite-cmp-vc/00-basic/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-cmp-vc/00-basic/suite.rc b/t/rose-suite-cmp-vc/00-basic/suite.rc deleted file mode 100644 index 5dd6a9dbe3..0000000000 --- a/t/rose-suite-cmp-vc/00-basic/suite.rc +++ /dev/null @@ -1,6 +0,0 @@ -[scheduling] - [[dependencies]] - graph = meow -[runtime] - [[meow]] - script = nice man cat head tail diff --git a/t/rose-suite-cmp-vc/01-no-vc b/t/rose-suite-cmp-vc/01-no-vc deleted file mode 120000 index 2293ed496a..0000000000 --- a/t/rose-suite-cmp-vc/01-no-vc +++ /dev/null @@ -1 +0,0 @@ -00-basic \ No newline at end of file diff --git a/t/rose-suite-cmp-vc/01-no-vc.t b/t/rose-suite-cmp-vc/01-no-vc.t deleted file mode 100755 index 78c1f80dcb..0000000000 --- a/t/rose-suite-cmp-vc/01-no-vc.t +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Tests for "rose suite-cmp-vc" with VC. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" -tests 2 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -cp -pr "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}/" 'source' -mkdir -p "${HOME}/cylc-run" -RUND="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${RUND}")" -rose suite-run -C './source' --debug -q --name="${NAME}" -l - -TEST_KEY="${TEST_KEY_BASE}" -run_fail "${TEST_KEY}" rose suite-cmp-vc "${NAME}" -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] ${NAME}: rose-suite-run.version: VC info not found -__ERR__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-cmp-vc/test_header b/t/rose-suite-cmp-vc/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-suite-cmp-vc/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-suite-log/00-update-task.t b/t/rose-suite-log/00-update-task.t deleted file mode 100755 index b5166e4a77..0000000000 --- a/t/rose-suite-log/00-update-task.t +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-log --update TASK", without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -if [[ $TEST_KEY_BASE == *-remote* ]]; then - JOB_HOST=$(rose config 't' 'job-host') - if [[ -z $JOB_HOST ]]; then - skip_all '"[t]job-host" not defined' - fi - JOB_HOST=$(rose host-select -q $JOB_HOST) -fi -#------------------------------------------------------------------------------- -tests 4 -#------------------------------------------------------------------------------- -# Run the suite. -export CYLC_CONF_PATH= -export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -if [[ -n ${JOB_HOST:-} ]]; then - rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" -- --no-detach --debug -else - rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug -fi -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE-before-log.out -if [[ -n ${JOB_HOST:-} ]]; then - run_fail "$TEST_KEY-log.out" \ - test -f $SUITE_RUN_DIR/log/job/1/my_task_2/01/job.out -else - pass "$TEST_KEY-log.out" -fi -TEST_KEY=$TEST_KEY_BASE-command -run_pass "$TEST_KEY" rose suite-log -n $NAME -U 'my_task_2' --debug -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" . -#------------------------------------------------------------------------------- -# Test "rose suite-log --update CYCLE", without site/user configurations. -# Test "rose suite-log --archive CYCLE", without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -if [[ $TEST_KEY_BASE == *-remote* ]]; then - JOB_HOST=$(rose config 't' 'job-host') - if [[ -z $JOB_HOST ]]; then - skip_all '"[t]job-host" not defined' - fi - JOB_HOST=$(rose host-select -q $JOB_HOST) -fi -#------------------------------------------------------------------------------- -tests 14 -#------------------------------------------------------------------------------- -# Run the suite. -export CYLC_CONF_PATH= -export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -if [[ -n ${JOB_HOST:-} ]]; then - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" -- --no-detach --debug -else - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug -fi -#------------------------------------------------------------------------------- -# Test --archive. -CYCLE=20130101T0000Z -TEST_KEY="$TEST_KEY_BASE-archive-$CYCLE" -(cd $SUITE_RUN_DIR/log; ls job/$CYCLE/*/01/{job,job-activity.log,job.err,job.out,job.xtrace}) \ - >"$TEST_KEY-list-job-logs-before.out" -if [[ -n ${JOB_HOST:-} ]]; then - ssh -oBatchMode=yes $JOB_HOST \ - test -f cylc-run/$NAME/log/job/$CYCLE/my_task_2/01/job.out - ! test -f $SUITE_RUN_DIR/log/job/$CYCLE/my_task_2/01/job.out -fi -# N_JOB_LOGS should be 4, my_task_1 script, err, out and my_task_2 script -N_JOB_LOGS=$(wc -l "$TEST_KEY-list-job-logs-before.out" | cut -d' ' -f1) -run_pass "$TEST_KEY-command" rose suite-log -n $NAME --archive $CYCLE --debug -run_fail "$TEST_KEY-list-job-logs-after" ls $SUITE_RUN_DIR/log/job/$CYCLE/* -if [[ -n ${JOB_HOST:-} ]]; then - ((N_JOB_LOGS += 3)) # my_task_2 job.out, job.err, job.xtrace - run_fail "$TEST_KEY-job-log.out-after-jobhost" \ - ssh -oBatchMode=yes $JOB_HOST \ - test -f cylc-run/$NAME/log/job/$CYCLE/my_task_2/01/job.out -else - pass "$TEST_KEY-job-log.out-after-jobhost" -fi -file_test "$TEST_KEY-tar-exist" $SUITE_RUN_DIR/log/job-$CYCLE.tar.gz -JOB_LOGS_ARCH=$(tar -tzf $SUITE_RUN_DIR/log/job-$CYCLE.tar.gz) -run_pass "$TEST_KEY-after-log.out" \ - grep -q "job/$CYCLE/my_task_2/../job.out" <<<"$JOB_LOGS_ARCH" -N_JOB_LOGS_ARCH=$(echo "$JOB_LOGS_ARCH" | wc -l | cut -d' ' -f1) -run_pass "$TEST_KEY-n-arch" test "${N_JOB_LOGS}" -eq "${N_JOB_LOGS_ARCH}" -file_cmp "$TEST_KEY-command.err" "$TEST_KEY-command.err" . -#------------------------------------------------------------------------------- -# Test "rose suite-log --force", without site/user configurations. -# Test "rose suite-log -U --prune-remote", without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -if [[ $TEST_KEY_BASE == *-remote* ]]; then - JOB_HOST=$(rose config 't' 'job-host') - if [[ -z $JOB_HOST ]]; then - skip_all '"[t]job-host" not defined' - fi - JOB_HOST=$(rose host-select -q $JOB_HOST) -fi -#------------------------------------------------------------------------------- -tests 15 -#------------------------------------------------------------------------------- -# Run the suite. -export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -if [[ -n ${JOB_HOST:-} ]]; then - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" -- --no-detach --debug -else - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug -fi -#------------------------------------------------------------------------------- -# Test --force. -CYCLES='20130101T0000Z 20130101T1200Z 20130102T0000Z' -for CYCLE in $CYCLES; do - TEST_KEY="$TEST_KEY_BASE-before-$CYCLE" - run_fail "$TEST_KEY" \ - test -f "$HOME/cylc-run/$NAME/log/rose-suite-log-$CYCLE.json" -done -TEST_KEY="$TEST_KEY_BASE-command" -run_pass "$TEST_KEY" rose suite-log -n $NAME -f --debug -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" "$TEST_KEY.out.expected" - file_cmp "$TEST_KEY.out" "$TEST_KEY.out.expected" <<__OUT__ -[INFO] delete: $JOB_HOST:log/job/20130101T0000Z/ -[INFO] delete: $JOB_HOST:log/job/20130101T1200Z/ -__OUT__ - ssh -oBatchMode=yes $JOB_HOST ls "~/cylc-run/$NAME/log/job" \ - | LANG=C sort >"$TEST_KEY.ls" - file_cmp "$TEST_KEY.ls" "$TEST_KEY.ls" <<<'20130102T0000Z' -else - skip 3 "$TEST_KEY: [t]job-host not defined" -fi -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-log/02-update-force/rose-suite.conf b/t/rose-suite-log/02-update-force/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-log/02-update-force/suite.rc b/t/rose-suite-log/02-update-force/suite.rc deleted file mode 100644 index a94b6b1e0e..0000000000 --- a/t/rose-suite-log/02-update-force/suite.rc +++ /dev/null @@ -1,23 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout = PT1M -[scheduling] - initial cycle point = 20130101 - final cycle point = 20130102 - [[dependencies]] - [[[T00, T12]]] - graph = my_task_1 & my_task_2 - -[runtime] - [[my_task_1, my_task_2]] - script = echo Hello - [[[job]]] - execution time limit = PT1M -{% if HOST is defined %} - [[[remote]]] - host = {{HOST}} -{% endif %} diff --git a/t/rose-suite-log/03-update-task-remote b/t/rose-suite-log/03-update-task-remote deleted file mode 120000 index a6a559a8cc..0000000000 --- a/t/rose-suite-log/03-update-task-remote +++ /dev/null @@ -1 +0,0 @@ -00-update-task \ No newline at end of file diff --git a/t/rose-suite-log/03-update-task-remote.t b/t/rose-suite-log/03-update-task-remote.t deleted file mode 120000 index 857ea57cc6..0000000000 --- a/t/rose-suite-log/03-update-task-remote.t +++ /dev/null @@ -1 +0,0 @@ -00-update-task.t \ No newline at end of file diff --git a/t/rose-suite-log/04-update-cycle-remote b/t/rose-suite-log/04-update-cycle-remote deleted file mode 120000 index d3d43f00c2..0000000000 --- a/t/rose-suite-log/04-update-cycle-remote +++ /dev/null @@ -1 +0,0 @@ -01-update-cycle \ No newline at end of file diff --git a/t/rose-suite-log/04-update-cycle-remote.t b/t/rose-suite-log/04-update-cycle-remote.t deleted file mode 120000 index 08e1d67a72..0000000000 --- a/t/rose-suite-log/04-update-cycle-remote.t +++ /dev/null @@ -1 +0,0 @@ -01-update-cycle.t \ No newline at end of file diff --git a/t/rose-suite-log/05-update-force-remote b/t/rose-suite-log/05-update-force-remote deleted file mode 120000 index 65ae64cf60..0000000000 --- a/t/rose-suite-log/05-update-force-remote +++ /dev/null @@ -1 +0,0 @@ -02-update-force \ No newline at end of file diff --git a/t/rose-suite-log/05-update-force-remote.t b/t/rose-suite-log/05-update-force-remote.t deleted file mode 120000 index b09108744d..0000000000 --- a/t/rose-suite-log/05-update-force-remote.t +++ /dev/null @@ -1 +0,0 @@ -02-update-force.t \ No newline at end of file diff --git a/t/rose-suite-log/06-archive-star b/t/rose-suite-log/06-archive-star deleted file mode 120000 index d3d43f00c2..0000000000 --- a/t/rose-suite-log/06-archive-star +++ /dev/null @@ -1 +0,0 @@ -01-update-cycle \ No newline at end of file diff --git a/t/rose-suite-log/06-archive-star.t b/t/rose-suite-log/06-archive-star.t deleted file mode 100644 index bd7adea359..0000000000 --- a/t/rose-suite-log/06-archive-star.t +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-log --archive *", without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -if [[ $TEST_KEY_BASE == *-remote* ]]; then - JOB_HOST=$(rose config 't' 'job-host') - if [[ -z $JOB_HOST ]]; then - skip_all '"[t]job-host" not defined' - fi - JOB_HOST=$(rose host-select -q $JOB_HOST) -fi -#------------------------------------------------------------------------------- -tests 12 -#------------------------------------------------------------------------------- -# Run the suite. -export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -if [[ -n ${JOB_HOST:-} ]]; then - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" -- --no-detach --debug -else - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug -fi -#------------------------------------------------------------------------------- -# Test --archive. -TEST_KEY="$TEST_KEY_BASE" - -set -e -(cd $SUITE_RUN_DIR/log/job; ls */*/01/{job,job-activity.log,job.err,job.out,job.xtrace}) \ - >"$TEST_KEY-list-job-logs-before.out" -if [[ -n ${JOB_HOST:-} ]]; then - ssh -oBatchMode=yes $JOB_HOST \ - test -f cylc-run/$NAME/log/job/*/my_task_2/01/job.out - ! test -f $SUITE_RUN_DIR/log/job/*/my_task_2/01/job.out -fi -set +e - -N_JOB_LOGS=$(wc -l "$TEST_KEY-list-job-logs-before.out" | cut -d' ' -f1) -run_pass "$TEST_KEY-command" rose suite-log -n $NAME --archive '*' --debug -run_fail "$TEST_KEY-list-job-logs-after" ls $SUITE_RUN_DIR/log/job/* -if [[ -n ${JOB_HOST:-} ]]; then - ((N_JOB_LOGS += 4)) # job, job.activity.log, job.out, job.err, job.xtrace files - run_fail "$TEST_KEY-job-log.out-after-jobhost" \ - ssh -oBatchMode=yes $JOB_HOST \ - test -f cylc-run/$NAME/log/job/*/my_task_2/01/job.out -else - pass "$TEST_KEY-job-log.out-after-jobhost" -fi -ALL_JOB_LOGS_ARCH= -for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do - file_test "$TEST_KEY-$CYCLE-tar-exist" $SUITE_RUN_DIR/log/job-$CYCLE.tar.gz - JOB_LOGS_ARCH=$(tar -tzf $SUITE_RUN_DIR/log/job-$CYCLE.tar.gz) - if [[ -n $ALL_JOB_LOGS_ARCH ]]; then - ALL_JOB_LOGS_ARCH=$(cat <<__LIST__ -${ALL_JOB_LOGS_ARCH} -${JOB_LOGS_ARCH} -__LIST__ - ) - else - ALL_JOB_LOGS_ARCH="$JOB_LOGS_ARCH" - fi - run_pass "$TEST_KEY-$CYCLE-after-log.out" \ - grep -q "job/$CYCLE/my_task_2/01/job.out" <<<"$JOB_LOGS_ARCH" -done - -N_JOB_LOGS_ARCH=$(echo "$ALL_JOB_LOGS_ARCH" | wc -l | cut -d' ' -f1) -run_pass "$TEST_KEY-n-arch" test "${N_JOB_LOGS}" -eq "${N_JOB_LOGS_ARCH}" - -file_cmp "$TEST_KEY-command.err" "$TEST_KEY-command.err" . -#------------------------------------------------------------------------------- -# Test "rose suite-log" on an uninstalled suite. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 3 -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" -NAME="$(uuidgen)" # Very unlikely to have a suite of this name installed -run_fail "${TEST_KEY}" rose suite-log "--name=${NAME}" -file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" . -#------------------------------------------------------------------------------- -# Test "rose suite-restart" on suites that don't exist. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -tests 7 -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-pwd" -ROSE_CONF_PATH= run_fail "${TEST_KEY}" rose suite-restart -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] ${PWD} - no suite found for this path. -__ERR__ -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-name-uuid" -NAME="$(uuidgen)" -ROSE_CONF_PATH= run_fail "${TEST_KEY}" rose suite-restart -n "${NAME}" -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] ${HOME}/cylc-run/${NAME} - no suite found for this path. -__ERR__ -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-name-uuid-installed" -NAME="$(uuidgen)" -mkdir -p "${HOME}/cylc-run/${NAME}" -ROSE_CONF_PATH= run_fail "${TEST_KEY}" rose suite-restart -n "${NAME}" \ - -- --no-detach --debug -# N.B. This relies on output of "cylc restart" -head -1 "${TEST_KEY}.err" >"${TEST_KEY}.err.head" -file_cmp "${TEST_KEY}.err.head" "${TEST_KEY}.err.head" <<__ERR__ -[FAIL] cylc restart ${NAME} --no-detach --debug # return-code=1, stderr= -__ERR__ -file_grep "${TEST_KEY}.err.grep"\ - "cylc.flow.exceptions.SuiteServiceFileError: no suite.rc in ${HOME}/cylc-run/${NAME}" \ - "${TEST_KEY}.err" -rm -fr "${HOME}/cylc-run/${NAME}" -#------------------------------------------------------------------------------- -exit diff --git a/t/rose-suite-restart/01-running.t b/t/rose-suite-restart/01-running.t deleted file mode 100755 index b7dbd1221f..0000000000 --- a/t/rose-suite-restart/01-running.t +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-restart" on suites that are still running. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -tests 4 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -SUITE_RUN_DIR="$(readlink -f ${SUITE_RUN_DIR})" -NAME="$(basename "${SUITE_RUN_DIR}")" -timeout 120 rose suite-run --debug -q \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - -- --no-detach & -ROSE_SUITE_RUN_PID=$! -CONTACT="${HOME}/cylc-run/${NAME}/.service/contact" -poll ! test -e "${CONTACT}" -poll ! test -e "${HOME}/cylc-run/${NAME}/log/job/1/foo/NN/job.status" -SUITE_HOST="$(sed -n 's/CYLC_SUITE_HOST=//p' "${CONTACT}")" -SUITE_OWNER="$(sed -n 's/CYLC_SUITE_OWNER=//p' "${CONTACT}")" -SUITE_PORT="$(sed -n 's/CYLC_SUITE_PORT=//p' "${CONTACT}")" -SUITE_PROCESS="$(sed -n 's/CYLC_SUITE_PROCESS=//p' "${CONTACT}")" - -TEST_KEY="${TEST_KEY_BASE}-name" -run_fail "${TEST_KEY}" rose suite-restart --name="${NAME}" -# Note: Error file CYLC_SUITE_* lines contain \t characters -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] Suite "${NAME}" appears to be running: -[FAIL] Contact info from: "${CONTACT}" -[FAIL] CYLC_SUITE_HOST=${SUITE_HOST} -[FAIL] CYLC_SUITE_OWNER=${SUITE_OWNER} -[FAIL] CYLC_SUITE_PORT=${SUITE_PORT} -[FAIL] CYLC_SUITE_PROCESS=${SUITE_PROCESS} -[FAIL] Try "cylc stop '${NAME}'" first? -__ERR__ - -TEST_KEY="${TEST_KEY_BASE}-cwd" -run_fail "${TEST_KEY}" bash -c "cd '${SUITE_RUN_DIR}'; rose suite-restart" -# Note: Error file CYLC_SUITE_* lines contain \t characters -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] Suite "${NAME}" appears to be running: -[FAIL] Contact info from: "${CONTACT}" -[FAIL] CYLC_SUITE_HOST=${SUITE_HOST} -[FAIL] CYLC_SUITE_OWNER=${SUITE_OWNER} -[FAIL] CYLC_SUITE_PORT=${SUITE_PORT} -[FAIL] CYLC_SUITE_PROCESS=${SUITE_PROCESS} -[FAIL] Try "cylc stop '${NAME}'" first? -__ERR__ -#------------------------------------------------------------------------------- -rm -f "${SUITE_RUN_DIR}/work/1/foo/file" -wait "${ROSE_SUITE_RUN_PID}" -poll test -e "${CONTACT}" -rm -f "${CONTACT}" # In case suite takes long time and is killed by timeout -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-restart/01-running/rose-suite.conf b/t/rose-suite-restart/01-running/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-restart/01-running/suite.rc b/t/rose-suite-restart/01-running/suite.rc deleted file mode 100644 index aea89f692b..0000000000 --- a/t/rose-suite-restart/01-running/suite.rc +++ /dev/null @@ -1,9 +0,0 @@ -[scheduling] - [[dependencies]] - graph = foo -[runtime] - [[foo]] - script = """ -touch 'file' -timeout 60 bash -c 'while test -e "file"; do sleep 1; done' -""" diff --git a/t/rose-suite-restart/02-basic.t b/t/rose-suite-restart/02-basic.t deleted file mode 100755 index 28654663e4..0000000000 --- a/t/rose-suite-restart/02-basic.t +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-restart", basic usage. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -tests 2 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run --debug -q \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - -- --no-detach --debug - -TEST_KEY="${TEST_KEY_BASE}" -run_pass "${TEST_KEY}" \ - rose suite-restart --debug --name="${NAME}" \ - -- --no-detach --debug -# N.B. This relies on output from "cylc restart" -file_grep "${TEST_KEY}.log" \ - "\\[jobs-submit cmd\\] cylc jobs-submit --debug -- ${SUITE_RUN_DIR}/log/job 1/t2/01" \ - "${SUITE_RUN_DIR}/log/suite/log" -sed -i '/no HTTPS.* support/d' "$TEST_KEY.err" -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-restart/02-basic/rose-suite.conf b/t/rose-suite-restart/02-basic/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-restart/02-basic/suite.rc b/t/rose-suite-restart/02-basic/suite.rc deleted file mode 100644 index 757f4bfa5f..0000000000 --- a/t/rose-suite-restart/02-basic/suite.rc +++ /dev/null @@ -1,8 +0,0 @@ -[scheduling] - [[dependencies]] - graph = "t1 => t2" -[runtime] - [[t1]] - script = cylc stop "${CYLC_SUITE_NAME}" - [[t2]] - script = true diff --git a/t/rose-suite-restart/03-host b/t/rose-suite-restart/03-host deleted file mode 120000 index 696ef1d9f8..0000000000 --- a/t/rose-suite-restart/03-host +++ /dev/null @@ -1 +0,0 @@ -02-basic \ No newline at end of file diff --git a/t/rose-suite-restart/03-host.t b/t/rose-suite-restart/03-host.t deleted file mode 100755 index 243203fb36..0000000000 --- a/t/rose-suite-restart/03-host.t +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-restart", --host=HOST option. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -HOSTS=$(rose config --default= 'rose-suite-run' 'hosts') -HOST= -if [[ -n "${HOSTS}" ]]; then - HOST="$(rose host-select -q "${HOSTS}")" -fi -if [[ -z "${HOST}" ]]; then - skip_all '"[rose-suite-run]hosts" not defined or no suite host available' -fi -tests 3 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run --debug -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ - --name="${NAME}" --host="${HOST}" \ - -- --no-detach --debug - -TEST_KEY="${TEST_KEY_BASE}" -run_pass "${TEST_KEY}" rose suite-restart \ - -v -v --debug --name="${NAME}" --host="${HOST}" \ - -- --no-detach --debug -file_grep "${TEST_KEY}.out" \ - "cylc restart --host=${HOST} ${NAME} --no-detach --debug" "${TEST_KEY}.out" -# N.B. This relies on output from "cylc restart" -file_grep "${TEST_KEY}.log" \ - "\\[jobs-submit cmd\\] cylc jobs-submit --debug -- ${SUITE_RUN_DIR}/log/job 1/t2/01" \ - "${SUITE_RUN_DIR}/log/suite/log" -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-restart/test_header b/t/rose-suite-restart/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-suite-restart/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-suite-run/00-run-basic.t b/t/rose-suite-run/00-run-basic.t deleted file mode 100755 index ea8596308f..0000000000 --- a/t/rose-suite-run/00-run-basic.t +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", with and without site/user configurations. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" -#------------------------------------------------------------------------------- -# Run the suite. -if [[ "${TEST_KEY_BASE}" == *conf ]]; then - if ! rose config -q 'rose-suite-run' 'hosts'; then - skip_all '"[rose-suite-run]hosts" not defined' - fi -else - export ROSE_CONF_PATH= -fi - -get_host_fqdn() { - python3 - "$@" <<'__PYTHON__' -import socket -import sys -sys.stdout.write(socket.gethostbyname_ex(sys.argv[1])[0] + "\n") -__PYTHON__ -} -#------------------------------------------------------------------------------- -tests 11 -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" -mkdir -p "${HOME}/cylc-run" -RUND="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${RUND}")" -run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - -CONTACT="${HOME}/cylc-run/${NAME}/.service/contact" -SUITE_HOST="$(sed -n 's/CYLC_SUITE_HOST=//p' "${CONTACT}")" -SUITE_OWNER="$(sed -n 's/CYLC_SUITE_OWNER=//p' "${CONTACT}")" -SUITE_PORT="$(sed -n 's/CYLC_SUITE_PORT=//p' "${CONTACT}")" -SUITE_PROCESS="$(sed -n 's/CYLC_SUITE_PROCESS=//p' "${CONTACT}")" -poll ! test -e "${RUND}/log/job/20130101T0000Z/my_task_1/01" -#------------------------------------------------------------------------------- -# "rose suite-run" should not work while suite is running. -# except --reload mode. -for OPTION in -i -l '' --restart; do - TEST_KEY="${TEST_KEY_BASE}-running${OPTION}" - run_fail "${TEST_KEY}" rose suite-run "${OPTION}" \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" - file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] Suite "${NAME}" appears to be running: -[FAIL] Contact info from: "${CONTACT}" -[FAIL] CYLC_SUITE_HOST=${SUITE_HOST} -[FAIL] CYLC_SUITE_OWNER=${SUITE_OWNER} -[FAIL] CYLC_SUITE_PORT=${SUITE_PORT} -[FAIL] CYLC_SUITE_PROCESS=${SUITE_PROCESS} -[FAIL] Try "cylc stop '${NAME}'" first? -__ERR__ -done -# Don't reload until tasks begin -TIMEOUT=$(($(date +%s) + 60)) # wait 1 minute -while (($(date +%s) < TIMEOUT)) && ! ( - cd "${RUND}/log/job/" - test -f "20130101T0000Z/my_task_1/01/job.out" && test -f "20130101T1200Z/my_task_1/01/job.out" -) -do - sleep 1 -done -TEST_KEY="${TEST_KEY_BASE}-running-reload" -run_pass "${TEST_KEY}" rose suite-run --reload \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --debug -sleep 1 -#------------------------------------------------------------------------------- -# Wait for the suite to complete -TEST_KEY="${TEST_KEY_BASE}-suite-run-wait" -touch "${RUND}/flag" # allow the task to die -TIMEOUT=$(($(date +%s) + 60)) # wait 1 minute -CONTACT="${HOME}/cylc-run/${NAME}/.service/contact" -while [[ -e "${CONTACT}" ]] && (($(date +%s) < TIMEOUT)) -do - sleep 1 -done -if [[ -e "${CONTACT}" ]]; then - fail "${TEST_KEY}" - exit 1 -else - pass "${TEST_KEY}" -fi -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit 0 diff --git a/t/rose-suite-run/00-run-basic/rose-suite.conf b/t/rose-suite-run/00-run-basic/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/00-run-basic/suite.rc b/t/rose-suite-run/00-run-basic/suite.rc deleted file mode 100644 index 3a7bf8fd74..0000000000 --- a/t/rose-suite-run/00-run-basic/suite.rc +++ /dev/null @@ -1,32 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout=PT2M -[scheduling] - initial cycle point=20130101T00Z - final cycle point=20130102T12Z - [[dependencies]] - [[[T00, T12]]] - graph=my_task_1 - -[runtime] - [[my_task_1]] - script = """ -I=0 -OLD_I= -while [[ ! -e $CYLC_SUITE_RUN_DIR/flag ]]; do - sleep 1 - ((++I)) - if [[ -n $OLD_I ]]; then - echo -n $(tr '[:print:]' '\b' <<<"$OLD_I") - fi - echo -n $I - OLD_I=$I -done -echo -""" - [[[job]]] - execution time limit = PT1M diff --git a/t/rose-suite-run/01-run-basic-conf b/t/rose-suite-run/01-run-basic-conf deleted file mode 120000 index 6907e44de1..0000000000 --- a/t/rose-suite-run/01-run-basic-conf +++ /dev/null @@ -1 +0,0 @@ -00-run-basic \ No newline at end of file diff --git a/t/rose-suite-run/01-run-basic-conf.t b/t/rose-suite-run/01-run-basic-conf.t deleted file mode 120000 index a0a623ac5f..0000000000 --- a/t/rose-suite-run/01-run-basic-conf.t +++ /dev/null @@ -1 +0,0 @@ -00-run-basic.t \ No newline at end of file diff --git a/t/rose-suite-run/02-install.t b/t/rose-suite-run/02-install.t deleted file mode 100755 index 9837a5675e..0000000000 --- a/t/rose-suite-run/02-install.t +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", with and without site/user configurations. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -#------------------------------------------------------------------------------- -JOB_HOST="$(rose config --default= 't' 'job-host')" -if [[ -n "${JOB_HOST}" ]]; then - JOB_HOST="$(rose host-select -q "${JOB_HOST}")" -fi -#------------------------------------------------------------------------------- -if [[ "${TEST_KEY_BASE}" == *conf ]]; then - if ! rose config -q 'rose-suite-run' 'hosts'; then - skip_all '"[rose-suite-run]hosts" not defined' - fi -else - export ROSE_CONF_PATH= -fi -#------------------------------------------------------------------------------- -tests 6 -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -case "${TEST_KEY_BASE}" in - *local*) - OPTION='-l';; - *validate*) - OPTION='--validate-suite-only';; - *host-localhost*) - OPTION='-i' - JOB_HOST=$HOSTNAME;; - *) - OPTION='-i';; -esac -mkdir "${TEST_KEY_BASE}" -cp -pr "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}/"* "${TEST_KEY_BASE}" -touch "${TEST_KEY_BASE}/colon:is:ok" -if [[ -n "${JOB_HOST}" ]]; then - run_pass "${TEST_KEY}" rose suite-run --debug \ - -C "${TEST_KEY_BASE}" "${OPTION}" \ - --name="${NAME}" \ - -S "HOST=\"${JOB_HOST}\"" -else - run_pass "${TEST_KEY}" rose suite-run --debug \ - -C "${TEST_KEY_BASE}" "${OPTION}" \ - --name="${NAME}" -fi -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-port-file" -run_fail "${TEST_KEY}" test -e "${HOME}/cylc-run/${NAME}/.service/contact" -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-items" -_run_mode=run_pass -if [[ "${TEST_KEY_BASE}" == *validate* ]]; then - _run_mode=run_fail -fi -"$_run_mode" "${TEST_KEY}" find "${SUITE_RUN_DIR}/"{app,colon:is:ok,etc} -type f -if [[ "${TEST_KEY_BASE}" != *validate* ]]; then - sort "${TEST_KEY}.out" >"${TEST_KEY}.out.sort" - file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out.sort" <<__OUT__ -${SUITE_RUN_DIR}/app/my_task_1/rose-app.conf -${SUITE_RUN_DIR}/app/my_task_2/rose-app.conf -${SUITE_RUN_DIR}/colon:is:ok -${SUITE_RUN_DIR}/etc/junk -__OUT__ -fi -#------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}-items-${JOB_HOST}" -if [[ "${TEST_KEY_BASE}" == *local* ]]; then - skip 2 "${TEST_KEY}: local-install-only" -elif [[ "${TEST_KEY_BASE}" == *validate* ]]; then - skip 3 "${TEST_KEY}: validate-suite-only" -elif [[ -n "${JOB_HOST}" ]]; then - run_pass "${TEST_KEY}" \ - ssh -oBatchMode=yes "${JOB_HOST}" \ - "find 'cylc-run/${NAME}/'{app,colon:is:ok,etc} -type f" - sort "${TEST_KEY}.out" >"${TEST_KEY}.out.sort" - file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out.sort" <<__OUT__ -cylc-run/${NAME}/app/my_task_1/rose-app.conf -cylc-run/${NAME}/colon:is:ok -cylc-run/${NAME}/etc/junk -__OUT__ -else - skip 2 "${TEST_KEY_BASE}-items: [t]job-host not defined" -fi -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit 0 diff --git a/t/rose-suite-run/02-install/app/my_task_1/rose-app.conf b/t/rose-suite-run/02-install/app/my_task_1/rose-app.conf deleted file mode 100644 index 7521154cea..0000000000 --- a/t/rose-suite-run/02-install/app/my_task_1/rose-app.conf +++ /dev/null @@ -1,2 +0,0 @@ -[command] -default=echo Hello World diff --git a/t/rose-suite-run/02-install/app/my_task_2/rose-app.conf b/t/rose-suite-run/02-install/app/my_task_2/rose-app.conf deleted file mode 100644 index a83a2918ff..0000000000 --- a/t/rose-suite-run/02-install/app/my_task_2/rose-app.conf +++ /dev/null @@ -1,2 +0,0 @@ -[command] -default=echo Hi World diff --git a/t/rose-suite-run/02-install/etc/junk b/t/rose-suite-run/02-install/etc/junk deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/02-install/rose-suite.conf b/t/rose-suite-run/02-install/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/02-install/suite.rc b/t/rose-suite-run/02-install/suite.rc deleted file mode 100644 index e71093f4d4..0000000000 --- a/t/rose-suite-run/02-install/suite.rc +++ /dev/null @@ -1,29 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout = PT2M -[scheduling] - initial cycle point = 20130101 - final cycle point = 20130102 - [[dependencies]] - [[[T00, T12]]] - graph = my_task_1 & my_task_2 - -[runtime] - [[root]] - script = rose task-run - [[[job]]] - execution time limit = PT1M - [[my_task_1]] -{% if HOST is defined %} - [[[remote]]] - host = {{HOST}} -{% endif %} - [[my_task_2]] -{% if HOST is defined %} - [[[remote]]] - host = $(echo {{HOST}}) -{% endif %} diff --git a/t/rose-suite-run/03-install-conf b/t/rose-suite-run/03-install-conf deleted file mode 120000 index e1b7ad15e0..0000000000 --- a/t/rose-suite-run/03-install-conf +++ /dev/null @@ -1 +0,0 @@ -02-install \ No newline at end of file diff --git a/t/rose-suite-run/03-install-conf.t b/t/rose-suite-run/03-install-conf.t deleted file mode 120000 index 3d0cca2073..0000000000 --- a/t/rose-suite-run/03-install-conf.t +++ /dev/null @@ -1 +0,0 @@ -02-install.t \ No newline at end of file diff --git a/t/rose-suite-run/04-install-local b/t/rose-suite-run/04-install-local deleted file mode 120000 index e1b7ad15e0..0000000000 --- a/t/rose-suite-run/04-install-local +++ /dev/null @@ -1 +0,0 @@ -02-install \ No newline at end of file diff --git a/t/rose-suite-run/04-install-local.t b/t/rose-suite-run/04-install-local.t deleted file mode 120000 index 3d0cca2073..0000000000 --- a/t/rose-suite-run/04-install-local.t +++ /dev/null @@ -1 +0,0 @@ -02-install.t \ No newline at end of file diff --git a/t/rose-suite-run/05-log.t b/t/rose-suite-run/05-log.t deleted file mode 100755 index 637fd5290a..0000000000 --- a/t/rose-suite-run/05-log.t +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test --*-log-* options of "rose suite-run". -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -#------------------------------------------------------------------------------- -tests 23 -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -ROSE_SUITE_RUN="rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE" -ROSE_SUITE_RUN="$ROSE_SUITE_RUN --name=$NAME" -ROSE_SUITE_RUN="$ROSE_SUITE_RUN " - -N_RUNS=6 -I_KEEP=$((RANDOM % N_RUNS)) # 0 to N_RUNS - 1 -I_NO_LOG_ARCHIVE=$((RANDOM % N_RUNS + 1)) -while ((I_KEEP == I_NO_LOG_ARCHIVE)); do - I_NO_LOG_ARCHIVE=$((RANDOM % N_RUNS + 1)) -done -OLD_LOG= -KEPT_LOG= -for I in $(seq 0 $N_RUNS); do - # Run the suite - TEST_KEY=$TEST_KEY_BASE-$I - if ((I == I_KEEP)); then - run_pass "$TEST_KEY-keep" $ROSE_SUITE_RUN --log-name=keep \ - -- --no-detach --debug - elif ((I == I_NO_LOG_ARCHIVE)); then - run_pass "$TEST_KEY-no-log-archive" \ - $ROSE_SUITE_RUN --no-log-archive -- --no-detach --debug - else - run_pass "$TEST_KEY" $ROSE_SUITE_RUN -- --no-detach --debug - fi - - TEST_KEY=$TEST_KEY_BASE-$I-log - file_test "$TEST_KEY" $SUITE_RUN_DIR/log -L - if ((I == I_KEEP)); then - file_test "$TEST_KEY-keep" $SUITE_RUN_DIR/log.keep -L - KEPT_LOG=$(readlink $SUITE_RUN_DIR/log.keep) - fi - if ((I - 1 == I_KEEP)) || ((I == I_NO_LOG_ARCHIVE)); then - file_test "$TEST_KEY-old-kept" $SUITE_RUN_DIR/$OLD_LOG - elif [[ -n $OLD_LOG ]]; then - file_test "$TEST_KEY-old" $SUITE_RUN_DIR/$OLD_LOG.tar.gz - fi - OLD_LOG=$(readlink $SUITE_RUN_DIR/log) -done - -TEST_KEY=$TEST_KEY_BASE-log-keep-0 -run_pass "$TEST_KEY" $ROSE_SUITE_RUN --log-keep=0 -- --no-detach --debug - -TEST_KEY=$TEST_KEY_BASE-log-keep-0-ls -ls -d $SUITE_RUN_DIR/log* >$TEST_KEY.out -THIS_LOG=$(readlink $SUITE_RUN_DIR/log) -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -$SUITE_RUN_DIR/log -$SUITE_RUN_DIR/$KEPT_LOG -$SUITE_RUN_DIR/$THIS_LOG -$SUITE_RUN_DIR/log.keep -__OUT__ - -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-run/05-log/rose-suite.conf b/t/rose-suite-run/05-log/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/05-log/suite.rc b/t/rose-suite-run/05-log/suite.rc deleted file mode 100644 index 8aaeb128b9..0000000000 --- a/t/rose-suite-run/05-log/suite.rc +++ /dev/null @@ -1,16 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on timeout = True - timeout = PT1M -[scheduling] - initial cycle point = 20130101T00Z - final cycle point = 20130101T00Z - [[dependencies]] - [[[T00]]] - graph = my_task_1 - -[runtime] - [[my_task_1]] - script = true diff --git a/t/rose-suite-run/06-opt-conf.t b/t/rose-suite-run/06-opt-conf.t deleted file mode 100755 index 0808a310ba..0000000000 --- a/t/rose-suite-run/06-opt-conf.t +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run -O KEY". -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -#------------------------------------------------------------------------------- -tests 6 -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -cat tests -mkdir -p $HOME/cylc-run -for OPT_KEY in world earth neutron-star; do - SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') - echo "$OPT_KEY $SUITE_RUN_DIR" >>tests -done -# Run the suites -while read OPT_KEY SUITE_RUN_DIR; do - NAME=$(basename $SUITE_RUN_DIR) - ROSE_SUITE_RUN="rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE" - ROSE_SUITE_RUN="$ROSE_SUITE_RUN --name=$NAME " - if [[ $OPT_KEY != 'world' ]]; then - ROSE_SUITE_RUN="$ROSE_SUITE_RUN -O $OPT_KEY" - fi - ROSE_SUITE_RUN="$ROSE_SUITE_RUN -- --no-detach --debug" - TEST_KEY=$TEST_KEY_BASE-$OPT_KEY - run_pass "$TEST_KEY" $ROSE_SUITE_RUN -done $CYLC_TASK_LOG_ROOT.txt""" diff --git a/t/rose-suite-run/09-pgrep-conf b/t/rose-suite-run/09-pgrep-conf deleted file mode 120000 index 6851810496..0000000000 --- a/t/rose-suite-run/09-pgrep-conf +++ /dev/null @@ -1 +0,0 @@ -08-pgrep \ No newline at end of file diff --git a/t/rose-suite-run/09-pgrep-conf.t b/t/rose-suite-run/09-pgrep-conf.t deleted file mode 120000 index d8388f0486..0000000000 --- a/t/rose-suite-run/09-pgrep-conf.t +++ /dev/null @@ -1 +0,0 @@ -08-pgrep.t \ No newline at end of file diff --git a/t/rose-suite-run/10-import-1.t b/t/rose-suite-run/10-import-1.t deleted file mode 100755 index a8c4ac22e7..0000000000 --- a/t/rose-suite-run/10-import-1.t +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", import. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 5 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -# Install the "hello_earth" suite -TEST_KEY="$TEST_KEY_BASE-local-install" -run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE/hello_earth -n $NAME -l -(cd ~/cylc-run/$NAME; find app bin -type f | LANG=C sort) >"$TEST_KEY.find" -file_cmp "$TEST_KEY.find" "$TEST_KEY.find" <<'__FIND__' -app/hello/rose-app.conf -bin/my-hello -__FIND__ -{ - CONF=$HOME/cylc-run/$NAME/log/rose-suite-run.conf - rose config -f $CONF jinja2:suite.rc hello - rose config -f $CONF jinja2:suite.rc worlds -} >"$TEST_KEY.conf" -file_cmp "$TEST_KEY.conf" "$TEST_KEY.conf" <<'__FIND__' -"Hello" -["Earth", "Moon"] -__FIND__ -#------------------------------------------------------------------------------- -# Start the "hello_earth" suite -TEST_KEY="$TEST_KEY_BASE-suite-run" -run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE/hello_earth \ - -n $NAME -- --no-detach --debug -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-suite-run-my-hello.log" -LANG=C sort $SUITE_RUN_DIR/my-hello.log >"$TEST_KEY" -file_cmp "$TEST_KEY" "$TEST_KEY" <<'__LOG__' -[20130101T0000Z] Hello Earth -[20130101T0000Z] Hello Moon -[20130101T1200Z] Hello Earth -[20130101T1200Z] Hello Moon -[20130102T0000Z] Hello Earth -[20130102T0000Z] Hello Moon -__LOG__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-run/10-import-1/hello/app/hello/rose-app.conf b/t/rose-suite-run/10-import-1/hello/app/hello/rose-app.conf deleted file mode 100644 index acfcc551f6..0000000000 --- a/t/rose-suite-run/10-import-1/hello/app/hello/rose-app.conf +++ /dev/null @@ -1,2 +0,0 @@ -[command] -default=my-hello diff --git a/t/rose-suite-run/10-import-1/hello/bin/my-hello b/t/rose-suite-run/10-import-1/hello/bin/my-hello deleted file mode 100755 index 274565695a..0000000000 --- a/t/rose-suite-run/10-import-1/hello/bin/my-hello +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -eu -echo "[$ROSE_TASK_CYCLE_TIME] ${HELLO:-hello} ${ROSE_TASK_NAME#hello_}" \ - | tee -a $ROSE_SUITE_DIR/$(basename $0).log diff --git a/t/rose-suite-run/10-import-1/hello/rose-suite.conf b/t/rose-suite-run/10-import-1/hello/rose-suite.conf deleted file mode 100644 index 0b399a1463..0000000000 --- a/t/rose-suite-run/10-import-1/hello/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -[jinja2:suite.rc] -hello="Hello" diff --git a/t/rose-suite-run/10-import-1/hello/suite.rc b/t/rose-suite-run/10-import-1/hello/suite.rc deleted file mode 100644 index 54b42898d2..0000000000 --- a/t/rose-suite-run/10-import-1/hello/suite.rc +++ /dev/null @@ -1,28 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout=PT2M -[scheduling] - initial cycle point=20130101 - final cycle point=20130102 - [[dependencies]] - [[[T00, T12]]] - graph = """ -{% for world in worlds|default(["worlds"]) -%} -hello_{{world}} -{% endfor -%} -""" - -[runtime] - [[root]] - script=rose task-run --debug --app-key=hello - [[[environment]]] - HELLO={{hello|default("hello")}} - [[[job]]] - execution time limit = PT1M -{% for world in worlds -%} - [[hello_{{world}}]] -{% endfor -%} diff --git a/t/rose-suite-run/10-import-1/hello_earth/rose-suite.conf b/t/rose-suite-run/10-import-1/hello_earth/rose-suite.conf deleted file mode 100644 index a9e2a87c1c..0000000000 --- a/t/rose-suite-run/10-import-1/hello_earth/rose-suite.conf +++ /dev/null @@ -1,4 +0,0 @@ -import=hello - -[jinja2:suite.rc] -worlds=["Earth", "Moon"] diff --git a/t/rose-suite-run/11-import-2.t b/t/rose-suite-run/11-import-2.t deleted file mode 100755 index 2f5d111109..0000000000 --- a/t/rose-suite-run/11-import-2.t +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", multiple imports. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 5 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -# Install the "greet_earth" suite -TEST_KEY="$TEST_KEY_BASE-local-install" -run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE/greet_earth -n $NAME -l -(cd ~/cylc-run/$NAME; find app bin -type f | LANG=C sort) >"$TEST_KEY.find" -file_cmp "$TEST_KEY.find" "$TEST_KEY.find" <<'__FIND__' -app/hello/rose-app.conf -bin/my-hello -__FIND__ -{ - CONF=$HOME/cylc-run/$NAME/log/rose-suite-run.conf - rose config -f $CONF jinja2:suite.rc hello - rose config -f $CONF jinja2:suite.rc worlds -} >"$TEST_KEY.conf" -file_cmp "$TEST_KEY.conf" "$TEST_KEY.conf" <<'__FIND__' -"Greet" -["Earth", "Moon"] -__FIND__ -#------------------------------------------------------------------------------- -# Start the "greet_earth" suite -TEST_KEY="$TEST_KEY_BASE-suite-run" -run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE/greet_earth \ - -n $NAME -- --no-detach --debug -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-suite-run-my-hello.log" -LANG=C sort $SUITE_RUN_DIR/my-hello.log >"$TEST_KEY" -file_cmp "$TEST_KEY" "$TEST_KEY" <<'__LOG__' -[20130101T0000Z] Greet Earth -[20130101T0000Z] Greet Moon -[20130101T1200Z] Greet Earth -[20130101T1200Z] Greet Moon -[20130102T0000Z] Greet Earth -[20130102T0000Z] Greet Moon -__LOG__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-run/11-import-2/greet/rose-suite.conf b/t/rose-suite-run/11-import-2/greet/rose-suite.conf deleted file mode 100644 index e8c25ca703..0000000000 --- a/t/rose-suite-run/11-import-2/greet/rose-suite.conf +++ /dev/null @@ -1,4 +0,0 @@ -import=../10-import-1/hello - -[jinja2:suite.rc] -hello="Greet" diff --git a/t/rose-suite-run/11-import-2/greet_earth/rose-suite.conf b/t/rose-suite-run/11-import-2/greet_earth/rose-suite.conf deleted file mode 100644 index 1135417119..0000000000 --- a/t/rose-suite-run/11-import-2/greet_earth/rose-suite.conf +++ /dev/null @@ -1 +0,0 @@ -import=greet ../10-import-1/hello_earth diff --git a/t/rose-suite-run/12-run-dir-root.t b/t/rose-suite-run/12-run-dir-root.t deleted file mode 100755 index af6ea3593d..0000000000 --- a/t/rose-suite-run/12-run-dir-root.t +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", modification of the suite run root directory. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == $HOME ]]; then - skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"' -fi -#------------------------------------------------------------------------------- -N_TESTS=7 -tests $N_TESTS -#------------------------------------------------------------------------------- -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE/* . -cat >rose-suite.conf <<__CONF__ -root-dir=*=\$TMPDIR/\$USER -__CONF__ -JOB_HOST=$(rose config 't' 'job-host') -JOB_HOST_RUN_ROOT=$(rose config 't' 'job-host-run-root') -JOB_HOST_OPT= -if [[ -n $JOB_HOST && -n $JOB_HOST_RUN_ROOT ]]; then - export JOB_HOST=$(rose host-select -q $JOB_HOST) - export JOB_HOST_RUN_ROOT - JOB_HOST_OPT='-O job-host' - mkdir opt - cat >opt/rose-suite-job-host.conf <<__CONF__ -root-dir=$JOB_HOST=$JOB_HOST_RUN_ROOT - =*=\$TMPDIR/\$USER - -[jinja2:suite.rc] -JOB_HOST="$JOB_HOST" -__CONF__ -fi -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE-install -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -run_pass "$TEST_KEY" rose suite-run -n $NAME -i $JOB_HOST_OPT -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-locs" -rose config -f $SUITE_RUN_DIR/log/rose-suite-run.locs localhost root-dir \ - >"$TEST_KEY.localhost" -file_cmp "$TEST_KEY.localhost" "$TEST_KEY.localhost" <<<'$TMPDIR/$USER' -if [[ -n $JOB_HOST_OPT ]]; then - rose config -f $SUITE_RUN_DIR/log/rose-suite-run.locs $JOB_HOST root-dir \ - >"$TEST_KEY.$JOB_HOST" - file_cmp "$TEST_KEY.$JOB_HOST" "$TEST_KEY.$JOB_HOST" <<<$JOB_HOST_RUN_ROOT -else - skip 1 "[t]job-host not defined" -fi -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-root-symlink" -if [[ $(readlink $HOME/cylc-run/$NAME) == $TMPDIR/$USER/cylc-run/$NAME ]]; then - pass "$TEST_KEY.localhost" -else - fail "$TEST_KEY.localhost" -fi -if [[ -n $JOB_HOST_OPT ]]; then - RUN_ROOT=$(ssh $JOB_HOST "bash -l -c echo\\ \\$JOB_HOST_RUN_ROOT" | tail -1) - RUN_DIR=$(ssh $JOB_HOST "readlink ~/cylc-run/$NAME" | tail -1) - if [[ $RUN_DIR == $RUN_ROOT/cylc-run/$NAME ]]; then - pass "$TEST_KEY.$JOB_HOST" - else - fail "$TEST_KEY.$JOB_HOST" - fi -else - skip 1 "[t]job-host not defined" -fi -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-clean" -run_pass "$TEST_KEY" rose suite-clean -y $NAME -if [[ -e $HOME/cylc-run/$NAME || -e $TMPDIR/$USER/cylc-run/$NAME ]]; then - fail "$TEST_KEY.localhost" -else - pass "$TEST_KEY.localhost" -fi -#------------------------------------------------------------------------------- -exit 0 diff --git a/t/rose-suite-run/12-run-dir-root/suite.rc b/t/rose-suite-run/12-run-dir-root/suite.rc deleted file mode 100644 index d1d516a1a2..0000000000 --- a/t/rose-suite-run/12-run-dir-root/suite.rc +++ /dev/null @@ -1,25 +0,0 @@ -#!jinja2 -[cylc] -UTC mode=True - [[events]] - abort on timeout = True - timeout=PT1M - -[scheduling] - [[dependencies]] - graph=""" -my_task_1 -{% if JOB_HOST is defined %} -my_task_2 -{% endif %} -""" - -[runtime] - [[root]] - script=true - [[my_task_1]] -{% if JOB_HOST is defined %} - [[my_task_2]] - [[[remote]]] - host={{JOB_HOST}} -{% endif %} diff --git a/t/rose-suite-run/13-ignore-cylc-version.t b/t/rose-suite-run/13-ignore-cylc-version.t deleted file mode 100755 index f70418cd1c..0000000000 --- a/t/rose-suite-run/13-ignore-cylc-version.t +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", reload with !CYLC_VERSION. -# See issue metomi/rose#1143. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 1 -export ROSE_CONF_PATH= -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE/* . -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -n $NAME -# Wait for the only task to fail, before reload -ST_FILE=$SUITE_RUN_DIR/log/job/1/t1/01/job.status -TIMEOUT=$(($(date +%s) + 60)) # wait 1 minute -while (($(date +%s) < TIMEOUT)) \ - && ! grep -q 'CYLC_JOB_EXIT_TIME=' $ST_FILE 2>/dev/null -do - sleep 1 -done -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -run_pass "$TEST_KEY" rose suite-run --reload -q -n $NAME -#------------------------------------------------------------------------------- -rose suite-stop -q -y -n $NAME -- --max-polls=12 --interval=5 -rose suite-clean -q -y $NAME -exit diff --git a/t/rose-suite-run/13-ignore-cylc-version/rose-suite.conf b/t/rose-suite-run/13-ignore-cylc-version/rose-suite.conf deleted file mode 100644 index b4b4dcb614..0000000000 --- a/t/rose-suite-run/13-ignore-cylc-version/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -[env] -!CYLC_VERSION=5.4.8 diff --git a/t/rose-suite-run/13-ignore-cylc-version/suite.rc b/t/rose-suite-run/13-ignore-cylc-version/suite.rc deleted file mode 100644 index f2706b118e..0000000000 --- a/t/rose-suite-run/13-ignore-cylc-version/suite.rc +++ /dev/null @@ -1,12 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - abort on timeout=True - timeout=PT2M -[scheduling] - [[dependencies]] - graph=t1 -[runtime] - [[t1]] - script=false diff --git a/t/rose-suite-run/14-reload-null.t b/t/rose-suite-run/14-reload-null.t deleted file mode 100755 index df4d718197..0000000000 --- a/t/rose-suite-run/14-reload-null.t +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", null reloads. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 7 -export ROSE_CONF_PATH= -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE src -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -n $NAME -C src -poll ! test -e "$SUITE_RUN_DIR/log/job/20130101T0000Z/t1/01/job.status" -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-0" -run_pass "$TEST_KEY" rose suite-run --run=reload -n $NAME -C src -sed -n '/reload complete/p' "$TEST_KEY.out" >"$TEST_KEY.out.tail" -file_cmp "$TEST_KEY.out" "$TEST_KEY.out.tail" <<__OUT__ -[INFO] $NAME: reload complete. "suite.rc" unchanged -__OUT__ -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" /dev/null -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-1" -# Add file that allows the jobs to proceed -cat >"src/hello.txt" <<'__TXT__' -hello world -hello earth -__TXT__ -run_pass "$TEST_KEY" rose suite-run --run=reload -n $NAME -C src -sed -n '/hello\.txt/p; /reload complete/p' "$TEST_KEY.out" >"$TEST_KEY.out.tail" -file_cmp "$TEST_KEY.out" "$TEST_KEY.out.tail" <<__OUT__ -[INFO] install: hello.txt -[INFO] source: $PWD/src/hello.txt -[INFO] $NAME: reload complete. "suite.rc" unchanged -__OUT__ -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" /dev/null -# Wait for the suite to complete -poll test -e "$HOME/cylc-run/$NAME/.service/contact" -grep '^hello ' $SUITE_RUN_DIR/log/job/*/t1/01/job.out >"$TEST_KEY.job.out" -file_cmp "$TEST_KEY.job.out" "$TEST_KEY.job.out" <<__OUT__ -$SUITE_RUN_DIR/log/job/20130101T0000Z/t1/01/job.out:hello world -$SUITE_RUN_DIR/log/job/20130101T1200Z/t1/01/job.out:hello world -__OUT__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit diff --git a/t/rose-suite-run/14-reload-null/app/t1/opt/rose-app-earth.conf b/t/rose-suite-run/14-reload-null/app/t1/opt/rose-app-earth.conf deleted file mode 100644 index 3d2683f6b6..0000000000 --- a/t/rose-suite-run/14-reload-null/app/t1/opt/rose-app-earth.conf +++ /dev/null @@ -1,2 +0,0 @@ -[env] -WORLD=earth diff --git a/t/rose-suite-run/14-reload-null/app/t1/rose-app.conf b/t/rose-suite-run/14-reload-null/app/t1/rose-app.conf deleted file mode 100644 index baa3953fc2..0000000000 --- a/t/rose-suite-run/14-reload-null/app/t1/rose-app.conf +++ /dev/null @@ -1,9 +0,0 @@ -[command] -default=grep $WORLD $ROSE_SUITE_DIR/hello.txt - -[env] -WORLD=world - -[poll] -all-files=$ROSE_SUITE_DIR/hello.txt -delays=120*1s diff --git a/t/rose-suite-run/14-reload-null/rose-suite.conf b/t/rose-suite-run/14-reload-null/rose-suite.conf deleted file mode 100644 index 50dbc24094..0000000000 --- a/t/rose-suite-run/14-reload-null/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -[jinja2:suite.rc] -ROSE_TASK_RUN_ARGS="" diff --git a/t/rose-suite-run/14-reload-null/suite.rc b/t/rose-suite-run/14-reload-null/suite.rc deleted file mode 100644 index c4b9b06837..0000000000 --- a/t/rose-suite-run/14-reload-null/suite.rc +++ /dev/null @@ -1,16 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - abort if any task fails=True - [[events]] - abort on timeout=True - timeout=PT2M -[scheduling] - initial cycle point=20130101T00Z - final cycle point=20130101T12Z - [[dependencies]] - [[[T00, T12]]] - graph="""t1[-PT12H]=>t1""" -[runtime] - [[t1]] - script=rose task-run {{ROSE_TASK_RUN_ARGS}} diff --git a/t/rose-suite-run/15-reload-rc b/t/rose-suite-run/15-reload-rc deleted file mode 120000 index 56966bd3d1..0000000000 --- a/t/rose-suite-run/15-reload-rc +++ /dev/null @@ -1 +0,0 @@ -14-reload-null \ No newline at end of file diff --git a/t/rose-suite-run/15-reload-rc.t b/t/rose-suite-run/15-reload-rc.t deleted file mode 100755 index 0154c8858a..0000000000 --- a/t/rose-suite-run/15-reload-rc.t +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", reload "suite.rc". -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 3 -export ROSE_CONF_PATH= -mkdir -p src -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE/* src -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -n $NAME -C src -poll ! test -e "$SUITE_RUN_DIR/log/job/20130101T0000Z/t1/01/job.status" -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE" -cat >src/rose-suite.conf <<'__CONF__' -[jinja2:suite.rc] -ROSE_TASK_RUN_ARGS="-O earth" -__CONF__ -run_pass "$TEST_KEY" rose suite-run --run=reload -n $NAME -C src -sed -n '/\(delete\|install\): suite\.rc/p' "${TEST_KEY}.out" \ - >"${TEST_KEY}.out.edited" -file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out.edited" <<'__OUT__' -[INFO] delete: suite.rc -[INFO] install: suite.rc -__OUT__ -poll ! grep -q \ - -e 'RELOADING.TASK.DEFINITION.FOR.t1\.20130101T0000Z' \ - -e 'RELOADING.TASK.DEFINITION.FOR.t1\.20130101T1200Z' \ - "$SUITE_RUN_DIR/log/suite/log" -# Add file that allows the jobs to proceed -cat >"$SUITE_RUN_DIR/hello.txt" <<'__TXT__' -hello world -hello earth -__TXT__ -# Wait for the suite to complete -poll test -e "$HOME/cylc-run/$NAME/.service/contact" -grep '^hello ' $SUITE_RUN_DIR/log/job/*/t1/01/job.out >"$TEST_KEY.job.out" -file_cmp "$TEST_KEY.job.out" "$TEST_KEY.job.out" <<__OUT__ -$SUITE_RUN_DIR/log/job/20130101T0000Z/t1/01/job.out:hello world -$SUITE_RUN_DIR/log/job/20130101T1200Z/t1/01/job.out:hello earth -__OUT__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit diff --git a/t/rose-suite-run/16-suite-rc-not-writable.t b/t/rose-suite-run/16-suite-rc-not-writable.t deleted file mode 100755 index 4b48d85711..0000000000 --- a/t/rose-suite-run/16-suite-rc-not-writable.t +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", should work whether "suite.rc" is writable or not. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 1 -export ROSE_CONF_PATH= -mkdir -p src -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE/* src -chmod -w src/suite.rc -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -run_pass "$TEST_KEY" rose suite-run -q -n $NAME -C src \ - -- --no-detach --debug -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit diff --git a/t/rose-suite-run/16-suite-rc-not-writable/rose-suite.conf b/t/rose-suite-run/16-suite-rc-not-writable/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/16-suite-rc-not-writable/suite.rc b/t/rose-suite-run/16-suite-rc-not-writable/suite.rc deleted file mode 100644 index 66d0769878..0000000000 --- a/t/rose-suite-run/16-suite-rc-not-writable/suite.rc +++ /dev/null @@ -1,12 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - abort on timeout=True - timeout=PT1M -[scheduling] - [[dependencies]] - graph=t1 -[runtime] - [[t1]] - script=true diff --git a/t/rose-suite-run/17-install-overlap.t b/t/rose-suite-run/17-install-overlap.t deleted file mode 100755 index fd5cf0a752..0000000000 --- a/t/rose-suite-run/17-install-overlap.t +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", file install targets overlap. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 3 -export ROSE_CONF_PATH= - -mkdir -p src -echo 'yummie' >src/bacon.txt - -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -export TEST_DIR -run_pass "$TEST_KEY-1" rose suite-run \ - -n $NAME -C $TEST_SOURCE_DIR/$TEST_KEY_BASE -i -run_pass "$TEST_KEY-2" rose suite-run \ - -n $NAME -C $TEST_SOURCE_DIR/$TEST_KEY_BASE -i -(cd $SUITE_RUN_DIR/etc; find -type f) | LANG=C sort >"$TEST_KEY.find" -file_cmp "$TEST_KEY.find" "$TEST_KEY.find" <<'__FIND__' -./foo/bar/baz/bacon.txt -./foo/bar/egg/humpty.txt -__FIND__ -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit diff --git a/t/rose-suite-run/17-install-overlap/etc/foo/bar/egg/humpty.txt b/t/rose-suite-run/17-install-overlap/etc/foo/bar/egg/humpty.txt deleted file mode 100644 index 1e7074c4a8..0000000000 --- a/t/rose-suite-run/17-install-overlap/etc/foo/bar/egg/humpty.txt +++ /dev/null @@ -1 +0,0 @@ -Dumpty diff --git a/t/rose-suite-run/17-install-overlap/rose-suite.conf b/t/rose-suite-run/17-install-overlap/rose-suite.conf deleted file mode 100644 index c18d7b25e4..0000000000 --- a/t/rose-suite-run/17-install-overlap/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -[file:etc/foo/bar/baz] -source=$TEST_DIR/src diff --git a/t/rose-suite-run/17-install-overlap/suite.rc b/t/rose-suite-run/17-install-overlap/suite.rc deleted file mode 100644 index 66d0769878..0000000000 --- a/t/rose-suite-run/17-install-overlap/suite.rc +++ /dev/null @@ -1,12 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - abort on timeout=True - timeout=PT1M -[scheduling] - [[dependencies]] - graph=t1 -[runtime] - [[t1]] - script=true diff --git a/t/rose-suite-run/18-restart-init-dir.t b/t/rose-suite-run/18-restart-init-dir.t deleted file mode 100755 index e72fffb211..0000000000 --- a/t/rose-suite-run/18-restart-init-dir.t +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run --restart" does not re-initialise run directory. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -tests 3 - -export ROSE_CONF_PATH= -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE src -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -n $NAME -C src -- --no-detach --debug -cat >'src/rose-suite.conf' <<__CONF__ -root-dir=*=$PWD -__CONF__ -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-restart" -run_pass "$TEST_KEY" \ - rose suite-run -q -n $NAME -C src --restart \ - -- --no-detach --debug -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-dir" -run_pass "$TEST_KEY" test -d "$SUITE_RUN_DIR" -TEST_KEY="$TEST_KEY_BASE-symlink" -run_fail "$TEST_KEY" test -L "$SUITE_RUN_DIR" -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit diff --git a/t/rose-suite-run/18-restart-init-dir/rose-suite.conf b/t/rose-suite-run/18-restart-init-dir/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/18-restart-init-dir/suite.rc b/t/rose-suite-run/18-restart-init-dir/suite.rc deleted file mode 100644 index 66d0769878..0000000000 --- a/t/rose-suite-run/18-restart-init-dir/suite.rc +++ /dev/null @@ -1,12 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - abort on timeout=True - timeout=PT1M -[scheduling] - [[dependencies]] - graph=t1 -[runtime] - [[t1]] - script=true diff --git a/t/rose-suite-run/19-restart-init-dir-remote.t b/t/rose-suite-run/19-restart-init-dir-remote.t deleted file mode 100755 index 7ff294f8a3..0000000000 --- a/t/rose-suite-run/19-restart-init-dir-remote.t +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run --restart" does not re-initialise run directory, -# on remote host. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -T_HOST=$(rose config --default= t job-host) -T_HOST_RUN_ROOT=$(rose config --default= t job-host-run-root) -if [[ -z "$T_HOST" || -z "$T_HOST_RUN_ROOT" ]]; then - skip_all '"[t]job-host" or "[t]job-host-run-root" not defined' -fi -T_HOST=$(rose host-select -q $T_HOST) -#------------------------------------------------------------------------------- -SSH='ssh -oBatchMode=yes' -function ssh_mkdtemp() { - local T_HOST=$1 - $SSH $T_HOST python3 - <<'__PYTHON__' -import os -from tempfile import mkdtemp -print mkdtemp(dir=os.path.expanduser("~"), prefix="rose-") -__PYTHON__ -} - -T_HOST_ROSE_HOME=$(ssh_mkdtemp $T_HOST) -rsync -a --exclude=*.pyc $ROSE_HOME/* $T_HOST:$T_HOST_ROSE_HOME/ - -mkdir -p 'conf' -cat >'conf/rose.conf' <<__CONF__ -[rose-suite-run] -remote-no-login-shell=${T_HOST}=true -remote-rose-bin=${T_HOST}=${T_HOST_ROSE_HOME}/bin/rose -__CONF__ -export ROSE_CONF_PATH="${PWD}/conf" - -tests 3 - -cp -r $TEST_SOURCE_DIR/$TEST_KEY_BASE src -cat >'src/rose-suite.conf' <<__CONF__ -[jinja2:suite.rc] -T_HOST="$T_HOST" -__CONF__ -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -n $NAME -C src -- --no-detach --debug -cat >'src/rose-suite.conf' <<__CONF__ -root-dir=$T_HOST=$T_HOST_RUN_ROOT -[jinja2:suite.rc] -T_HOST="$T_HOST" -__CONF__ -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-restart" -run_pass "$TEST_KEY" \ - rose suite-run -q -n $NAME -C src --restart \ - -- --no-detach --debug -#------------------------------------------------------------------------------- -TEST_KEY="$TEST_KEY_BASE-dir" -run_pass "$TEST_KEY" ssh -oBatchMode=yes $T_HOST test -d "cylc-run/$NAME" -TEST_KEY="$TEST_KEY_BASE-symlink" -run_fail "$TEST_KEY" ssh -oBatchMode=yes $T_HOST test -L "cylc-run/$NAME" -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -$SSH $T_HOST "rm -fr '$T_HOST_ROSE_HOME'" -exit diff --git a/t/rose-suite-run/19-restart-init-dir-remote/rose-suite.conf b/t/rose-suite-run/19-restart-init-dir-remote/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/19-restart-init-dir-remote/suite.rc b/t/rose-suite-run/19-restart-init-dir-remote/suite.rc deleted file mode 100644 index a226d2443b..0000000000 --- a/t/rose-suite-run/19-restart-init-dir-remote/suite.rc +++ /dev/null @@ -1,14 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - abort on timeout = True - timeout=PT1M -[scheduling] - [[dependencies]] - graph=t1 -[runtime] - [[t1]] - script=true - [[[remote]]] - host={{T_HOST}} diff --git a/t/rose-suite-run/21-sharecycle-back-compat.t b/t/rose-suite-run/21-sharecycle-back-compat.t deleted file mode 100755 index f266c60eb4..0000000000 --- a/t/rose-suite-run/21-sharecycle-back-compat.t +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run" installation of share/cycle/ sub-directory using the -# backward compatable configuration setting of -# "root-dir{share/cycle}=HOST=share/data". -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -tests 2 -#------------------------------------------------------------------------------- -mkdir -p 'conf' -cat >'conf/rose.conf' <<'__CONF__' -[rose-suite-run] -root-dir{share/cycle}=*=share/data -__CONF__ -export ROSE_CONF_PATH="${PWD}/conf" - -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q \ - -n "${NAME}" -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" -- --no-detach --debug -# Check contents of files written to ROSE_DATAC is going to historic location -for DATETIME in '20200101T0000Z' '20200101T0000Z'; do - file_cmp "nl-${DATETIME}" \ - "${SUITE_RUN_DIR}/share/data/${DATETIME}/share.nl" <<__NL__ -&share_nl -date=${DATETIME}, -index=ftse, -market=lse, -/ -__NL__ -done -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-run/21-sharecycle-back-compat/app/t1/rose-app.conf b/t/rose-suite-run/21-sharecycle-back-compat/app/t1/rose-app.conf deleted file mode 100644 index 16c26edf75..0000000000 --- a/t/rose-suite-run/21-sharecycle-back-compat/app/t1/rose-app.conf +++ /dev/null @@ -1,10 +0,0 @@ -[command] -default=true - -[file:$ROSE_DATAC/share.nl] -source=namelist:share_nl - -[namelist:share_nl] -market=lse -index=ftse -date=$ROSE_TASK_CYCLE_TIME diff --git a/t/rose-suite-run/21-sharecycle-back-compat/rose-suite.conf b/t/rose-suite-run/21-sharecycle-back-compat/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/21-sharecycle-back-compat/suite.rc b/t/rose-suite-run/21-sharecycle-back-compat/suite.rc deleted file mode 100644 index 883e0fde86..0000000000 --- a/t/rose-suite-run/21-sharecycle-back-compat/suite.rc +++ /dev/null @@ -1,15 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on timeout = True - timeout = PT1M -[scheduling] - initial cycle point = 2020 - final cycle point = 2021 - [[dependencies]] - [[[P1Y]]] - graph = t1 -[runtime] - [[t1]] - script = rose task-run diff --git a/t/rose-suite-run/22-sharecycle.t b/t/rose-suite-run/22-sharecycle.t deleted file mode 100755 index 643dd560c6..0000000000 --- a/t/rose-suite-run/22-sharecycle.t +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run" installation of share/cycle/ sub-directory. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -tests 2 -#------------------------------------------------------------------------------- -export CYLC_CONF_PATH= -export ROSE_CONF_PATH="${PWD}/conf" -export ROOT_DIR_WORK="${PWD}" -mkdir -p 'conf' -cat >'conf/rose.conf' <<'__CONF__' -[rose-suite-run] -root-dir{share/cycle}=*=$ROOT_DIR_WORK -__CONF__ - -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q \ - -n "${NAME}" -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" -- --no-detach --debug -# Check contents of files written to ROSE_DATAC is going to $PWD/share/cycle/ -for DATETIME in '20200101T0000Z' '20200101T0000Z'; do - file_cmp "nl-${DATETIME}" \ - "${PWD}/cylc-run/${NAME}/share/cycle/${DATETIME}/share.nl" <<__NL__ -&share_nl -date=${DATETIME}, -index=ftse, -market=lse, -/ -__NL__ -done -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-run/22-sharecycle/app/t1/rose-app.conf b/t/rose-suite-run/22-sharecycle/app/t1/rose-app.conf deleted file mode 100644 index 16c26edf75..0000000000 --- a/t/rose-suite-run/22-sharecycle/app/t1/rose-app.conf +++ /dev/null @@ -1,10 +0,0 @@ -[command] -default=true - -[file:$ROSE_DATAC/share.nl] -source=namelist:share_nl - -[namelist:share_nl] -market=lse -index=ftse -date=$ROSE_TASK_CYCLE_TIME diff --git a/t/rose-suite-run/22-sharecycle/rose-suite.conf b/t/rose-suite-run/22-sharecycle/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/22-sharecycle/suite.rc b/t/rose-suite-run/22-sharecycle/suite.rc deleted file mode 100644 index 883e0fde86..0000000000 --- a/t/rose-suite-run/22-sharecycle/suite.rc +++ /dev/null @@ -1,15 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on timeout = True - timeout = PT1M -[scheduling] - initial cycle point = 2020 - final cycle point = 2021 - [[dependencies]] - [[[P1Y]]] - graph = t1 -[runtime] - [[t1]] - script = rose task-run diff --git a/t/rose-suite-run/23-reload-host.t b/t/rose-suite-run/23-reload-host.t deleted file mode 100755 index faab76330c..0000000000 --- a/t/rose-suite-run/23-reload-host.t +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", reload with a new job host. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -JOB_HOST="$(rose config --default= 't' 'job-host')" -if [[ -n "${JOB_HOST}" ]]; then - JOB_HOST="$(rose host-select -q "${JOB_HOST}")" -fi -if [[ -z "${JOB_HOST}" ]]; then - skip_all '"[t]job-host" not defined' -fi -tests 2 -export ROSE_CONF_PATH= -rsync -a "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}/" '.' -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename ${SUITE_RUN_DIR})" -rose suite-run --debug --name="${NAME}" \ - -S "HOST=\"${JOB_HOST}\"" -- --no-detach --debug --hold 1>'/dev/null' 2>&1 & -ROSE_SUITE_RUN_PID=$! - -timeout 60 bash -c \ - "while ! test -e '${HOME}/cylc-run/${NAME}/.service/contact'; do sleep 1; done" -sed -i "s/host = localhost/host = ${JOB_HOST}/" 'suite.rc' - -TEST_KEY="${TEST_KEY_BASE}" -run_pass "${TEST_KEY}" \ - rose suite-run --debug --reload --name="${NAME}" \ - -S "HOST=\"${JOB_HOST}\"" -sed -n '/\(delete\|install\): suite\.rc/p' \ - "${TEST_KEY}.out" >"${TEST_KEY}.out.edited" -file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out.edited" <<'__OUT__' -[INFO] delete: suite.rc -[INFO] install: suite.rc -__OUT__ - -cylc release "${NAME}" - -wait "${ROSE_SUITE_RUN_PID}" -rose suite-clean -q -y "${NAME}" -exit 0 diff --git a/t/rose-suite-run/23-reload-host/rose-suite.conf b/t/rose-suite-run/23-reload-host/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/23-reload-host/suite.rc b/t/rose-suite-run/23-reload-host/suite.rc deleted file mode 100644 index da227cb1a9..0000000000 --- a/t/rose-suite-run/23-reload-host/suite.rc +++ /dev/null @@ -1,17 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on timeout = True - timeout = PT1M -[scheduling] - initial cycle point = 2020 - final cycle point = 2021 - [[dependencies]] - [[[P1Y]]] - graph = t1 -[runtime] - [[t1]] - script = true - [[[remote]]] - host = localhost diff --git a/t/rose-suite-run/24-host-log-timestamp.t b/t/rose-suite-run/24-host-log-timestamp.t deleted file mode 100755 index dafb746c79..0000000000 --- a/t/rose-suite-run/24-host-log-timestamp.t +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", match TIMESTAMP of local and remote log.TIMESTAMP/ -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -JOB_HOST="$(rose config --default= 't' 'job-host')" -if [[ -n "${JOB_HOST}" ]]; then - JOB_HOST="$(rose host-select -q "${JOB_HOST}")" -fi -if [[ -z "${JOB_HOST}" ]]; then - skip_all '"[t]job-host" not defined' -fi -tests 3 -export ROSE_CONF_PATH="${PWD}/conf" -export PATH="${PWD}/bin:${PATH}" -mkdir 'bin' 'conf' -cat >'bin/myssh' <<'__BASH__' -#!/bin/bash -# Make sure that local and remote log cannot be created in the same second -sleep 1 -# Print arguments to log file -echo "$@" >"$(dirname "$0")/../myssh.log" -# Invoke real SSH -exec ssh "$@" -__BASH__ -chmod +x 'bin/myssh' -cat >'conf/rose.conf' <<'__CONF__' -[external] -ssh=myssh -oBatchMode=yes -oConnectTimeout=10 -__CONF__ -rsync -a "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}/" '.' -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename ${SUITE_RUN_DIR})" -run_pass "${TEST_KEY_BASE}" \ - rose suite-run --debug --name="${NAME}" \ - -S "HOST=\"${JOB_HOST}\"" -- --no-detach -NOW_STR="$(sed 's/^.*now-str=\([^,]*\),\?.*$/\1/' 'myssh.log')" -run_pass "${TEST_KEY_BASE}-log-timestamp-local" \ - test -d "${SUITE_RUN_DIR}/log.${NOW_STR}" -run_pass "${TEST_KEY_BASE}-log-timestamp-remote" \ - ssh -n -oBatchMode='yes' "${JOB_HOST}" \ - "test -d cylc-run/${NAME}/log.${NOW_STR}" -rose suite-clean -q -y "${NAME}" -exit 0 diff --git a/t/rose-suite-run/24-host-log-timestamp/rose-suite.conf b/t/rose-suite-run/24-host-log-timestamp/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-run/24-host-log-timestamp/suite.rc b/t/rose-suite-run/24-host-log-timestamp/suite.rc deleted file mode 100644 index e37d6423c8..0000000000 --- a/t/rose-suite-run/24-host-log-timestamp/suite.rc +++ /dev/null @@ -1,14 +0,0 @@ -#!jinja2 -[cylc] - UTC mode = True - [[events]] - abort on timeout = True - timeout = PT1M -[scheduling] - [[dependencies]] - graph = t1 -[runtime] - [[t1]] - script = true - [[[remote]]] - host = {{HOST}} diff --git a/t/rose-suite-run/25-install-validate-only b/t/rose-suite-run/25-install-validate-only deleted file mode 120000 index e1b7ad15e0..0000000000 --- a/t/rose-suite-run/25-install-validate-only +++ /dev/null @@ -1 +0,0 @@ -02-install \ No newline at end of file diff --git a/t/rose-suite-run/25-install-validate-only.t b/t/rose-suite-run/25-install-validate-only.t deleted file mode 120000 index 3d0cca2073..0000000000 --- a/t/rose-suite-run/25-install-validate-only.t +++ /dev/null @@ -1 +0,0 @@ -02-install.t \ No newline at end of file diff --git a/t/rose-suite-run/26-jinja2-insert.t b/t/rose-suite-run/26-jinja2-insert.t deleted file mode 100755 index 7df1416b59..0000000000 --- a/t/rose-suite-run/26-jinja2-insert.t +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run" in $HOME/cylc-run to ensure that insertion of jinja2 -# variable declarations to "suite.rc" do not get repeated. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - - -#------------------------------------------------------------------------------- -N_TESTS=3 -tests $N_TESTS -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -cat >$SUITE_RUN_DIR/rose-suite.conf <<__ROSE_SUITE_CONF__ -[jinja2:suite.rc] -foo="food store" -bar="barley drink" -__ROSE_SUITE_CONF__ -cat >"$SUITE_RUN_DIR/suite.rc" <<__SUITE_RC__ -#!jinja2 -{% set egg="egg sandwich" %} -{% set ham="hamburger" %} -[cylc] -UTC mode=True -[scheduling] -[[dependencies]] -graph=x -[runtime] -[[x]] -__SUITE_RC__ -NAME=$(basename $SUITE_RUN_DIR) -CYLC_VERSION=$(cylc --version) -ROSE_ORIG_HOST=$(hostname) -ROSE_VERSION=$(rose --version | cut -d' ' -f2) -for I in $(seq 1 $N_TESTS); do - rose suite-run -C$SUITE_RUN_DIR --name=$NAME -l -q --debug || break - file_cmp "${TEST_KEY}-${I}" "$SUITE_RUN_DIR/suite.rc" <<__SUITE_RC__ -#!jinja2 -{# Rose Configuration Insertion: Init #} -{% set CYLC_VERSION="$CYLC_VERSION" %} -{% set ROSE_ORIG_HOST="$ROSE_ORIG_HOST" %} -{% set ROSE_SITE="" %} -{% set ROSE_VERSION="$ROSE_VERSION" %} -{% set bar="barley drink" %} -{% set foo="food store" %} -{% set ROSE_SUITE_VARIABLES={ - 'CYLC_VERSION': CYLC_VERSION, - 'ROSE_ORIG_HOST': ROSE_ORIG_HOST, - 'ROSE_SITE': ROSE_SITE, - 'ROSE_VERSION': ROSE_VERSION, - 'bar': bar, - 'foo': foo, -} %} -[cylc] - [[environment]] - CYLC_VERSION=${CYLC_VERSION} - ROSE_ORIG_HOST=${ROSE_ORIG_HOST} - ROSE_SITE= - ROSE_VERSION=${ROSE_VERSION} -{# Rose Configuration Insertion: Done #} -{% set egg="egg sandwich" %} -{% set ham="hamburger" %} -[cylc] -UTC mode=True -[scheduling] -[[dependencies]] -graph=x -[runtime] -[[x]] -__SUITE_RC__ -done -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME --debug -exit 0 diff --git a/t/rose-suite-run/27-empy-insert.t b/t/rose-suite-run/27-empy-insert.t deleted file mode 100755 index e57b47b3bb..0000000000 --- a/t/rose-suite-run/27-empy-insert.t +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run" in $HOME/cylc-run to ensure that insertion of jinja2 -# variable declarations to "suite.rc" do not get repeated. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header - -if ! cylc check-software 2>/dev/null | grep '^Python:EmPy.*([^-]*)$' >/dev/null; then - skip_all '"EmPy" not installed' -fi -#------------------------------------------------------------------------------- -N_TESTS=3 -tests $N_TESTS -export ROSE_CONF_PATH= -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -cat >$SUITE_RUN_DIR/rose-suite.conf <<__ROSE_SUITE_CONF__ -[empy:suite.rc] -foo="food store" -bar="barley drink" -__ROSE_SUITE_CONF__ -cat >"$SUITE_RUN_DIR/suite.rc" <<__SUITE_RC__ -#!empy -@# Rose Configuration Insertion: Init -# Anything here is to be replaced -@# Rose Configuration Insertion: Done -@{ egg="egg sandwich" }@ -@{ ham="hamburger" }@ -[cylc] -UTC mode=True -[scheduling] -[[dependencies]] -graph=x -[runtime] -[[x]] -__SUITE_RC__ -NAME=$(basename $SUITE_RUN_DIR) -CYLC_VERSION=$(cylc --version) -ROSE_ORIG_HOST=$(hostname) -ROSE_VERSION=$(rose --version | cut -d' ' -f2) -for I in $(seq 1 $N_TESTS); do - rose suite-run -C$SUITE_RUN_DIR --name=$NAME -l -q --debug -S "!bar" -S baz=True || break - file_cmp "$TEST_KEY" "$SUITE_RUN_DIR/suite.rc" <<__SUITE_RC__ -#!empy -@# Rose Configuration Insertion: Init -@{CYLC_VERSION="$CYLC_VERSION"}@ -@{ROSE_ORIG_HOST="$ROSE_ORIG_HOST"}@ -@{ROSE_SITE=""}@ -@{ROSE_VERSION="$ROSE_VERSION"}@ -@{baz=True}@ -@{foo="food store"}@ -@{ROSE_SUITE_VARIABLES={ - 'CYLC_VERSION': CYLC_VERSION, - 'ROSE_ORIG_HOST': ROSE_ORIG_HOST, - 'ROSE_SITE': ROSE_SITE, - 'ROSE_VERSION': ROSE_VERSION, - 'baz': baz, - 'foo': foo, -}}@ -[cylc] - [[environment]] - CYLC_VERSION=${CYLC_VERSION} - ROSE_ORIG_HOST=${ROSE_ORIG_HOST} - ROSE_SITE= - ROSE_VERSION=${ROSE_VERSION} -@# Rose Configuration Insertion: Done -@{ egg="egg sandwich" }@ -@{ ham="hamburger" }@ -[cylc] -UTC mode=True -[scheduling] -[[dependencies]] -graph=x -[runtime] -[[x]] -__SUITE_RC__ -done -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME --debug -exit 0 diff --git a/t/rose-suite-run/28-rose-site-env.t b/t/rose-suite-run/28-rose-site-env.t deleted file mode 100755 index 59d0a1d40a..0000000000 --- a/t/rose-suite-run/28-rose-site-env.t +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run" with "site=SITE" setting in site/user conf. -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - - -N_TESTS=3 -tests "${N_TESTS}" -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH="${PWD}/conf" -export ROOT_DIR_WORK="${PWD}" -mkdir -p 'conf' -cat >'conf/rose.conf' <<'__CONF__' -site=my-site -__CONF__ - -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -touch "${SUITE_RUN_DIR}/rose-suite.conf" -cat >"$SUITE_RUN_DIR/suite.rc" <<'__SUITE_RC__' -#!jinja2 -[cylc] -UTC mode=True -[scheduling] -[[dependencies]] -graph=x -[runtime] -[[x]] -__SUITE_RC__ -NAME="$(basename "${SUITE_RUN_DIR}")" -CYLC_VERSION="$(cylc --version)" -ROSE_ORIG_HOST="$(hostname)" -ROSE_VERSION="$(rose --version | cut -d' ' -f2)" -for I in $(seq 1 "${N_TESTS}"); do - rose suite-run -C"${SUITE_RUN_DIR}" --name="${NAME}" -l -q --debug || break - file_cmp "${TEST_KEY}-${I}" "${SUITE_RUN_DIR}/suite.rc" <<__SUITE_RC__ -#!jinja2 -{# Rose Configuration Insertion: Init #} -{% set CYLC_VERSION="${CYLC_VERSION}" %} -{% set ROSE_ORIG_HOST="${ROSE_ORIG_HOST}" %} -{% set ROSE_SITE="my-site" %} -{% set ROSE_VERSION="${ROSE_VERSION}" %} -{% set ROSE_SUITE_VARIABLES={ - 'CYLC_VERSION': CYLC_VERSION, - 'ROSE_ORIG_HOST': ROSE_ORIG_HOST, - 'ROSE_SITE': ROSE_SITE, - 'ROSE_VERSION': ROSE_VERSION, -} %} -[cylc] - [[environment]] - CYLC_VERSION=${CYLC_VERSION} - ROSE_ORIG_HOST=${ROSE_ORIG_HOST} - ROSE_SITE=my-site - ROSE_VERSION=${ROSE_VERSION} -{# Rose Configuration Insertion: Done #} -[cylc] -UTC mode=True -[scheduling] -[[dependencies]] -graph=x -[runtime] -[[x]] -__SUITE_RC__ -done -#------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" -exit diff --git a/t/rose-suite-run/29-install-host-localhost b/t/rose-suite-run/29-install-host-localhost deleted file mode 120000 index e1b7ad15e0..0000000000 --- a/t/rose-suite-run/29-install-host-localhost +++ /dev/null @@ -1 +0,0 @@ -02-install \ No newline at end of file diff --git a/t/rose-suite-run/29-install-host-localhost.t b/t/rose-suite-run/29-install-host-localhost.t deleted file mode 120000 index 3d0cca2073..0000000000 --- a/t/rose-suite-run/29-install-host-localhost.t +++ /dev/null @@ -1 +0,0 @@ -02-install.t \ No newline at end of file diff --git a/t/rose-suite-run/test_header b/t/rose-suite-run/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-suite-run/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-suite-shutdown/00-run-basic.t b/t/rose-suite-shutdown/00-run-basic.t deleted file mode 100755 index 7689283e8b..0000000000 --- a/t/rose-suite-shutdown/00-run-basic.t +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", with and without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -#------------------------------------------------------------------------------- -if [[ $TEST_KEY_BASE == *conf ]]; then - if ! rose config -q 'rose-suite-run' 'hosts'; then - skip_all '"[rose-suite-run]hosts" not defined' - fi -else - export ROSE_CONF_PATH= -fi -#------------------------------------------------------------------------------- -set -e -N_TESTS=1 -tests $N_TESTS -#------------------------------------------------------------------------------- -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -sleep 1 -run_pass "$TEST_KEY" rose suite-stop -y -n $NAME -- --max-polls=12 --interval=5 -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-shutdown/00-run-basic/rose-suite.conf b/t/rose-suite-shutdown/00-run-basic/rose-suite.conf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/t/rose-suite-shutdown/00-run-basic/suite.rc b/t/rose-suite-shutdown/00-run-basic/suite.rc deleted file mode 100644 index 168bdc622e..0000000000 --- a/t/rose-suite-shutdown/00-run-basic/suite.rc +++ /dev/null @@ -1,14 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - abort on timeout = True - timeout=PT1M -[scheduling] - [[dependencies]] - graph=my_task_1 - -[runtime] - [[root]] - script=false - [[my_task_1]] diff --git a/t/rose-suite-shutdown/01-run-conf b/t/rose-suite-shutdown/01-run-conf deleted file mode 120000 index 6907e44de1..0000000000 --- a/t/rose-suite-shutdown/01-run-conf +++ /dev/null @@ -1 +0,0 @@ -00-run-basic \ No newline at end of file diff --git a/t/rose-suite-shutdown/01-run-conf.t b/t/rose-suite-shutdown/01-run-conf.t deleted file mode 120000 index a0a623ac5f..0000000000 --- a/t/rose-suite-shutdown/01-run-conf.t +++ /dev/null @@ -1 +0,0 @@ -00-run-basic.t \ No newline at end of file diff --git a/t/rose-suite-shutdown/02-rose-stem-name b/t/rose-suite-shutdown/02-rose-stem-name deleted file mode 120000 index 6907e44de1..0000000000 --- a/t/rose-suite-shutdown/02-rose-stem-name +++ /dev/null @@ -1 +0,0 @@ -00-run-basic \ No newline at end of file diff --git a/t/rose-suite-shutdown/02-rose-stem-name.t b/t/rose-suite-shutdown/02-rose-stem-name.t deleted file mode 100755 index 31f9ad882c..0000000000 --- a/t/rose-suite-shutdown/02-rose-stem-name.t +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", with and without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -set -e -#------------------------------------------------------------------------------- -tests 1 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -mkdir -p $NAME -ln -s $TEST_SOURCE_DIR/$TEST_KEY_BASE $NAME/rose-stem -cd $NAME -TIMEOUT=$(($(date +%s) + 60)) -while (($(date +%s) < $TIMEOUT)) \ - && [[ ! -e $SUITE_RUN_DIR/log/job/1/my_task_1/01/job.status ]] -do - sleep 1 -done -run_pass "$TEST_KEY" rose suite-stop -y -- --max-polls=12 --interval=5 -cd $OLDPWD -#------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-shutdown/03-normal-name b/t/rose-suite-shutdown/03-normal-name deleted file mode 120000 index 6907e44de1..0000000000 --- a/t/rose-suite-shutdown/03-normal-name +++ /dev/null @@ -1 +0,0 @@ -00-run-basic \ No newline at end of file diff --git a/t/rose-suite-shutdown/03-normal-name.t b/t/rose-suite-shutdown/03-normal-name.t deleted file mode 100755 index c59d3c829f..0000000000 --- a/t/rose-suite-shutdown/03-normal-name.t +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------------------- -# Copyright (C) British Crown (Met Office) & Contributors. -# -# This file is part of Rose, a framework for meteorological suites. -# -# Rose is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Rose is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Rose. If not, see . -#------------------------------------------------------------------------------- -# Test "rose suite-run", with and without site/user configurations. -#------------------------------------------------------------------------------- -. $(dirname $0)/test_header -set -e -#------------------------------------------------------------------------------- -tests 1 -#------------------------------------------------------------------------------- -export ROSE_CONF_PATH= -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME -HOST="$(awk -F= '$1 == "CYLC_SUITE_HOST" {print $2}' "${SUITE_RUN_DIR}/.service/contact")" -#------------------------------------------------------------------------------- -TEST_KEY=$TEST_KEY_BASE -mkdir -p "$TEST_KEY/$NAME" -ln -s $TEST_SOURCE_DIR/$TEST_KEY_BASE/rose-suite.conf $TEST_KEY/$NAME/ -cd "$TEST_KEY/$NAME" -run_pass "$TEST_KEY" rose suite-stop -y -- --max-polls=12 --interval=5 -cd $OLDPWD -#------------------------------------------------------------------------------- -sleep 1 -rose suite-clean -q -y $NAME -exit 0 diff --git a/t/rose-suite-shutdown/test_header b/t/rose-suite-shutdown/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/rose-suite-shutdown/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/t/rose-task-env/00-non-cycle.t b/t/rose-task-env/00-non-cycle.t index aa30d42ca9..5377e9baf5 100755 --- a/t/rose-task-env/00-non-cycle.t +++ b/t/rose-task-env/00-non-cycle.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,17 +24,25 @@ export ROSE_CONF_PATH= #------------------------------------------------------------------------------- -tests 1 +tests 3 #------------------------------------------------------------------------------- -# Run the suite. -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME=$(basename "${SUITE_RUN_DIR}") -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug -sqlite3 "${SUITE_RUN_DIR}/log/db" \ +sqlite3 "${FLOW_RUN_DIR}/log/db" \ 'select distinct status from task_states;' >"$TEST_KEY_BASE-db.out" file_cmp "$TEST_KEY_BASE-db.out" "$TEST_KEY_BASE-db.out" <<<'succeeded' #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-env/00-non-cycle/suite.rc b/t/rose-task-env/00-non-cycle/flow.cylc similarity index 90% rename from t/rose-task-env/00-non-cycle/suite.rc rename to t/rose-task-env/00-non-cycle/flow.cylc index 19faad0a85..3baa8beb32 100644 --- a/t/rose-task-env/00-non-cycle/suite.rc +++ b/t/rose-task-env/00-non-cycle/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT1M diff --git a/t/rose-task-env/01-integer-cycling.t b/t/rose-task-env/01-integer-cycling.t index d9db984d28..f8b2a26cd2 100755 --- a/t/rose-task-env/01-integer-cycling.t +++ b/t/rose-task-env/01-integer-cycling.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,21 +24,30 @@ export ROSE_CONF_PATH= #------------------------------------------------------------------------------- -tests 7 +tests 9 #------------------------------------------------------------------------------- # Run the suite. -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME=$(basename "${SUITE_RUN_DIR}") -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug -run_pass "$TEST_KEY_BASE-0" ls -d $HOME/cylc-run/${NAME}/share/cycle/0 -run_pass "$TEST_KEY_BASE-1" ls -d $HOME/cylc-run/${NAME}/share/cycle/1 -run_pass "$TEST_KEY_BASE-2" ls -d $HOME/cylc-run/${NAME}/share/cycle/2 -run_pass "$TEST_KEY_BASE-3" ls -d $HOME/cylc-run/${NAME}/share/cycle/3 -run_pass "$TEST_KEY_BASE-6" ls -d $HOME/cylc-run/${NAME}/share/cycle/6 -run_pass "$TEST_KEY_BASE-7" ls -d $HOME/cylc-run/${NAME}/share/cycle/7 -run_pass "$TEST_KEY_BASE-8" ls -d $HOME/cylc-run/${NAME}/share/cycle/8 +run_pass "$TEST_KEY_BASE-0" ls -d "$FLOW_RUN_DIR/share/cycle/0" +run_pass "$TEST_KEY_BASE-1" ls -d "$FLOW_RUN_DIR/share/cycle/1" +run_pass "$TEST_KEY_BASE-2" ls -d "$FLOW_RUN_DIR/share/cycle/2" +run_pass "$TEST_KEY_BASE-3" ls -d "$FLOW_RUN_DIR/share/cycle/3" +run_pass "$TEST_KEY_BASE-6" ls -d "$FLOW_RUN_DIR/share/cycle/6" +run_pass "$TEST_KEY_BASE-7" ls -d "$FLOW_RUN_DIR/share/cycle/7" +run_pass "$TEST_KEY_BASE-8" ls -d "$FLOW_RUN_DIR/share/cycle/8" #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-env/01-integer-cycling/suite.rc b/t/rose-task-env/01-integer-cycling/flow.cylc similarity index 92% rename from t/rose-task-env/01-integer-cycling/suite.rc rename to t/rose-task-env/01-integer-cycling/flow.cylc index af29f0982e..f43a252a40 100644 --- a/t/rose-task-env/01-integer-cycling/suite.rc +++ b/t/rose-task-env/01-integer-cycling/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT1M diff --git a/t/rose-task-env/02-360day-cycling.t b/t/rose-task-env/02-360day-cycling.t index eece22c5a6..87cab7fa08 100755 --- a/t/rose-task-env/02-360day-cycling.t +++ b/t/rose-task-env/02-360day-cycling.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,12 +23,20 @@ export ROSE_CONF_PATH= -tests 1 +tests 3 -RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rtb-rose-task-env-02.XXXXXX')" -NAME="$(basename "${RUN_DIR}")" -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --host='localhost' \ + --no-detach \ + --debug for CYCLE in \ '20200227T0000Z' \ '20200228T0000Z' \ @@ -38,7 +46,7 @@ for CYCLE in \ '20200302T0000Z' do echo "${CYCLE}:" - sed "s?^${RUN_DIR}??" "${RUN_DIR}/work/${CYCLE}/foo/my-datac.txt" + sed "s?^${FLOW_RUN_DIR}??" "${FLOW_RUN_DIR}/work/${CYCLE}/foo/my-datac.txt" done >'expected-my-datac.txt' file_cmp "${TEST_KEY_BASE}-my-datac" 'expected-my-datac.txt' <<'__TXT__' @@ -68,5 +76,5 @@ file_cmp "${TEST_KEY_BASE}-my-datac" 'expected-my-datac.txt' <<'__TXT__' /share/cycle/20200303T0000Z __TXT__ -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-env/02-360day-cycling/suite.rc b/t/rose-task-env/02-360day-cycling/flow.cylc similarity index 100% rename from t/rose-task-env/02-360day-cycling/suite.rc rename to t/rose-task-env/02-360day-cycling/flow.cylc diff --git a/t/rose-task-run/00-run-basic.t b/t/rose-task-run/00-run-basic.t index 2cca47e1be..35e913ffbd 100755 --- a/t/rose-task-run/00-run-basic.t +++ b/t/rose-task-run/00-run-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,39 +24,48 @@ export ROSE_CONF_PATH= #------------------------------------------------------------------------------- -tests 44 +tests 46 #------------------------------------------------------------------------------- # Run the suite. -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- MY_PATH= -for P in $(ls -d $SUITE_RUN_DIR/etc/my-path/*); do +for P in $(ls -d $FLOW_RUN_DIR/etc/my-path/*); do if [[ -n $MY_PATH ]]; then MY_PATH="$P:$MY_PATH" else MY_PATH="$P" fi done -if [[ -d $SUITE_RUN_DIR/etc/your-path ]]; then +if [[ -d $FLOW_RUN_DIR/etc/your-path ]]; then if [[ -n $MY_PATH ]]; then - MY_PATH="$SUITE_RUN_DIR/etc/your-path:$MY_PATH" + MY_PATH="$FLOW_RUN_DIR/etc/your-path:$MY_PATH" else - MY_PATH="$SUITE_RUN_DIR/etc/your-path" + MY_PATH="$FLOW_RUN_DIR/etc/your-path" fi fi PREV_CYCLE= for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do TEST_KEY=$TEST_KEY_BASE-file-$CYCLE TASK=my_task_1 - FILE=$HOME/cylc-run/$NAME/log/job/$CYCLE/$TASK/01/job.txt + FILE="$FLOW_RUN_DIR/log/job/$CYCLE/$TASK/01/job.txt" file_test "$TEST_KEY" $FILE - file_grep "$TEST_KEY-ROSE_SUITE_DIR" "ROSE_SUITE_DIR=$SUITE_RUN_DIR" $FILE + file_grep "$TEST_KEY-ROSE_SUITE_DIR" "ROSE_SUITE_DIR=$FLOW_RUN_DIR" $FILE file_grep "$TEST_KEY-ROSE_SUITE_DIR_REL" \ - "ROSE_SUITE_DIR_REL=${SUITE_RUN_DIR#$HOME/}" $FILE - file_grep "$TEST_KEY-ROSE_SUITE_NAME" "ROSE_SUITE_NAME=$NAME" $FILE + "ROSE_SUITE_DIR_REL=${FLOW_RUN_DIR#$HOME/}" $FILE + file_grep "$TEST_KEY-ROSE_SUITE_NAME" "ROSE_SUITE_NAME=$FLOW" $FILE file_grep "$TEST_KEY-ROSE_TASK_NAME" "ROSE_TASK_NAME=$TASK" $FILE file_grep "$TEST_KEY-ROSE_TASK_CYCLE_TIME" \ "ROSE_TASK_CYCLE_TIME=$CYCLE" $FILE @@ -64,19 +73,19 @@ for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do "ROSE_TASK_LOG_DIR=${FILE%/job.txt}" $FILE file_grep "$TEST_KEY-ROSE_TASK_LOG_ROOT" \ "ROSE_TASK_LOG_ROOT=${FILE%/job.txt}/job" $FILE - file_grep "$TEST_KEY-ROSE_DATA" "ROSE_DATA=$SUITE_RUN_DIR/share/data" $FILE + file_grep "$TEST_KEY-ROSE_DATA" "ROSE_DATA=$FLOW_RUN_DIR/share/data" $FILE file_grep "$TEST_KEY-ROSE_DATAC" \ - "ROSE_DATAC=$SUITE_RUN_DIR/share/cycle/$CYCLE" $FILE + "ROSE_DATAC=$FLOW_RUN_DIR/share/cycle/$CYCLE" $FILE if [[ -n $PREV_CYCLE ]]; then file_grep "$TEST_KEY-ROSE_DATACT12H" \ - "ROSE_DATACT12H=$SUITE_RUN_DIR/share/cycle/$PREV_CYCLE" $FILE + "ROSE_DATACT12H=$FLOW_RUN_DIR/share/cycle/$PREV_CYCLE" $FILE fi - file_grep "$TEST_KEY-ROSE_ETC" "ROSE_ETC=$SUITE_RUN_DIR/etc" $FILE + file_grep "$TEST_KEY-ROSE_ETC" "ROSE_ETC=$FLOW_RUN_DIR/etc" $FILE file_grep "$TEST_KEY-ROSE_TASK_PREFIX" "ROSE_TASK_PREFIX=my" $FILE file_grep "$TEST_KEY-ROSE_TASK_SUFFIX" "ROSE_TASK_SUFFIX=1" $FILE file_grep "$TEST_KEY-MY_PATH" "MY_PATH=$MY_PATH" $FILE PREV_CYCLE=$CYCLE done #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/00-run-basic/suite.rc b/t/rose-task-run/00-run-basic/flow.cylc similarity index 93% rename from t/rose-task-run/00-run-basic/suite.rc rename to t/rose-task-run/00-run-basic/flow.cylc index 01dedfe8e1..92b9a26a7c 100644 --- a/t/rose-task-run/00-run-basic/suite.rc +++ b/t/rose-task-run/00-run-basic/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT2M diff --git a/t/rose-task-run/01-run-basic-iso.t b/t/rose-task-run/01-run-basic-iso.t index 0098c17718..989a22a6eb 100755 --- a/t/rose-task-run/01-run-basic-iso.t +++ b/t/rose-task-run/01-run-basic-iso.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,47 +23,49 @@ export ROSE_CONF_PATH= +tests 48 + #------------------------------------------------------------------------------- # Run the suite. -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME -l \ - 1>/dev/null 2>&1 -if (($? != 0)); then - skip_all "cylc version not compatible with ISO 8601" - exit 0 -fi -#------------------------------------------------------------------------------- -tests 46 -#------------------------------------------------------------------------------- -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- MY_PATH= -for P in $(ls -d $SUITE_RUN_DIR/etc/my-path/*); do +for P in $(ls -d $FLOW_RUN_DIR/etc/my-path/*); do if [[ -n $MY_PATH ]]; then MY_PATH="$P:$MY_PATH" else MY_PATH="$P" fi done -if [[ -d $SUITE_RUN_DIR/etc/your-path ]]; then +if [[ -d $FLOW_RUN_DIR/etc/your-path ]]; then if [[ -n $MY_PATH ]]; then - MY_PATH="$SUITE_RUN_DIR/etc/your-path:$MY_PATH" + MY_PATH="$FLOW_RUN_DIR/etc/your-path:$MY_PATH" else - MY_PATH="$SUITE_RUN_DIR/etc/your-path" + MY_PATH="$FLOW_RUN_DIR/etc/your-path" fi fi PREV_CYCLE= for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do TEST_KEY=$TEST_KEY_BASE-file-$CYCLE TASK=my_task_1 - FILE=$HOME/cylc-run/$NAME/log/job/$CYCLE/$TASK/01/job.txt + FILE="$FLOW_RUN_DIR/log/job/$CYCLE/$TASK/01/job.txt" file_test "$TEST_KEY" $FILE - file_grep "$TEST_KEY-ROSE_SUITE_DIR" "ROSE_SUITE_DIR=$SUITE_RUN_DIR" $FILE + file_grep "$TEST_KEY-ROSE_SUITE_DIR" "ROSE_SUITE_DIR=$FLOW_RUN_DIR" $FILE file_grep "$TEST_KEY-ROSE_SUITE_DIR_REL" \ - "ROSE_SUITE_DIR_REL=${SUITE_RUN_DIR#$HOME/}" $FILE - file_grep "$TEST_KEY-ROSE_SUITE_NAME" "ROSE_SUITE_NAME=$NAME" $FILE + "ROSE_SUITE_DIR_REL=${FLOW_RUN_DIR#$HOME/}" $FILE + file_grep "$TEST_KEY-ROSE_SUITE_NAME" "ROSE_SUITE_NAME=${FLOW}" $FILE file_grep "$TEST_KEY-ROSE_TASK_NAME" "ROSE_TASK_NAME=$TASK" $FILE file_grep "$TEST_KEY-ROSE_TASK_CYCLE_TIME" \ "ROSE_TASK_CYCLE_TIME=$CYCLE" $FILE @@ -71,14 +73,14 @@ for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do "ROSE_TASK_LOG_DIR=${FILE%/job.txt}" $FILE file_grep "$TEST_KEY-ROSE_TASK_LOG_ROOT" \ "ROSE_TASK_LOG_ROOT=${FILE%job.txt}job" $FILE - file_grep "$TEST_KEY-ROSE_DATA" "ROSE_DATA=$SUITE_RUN_DIR/share/data" $FILE + file_grep "$TEST_KEY-ROSE_DATA" "ROSE_DATA=$FLOW_RUN_DIR/share/data" $FILE file_grep "$TEST_KEY-ROSE_DATAC" \ - "ROSE_DATAC=$SUITE_RUN_DIR/share/cycle/$CYCLE" $FILE + "ROSE_DATAC=$FLOW_RUN_DIR/share/cycle/$CYCLE" $FILE if [[ -n $PREV_CYCLE ]]; then file_grep "$TEST_KEY-ROSE_DATACPT12H" \ - "ROSE_DATACPT12H=$SUITE_RUN_DIR/share/cycle/$PREV_CYCLE" $FILE + "ROSE_DATACPT12H=$FLOW_RUN_DIR/share/cycle/$PREV_CYCLE" $FILE fi - file_grep "$TEST_KEY-ROSE_ETC" "ROSE_ETC=$SUITE_RUN_DIR/etc" $FILE + file_grep "$TEST_KEY-ROSE_ETC" "ROSE_ETC=$FLOW_RUN_DIR/etc" $FILE file_grep "$TEST_KEY-ROSE_TASK_PREFIX" "ROSE_TASK_PREFIX=my" $FILE file_grep "$TEST_KEY-ROSE_TASK_SUFFIX" "ROSE_TASK_SUFFIX=1" $FILE file_grep "$TEST_KEY-MY_PATH" "MY_PATH=$MY_PATH" $FILE @@ -89,13 +91,13 @@ NEXT_CYCLE= for CYCLE in 20130102T0000Z 20130101T1200Z 20130101T0000Z; do if [[ -n "${NEXT_CYCLE}" ]]; then TEST_KEY="${TEST_KEY_BASE}-file-${CYCLE}" - FILE="${HOME}/cylc-run/${NAME}/log/job/${CYCLE}/my_task_1/01/job.txt" + FILE="${HOME}/cylc-run/${FLOW}/log/job/${CYCLE}/my_task_1/01/job.txt" file_grep "${TEST_KEY}-ROSE_DATAC__PT12H" \ - "ROSE_DATAC__PT12H=${SUITE_RUN_DIR}/share/cycle/${NEXT_CYCLE}" \ + "ROSE_DATAC__PT12H=${FLOW_RUN_DIR}/share/cycle/${NEXT_CYCLE}" \ "${FILE}" fi NEXT_CYCLE="${CYCLE}" done #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/01-run-basic-iso/suite.rc b/t/rose-task-run/01-run-basic-iso/flow.cylc similarity index 92% rename from t/rose-task-run/01-run-basic-iso/suite.rc rename to t/rose-task-run/01-run-basic-iso/flow.cylc index 0026b5a4ca..67f86a67fd 100644 --- a/t/rose-task-run/01-run-basic-iso/suite.rc +++ b/t/rose-task-run/01-run-basic-iso/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT2M diff --git a/t/rose-task-run/02-env-basic/suite.rc b/t/rose-task-run/02-env-basic/flow.cylc similarity index 93% rename from t/rose-task-run/02-env-basic/suite.rc rename to t/rose-task-run/02-env-basic/flow.cylc index db616fbd1e..7bf0cebae0 100644 --- a/t/rose-task-run/02-env-basic/suite.rc +++ b/t/rose-task-run/02-env-basic/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT2M diff --git a/t/rose-task-run/03-env-basic-iso/suite.rc b/t/rose-task-run/03-env-basic-iso/flow.cylc similarity index 93% rename from t/rose-task-run/03-env-basic-iso/suite.rc rename to t/rose-task-run/03-env-basic-iso/flow.cylc index 10518d4a83..535a0d18b0 100644 --- a/t/rose-task-run/03-env-basic-iso/suite.rc +++ b/t/rose-task-run/03-env-basic-iso/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT2M diff --git a/t/rose-task-run/04-run-path-empty.t b/t/rose-task-run/04-run-path-empty.t index cda05f7ce5..32ef3842d8 100755 --- a/t/rose-task-run/04-run-path-empty.t +++ b/t/rose-task-run/04-run-path-empty.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,23 +24,32 @@ export ROSE_CONF_PATH= #------------------------------------------------------------------------------- -tests 3 +tests 5 #------------------------------------------------------------------------------- # Run the suite. -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- PREV_CYCLE= for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do TEST_KEY=$TEST_KEY_BASE-file-$CYCLE TASK=my_task_1 - FILE=$HOME/cylc-run/$NAME/log/job/$CYCLE/$TASK/01/job.txt + FILE="$FLOW_RUN_DIR/log/job/$CYCLE/$TASK/01/job.txt" file_grep "$TEST_KEY-PATH" \ - "PATH=$SUITE_RUN_DIR/app/$TASK/bin:$SUITE_RUN_DIR/etc/your-path" $FILE + "PATH=$FLOW_RUN_DIR/app/$TASK/bin:$FLOW_RUN_DIR/etc/your-path" $FILE PREV_CYCLE=$CYCLE done #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/04-run-path-empty/suite.rc b/t/rose-task-run/04-run-path-empty/flow.cylc similarity index 93% rename from t/rose-task-run/04-run-path-empty/suite.rc rename to t/rose-task-run/04-run-path-empty/flow.cylc index 717e782ce4..6a6f2e8504 100644 --- a/t/rose-task-run/04-run-path-empty/suite.rc +++ b/t/rose-task-run/04-run-path-empty/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT2M diff --git a/t/rose-task-run/06-app-prune-iso.t b/t/rose-task-run/06-app-prune-iso.t index dbefc972ba..3047bdb476 100755 --- a/t/rose-task-run/06-app-prune-iso.t +++ b/t/rose-task-run/06-app-prune-iso.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -20,38 +20,29 @@ # Test rose_prune built-in application, basic cycle housekeep usage. #------------------------------------------------------------------------------- . $(dirname $0)/test_header - - -#------------------------------------------------------------------------------- -# Test the suite. +skip_all 'TODO: #2445' JOB_HOST=$(rose config --default= 't' 'job-host') -if [[ -n $JOB_HOST ]]; then - JOB_HOST=$(rose host-select -q $JOB_HOST) +tests 9 +if [[ -z $JOB_HOST ]]; then + JOB_HOST=localhost fi -export CYLC_CONF_PATH= export ROSE_CONF_PATH= TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME -l \ - 1>/dev/null 2>&1 -if (($? != 0)); then - skip_all "cylc version not compatible with ISO 8601" - exit 0 -fi -#------------------------------------------------------------------------------- -tests 7 #------------------------------------------------------------------------------- -# Run the suite. -if [[ -n ${JOB_HOST:-} ]]; then - rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name \ + -S "HOST='${JOB_HOST}'" +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "$FLOW" \ + --abort-if-any-task-fails \ --host=localhost \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" -- --no-detach --debug -else - rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug -fi + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-prune-log sed '/^\[INFO\] \(create\|delete\|update\)/!d; @@ -59,8 +50,8 @@ sed '/^\[INFO\] \(create\|delete\|update\)/!d; /^\[INFO\] delete: \.rose-suite-log.lock/d; /\.json/d; /[0-9a-h]\{8\}\(-[0-9a-h]\{4\}\)\{3\}-[0-9a-h]\{12\}$/d' \ - $SUITE_RUN_DIR/prune.log >edited-prune.log -if [[ -n $JOB_HOST ]]; then + $FLOW_RUN_DIR/prune.log >edited-prune.log +if [[ $JOB_HOST != localhost ]]; then sed "s/\\\$JOB_HOST/$JOB_HOST/g" \ $TEST_SOURCE_DIR/$TEST_KEY_BASE.log >expected-prune.log else @@ -71,20 +62,20 @@ file_cmp "$TEST_KEY" expected-prune.log edited-prune.log #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-ls run_pass "$TEST_KEY" \ - ls $SUITE_RUN_DIR/log/job-*.tar.gz $SUITE_RUN_DIR/{log/job,share/cycle,work} -sed "s?\\\$SUITE_RUN_DIR?$SUITE_RUN_DIR?g" \ + ls $FLOW_RUN_DIR/log/job-*.tar.gz $FLOW_RUN_DIR/{log/job,share/cycle,work} +sed "s?\\\$SUITE_RUN_DIR?$FLOW_RUN_DIR?g" \ $TEST_SOURCE_DIR/$TEST_KEY.out >expected-ls.out -if [[ -z $JOB_HOST ]]; then +if [[ $JOB_HOST != localhost ]]; then sed -i "/my_task_2/d" expected-ls.out fi file_cmp "$TEST_KEY.out" "$TEST_KEY.out" expected-ls.out file_cmp "$TEST_KEY.err" "$TEST_KEY.err" expected-host-ls.out file_cmp "$TEST_KEY.out" "$TEST_KEY.out" expected-host-ls.out file_cmp "$TEST_KEY.err" "$TEST_KEY.err" rose_prune @@ -26,7 +25,7 @@ WARM[-PT12H]:finish-all => rose_prune [[WARM]] [[my_task_1]] inherit = WARM -{% if HOST is defined %} +{% if HOST != 'localhost' %} [[my_task_2]] inherit = WARM [[[remote]]] diff --git a/t/rose-task-run/07-app-arch.t b/t/rose-task-run/07-app-arch.t index bda24a78b6..291dd7a1ec 100755 --- a/t/rose-task-run/07-app-arch.t +++ b/t/rose-task-run/07-app-arch.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,38 +23,46 @@ #------------------------------------------------------------------------------- -tests 48 +tests 49 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "$TEST_KEY" \ + cylc play \ + "$FLOW" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug cp ${TEST_KEY}.err ~/temp cp ${TEST_KEY}.out ~/temp #------------------------------------------------------------------------------- # Results, good ones TEST_KEY="$TEST_KEY_BASE-find-foo" -(cd $SUITE_RUN_DIR; find foo -type f | LANG=C sort) >"$TEST_KEY.out" +(cd $FLOW_RUN_DIR; find foo -type f | LANG=C sort) >"$TEST_KEY.out" file_cmp "$TEST_KEY.out" "$TEST_SOURCE_DIR/$TEST_KEY.out" "$TEST_KEY.out" for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do TEST_KEY="$TEST_KEY_BASE-$CYCLE.out" sed '/^\[INFO\] [=!+]/!d; s/\(t(init)=\)[^Z]*Z/\1YYYY-mm-DDTHH:MM:SSZ/; s/\(dt(\(tran\|arch\))=\)[^s]*s/\1SSSSs/g' \ - $SUITE_RUN_DIR/log/job/$CYCLE/archive/0*/job.out >"$TEST_KEY" + $FLOW_RUN_DIR/log/job/$CYCLE/archive/0*/job.out >"$TEST_KEY" file_cmp "$TEST_KEY" "$TEST_KEY" $TEST_SOURCE_DIR/$TEST_KEY_BASE-$CYCLE.out TEST_KEY="$TEST_KEY_BASE-planet-n" - tar -tzf $SUITE_RUN_DIR/foo/$CYCLE/hello/worlds/planet-n.tar.gz | \ + tar -tzf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/planet-n.tar.gz | \ LANG=C sort >"$TEST_KEY-$CYCLE.out" file_cmp "$TEST_KEY-$CYCLE.out" \ "$TEST_KEY-$CYCLE.out" "$TEST_SOURCE_DIR/$TEST_KEY.out" TEST_KEY="$TEST_KEY_BASE-unknown-stuff" - tar -tf $SUITE_RUN_DIR/foo/$CYCLE/hello/worlds/unknown/stuff.pax | \ + tar -tf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/unknown/stuff.pax | \ LANG=C sort >"$TEST_KEY-$CYCLE.out" sed "s/\\\$CYCLE/$CYCLE/" "$TEST_SOURCE_DIR/$TEST_KEY.out" \ >"$TEST_KEY-$CYCLE.out.expected" @@ -62,27 +70,27 @@ for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do "$TEST_KEY-$CYCLE.out" "$TEST_KEY-$CYCLE.out.expected" TEST_KEY="$TEST_KEY_BASE-db" for TRY in 1 2; do - FILE=$SUITE_RUN_DIR/work/$CYCLE/archive/rose-arch-db-$TRY.out - sed "s?\\\$ROSE_DATAC?$SUITE_RUN_DIR/share/cycle/$CYCLE?" \ + FILE=$FLOW_RUN_DIR/work/$CYCLE/archive/rose-arch-db-$TRY.out + sed "s?\\\$ROSE_DATAC?$FLOW_RUN_DIR/share/cycle/$CYCLE?" \ "$TEST_SOURCE_DIR/$TEST_KEY-$CYCLE-$TRY.out" >$FILE.expected file_cmp "$TEST_KEY-$CYCLE.out" $FILE.expected $FILE done for KEY in dark-matter.txt jupiter.txt try.nl uranus.txt; do TEST_KEY="$TEST_KEY_BASE-$CYCLE-grep-$KEY-foo-log-2" - file_grep "$TEST_KEY" $KEY $SUITE_RUN_DIR/foo.log.$CYCLE.2 + file_grep "$TEST_KEY" $KEY $FLOW_RUN_DIR/foo.log.$CYCLE.2 done - if test $(wc -l <$SUITE_RUN_DIR/foo.log.$CYCLE.2) -eq 4; then + if test $(wc -l <$FLOW_RUN_DIR/foo.log.$CYCLE.2) -eq 4; then pass "$TEST_KEY_BASE-$CYCLE-foo-log-2-wc-l" else fail "$TEST_KEY_BASE-$CYCLE-foo-log-2-wc-l" fi TEST_KEY="$TEST_KEY_BASE-$CYCLE-neptune-1.txt" file_cmp "$TEST_KEY" \ - $SUITE_RUN_DIR/foo/$CYCLE/hello/worlds/neptune-1.txt <<__TXT__ + $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/neptune-1.txt <<__TXT__ [$CYCLE] Greet Triton __TXT__ TEST_KEY="$TEST_KEY_BASE-$CYCLE-jupiter-moons.tar.gz" - tar -xzf $SUITE_RUN_DIR/foo/$CYCLE/hello/worlds/jupiter-moons.tar.gz -O \ + tar -xzf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/jupiter-moons.tar.gz -O \ >"$TEST_KEY.out" file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__TXT__ [$CYCLE] Greet Io @@ -92,7 +100,7 @@ done # Results, bad ones CYCLE=20130101T1200Z TEST_KEY="$TEST_KEY_BASE-bad-archive-1" -FILE_PREFIX="$SUITE_RUN_DIR/log/job/$CYCLE/archive_bad_" +FILE_PREFIX="$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_" sed '/^\[FAIL\] /!d' "${FILE_PREFIX}1/01/job.err" >"${TEST_KEY}.err" file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' [FAIL] foo://20130101T1200Z/hello/worlds/planet-n.tar.gz: bad command-format: foo put %(target)s %(source)s: KeyError: 'source' @@ -125,15 +133,20 @@ __ERR__ TEST_KEY="$TEST_KEY_BASE-bad-archive-5" sed '/^\[FAIL\] /!d' "${FILE_PREFIX}5/01/job.err" >"${TEST_KEY}.err" file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ -[FAIL] [arch:inner.tar.gz]source=hello/mercurry.txt: configuration value error: [Errno 2] No such file or directory: '${SUITE_RUN_DIR}/share/cycle/20130101T1200Z/hello/mercurry.txt' +[FAIL] [arch:inner.tar.gz]source=hello/mercurry.txt: configuration value error: [Errno 2] No such file or directory: '${FLOW_RUN_DIR}/share/cycle/20130101T1200Z/hello/mercurry.txt' [FAIL] ! foo://20130101T1200Z/hello/worlds/inner.tar.gz [compress=tar.gz] [FAIL] ! hello/venus.txt (hello/venus.txt) __ERR__ TEST_KEY="$TEST_KEY_BASE-bad-archive-6" -sed '/^\[FAIL\] /!d; s/ \[compress.*\]$//' "$TEST_KEY.err" \ +# NOTE: /BASH_XTRACEFD/d is to remove any Cylc errors of the form: +# BASH_XTRACEFD: 19: invalid value for trace file descriptor +sed \ + -e '/^\[FAIL\] /!d; s/ \[compress.*\]$//' \ + -e '/BASH_XTRACEFD/d' \ + "$TEST_KEY.err" \ "${FILE_PREFIX}6/01/job.err" >"$TEST_KEY.err" file_cmp "$TEST_KEY.err" "$TEST_KEY.err" <<__ERR__ -[FAIL] foo push foo://20130101T1200Z/hello/worlds/mars.txt.gz $SUITE_RUN_DIR/share/cycle/20130101T1200Z/hello/mars.txt # return-code=1, stderr= +[FAIL] foo push foo://20130101T1200Z/hello/worlds/mars.txt.gz $FLOW_RUN_DIR/share/cycle/20130101T1200Z/hello/mars.txt # return-code=1, stderr= [FAIL] foo: push: unknown action [FAIL] ! foo://20130101T1200Z/hello/worlds/mars.txt.gz [FAIL] ! hello/mars.txt (hello/mars.txt) @@ -152,14 +165,15 @@ TEST_KEY="$TEST_KEY_BASE-bad-archive-9" sed '/^\[INFO\] [=!+]/!d; s/\(t(init)=\)[^Z]*Z/\1YYYY-mm-DDTHH:MM:SSZ/; s/\(dt(\(tran\|arch\))=\)[^s]*s/\1SSSSs/g' \ - "$SUITE_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.out" >"$TEST_KEY.out" + "$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.out" >"$TEST_KEY.out" file_cmp "$TEST_KEY.out" \ "$TEST_SOURCE_DIR/$TEST_KEY_BASE-bad-9.out" "$TEST_KEY.out" sed -e '/^\[FAIL\] /!d' \ -e 's/^\(\[FAIL\] my-bad-command\) .*\( # return-code=1, stderr=\)$/\1\2/' \ -e '/^\[FAIL\] \[my-bad-command\]/d' \ -e 's/ \[compress.*]$//' \ - "$SUITE_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.err" >"$TEST_KEY.err" + -e '/BASH_XTRACEFD/d' \ + "$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.err" >"$TEST_KEY.err" file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ [FAIL] ! foo://20130101T1200Z/planet-n.tar.gz [FAIL] ! hello/dark-matter.txt (hello/dark-matter.txt) @@ -194,5 +208,5 @@ file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ __ERR__ #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/07-app-arch/app/archive_bad_9/bin/my-bad-command b/t/rose-task-run/07-app-arch/app/archive_bad_9/bin/my-bad-command index 6d7f602b62..7abc5524da 100755 --- a/t/rose-task-run/07-app-arch/app/archive_bad_9/bin/my-bad-command +++ b/t/rose-task-run/07-app-arch/app/archive_bad_9/bin/my-bad-command @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash THIS=$(basename $0) echo "[$THIS] $*" >&2 exit 1 diff --git a/t/rose-task-run/07-app-arch/app/install/bin/my-install b/t/rose-task-run/07-app-arch/app/install/bin/my-install index f821bbbe5f..4db149c402 100755 --- a/t/rose-task-run/07-app-arch/app/install/bin/my-install +++ b/t/rose-task-run/07-app-arch/app/install/bin/my-install @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu PREFIX= if [[ ${ROSE_TASK_CYCLE_TIME:-} ]]; then diff --git a/t/rose-task-run/07-app-arch/bin/foo b/t/rose-task-run/07-app-arch/bin/foo index 678b844e3f..532940fe39 100755 --- a/t/rose-task-run/07-app-arch/bin/foo +++ b/t/rose-task-run/07-app-arch/bin/foo @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-task-run/07-app-arch/suite.rc b/t/rose-task-run/07-app-arch/flow.cylc similarity index 97% rename from t/rose-task-run/07-app-arch/suite.rc rename to t/rose-task-run/07-app-arch/flow.cylc index 55c6a6cfc2..0787739379 100644 --- a/t/rose-task-run/07-app-arch/suite.rc +++ b/t/rose-task-run/07-app-arch/flow.cylc @@ -2,7 +2,6 @@ {% set bad_nums=["1", "2", "3", "4", "5", "6", "7", "8", "9"] %} [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/08-app-fcm-make.t b/t/rose-task-run/08-app-fcm-make.t index d23e732a70..9675d6f019 100755 --- a/t/rose-task-run/08-app-fcm-make.t +++ b/t/rose-task-run/08-app-fcm-make.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -20,6 +20,7 @@ # Test fcm_make built-in application, basic usages. #------------------------------------------------------------------------------- . $(dirname $0)/test_header +skip_all 'TODO: #2445' if ! fcm help make 1>/dev/null 2>&1; then skip_all '"fcm make" unavailable' @@ -28,7 +29,7 @@ if ! gfortran --version 1>/dev/null 2>&1; then skip_all '"gfortran" unavailable' fi #------------------------------------------------------------------------------- -tests 8 +tests 10 #------------------------------------------------------------------------------- JOB_HOST=$(rose config --default= 't' 'job-host') if [[ -n $JOB_HOST ]]; then @@ -37,22 +38,28 @@ fi #------------------------------------------------------------------------------- # Run the suite. export ROSE_CONF_PATH= -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) +get_reg +OPTS=( + "--flow-name=${FLOW}" + "--no-run-name" +) if [[ -n ${JOB_HOST:-} ]]; then - timeout 60 rose suite-run -q \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" -- --no-detach --debug -else - timeout 60 rose suite-run -q \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' -- --no-detach --debug + OPTS+=(-S "HOST='$JOB_HOST'") fi +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + "${OPTS[@]}" +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-status" -sqlite3 $SUITE_RUN_DIR/log/db \ +sqlite3 $FLOW_RUN_DIR/log/db \ 'SELECT name,status FROM task_states ORDER BY name;' \ >"$TEST_KEY" if [[ -n $JOB_HOST ]]; then @@ -75,40 +82,40 @@ fi #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-t1" # normal file_grep "$TEST_KEY.out" \ - "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$SUITE_RUN_DIR/work/1/fcm_make_t1/fcm-make.cfg -C .*$SUITE_RUN_DIR/share/fcm_make_t1 -j 4" \ - $SUITE_RUN_DIR/log/job/1/fcm_make_t1/01/job.out + "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$FLOW_RUN_DIR/work/1/fcm_make_t1/fcm-make.cfg -C .*$FLOW_RUN_DIR/share/fcm_make_t1 -j 4" \ + $FLOW_RUN_DIR/log/job/1/fcm_make_t1/01/job.out #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-t2" # use-pwd file_grep "$TEST_KEY.out" "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -j 4" \ - $SUITE_RUN_DIR/log/job/1/fcm_make_t2/01/job.out + $FLOW_RUN_DIR/log/job/1/fcm_make_t2/01/job.out #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-t3" # opt.jobs file_grep "$TEST_KEY.out" \ - "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$SUITE_RUN_DIR/work/1/fcm_make_t3/fcm-make.cfg -C .*$SUITE_RUN_DIR/share/fcm_make_t3 -j 1" \ - $SUITE_RUN_DIR/log/job/1/fcm_make_t3/01/job.out + "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$FLOW_RUN_DIR/work/1/fcm_make_t3/fcm-make.cfg -C .*$FLOW_RUN_DIR/share/fcm_make_t3 -j 1" \ + $FLOW_RUN_DIR/log/job/1/fcm_make_t3/01/job.out #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-t4" # args file_grep "$TEST_KEY.out" \ - "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$SUITE_RUN_DIR/work/1/fcm_make_t4/fcm-make.cfg -C .*$SUITE_RUN_DIR/share/fcm_make_t4 -j 4 -v -v" \ - $SUITE_RUN_DIR/log/job/1/fcm_make_t4/01/job.out + "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$FLOW_RUN_DIR/work/1/fcm_make_t4/fcm-make.cfg -C .*$FLOW_RUN_DIR/share/fcm_make_t4 -j 4 -v -v" \ + $FLOW_RUN_DIR/log/job/1/fcm_make_t4/01/job.out #------------------------------------------------------------------------------- if [[ -z $JOB_HOST ]]; then skip 3 '[t]job-host not defined' else TEST_KEY="$TEST_KEY_BASE-t5" # mirror file_grep "$TEST_KEY.out.env" \ - "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* export ROSE_TASK_MIRROR_TARGET=$JOB_HOST:cylc-run/$NAME/share/fcm_make_t5" \ - $SUITE_RUN_DIR/log/job/1/fcm_make_t5/01/job.out + "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* export ROSE_TASK_MIRROR_TARGET=$JOB_HOST:cylc-run/$FLOW/share/fcm_make_t5" \ + $FLOW_RUN_DIR/log/job/1/fcm_make_t5/01/job.out file_grep "$TEST_KEY.out.cmd" \ - "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$SUITE_RUN_DIR/work/1/fcm_make_t5/fcm-make.cfg -C .*$SUITE_RUN_DIR/share/fcm_make_t5 -j 4 mirror\\.target=${JOB_HOST}:cylc-run/${NAME}/share/fcm_make_t5" \ - $SUITE_RUN_DIR/log/job/1/fcm_make_t5/01/job.out + "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -f .*$FLOW_RUN_DIR/work/1/fcm_make_t5/fcm-make.cfg -C .*$FLOW_RUN_DIR/share/fcm_make_t5 -j 4 mirror\\.target=${JOB_HOST}:cylc-run/${FLOW}/share/fcm_make_t5" \ + $FLOW_RUN_DIR/log/job/1/fcm_make_t5/01/job.out TEST_KEY="$TEST_KEY_BASE-t5-part-2" - rose suite-log -q --name=$NAME --update fcm_make2_t5 + rose suite-log -q --name=$FLOW --update fcm_make2_t5 file_grep "$TEST_KEY.out" \ - "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -C .*/cylc-run/${NAME}/share/fcm_make_t5 -n 2 -j 4" \ - $SUITE_RUN_DIR/log/job/1/fcm_make2_t5/01/job.out + "\\[INFO\\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]* fcm make -C .*/cylc-run/${FLOW}/share/fcm_make_t5 -n 2 -j 4" \ + $FLOW_RUN_DIR/log/job/1/fcm_make2_t5/01/job.out fi #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/08-app-fcm-make/suite.rc b/t/rose-task-run/08-app-fcm-make/flow.cylc similarity index 94% rename from t/rose-task-run/08-app-fcm-make/suite.rc rename to t/rose-task-run/08-app-fcm-make/flow.cylc index d555cd102b..d7e98a3eed 100644 --- a/t/rose-task-run/08-app-fcm-make/suite.rc +++ b/t/rose-task-run/08-app-fcm-make/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT5M diff --git a/t/rose-task-run/09-app-prune-work.t b/t/rose-task-run/09-app-prune-work.t index 04a6a44082..211566d795 100755 --- a/t/rose-task-run/09-app-prune-work.t +++ b/t/rose-task-run/09-app-prune-work.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,24 +23,32 @@ #------------------------------------------------------------------------------- -tests 2 +tests 3 #------------------------------------------------------------------------------- export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "$TEST_KEY" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY="$TEST_KEY_BASE-prune.log" sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0-9]*/YYYY-MM-DDTHHMM/g'\ - "${SUITE_RUN_DIR}/prune.log" > stamp-removed.log + "${FLOW_RUN_DIR}/prune.log" > stamp-removed.log sed '/^\[INFO\] YYYY-MM-DDTHHMM export ROSE_TASK_CYCLE_TIME=/p; /^\[INFO\] YYYY-MM-DDTHHMM delete: /!d' \ stamp-removed.log >edited-prune.log file_cmp "$TEST_KEY" "$TEST_SOURCE_DIR/$TEST_KEY_BASE.log" edited-prune.log #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/09-app-prune-work/suite.rc b/t/rose-task-run/09-app-prune-work/flow.cylc similarity index 94% rename from t/rose-task-run/09-app-prune-work/suite.rc rename to t/rose-task-run/09-app-prune-work/flow.cylc index e7813e2be8..917c352398 100644 --- a/t/rose-task-run/09-app-prune-work/suite.rc +++ b/t/rose-task-run/09-app-prune-work/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/10-specify-cycle.t b/t/rose-task-run/10-specify-cycle.t index 7a9e91fc0d..4e35070423 100755 --- a/t/rose-task-run/10-specify-cycle.t +++ b/t/rose-task-run/10-specify-cycle.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,15 +24,24 @@ export ROSE_CONF_PATH= #------------------------------------------------------------------------------- -tests 1 +tests 3 #------------------------------------------------------------------------------- # Run the suite. -TEST_KEY=$TEST_KEY_BASE -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug +TEST_KEY="$TEST_KEY_BASE" +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name=$FLOW \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +file_cmp "$TEST_KEY" "$FLOW_RUN_DIR/file" <<<'20121231T1200Z' #------------------------------------------------------------------------------- -file_cmp "$TEST_KEY" "$SUITE_RUN_DIR/file" <<<'20121231T1200Z' -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/10-specify-cycle/suite.rc b/t/rose-task-run/10-specify-cycle/flow.cylc similarity index 91% rename from t/rose-task-run/10-specify-cycle/suite.rc rename to t/rose-task-run/10-specify-cycle/flow.cylc index dbc8660c52..c1cbbcffd0 100644 --- a/t/rose-task-run/10-specify-cycle/suite.rc +++ b/t/rose-task-run/10-specify-cycle/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails = True [[events]] abort on timeout = True timeout=PT1M diff --git a/t/rose-task-run/11-specify-cycle-iso.t b/t/rose-task-run/11-specify-cycle-iso.t index 73aab90d7b..0b5d5c14bf 100755 --- a/t/rose-task-run/11-specify-cycle-iso.t +++ b/t/rose-task-run/11-specify-cycle-iso.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -25,21 +25,22 @@ export ROSE_CONF_PATH= #------------------------------------------------------------------------------- # Run the suite. +tests 3 TEST_KEY=$TEST_KEY_BASE -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME -l \ - 1>/dev/null 2>&1 -if (($? != 0)); then - skip_all "cylc version not compatible with ISO 8601" - exit 0 -fi +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- -tests 1 -#------------------------------------------------------------------------------- -rose suite-run -q -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug -#------------------------------------------------------------------------------- -file_cmp "$TEST_KEY" "$SUITE_RUN_DIR/file" <<<'20121231T1200Z' -rose suite-clean -q -y $NAME +file_cmp "$TEST_KEY" "$FLOW_RUN_DIR/file" <<<'20121231T1200Z' +cylc clean "$FLOW" exit 0 diff --git a/t/rose-task-run/11-specify-cycle-iso/suite.rc b/t/rose-task-run/11-specify-cycle-iso/flow.cylc similarity index 91% rename from t/rose-task-run/11-specify-cycle-iso/suite.rc rename to t/rose-task-run/11-specify-cycle-iso/flow.cylc index 6a7b1305aa..cef73734ac 100644 --- a/t/rose-task-run/11-specify-cycle-iso/suite.rc +++ b/t/rose-task-run/11-specify-cycle-iso/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/12-app-prune-integer.t b/t/rose-task-run/12-app-prune-integer.t index 11ec65c684..16f7e3d47d 100755 --- a/t/rose-task-run/12-app-prune-integer.t +++ b/t/rose-task-run/12-app-prune-integer.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,60 +21,68 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header +skip_all 'TODO: #2445' + #------------------------------------------------------------------------------- JOB_HOST=$(rose config --default= 't' 'job-host') if [[ -n $JOB_HOST ]]; then JOB_HOST=$(rose host-select -q $JOB_HOST) - tests 15 + tests 16 else - tests 12 + tests 13 fi #------------------------------------------------------------------------------- # Run the suite. export CYLC_CONF_PATH= export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) +get_reg +OPTS=( + "--flow-name=$FLOW" + --no-run-name +) if [[ -n ${JOB_HOST:-} ]]; then - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -D "[jinja2:suite.rc]HOST=\"$JOB_HOST\"" \ - -- --no-detach --debug -else - run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -- --no-detach --debug + OPTS+=( + -S "HOST='$JOB_HOST'" + ) fi +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "${TEST_KEY}" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + "${OPTS[@]}" +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-work -run_fail "$TEST_KEY.1" ls -d $HOME/cylc-run/$NAME/work/1 -run_fail "$TEST_KEY.2" ls -d $HOME/cylc-run/$NAME/work/2 -run_pass "$TEST_KEY.3" ls -d $HOME/cylc-run/$NAME/work/3 +run_fail "$TEST_KEY.1" ls -d "$FLOW_RUN_DIR/work/1" +run_fail "$TEST_KEY.2" ls -d "$FLOW_RUN_DIR/work/2" +run_pass "$TEST_KEY.3" ls -d "$FLOW_RUN_DIR/work/3" #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-share -run_fail "$TEST_KEY.1" ls -d $HOME/cylc-run/$NAME/share/cycle/1 -run_fail "$TEST_KEY.2" ls -d $HOME/cylc-run/$NAME/share/cycle/2 -run_pass "$TEST_KEY.3" ls -d $HOME/cylc-run/$NAME/share/cycle/3 +run_fail "$TEST_KEY.1" ls -d "$FLOW_RUN_DIR/share/cycle/1" +run_fail "$TEST_KEY.2" ls -d "$FLOW_RUN_DIR/share/cycle/2" +run_pass "$TEST_KEY.3" ls -d "$FLOW_RUN_DIR/share/cycle/3" #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-archive TEST_KEY=$TEST_KEY_BASE-share -run_fail "$TEST_KEY.1" ls -d $HOME/cylc-run/$NAME/log/job/1 -run_pass "$TEST_KEY.1-tar" ls -d $HOME/cylc-run/$NAME/log/job-1.tar.gz -run_fail "$TEST_KEY.2" ls -d $HOME/cylc-run/$NAME/log/job/2 -run_pass "$TEST_KEY.2-tar" ls -d $HOME/cylc-run/$NAME/log/job-2.tar.gz -run_pass "$TEST_KEY.3" ls -d $HOME/cylc-run/$NAME/log/job/3 +run_fail "$TEST_KEY.1" ls -d "$FLOW_RUN_DIR/log/job/1" +run_pass "$TEST_KEY.1-tar" ls -d "$FLOW_RUN_DIR/log/job-1.tar.gz" +run_fail "$TEST_KEY.2" ls -d "$FLOW_RUN_DIR/log/job/2" +run_pass "$TEST_KEY.2-tar" ls -d "$FLOW_RUN_DIR/log/job-2.tar.gz" +run_pass "$TEST_KEY.3" ls -d "$FLOW_RUN_DIR/log/job/3" #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-remote if [[ -n "$JOB_HOST" ]]; then - run_fail "$TEST_KEY.1" ssh "$JOB_HOST" "ls -d ~/cylc-run/$NAME/log/job/1" - run_fail "$TEST_KEY.2" ssh "$JOB_HOST" "ls -d ~/cylc-run/$NAME/log/job/2" - run_pass "$TEST_KEY.3" ssh "$JOB_HOST" "ls -d ~/cylc-run/$NAME/log/job/3" + run_fail "$TEST_KEY.1" ssh "$JOB_HOST" "ls -d $FLOW_RUN_DIR/log/job/1" + run_fail "$TEST_KEY.2" ssh "$JOB_HOST" "ls -d $FLOW_RUN_DIW/log/job/2" + run_pass "$TEST_KEY.3" ssh "$JOB_HOST" "ls -d $FLOW_RUN_DIW/log/job/3" fi #------------------------------------------------------------------------------- -rose suite-clean -y --name=$NAME -#------------------------------------------------------------------------------- +purge exit 0 diff --git a/t/rose-task-run/12-app-prune-integer/suite.rc b/t/rose-task-run/12-app-prune-integer/flow.cylc similarity index 93% rename from t/rose-task-run/12-app-prune-integer/suite.rc rename to t/rose-task-run/12-app-prune-integer/flow.cylc index 803c65b538..ff953af4fd 100644 --- a/t/rose-task-run/12-app-prune-integer/suite.rc +++ b/t/rose-task-run/12-app-prune-integer/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT1M diff --git a/t/rose-task-run/13-app-arch-cmd-out.t b/t/rose-task-run/13-app-arch-cmd-out.t index a55d778901..0b35a7f830 100755 --- a/t/rose-task-run/13-app-arch-cmd-out.t +++ b/t/rose-task-run/13-app-arch-cmd-out.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,31 +23,39 @@ #------------------------------------------------------------------------------- -tests 2 +tests 4 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export CYLC_CONF_PATH= export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d "${HOME}/cylc-run/rose-test-battery.XXXXXX")" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- -FOO_UUID=$(<"${SUITE_RUN_DIR}/foo-uuid") -grep -F "${FOO_UUID}" "${SUITE_RUN_DIR}/log/job/1/archive/NN/job.out" \ +FOO_UUID=$(<"${FLOW_RUN_DIR}/foo-uuid") +grep -F "${FOO_UUID}" "${FLOW_RUN_DIR}/log/job/1/archive/NN/job.out" \ >"${TEST_KEY_BASE}.out" # This ensures that the STDOUT of the "foo" command is only printed out once. file_cmp "${TEST_KEY_BASE}.out" "${TEST_KEY_BASE}.out" <<__OUT__ -[INFO] ${FOO_UUID} share/namelist/x.nl ${SUITE_RUN_DIR}/share/backup/x.nl +[INFO] ${FOO_UUID} share/namelist/x.nl ${FLOW_RUN_DIR}/share/backup/x.nl __OUT__ # This tests that the "foo" command has done what it is asked to do. file_cmp "${TEST_KEY_BASE}.content" \ - "${SUITE_RUN_DIR}/share/backup/x.nl" <<'__NL__' + "${FLOW_RUN_DIR}/share/backup/x.nl" <<'__NL__' &x MMXIV=2014, / __NL__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/13-app-arch-cmd-out/bin/foo b/t/rose-task-run/13-app-arch-cmd-out/bin/foo index 8e0a6b2aba..585fea083d 100755 --- a/t/rose-task-run/13-app-arch-cmd-out/bin/foo +++ b/t/rose-task-run/13-app-arch-cmd-out/bin/foo @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose-task-run/13-app-arch-cmd-out/suite.rc b/t/rose-task-run/13-app-arch-cmd-out/flow.cylc similarity index 89% rename from t/rose-task-run/13-app-arch-cmd-out/suite.rc rename to t/rose-task-run/13-app-arch-cmd-out/flow.cylc index 6218a9ec50..36e28dcd07 100644 --- a/t/rose-task-run/13-app-arch-cmd-out/suite.rc +++ b/t/rose-task-run/13-app-arch-cmd-out/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/14-app-prune-remove.t b/t/rose-task-run/14-app-prune-remove.t index cedbce1118..16d8977838 100755 --- a/t/rose-task-run/14-app-prune-remove.t +++ b/t/rose-task-run/14-app-prune-remove.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,33 +23,39 @@ #------------------------------------------------------------------------------- JOB_HOST=$(rose config --default= 't' 'job-host') -tests 10 +tests 11 #------------------------------------------------------------------------------- # Run the suite. export CYLC_CONF_PATH= export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost \ - -- --no-detach --debug + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "$TEST_KEY" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-log -run_fail "$TEST_KEY.1" ls -d $HOME/cylc-run/$NAME/log/job/20100101T0000Z -run_fail "$TEST_KEY.2" ls -d $HOME/cylc-run/$NAME/log/job/20100102T0000Z -run_fail "$TEST_KEY.3" ls -d $HOME/cylc-run/$NAME/log/job/20100103T0000Z -run_fail "$TEST_KEY.4" ls -d $HOME/cylc-run/$NAME/log/job/20100104T0000Z -run_pass "$TEST_KEY.5" ls -d $HOME/cylc-run/$NAME/log/job/20100105T0000Z +run_fail "$TEST_KEY.1" ls -d "$FLOW_RUN_DIR/log/job/20100101T0000Z" +run_fail "$TEST_KEY.2" ls -d "$FLOW_RUN_DIR/log/job/20100102T0000Z" +run_fail "$TEST_KEY.3" ls -d "$FLOW_RUN_DIR/log/job/20100103T0000Z" +run_fail "$TEST_KEY.4" ls -d "$FLOW_RUN_DIR/log/job/20100104T0000Z" +run_pass "$TEST_KEY.5" ls -d "$FLOW_RUN_DIR/log/job/20100105T0000Z" #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-archived -run_fail "$TEST_KEY.1" ls -d $HOME/cylc-run/$NAME/log/job-20100101T0000Z.tar.gz -run_fail "$TEST_KEY.2" ls -d $HOME/cylc-run/$NAME/log/job-20100102T0000Z.tar.gz -run_fail "$TEST_KEY.3" ls -d $HOME/cylc-run/$NAME/log/job-20100103T0000Z.tar.gz -run_pass "$TEST_KEY.4" ls -d $HOME/cylc-run/$NAME/log/job-20100104T0000Z.tar.gz -#------------------------------------------------------------------------------- -rose suite-clean -q -y --name=$NAME +run_fail "$TEST_KEY.1" ls -d "$FLOW_RUN_DIR/log/job-20100101T0000Z.tar.gz" +run_fail "$TEST_KEY.2" ls -d "$FLOW_RUN_DIR/log/job-20100102T0000Z.tar.gz" +run_fail "$TEST_KEY.3" ls -d "$FLOW_RUN_DIR/log/job-20100103T0000Z.tar.gz" +run_pass "$TEST_KEY.4" ls -d "$FLOW_RUN_DIR/log/job-20100104T0000Z.tar.gz" #------------------------------------------------------------------------------- +purge exit 0 diff --git a/t/rose-task-run/14-app-prune-remove/suite.rc b/t/rose-task-run/14-app-prune-remove/flow.cylc similarity index 92% rename from t/rose-task-run/14-app-prune-remove/suite.rc rename to t/rose-task-run/14-app-prune-remove/flow.cylc index 0eb0a0213d..733f1b5b7c 100644 --- a/t/rose-task-run/14-app-prune-remove/suite.rc +++ b/t/rose-task-run/14-app-prune-remove/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT1M diff --git a/t/rose-task-run/15-app-prune-extglob.t b/t/rose-task-run/15-app-prune-extglob.t index dbdeb0153c..6d52455b5b 100755 --- a/t/rose-task-run/15-app-prune-extglob.t +++ b/t/rose-task-run/15-app-prune-extglob.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,20 +21,29 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header -tests 2 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name \ + -S "JOB_HOST='localhost'" +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --no-detach \ + --debug TEST_KEY="${TEST_KEY_BASE}-prune.log" sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0-9]*/YYYY-MM-DDTHHMM/g'\ - "${SUITE_RUN_DIR}/prune.log" > stamp-removed.log + "${FLOW_RUN_DIR}/prune.log" > stamp-removed.log sed '/^\[INFO\] YYYY-MM-DDTHHMM export ROSE_TASK_CYCLE_TIME=/p; /^\[INFO\] YYYY-MM-DDTHHMM delete: /!d' \ "stamp-removed.log" >'edited-prune.log' @@ -50,5 +59,5 @@ file_cmp "${TEST_KEY}" 'edited-prune.log' <<__LOG__ [INFO] YYYY-MM-DDTHHMM delete: work/20150102T0000Z/creator/rose-app-run.conf __LOG__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/15-app-prune-extglob/flow.cylc b/t/rose-task-run/15-app-prune-extglob/flow.cylc new file mode 100644 index 0000000000..883d474049 --- /dev/null +++ b/t/rose-task-run/15-app-prune-extglob/flow.cylc @@ -0,0 +1,30 @@ +#!jinja2 + +[cylc] + UTC mode = True + [[events]] + abort on timeout = True + timeout = PT1M + +[scheduling] + initial cycle point = 20150101 + final cycle point = 20150103 + [[dependencies]] + [[[P1D]]] + graph = """ + creator[-P1D] => pruner => creator + """ + +[runtime] + [[root]] + execution time limit = PT1M + [[creator]] + script = rose task-run +{% if JOB_HOST != 'localhost' %} + [[[remote]]] + host = {{JOB_HOST}} +{% endif %} + [[pruner]] + script = """ + rose task-run -v -v --debug | tee -a "${CYLC_SUITE_RUN_DIR}/prune.log" + """ diff --git a/t/rose-task-run/15-app-prune-extglob/suite.rc b/t/rose-task-run/15-app-prune-extglob/suite.rc deleted file mode 100644 index 248885b806..0000000000 --- a/t/rose-task-run/15-app-prune-extglob/suite.rc +++ /dev/null @@ -1,30 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - abort if any task fails = True - [[events]] - abort on timeout = True - timeout=PT1M -[scheduling] - initial cycle point=20150101 - final cycle point=20150103 - [[dependencies]] - [[[P1D]]] - graph=""" -creator[-P1D] => pruner => creator -""" - -[runtime] - [[root]] - [[[job]]] - execution time limit=PT1M - [[creator]] - script=rose task-run -{% if JOB_HOST is defined %} - [[[remote]]] - host = {{JOB_HOST}} -{% endif %} - [[pruner]] - script=""" -rose task-run -v -v --debug | tee -a "${CYLC_SUITE_RUN_DIR}/prune.log" -""" diff --git a/t/rose-task-run/16-app-prune-point.t b/t/rose-task-run/16-app-prune-point.t index d2a3e6d381..19534f78cb 100755 --- a/t/rose-task-run/16-app-prune-point.t +++ b/t/rose-task-run/16-app-prune-point.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,21 +23,30 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header -tests 2 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' --debug -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name + +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --debug \ + --no-detach TEST_KEY="${TEST_KEY_BASE}-prune.log" sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0-9]*/YYYY-MM-DDTHHMM/g'\ - "${SUITE_RUN_DIR}/prune.log" > stamp-removed.log + "${FLOW_RUN_DIR}/prune.log" > stamp-removed.log sed '/^\[INFO\] YYYY-MM-DDTHHMM export ROSE_TASK_CYCLE_TIME=/p; /^\[INFO\] YYYY-MM-DDTHHMM delete: /!d' \ @@ -52,5 +61,5 @@ file_cmp "${TEST_KEY}" 'edited-prune.log' <<__LOG__ [INFO] YYYY-MM-DDTHHMM delete: work/19800101T0000Z __LOG__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/16-app-prune-point/suite.rc b/t/rose-task-run/16-app-prune-point/flow.cylc similarity index 95% rename from t/rose-task-run/16-app-prune-point/suite.rc rename to t/rose-task-run/16-app-prune-point/flow.cylc index 76e9de7701..3b332ad652 100644 --- a/t/rose-task-run/16-app-prune-point/suite.rc +++ b/t/rose-task-run/16-app-prune-point/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/17-app-prune-glob-as-cycle-fmt.t b/t/rose-task-run/17-app-prune-glob-as-cycle-fmt.t index cc17f56aca..a717b0d200 100755 --- a/t/rose-task-run/17-app-prune-glob-as-cycle-fmt.t +++ b/t/rose-task-run/17-app-prune-glob-as-cycle-fmt.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -22,21 +22,29 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header -tests 2 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' --debug -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --debug \ + --no-detach TEST_KEY="${TEST_KEY_BASE}-prune.log" sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]*/YYYY-MM-DDTHHMM/g'\ - "${SUITE_RUN_DIR}/prune.log" > stamp-removed.log + "${FLOW_RUN_DIR}/prune.log" > stamp-removed.log sed '/^\[INFO\] YYYY-MM-DDTHHMM export ROSE_TASK_CYCLE_TIME=/p; /^\[INFO\] YYYY-MM-DDTHHMM delete: /!d' \ "stamp-removed.log" >'edited-prune.log' @@ -47,5 +55,5 @@ file_cmp "${TEST_KEY}" 'edited-prune.log' <<__LOG__ [INFO] YYYY-MM-DDTHHMM delete: share/hello-venus-at-19700101T0000Z.txt __LOG__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/17-app-prune-glob-as-cycle-fmt/suite.rc b/t/rose-task-run/17-app-prune-glob-as-cycle-fmt/flow.cylc similarity index 95% rename from t/rose-task-run/17-app-prune-glob-as-cycle-fmt/suite.rc rename to t/rose-task-run/17-app-prune-glob-as-cycle-fmt/flow.cylc index f72a4962db..2cc6bf9b9c 100644 --- a/t/rose-task-run/17-app-prune-glob-as-cycle-fmt/suite.rc +++ b/t/rose-task-run/17-app-prune-glob-as-cycle-fmt/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/18-app-fcm-make-ctx-name.t b/t/rose-task-run/18-app-fcm-make-ctx-name.t index 6261591a02..c858347848 100755 --- a/t/rose-task-run/18-app-fcm-make-ctx-name.t +++ b/t/rose-task-run/18-app-fcm-make-ctx-name.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -27,6 +27,8 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header +skip_all 'TODO: #2445' + if ! fcm help make 1>/dev/null 2>&1; then skip_all '"fcm make" unavailable' fi @@ -45,21 +47,28 @@ if [[ -z "${JOB_HOST}" ]]; then skip_all '"[t]job-host" not defined or not available' fi #------------------------------------------------------------------------------- -tests 1 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" #------------------------------------------------------------------------------- -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -timeout 120 rose suite-run -v -v --debug \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' \ - -D "[jinja2:suite.rc]HOST=\"${JOB_HOST}\"" \ - -- --no-detach --debug 1>'/dev/null' 2>&1 +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name \ + -S "HOST='${JOB_HOST}'" +run_pass "${TEST_KEY_BASE}-play" \ + timeout 120 \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --no-detach \ + --debug #------------------------------------------------------------------------------- ssh -n -oBatchMode=yes "${JOB_HOST}" \ - cat "cylc-run/${NAME}/share/hello.txt" >'hello.txt' + cat "cylc-run/${FLOW}/share/hello.txt" >'hello.txt' file_cmp "${TEST_KEY_BASE}" 'hello.txt' <<<'Hello World!' #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/18-app-fcm-make-ctx-name/suite.rc b/t/rose-task-run/18-app-fcm-make-ctx-name/flow.cylc similarity index 93% rename from t/rose-task-run/18-app-fcm-make-ctx-name/suite.rc rename to t/rose-task-run/18-app-fcm-make-ctx-name/flow.cylc index 33119f068c..8f02021fca 100644 --- a/t/rose-task-run/18-app-fcm-make-ctx-name/suite.rc +++ b/t/rose-task-run/18-app-fcm-make-ctx-name/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [scheduling] [[dependencies]] graph = """hello-make => hello-make-bin => hello-run""" diff --git a/t/rose-task-run/20-app-fcm-make-dest.t b/t/rose-task-run/20-app-fcm-make-dest.t index 45c388ec11..37a06ffd0c 100755 --- a/t/rose-task-run/20-app-fcm-make-dest.t +++ b/t/rose-task-run/20-app-fcm-make-dest.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,6 +24,7 @@ # host, as well as "gfortran" being installed and available there. #------------------------------------------------------------------------------- . $(dirname $0)/test_header +skip_all 'TODO: #2445' if ! fcm help make 1>/dev/null 2>&1; then skip_all '"fcm make" unavailable' @@ -48,31 +49,40 @@ else GREET= fi #------------------------------------------------------------------------------- -tests 1 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" #------------------------------------------------------------------------------- -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -timeout 120 rose suite-run -q --debug \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' \ - -S "HOST=\"${JOB_HOST}\"" -S "GREET=\"${GREET}\"" -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name \ + -S "HOST='${JOB_HOST}'" \ + -S "GREET='${GREET}'" +run_pass "${TEST_KEY_BASE}-play" \ + timeout 120 \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --no-detach \ + --debug \ + --host='localhost' #------------------------------------------------------------------------------- JOB_HOST_HOME=$(ssh -n -oBatchMode=yes "${JOB_HOST}" 'echo "${HOME}"' | tail -1) ssh -n -oBatchMode=yes "${JOB_HOST}" \ - cat "cylc-run/${NAME}/share/hello.txt" >'hello.txt' + cat "cylc-run/${FLOW}/share/hello.txt" >'hello.txt' if [[ -n "${GREET}" ]]; then file_cmp "${TEST_KEY_BASE}" 'hello.txt' <<__TXT__ -${JOB_HOST_HOME}/cylc-run/${NAME}/opt/greet/build/bin/hello +${JOB_HOST_HOME}/cylc-run/${FLOW}/opt/greet/build/bin/hello Hello World! __TXT__ else file_cmp "${TEST_KEY_BASE}" 'hello.txt' <<__TXT__ -${JOB_HOST_HOME}/cylc-run/${NAME}/opt/hello/build/bin/hello +${JOB_HOST_HOME}/cylc-run/${FLOW}/opt/hello/build/bin/hello Hello World! __TXT__ fi #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/20-app-fcm-make-dest/suite.rc b/t/rose-task-run/20-app-fcm-make-dest/flow.cylc similarity index 94% rename from t/rose-task-run/20-app-fcm-make-dest/suite.rc rename to t/rose-task-run/20-app-fcm-make-dest/flow.cylc index 1ea6092e94..5128a9b97b 100644 --- a/t/rose-task-run/20-app-fcm-make-dest/suite.rc +++ b/t/rose-task-run/20-app-fcm-make-dest/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [scheduling] [[dependencies]] graph = """hello-make => hello-make-bin => hello-run""" diff --git a/t/rose-task-run/24-app-fcm-make-fast.t b/t/rose-task-run/24-app-fcm-make-fast.t index f408e92205..4928975181 100755 --- a/t/rose-task-run/24-app-fcm-make-fast.t +++ b/t/rose-task-run/24-app-fcm-make-fast.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -32,43 +32,51 @@ if ! gfortran --version 1>/dev/null 2>&1; then skip_all '"gfortran" unavailable' fi #------------------------------------------------------------------------------- -tests 5 +tests 7 export CYLC_CONF_PATH= export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" mkdir 'fast' MTIME_OF_FAST_BEFORE=$(stat '-c%y' 'fast') #------------------------------------------------------------------------------- -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" -timeout 120 rose suite-run -q --debug \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' -S "FAST_DEST_ROOT=\"${PWD}/fast\"" \ - -- --no-detach --debug +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + timeout 120 \ + cylc play \ + "${FLOW}" \ + -s "FAST_DEST_ROOT='${PWD}/fast'" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --no-detach \ + --debug #------------------------------------------------------------------------------- # Permission modes of make directory should be the same as a normal directory -mkdir "${SUITE_RUN_DIR}/share/hello-make-perm-mode-test" +mkdir "${FLOW_RUN_DIR}/share/hello-make-perm-mode-test" run_pass "${TEST_KEY_BASE}-perm-mode" test \ - "$(stat -c'%a' "${SUITE_RUN_DIR}/share/hello-make-perm-mode-test")" = \ - "$(stat -c'%a' "${SUITE_RUN_DIR}/share/hello-make")" -rmdir "${SUITE_RUN_DIR}/share/hello-make-perm-mode-test" + "$(stat -c'%a' "${FLOW_RUN_DIR}/share/hello-make-perm-mode-test")" = \ + "$(stat -c'%a' "${FLOW_RUN_DIR}/share/hello-make")" +rmdir "${FLOW_RUN_DIR}/share/hello-make-perm-mode-test" # Executable runs OK -file_cmp "${TEST_KEY_BASE}" "${SUITE_RUN_DIR}/share/hello.txt" <<__TXT__ -${SUITE_RUN_DIR}/share/hello-make/build/bin/hello +file_cmp "${TEST_KEY_BASE}" "${FLOW_RUN_DIR}/share/hello.txt" <<__TXT__ +${FLOW_RUN_DIR}/share/hello-make/build/bin/hello Hello World! __TXT__ # Logs HOST=$(hostname) file_grep "${TEST_KEY_BASE}.log" \ - "\\[info\\] dest=${USER}@${HOST}:${PWD}/fast/hello-make.1.${NAME}" \ - "${SUITE_RUN_DIR}/share/hello-make/fcm-make.log" + "\\[info\\] dest=${USER}@${HOST}:${PWD}/fast/hello-make.1.${FLOW//\//_}" \ + "${FLOW_RUN_DIR}/share/hello-make/fcm-make.log" file_grep "${TEST_KEY_BASE}-bin.log" \ - "\\[info\\] dest=${USER}@${HOST}:${PWD}/fast/hello-make-bin.1.${NAME}" \ - "${SUITE_RUN_DIR}/share/hello-make/fcm-make-bin.log" + "\\[info\\] dest=${USER}@${HOST}:${PWD}/fast/hello-make-bin.1.${FLOW//\//_}" \ + "${FLOW_RUN_DIR}/share/hello-make/fcm-make-bin.log" # Prove that the fast directory has been modified MTIME_OF_FAST_AFTER=$(stat '-c%y' 'fast') run_pass "${TEST_KEY_BASE}-mtime-of-fast" \ test "${MTIME_OF_FAST_BEFORE}" '!=' "${MTIME_OF_FAST_AFTER}" #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/24-app-fcm-make-fast/suite.rc b/t/rose-task-run/24-app-fcm-make-fast/flow.cylc similarity index 88% rename from t/rose-task-run/24-app-fcm-make-fast/suite.rc rename to t/rose-task-run/24-app-fcm-make-fast/flow.cylc index 42516edbe7..0a62be3a06 100644 --- a/t/rose-task-run/24-app-fcm-make-fast/suite.rc +++ b/t/rose-task-run/24-app-fcm-make-fast/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [scheduling] [[dependencies]] graph = """hello-make => hello-make-bin => hello-run""" @@ -10,7 +9,7 @@ [[root]] [[MAKE]] script = """ -rose task-run --app-key=make \ +rose task-run --app-key=make -v -v --debug \ --define=fast-dest-root-orig={{FAST_DEST_ROOT}} \ --define=fast-dest-root-cont={{FAST_DEST_ROOT}} """ diff --git a/t/rose-task-run/25-app-fcm-make-new-mode.t b/t/rose-task-run/25-app-fcm-make-new-mode.t index f1741bd6fd..d465f264f8 100755 --- a/t/rose-task-run/25-app-fcm-make-new-mode.t +++ b/t/rose-task-run/25-app-fcm-make-new-mode.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -32,30 +32,38 @@ if ! gfortran --version 1>/dev/null 2>&1; then skip_all '"gfortran" unavailable' fi #------------------------------------------------------------------------------- -tests 3 +tests 5 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name # Add some garbage before running the suite -mkdir -p "${SUITE_RUN_DIR}/share/hello-make/junk2" -touch "${SUITE_RUN_DIR}/share/hello-make/junk1" +mkdir -p "${FLOW_RUN_DIR}/share/hello-make/junk2" +touch "${FLOW_RUN_DIR}/share/hello-make/junk1" -timeout 120 rose suite-run -q --debug \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' -- --no-detach --debug +run_pass "${TEST_KEY_BASE}-play" \ + timeout 120 \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --no-detach \ + --debug #------------------------------------------------------------------------------- -file_cmp "${TEST_KEY_BASE}" "${SUITE_RUN_DIR}/share/hello.txt" <<__TXT__ -${SUITE_RUN_DIR}/share/hello-make/build/bin/hello +file_cmp "${TEST_KEY_BASE}" "${FLOW_RUN_DIR}/share/hello.txt" <<__TXT__ +${FLOW_RUN_DIR}/share/hello-make/build/bin/hello Hello World! __TXT__ file_grep "${TEST_KEY_BASE}-fcm-make.log" \ - '\[info\] mode=new' "${SUITE_RUN_DIR}/share/hello-make/fcm-make.log" + '\[info\] mode=new' "${FLOW_RUN_DIR}/share/hello-make/fcm-make.log" run_fail "${TEST_KEY_BASE}" ls \ - "${SUITE_RUN_DIR}/share/hello-make/junk1" \ - "${SUITE_RUN_DIR}/share/hello-make/junk2" + "${FLOW_RUN_DIR}/share/hello-make/junk1" \ + "${FLOW_RUN_DIR}/share/hello-make/junk2" #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/25-app-fcm-make-new-mode/suite.rc b/t/rose-task-run/25-app-fcm-make-new-mode/flow.cylc similarity index 89% rename from t/rose-task-run/25-app-fcm-make-new-mode/suite.rc rename to t/rose-task-run/25-app-fcm-make-new-mode/flow.cylc index fb74b450d6..7cd1a3c5cd 100644 --- a/t/rose-task-run/25-app-fcm-make-new-mode/suite.rc +++ b/t/rose-task-run/25-app-fcm-make-new-mode/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [scheduling] [[dependencies]] graph = """hello-make => hello-run""" diff --git a/t/rose-task-run/26-app-fcm-make-new-mode-with-cont.t b/t/rose-task-run/26-app-fcm-make-new-mode-with-cont.t index cbc313d77d..59b194451c 100755 --- a/t/rose-task-run/26-app-fcm-make-new-mode-with-cont.t +++ b/t/rose-task-run/26-app-fcm-make-new-mode-with-cont.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -24,6 +24,7 @@ # "gfortran" being installed and available. #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" +skip_all 'TODO: #2445' if ! fcm help make 1>/dev/null 2>&1; then skip_all '"fcm make" unavailable' @@ -43,30 +44,37 @@ if [[ -z "${JOB_HOST}" ]]; then skip_all '"[t]job-host" not defined or not available' fi #------------------------------------------------------------------------------- -tests 2 +tests 4 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" - -SUITE_RUN_DIR="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${SUITE_RUN_DIR}")" +get_reg # Add some garbage before running the suite ssh -n -oBatchMode=yes "${JOB_HOST}" \ - "mkdir -p 'cylc-run/${NAME}/share/hello-make/junk2'" + "mkdir -p 'cylc-run/${FLOW}/share/hello-make/junk2'" ssh -n -oBatchMode=yes "${JOB_HOST}" \ - "touch 'cylc-run/${NAME}/share/hello-make/junk1'" + "touch 'cylc-run/${FLOW}/share/hello-make/junk1'" -timeout 120 rose suite-run -q --debug \ - -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' \ - -D "[jinja2:suite.rc]HOST=\"${JOB_HOST}\"" -- --no-detach --debug +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + timeout 120 \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + -S "HOST='${JOB_HOST}'" \ + --no-detach \ + --debug #------------------------------------------------------------------------------- ssh -n -oBatchMode=yes "${JOB_HOST}" \ - cat "cylc-run/${NAME}/share/hello.txt" >'hello.txt' + cat "cylc-run/${FLOW}/share/hello.txt" >'hello.txt' file_cmp "${TEST_KEY_BASE}" 'hello.txt' <<<'Hello World!' run_fail "${TEST_KEY_BASE}" \ ssh -n -oBatchMode=yes "${JOB_HOST}" \ - "ls 'cylc-run/${NAME}/share/hello-make/junk'*" + "ls 'cylc-run/${FLOW}/share/hello-make/junk'*" #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/26-app-fcm-make-new-mode-with-cont/suite.rc b/t/rose-task-run/26-app-fcm-make-new-mode-with-cont/flow.cylc similarity index 93% rename from t/rose-task-run/26-app-fcm-make-new-mode-with-cont/suite.rc rename to t/rose-task-run/26-app-fcm-make-new-mode-with-cont/flow.cylc index 9be2702803..90524113b4 100644 --- a/t/rose-task-run/26-app-fcm-make-new-mode-with-cont/suite.rc +++ b/t/rose-task-run/26-app-fcm-make-new-mode-with-cont/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [scheduling] [[dependencies]] graph = """hello-make => hello-make-bin => hello-run""" diff --git a/t/rose-task-run/28-env-path-run-path.t b/t/rose-task-run/28-env-path-run-path.t index e01fbf8771..7e66ea2d48 100755 --- a/t/rose-task-run/28-env-path-run-path.t +++ b/t/rose-task-run/28-env-path-run-path.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,21 +21,29 @@ #------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" -tests 2 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --no-detach \ + --debug TEST_KEY="${TEST_KEY_BASE}-hello.txt" -file_cmp "${TEST_KEY}" "${SUITE_RUN_DIR}/hello.txt" <<__HELLO__ +file_cmp "${TEST_KEY}" "${FLOW_RUN_DIR}/hello.txt" <<__HELLO__ Hello Earth! __HELLO__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/28-env-path-run-path/suite.rc b/t/rose-task-run/28-env-path-run-path/flow.cylc similarity index 92% rename from t/rose-task-run/28-env-path-run-path/suite.rc rename to t/rose-task-run/28-env-path-run-path/flow.cylc index dd4ccce23d..8529f9497a 100644 --- a/t/rose-task-run/28-env-path-run-path/suite.rc +++ b/t/rose-task-run/28-env-path-run-path/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT1M diff --git a/t/rose-task-run/28-env-path-run-path/opt/earth/bin/hello b/t/rose-task-run/28-env-path-run-path/opt/earth/bin/hello index 45fdb3d7e7..792a51b7dc 100755 --- a/t/rose-task-run/28-env-path-run-path/opt/earth/bin/hello +++ b/t/rose-task-run/28-env-path-run-path/opt/earth/bin/hello @@ -1,2 +1,2 @@ -#!/bin/bash +#!/usr/bin/env bash echo "Hello Earth!" diff --git a/t/rose-task-run/28-env-path-run-path/opt/world/bin/hello b/t/rose-task-run/28-env-path-run-path/opt/world/bin/hello index 3b81b522ff..9bfbdeed86 100755 --- a/t/rose-task-run/28-env-path-run-path/opt/world/bin/hello +++ b/t/rose-task-run/28-env-path-run-path/opt/world/bin/hello @@ -1,2 +1,2 @@ -#!/bin/bash +#!/usr/bin/env bash echo 'Hello World!' diff --git a/t/rose-task-run/29-app-prune-extglob-remote.t b/t/rose-task-run/29-app-prune-extglob-remote.t index 8a3fab9583..69cb5a2e9a 100755 --- a/t/rose-task-run/29-app-prune-extglob-remote.t +++ b/t/rose-task-run/29-app-prune-extglob-remote.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -20,9 +20,8 @@ # Test "rose_prune" built-in application, with bash extglob, using not glob. #------------------------------------------------------------------------------- . $(dirname $0)/test_header +skip_all 'TODO: #2445' - -JOB_HOST_OPT= if [[ "${TEST_KEY_BASE}" == *-remote ]]; then JOB_HOST=$(rose config --default= 't' 'job-host') if [[ -z "${JOB_HOST}" ]]; then @@ -32,24 +31,30 @@ if [[ "${TEST_KEY_BASE}" == *-remote ]]; then if [[ -z "${JOB_HOST}" ]]; then skip_all '"[t]job-host" not available' fi - JOB_HOST_OPT="-S JOB_HOST=\"${JOB_HOST}\"" fi -tests 2 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' ${JOB_HOST_OPT} -- --no-detach --debug - + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + -S "JOB_HOST='$JOB_HOST'" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --host='localhost' \ + --no-detach \ + --debug TEST_KEY="${TEST_KEY_BASE}-prune.log" sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0-9]*/YYYY-MM-DDTHHMM/g'\ - "${SUITE_RUN_DIR}/prune.log" > stamp-removed.log + "${FLOW_RUN_DIR}/prune.log" > stamp-removed.log sed '/^\[INFO\] YYYY-MM-DDTHHMM export ROSE_TASK_CYCLE_TIME=/p; /^\[INFO\] YYYY-MM-DDTHHMM delete: /!d' \ "stamp-removed.log" >'edited-prune.log' @@ -65,5 +70,5 @@ file_cmp "${TEST_KEY}" 'edited-prune.log' <<__LOG__ [INFO] YYYY-MM-DDTHHMM delete: work/20150101T0000Z __LOG__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/30-app-arch-opt-source.t b/t/rose-task-run/30-app-arch-opt-source.t index 8e70fe4a59..d673e15f0b 100755 --- a/t/rose-task-run/30-app-arch-opt-source.t +++ b/t/rose-task-run/30-app-arch-opt-source.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -18,56 +18,61 @@ # along with Rose. If not, see . #------------------------------------------------------------------------------- # Test "rose_arch" built-in application, archive with optional sources. -#------------------------------------------------------------------------------- . "$(dirname "$0")/test_header" - - -#------------------------------------------------------------------------------- -tests 6 +tests 8 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export CYLC_CONF_PATH= export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d "${HOME}/cylc-run/rose-test-battery.XXXXXX")" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach --debug + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY="${TEST_KEY_BASE}-job.status" file_grep "${TEST_KEY}-archive1-01" \ 'CYLC_JOB_EXIT=SUCCEEDED' \ - "${SUITE_RUN_DIR}/log/job/1/archive1/01/job.status" + "${FLOW_RUN_DIR}/log/job/1/archive1/01/job.status" file_grep "${TEST_KEY}-archive2-01" \ - 'CYLC_JOB_EXIT=EXIT' \ - "${SUITE_RUN_DIR}/log/job/1/archive2/01/job.status" + 'CYLC_JOB_EXIT=ERR' \ + "${FLOW_RUN_DIR}/log/job/1/archive2/01/job.status" file_grep "${TEST_KEY}-archive2-02" \ 'CYLC_JOB_EXIT=SUCCEEDED' \ - "${SUITE_RUN_DIR}/log/job/1/archive2/02/job.status" + "${FLOW_RUN_DIR}/log/job/1/archive2/02/job.status" TEST_KEY="${TEST_KEY_BASE}-find" -(cd "${SUITE_RUN_DIR}/share/backup" && find -type f) | sort >"${TEST_KEY}.out" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' ./archive1.d/2014.txt ./archive1.d/2016.txt ./archive2.d/2015.txt __FIND__ sed -n 's/^\[INFO\] \([+!=0] [^ ]*\) .*$/\1/p' \ - "${SUITE_RUN_DIR}/log/job/1/archive"*"/0"*"/job.out" \ + "${FLOW_RUN_DIR}/log/job/1/archive"*"/0"*"/job.out" \ | sort >'job.out.sorted' sort >'job.out.expected' <<__LOG__ -! ${SUITE_RUN_DIR}/share/backup/archive2.d/ -+ ${SUITE_RUN_DIR}/share/backup/archive1.d/ -+ ${SUITE_RUN_DIR}/share/backup/archive2.d/ -0 ${SUITE_RUN_DIR}/share/backup/nobody.d/ +! ${FLOW_RUN_DIR}/share/backup/archive2.d/ ++ ${FLOW_RUN_DIR}/share/backup/archive1.d/ ++ ${FLOW_RUN_DIR}/share/backup/archive2.d/ +0 ${FLOW_RUN_DIR}/share/backup/nobody.d/ __LOG__ file_cmp "${TEST_KEY}-job.out.sorted" 'job.out.sorted' 'job.out.expected' sed -n 's/^\[FAIL\] \(! [^ ]*\) .*$/\1/p' \ - "${SUITE_RUN_DIR}/log/job/1/archive"*"/0"*"/job.err" \ + "${FLOW_RUN_DIR}/log/job/1/archive"*"/0"*"/job.err" \ | sort >'job.err.sorted' sort >'job.err.expected' <<__LOG__ -! ${SUITE_RUN_DIR}/share/backup/archive2.d/ +! ${FLOW_RUN_DIR}/share/backup/archive2.d/ __LOG__ file_cmp "${TEST_KEY}-job.err.sorted" 'job.err.sorted' 'job.err.expected' #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/30-app-arch-opt-source/bin/foo b/t/rose-task-run/30-app-arch-opt-source/bin/foo index 82ca6b086b..0f9481f6d9 100755 --- a/t/rose-task-run/30-app-arch-opt-source/bin/foo +++ b/t/rose-task-run/30-app-arch-opt-source/bin/foo @@ -1,4 +1,6 @@ -#!/bin/bash +#!/bin/bash -l +# NOTE: not using /usr/bin/env bash here to allow the -l flag to be provided +# (required for MacOS to pick up GNU install rather than the BSD variant) #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -18,5 +20,6 @@ # along with Rose. If not, see . #------------------------------------------------------------------------------- set -eu +env >&2 mkdir -p "$1" exec install -t "$@" diff --git a/t/rose-task-run/30-app-arch-opt-source/suite.rc b/t/rose-task-run/30-app-arch-opt-source/flow.cylc similarity index 58% rename from t/rose-task-run/30-app-arch-opt-source/suite.rc rename to t/rose-task-run/30-app-arch-opt-source/flow.cylc index 26c39cdcbf..a17bcca0d1 100644 --- a/t/rose-task-run/30-app-arch-opt-source/suite.rc +++ b/t/rose-task-run/30-app-arch-opt-source/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails = True [[events]] timeout=PT1M abort on timeout=True @@ -18,16 +17,16 @@ archive2 execution time limit=PT1M [[archive1]] script=""" -echo 'MMXIV' >'2014.txt' -echo 'MMXVI' >'2016.txt' -rose task-run --debug -""" + echo 'MMXIV' >'2014.txt' + echo 'MMXVI' >'2016.txt' + rose task-run --debug + """ [[archive2]] script=""" -if ((${CYLC_TASK_TRY_NUMBER} > 1)); then - echo 'MMXV' >'2015.txt' -fi -rose task-run --debug -""" + if ((${CYLC_TASK_TRY_NUMBER} > 1)); then + echo 'MMXV' >'2015.txt' + fi + rose task-run --debug + """ [[[job]]] execution retry delays = PT0S diff --git a/t/rose-task-run/31-app-bunch.t b/t/rose-task-run/31-app-bunch.t index b5abd89c78..dbb2ca3a4e 100755 --- a/t/rose-task-run/31-app-bunch.t +++ b/t/rose-task-run/31-app-bunch.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -22,7 +22,7 @@ . $(dirname $0)/test_header #------------------------------------------------------------------------------- -tests 74 +tests 75 #------------------------------------------------------------------------------- # Define some constant patterns FAIL_PATTERN="\[FAIL\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]*" @@ -33,16 +33,24 @@ SKIP_PATTERN="\[SKIP\] [0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0:9]*" #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) + +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="${FLOW}" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc play \ + "${FLOW}" \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- CYCLE=20100101T0000Z -LOG_DIR="$SUITE_RUN_DIR/log/job/$CYCLE" +LOG_DIR="$FLOW_RUN_DIR/log/job/$CYCLE" #------------------------------------------------------------------------------- # Testing successful runs #------------------------------------------------------------------------------- @@ -242,5 +250,5 @@ file_grep "$TEST_KEY_PREFIX-TOTAL-RAN" \ "$INFO_PATTERN TOTAL: 8" \ "$FILE" #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/31-app-bunch/suite.rc b/t/rose-task-run/31-app-bunch/flow.cylc similarity index 100% rename from t/rose-task-run/31-app-bunch/suite.rc rename to t/rose-task-run/31-app-bunch/flow.cylc diff --git a/t/rose-task-run/32-app-arch-compressed.t b/t/rose-task-run/32-app-arch-compressed.t index 75495834f8..c8b1991554 100755 --- a/t/rose-task-run/32-app-arch-compressed.t +++ b/t/rose-task-run/32-app-arch-compressed.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,27 +23,36 @@ #------------------------------------------------------------------------------- -tests 2 +tests 4 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export CYLC_CONF_PATH= export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d "${HOME}/cylc-run/rose-test-battery.XXXXXX")" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach --debug + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY="${TEST_KEY_BASE}-job.status" file_grep "${TEST_KEY}-archive-01" \ 'CYLC_JOB_EXIT=SUCCEEDED' \ - "${SUITE_RUN_DIR}/log/job/1/archive/01/job.status" + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" TEST_KEY="${TEST_KEY_BASE}-find" -(cd "${SUITE_RUN_DIR}/share/backup" && find -type f) | sort >"${TEST_KEY}.out" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' ./archive.d/2016.txt.gz ./archive.d/whatever.tar.gz __FIND__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/32-app-arch-compressed/suite.rc b/t/rose-task-run/32-app-arch-compressed/flow.cylc similarity index 92% rename from t/rose-task-run/32-app-arch-compressed/suite.rc rename to t/rose-task-run/32-app-arch-compressed/flow.cylc index 1b0891c649..8a8d771596 100644 --- a/t/rose-task-run/32-app-arch-compressed/suite.rc +++ b/t/rose-task-run/32-app-arch-compressed/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails = True [[events]] timeout=PT1M abort on timeout=True diff --git a/t/rose-task-run/33-app-prune-cycle-format.t b/t/rose-task-run/33-app-prune-cycle-format.t index 84c8022ace..f7dff148dd 100755 --- a/t/rose-task-run/33-app-prune-cycle-format.t +++ b/t/rose-task-run/33-app-prune-cycle-format.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -22,21 +22,29 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header -tests 2 +tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' --debug -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --debug \ + --no-detach TEST_KEY="${TEST_KEY_BASE}-prune.log" sed 's/[0-9]*-[0-9]*-[0-9]*T[0-9]*:[0-9]*:[0-9]*+[0-9]*/YYYY-MM-DDTHHMM/g'\ - "${SUITE_RUN_DIR}/prune.log" > stamp-removed.log + "${FLOW_RUN_DIR}/prune.log" > stamp-removed.log sed '/^\[INFO\] YYYY-MM-DDTHHMM export ROSE_TASK_CYCLE_TIME=/p; /^\[INFO\] YYYY-MM-DDTHHMM delete: /!d' \ "stamp-removed.log" >'edited-prune.log' @@ -48,5 +56,5 @@ file_cmp "${TEST_KEY}" 'edited-prune.log' <<__LOG__ [INFO] YYYY-MM-DDTHHMM delete: share/hello-venus-in-1970.txt __LOG__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/33-app-prune-cycle-format/suite.rc b/t/rose-task-run/33-app-prune-cycle-format/flow.cylc similarity index 94% rename from t/rose-task-run/33-app-prune-cycle-format/suite.rc rename to t/rose-task-run/33-app-prune-cycle-format/flow.cylc index 858ff63dde..f76e960015 100644 --- a/t/rose-task-run/33-app-prune-cycle-format/suite.rc +++ b/t/rose-task-run/33-app-prune-cycle-format/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/34-app-prune-hosts-sharing-fs.t b/t/rose-task-run/34-app-prune-hosts-sharing-fs.t index 03455a1992..08bb523893 100755 --- a/t/rose-task-run/34-app-prune-hosts-sharing-fs.t +++ b/t/rose-task-run/34-app-prune-hosts-sharing-fs.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -31,41 +31,49 @@ if [[ -z "${JOB_HOST_1}" || -z "${JOB_HOST_2}" ]]; then skip_all '"[t]job-hosts-sharing-fs" not defined with 2 host names' fi -tests 4 +tests 5 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' --debug \ - -S "JOB_HOST_1=\"${JOB_HOST_1}\"" -S "JOB_HOST_2=\"${JOB_HOST_2}\"" \ - -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name \ + -S "JOB_HOST_1=\"${JOB_HOST_1}\"" \ + -S "JOB_HOST_2=\"${JOB_HOST_2}\"" +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --debug \ + --no-detach TEST_KEY="${TEST_KEY_BASE}-prune.log" grep \ "ssh .* \\(${JOB_HOST_1}\\|${JOB_HOST_2}\\) .* share/cycle/19700101T0000Z;" \ - "${SUITE_RUN_DIR}/prune.log" >'prune-ssh.log' + "${FLOW_RUN_DIR}/prune.log" >'prune-ssh.log' run_pass "${TEST_KEY}-prune-ssh-wc-l" test "$(wc -l <'prune-ssh.log')" -eq 2 if head -n 1 'prune-ssh.log' | grep ".* ${JOB_HOST_1} .*"; then run_pass "${TEST_KEY}-delete-1" \ grep -q "delete: ${JOB_HOST_1}:share/cycle/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" run_fail "${TEST_KEY}-delete-2" \ grep -q "delete: ${JOB_HOST_2}:share/cycle/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" else run_pass "${TEST_KEY}-delete-2" \ grep -q "delete: ${JOB_HOST_2}:share/cycle/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" run_fail "${TEST_KEY}-delete-1" \ grep -q "delete: ${JOB_HOST_1}:share/cycle/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" fi #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/34-app-prune-hosts-sharing-fs/suite.rc b/t/rose-task-run/34-app-prune-hosts-sharing-fs/flow.cylc similarity index 95% rename from t/rose-task-run/34-app-prune-hosts-sharing-fs/suite.rc rename to t/rose-task-run/34-app-prune-hosts-sharing-fs/flow.cylc index bb8a8a44f1..924a00a5bf 100644 --- a/t/rose-task-run/34-app-prune-hosts-sharing-fs/suite.rc +++ b/t/rose-task-run/34-app-prune-hosts-sharing-fs/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs.t b/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs.t index a65e29e797..b1179663c3 100755 --- a/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs.t +++ b/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -31,41 +31,49 @@ if [[ -z "${JOB_HOST_1}" || -z "${JOB_HOST_2}" ]]; then skip_all '"[t]job-hosts-sharing-fs" not defined with 2 host names' fi -tests 4 +tests 5 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR=$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX') -NAME="$(basename "${SUITE_RUN_DIR}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' --debug \ - -S "JOB_HOST_1=\"${JOB_HOST_1}\"" -S "JOB_HOST_2=\"${JOB_HOST_2}\"" \ - -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name \ + -S "JOB_HOST_1=\"${JOB_HOST_1}\"" \ + -S "JOB_HOST_2=\"${JOB_HOST_2}\"" +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --debug \ + --no-detach TEST_KEY="${TEST_KEY_BASE}-prune.log" grep \ "ssh .* \\(${JOB_HOST_1}\\|${JOB_HOST_2}\\).* ls. -d..* 19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" >'prune-ssh.log' + "${FLOW_RUN_DIR}/prune.log" >'prune-ssh.log' run_pass "${TEST_KEY}-prune-ssh-wc-l" test "$(wc -l <'prune-ssh.log')" -eq 2 if head -n 1 'prune-ssh.log' | grep ".* ${JOB_HOST_1} .*"; then run_pass "${TEST_KEY}-delete-1" \ grep -q "delete: ${JOB_HOST_1}:log/job/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" run_fail "${TEST_KEY}-delete-2" \ grep -q "delete: ${JOB_HOST_2}:log/job/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" else run_pass "${TEST_KEY}-delete-2" \ grep -q "delete: ${JOB_HOST_2}:log/job/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" run_fail "${TEST_KEY}-delete-1" \ grep -q "delete: ${JOB_HOST_1}:log/job/19700101T0000Z" \ - "${SUITE_RUN_DIR}/prune.log" + "${FLOW_RUN_DIR}/prune.log" fi #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs/suite.rc b/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs/flow.cylc similarity index 94% rename from t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs/suite.rc rename to t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs/flow.cylc index 857df13ff2..b673dbf974 100644 --- a/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs/suite.rc +++ b/t/rose-task-run/35-app-prune-log-on-hosts-sharing-fs/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/36-app-arch-interrupted.t b/t/rose-task-run/36-app-arch-interrupted.t index c436bcce26..dc6d526007 100755 --- a/t/rose-task-run/36-app-arch-interrupted.t +++ b/t/rose-task-run/36-app-arch-interrupted.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,29 +23,37 @@ #------------------------------------------------------------------------------- -tests 3 +tests 5 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export CYLC_CONF_PATH= export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d "${HOME}/cylc-run/rose-test-battery.XXXXXX")" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach --debug + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY="${TEST_KEY_BASE}-job.status" file_grep "${TEST_KEY}-archive-01" \ - 'CYLC_JOB_EXIT=EXIT' \ - "${SUITE_RUN_DIR}/log/job/1/archive/01/job.status" + 'CYLC_JOB_EXIT=ERR' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" file_grep "${TEST_KEY}-archive-02" \ 'CYLC_JOB_EXIT=SUCCEEDED' \ - "${SUITE_RUN_DIR}/log/job/1/archive/02/job.status" + "${FLOW_RUN_DIR}/log/job/1/archive/02/job.status" TEST_KEY="${TEST_KEY_BASE}-find" -(cd "${SUITE_RUN_DIR}/share" && find . -type f) | sort >"${TEST_KEY}.out" +(cd "${FLOW_RUN_DIR}/share" && find . -type f) | sort >"${TEST_KEY}.out" file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' ./new_whatever __FIND__ #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/36-app-arch-interrupted/suite.rc b/t/rose-task-run/36-app-arch-interrupted/flow.cylc similarity index 100% rename from t/rose-task-run/36-app-arch-interrupted/suite.rc rename to t/rose-task-run/36-app-arch-interrupted/flow.cylc diff --git a/t/rose-task-run/37-app-bunch-rm-old.t b/t/rose-task-run/37-app-bunch-rm-old.t index 291fdfecce..5c6c16e37d 100755 --- a/t/rose-task-run/37-app-bunch-rm-old.t +++ b/t/rose-task-run/37-app-bunch-rm-old.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -22,20 +22,29 @@ . $(dirname $0)/test_header #------------------------------------------------------------------------------- -tests 14 +tests 15 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) + +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play-1" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- CYCLE=20100101T0000Z -LOG_DIR="$SUITE_RUN_DIR/log/job/$CYCLE" +LOG_DIR="$FLOW_RUN_DIR/log/job/$CYCLE" #------------------------------------------------------------------------------- # Testing successful runs #------------------------------------------------------------------------------- @@ -61,9 +70,16 @@ for ARGVALUE in 0 1 2 3; do done #------------------------------------------------------------------------------- # Run suite a second time -run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug +# TODO: replace with cylc clean when required functionality is implemented +rm -rf "${FLOW_RUN_DIR}/log" +rm -rf "${FLOW_RUN_DIR}/.service/db" +run_pass "$TEST_KEY-play-2" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- # Confirm launching set of commands TEST_KEY_PREFIX=launch-ok-2nd-run @@ -75,6 +91,5 @@ for ARGVALUE in 0 1 2 3; do $FILE done #------------------------------------------------------------------------------- - -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/37-app-bunch-rm-old/suite.rc b/t/rose-task-run/37-app-bunch-rm-old/flow.cylc similarity index 91% rename from t/rose-task-run/37-app-bunch-rm-old/suite.rc rename to t/rose-task-run/37-app-bunch-rm-old/flow.cylc index 9c9468ed0b..58a33c3451 100644 --- a/t/rose-task-run/37-app-bunch-rm-old/suite.rc +++ b/t/rose-task-run/37-app-bunch-rm-old/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose-task-run/38-app-bunch-counts.t b/t/rose-task-run/38-app-bunch-counts.t index 5518371635..4e7fd6b65e 100755 --- a/t/rose-task-run/38-app-bunch-counts.t +++ b/t/rose-task-run/38-app-bunch-counts.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -22,20 +22,29 @@ . $(dirname $0)/test_header #------------------------------------------------------------------------------- -tests 16 +tests 17 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) + +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- CYCLE=1 -LOG_DIR="$SUITE_RUN_DIR/log/job/$CYCLE" +LOG_DIR="$FLOW_RUN_DIR/log/job/$CYCLE" #------------------------------------------------------------------------------- # Testing successful runs #------------------------------------------------------------------------------- @@ -104,5 +113,5 @@ TEST_KEY=${TEST_KEY_PREFIX}-TOTAL file_grep $TEST_KEY "\[INFO\] TOTAL: 5" $FILE #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/38-app-bunch-counts/suite.rc b/t/rose-task-run/38-app-bunch-counts/flow.cylc similarity index 88% rename from t/rose-task-run/38-app-bunch-counts/suite.rc rename to t/rose-task-run/38-app-bunch-counts/flow.cylc index b415b77a82..7df605ae7f 100644 --- a/t/rose-task-run/38-app-bunch-counts/suite.rc +++ b/t/rose-task-run/38-app-bunch-counts/flow.cylc @@ -1,6 +1,5 @@ [cylc] UTC mode = True - abort if any task fails = True [[events]] abort on timeout = True timeout = PT1M diff --git a/t/rose-task-run/39-app-prune-cycle-host.t b/t/rose-task-run/39-app-prune-cycle-host.t index 2065629d42..2a3fafe3ce 100755 --- a/t/rose-task-run/39-app-prune-cycle-host.t +++ b/t/rose-task-run/39-app-prune-cycle-host.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -33,16 +33,24 @@ fi tests 3 export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -RUND="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')" -NAME="$(basename "${RUND}")" #------------------------------------------------------------------------------- -TEST_KEY="${TEST_KEY_BASE}" +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" run_pass "${TEST_KEY}" \ - rose suite-run -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host='localhost' --debug \ - -S "JOB_HOST_1=\"${JOB_HOST_1}\"" -S "JOB_HOST_2=\"${JOB_HOST_2}\"" \ - -- --no-detach --debug + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name \ + -S "JOB_HOST_1=\"${JOB_HOST_1}\"" \ + -S "JOB_HOST_2=\"${JOB_HOST_2}\"" +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "${TEST_KEY}" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host='localhost' \ + --debug \ + --no-detach TEST_KEY="${TEST_KEY_BASE}-prune.log" run_pass "${TEST_KEY}-ssh-1" \ @@ -52,5 +60,5 @@ run_fail "${TEST_KEY}-ssh-2" \ grep -q "ssh .* ${JOB_HOST_2} .* share/cycle/19700101T0000Z;" \ "${RUND}/prune.log" #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/39-app-prune-cycle-host/suite.rc b/t/rose-task-run/39-app-prune-cycle-host/flow.cylc similarity index 95% rename from t/rose-task-run/39-app-prune-cycle-host/suite.rc rename to t/rose-task-run/39-app-prune-cycle-host/flow.cylc index 738ca517a0..fdf378652d 100644 --- a/t/rose-task-run/39-app-prune-cycle-host/suite.rc +++ b/t/rose-task-run/39-app-prune-cycle-host/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [scheduling] initial cycle point=1970 final cycle point=1990 diff --git a/t/rose-task-run/40-app-arch-duplicate.t b/t/rose-task-run/40-app-arch-duplicate.t index 16d09c1a35..14777bd5b6 100644 --- a/t/rose-task-run/40-app-arch-duplicate.t +++ b/t/rose-task-run/40-app-arch-duplicate.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -23,20 +23,28 @@ #------------------------------------------------------------------------------- -tests 1 +tests 3 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export CYLC_CONF_PATH= export ROSE_CONF_PATH= -mkdir -p "${HOME}/cylc-run" -SUITE_RUN_DIR="$(mktemp -d "${HOME}/cylc-run/rose-test-battery.XXXXXX")" -NAME="$(basename "${SUITE_RUN_DIR}")" -rose suite-run -q -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" --name="${NAME}" \ - --host=localhost -- --no-detach + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + -C "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --flow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- TEST_KEY="${TEST_KEY_BASE}" file_grep "${TEST_KEY}" 'duplicate archive target: "foo"' \ - "${SUITE_RUN_DIR}/log/job/1/archive_fail_duplicate/NN/job.err" + "${FLOW_RUN_DIR}/log/job/1/archive_fail_duplicate/NN/job.err" #------------------------------------------------------------------------------- -rose suite-clean -q -y "${NAME}" +purge exit 0 diff --git a/t/rose-task-run/40-app-arch-duplicate/suite.rc b/t/rose-task-run/40-app-arch-duplicate/flow.cylc similarity index 100% rename from t/rose-task-run/40-app-arch-duplicate/suite.rc rename to t/rose-task-run/40-app-arch-duplicate/flow.cylc diff --git a/t/rose-task-run/41-app-bunch-default-command.t b/t/rose-task-run/41-app-bunch-default-command.t index 89cc2556b9..fa035b2582 100755 --- a/t/rose-task-run/41-app-bunch-default-command.t +++ b/t/rose-task-run/41-app-bunch-default-command.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -22,20 +22,29 @@ . $(dirname $0)/test_header #------------------------------------------------------------------------------- -tests 26 +tests 27 #------------------------------------------------------------------------------- # Run the suite, and wait for it to complete export ROSE_CONF_PATH= -TEST_KEY=$TEST_KEY_BASE -mkdir -p $HOME/cylc-run -SUITE_RUN_DIR=$(mktemp -d --tmpdir=$HOME/cylc-run 'rose-test-battery.XXXXXX') -NAME=$(basename $SUITE_RUN_DIR) + +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + -C "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --flow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play-1" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- CYCLE=20100101T0000Z -LOG_DIR="$SUITE_RUN_DIR/log/job/$CYCLE" +LOG_DIR="$FLOW_RUN_DIR/log/job/$CYCLE" #------------------------------------------------------------------------------- # Testing successful runs #------------------------------------------------------------------------------- @@ -65,15 +74,20 @@ for TASK in buncher_default buncher_import; do done #------------------------------------------------------------------------------- # Run suite a second time +# TODO: replace with cylc clean when the required functionality is implemented +rm -rf "${FLOW_RUN_DIR}/log" +rm -rf "${FLOW_RUN_DIR}/.service/db" run_pass "$TEST_KEY" \ - rose suite-run -C $TEST_SOURCE_DIR/$TEST_KEY_BASE --name=$NAME \ - --host=localhost -- --no-detach --debug + cylc play "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug #------------------------------------------------------------------------------- # Testing successful rerun #------------------------------------------------------------------------------- -for TASK in buncher_default buncher_import; do -#------------------------------------------------------------------------------- # Confirm launching set of commands +for TASK in buncher_default buncher_import; do TEST_KEY_PREFIX=launch-ok-2nd-run FILE=$LOG_DIR/$TASK/NN/job.out for INSTANCE in 0 1 2 3; do @@ -82,8 +96,7 @@ for TASK in buncher_default buncher_import; do "\[INFO\] [-0-9]*T[+:0-9]* $INSTANCE: added to pool"\ $FILE done -#------------------------------------------------------------------------------- done #------------------------------------------------------------------------------- -rose suite-clean -q -y $NAME +purge exit 0 diff --git a/t/rose-task-run/41-app-bunch-default-command/suite.rc b/t/rose-task-run/41-app-bunch-default-command/flow.cylc similarity index 92% rename from t/rose-task-run/41-app-bunch-default-command/suite.rc rename to t/rose-task-run/41-app-bunch-default-command/flow.cylc index 8cf242c3ba..3a6e355859 100644 --- a/t/rose-task-run/41-app-bunch-default-command/suite.rc +++ b/t/rose-task-run/41-app-bunch-default-command/flow.cylc @@ -1,7 +1,6 @@ #!jinja2 [cylc] UTC mode=True - abort if any task fails=True [[events]] abort on timeout=True timeout=PT1M diff --git a/t/rose.env/00-export.t b/t/rose.env/00-export.t index 3a6f6ca114..2fbe5d3535 100755 --- a/t/rose.env/00-export.t +++ b/t/rose.env/00-export.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose.popen/00-self.t b/t/rose.popen/00-self.t index 634fab664b..9f7f2f686c 100755 --- a/t/rose.popen/00-self.t +++ b/t/rose.popen/00-self.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -20,6 +20,6 @@ D=$(cd $(dirname $0) && pwd) . $D/test_header tests 1 -POPEN_FILE=$(python -c "import metomi.rose.popen; print(metomi.rose.popen.__file__)") +POPEN_FILE=$(python3 -c "import metomi.rose.popen; print(metomi.rose.popen.__file__)") run_pass "$TEST_KEY_BASE" python3 $POPEN_FILE "$@" exit 0 diff --git a/t/rose.variable/00-trigger.py b/t/rose.variable/00-trigger.py index bd4b2cacc4..f59814b50e 100755 --- a/t/rose.variable/00-trigger.py +++ b/t/rose.variable/00-trigger.py @@ -1,8 +1,4 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/t/rose.variable/00-trigger.t b/t/rose.variable/00-trigger.t index c76fe2f239..c1edf4185b 100755 --- a/t/rose.variable/00-trigger.t +++ b/t/rose.variable/00-trigger.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose.variable/01-array-split.t b/t/rose.variable/01-array-split.t index 3086138488..0df940964f 100644 --- a/t/rose.variable/01-array-split.t +++ b/t/rose.variable/01-array-split.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rose.variable/t_array_split.py b/t/rose.variable/t_array_split.py index f6871b8a8c..ecb6d1aea6 100644 --- a/t/rose.variable/t_array_split.py +++ b/t/rose.variable/t_array_split.py @@ -1,8 +1,4 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. -# # This file is part of Rose, a framework for meteorological suites. # # Rose is free software: you can redistribute it and/or modify diff --git a/t/rose.variable/test_header b/t/rose.variable/test_header index 9e0a51f95a..bd787a8632 100644 --- a/t/rose.variable/test_header +++ b/t/rose.variable/test_header @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-checkout/00-basic.t b/t/rosie-checkout/00-basic.t index b4501cad3b..96ba6b4990 100755 --- a/t/rosie-checkout/00-basic.t +++ b/t/rosie-checkout/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-create/00-basic-edit b/t/rosie-create/00-basic-edit index c7e96a4ae8..2bc98bf1cc 100755 --- a/t/rosie-create/00-basic-edit +++ b/t/rosie-create/00-basic-edit @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-create/00-basic.t b/t/rosie-create/00-basic.t index a3e65b0aba..27e6d3a1ea 100755 --- a/t/rosie-create/00-basic.t +++ b/t/rosie-create/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-create/01-suite-info-meta.t b/t/rosie-create/01-suite-info-meta.t index 20966a5520..bff29701a4 100755 --- a/t/rosie-create/01-suite-info-meta.t +++ b/t/rosie-create/01-suite-info-meta.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-create/02-copy-inter.t b/t/rosie-create/02-copy-inter.t index ab5e963182..9ad7c997bd 100755 --- a/t/rosie-create/02-copy-inter.t +++ b/t/rosie-create/02-copy-inter.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-delete/00-basic.t b/t/rosie-delete/00-basic.t index fed1948404..af9b262280 100755 --- a/t/rosie-delete/00-basic.t +++ b/t/rosie-delete/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-disco/00-basic.t b/t/rosie-disco/00-basic.t index 6596eda2d4..65a59f47ee 100755 --- a/t/rosie-disco/00-basic.t +++ b/t/rosie-disco/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -59,7 +59,6 @@ URL_FOO_Q="${URL_FOO}query?" TEST_KEY=$TEST_KEY_BASE-curl-root-trailing-slash run_pass "$TEST_KEY" curl -i "${TEST_ROSE_WS_URL}/" # note: slash at end -cat $TEST_KEY.out >&2 file_grep "$TEST_KEY.out" 'HTTP/.* 200 OK' "$TEST_KEY.out" # The app has been set-up so that a trailing slash, as in the test directly diff --git a/t/rosie-graph/00-basic.t b/t/rosie-graph/00-basic.t index d126c775e3..f77b68f176 100755 --- a/t/rosie-graph/00-basic.t +++ b/t/rosie-graph/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -58,7 +58,7 @@ touch 'conf/opt/rose-port.conf' # Setup repository - create a suite. cat >repos/foo/hooks/post-commit <<__POST_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$ROSE_CONF_PATH rosa svn-post-commit --debug "\$@" \\ 1>$PWD/rosa-svn-post-commit.out 2>$PWD/rosa-svn-post-commit.err diff --git a/t/rosie-graph/test_header_extra b/t/rosie-graph/test_header_extra deleted file mode 120000 index 338abfca27..0000000000 --- a/t/rosie-graph/test_header_extra +++ /dev/null @@ -1 +0,0 @@ -../rose-metadata-graph/test_header_extra \ No newline at end of file diff --git a/t/rosie-graph/test_header_extra b/t/rosie-graph/test_header_extra new file mode 100644 index 0000000000..70979161a7 --- /dev/null +++ b/t/rosie-graph/test_header_extra @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) 2012-2019 British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Provides extra functions for rose metadata-graph testing. +#------------------------------------------------------------------------------- + +function filter_graphviz() { + FILTER_TMP_FILE=$(mktemp) + cat >"$FILTER_TMP_FILE" + # Sort and filter out non-essential properties from the output file. + # Get rid of line-broken newer graphviz output (replace ",\n"). + # Sort the file. + python3 << __PYTHON__ +import re +filename = '$FILTER_TMP_FILE' +f = open(filename, 'r') +text = f.read() +f.close() +text = text.replace(",\n", ", ") +text = re.sub("\s+\[", " [", text) +lines = text.splitlines() +f = open(filename, 'w') +for line in lines: + if '[' not in line: + if line.startswith("\t") and line.strip() != "];": + f.write(line.lstrip() + '\n') + continue + props = dict([_.strip().split('=', 1) for _ in + re.split(', ', + line.split('[', 1)[1].replace('];', ''))]) + new_prop_string = '' + for key in ['arrowhead', 'color', 'label', 'rankdir', 'shape', 'style']: + if key in props: + new_prop_string += key + '=' + props[key] + ', ' + new_prop_string = new_prop_string.rstrip().rstrip(',') + new_line = line.lstrip().split('[')[0] + '[' + new_prop_string + '\n' + if new_line.strip() != 'graph [': + f.write(new_line) +__PYTHON__ + LANG=C sort "$FILTER_TMP_FILE" + rm "$FILTER_TMP_FILE" +} diff --git a/t/rosie-hello/00-null.t b/t/rosie-hello/00-null.t index 7507aadedf..5e7eeb4951 100644 --- a/t/rosie-hello/00-null.t +++ b/t/rosie-hello/00-null.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/t/rosie-id/00-basic.t b/t/rosie-id/00-basic.t index 52e15f907c..107cbfc53e 100755 --- a/t/rosie-id/00-basic.t +++ b/t/rosie-id/00-basic.t @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # @@ -21,7 +21,7 @@ #------------------------------------------------------------------------------- . $(dirname $0)/test_header #------------------------------------------------------------------------------- -tests 45 +tests 36 #------------------------------------------------------------------------------- svnadmin create foo URL=file://$PWD/foo @@ -91,21 +91,6 @@ http://trac-host/foo/intertrac/source:/a/a/0/0/0/trunk@HEAD __OUT__ file_cmp "$TEST_KEY.err" "$TEST_KEY.err" 'foo-aa000/suite.rc' <<'__SUITE_RC__' -[scheduling] - [[dependencies]] - graph='t1' -[runtime] - [[t1]] -__SUITE_RC__ -rose suite-run -l -q -C "${PWD}/foo-aa000" --name="${SUITE_NAME}" -run_pass "$TEST_KEY" rosie id "${HOME}/cylc-run/${SUITE_NAME}" -file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ -foo-aa000 -__OUT__ -file_cmp "$TEST_KEY.err" "$TEST_KEY.err" 'foo-aa000/flow.cylc' <<'__SUITE_RC__' +#[scheduling] +# [[dependencies]] +# graph='t1' +#[runtime] +# [[t1]] +#__SUITE_RC__ +#cylc install \ +# -C "${PWD}/foo-aa000" \ +# --flow-name="${FLOW}" \ +# --no-run-name +#run_pass "$TEST_KEY" rosie id "${HOME}/cylc-run/${FLOW}" +#file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__ +#foo-aa000 +#__OUT__ +#file_cmp "$TEST_KEY.err" "$TEST_KEY.err" repos/foo/hooks/post-commit <<__POST_COMMIT__ -#!/bin/bash +#!/usr/bin/env bash export ROSE_CONF_PATH=$ROSE_CONF_PATH rosa svn-post-commit --debug "\$@" \\ 1>$PWD/rosa-svn-post-commit.out 2>$PWD/rosa-svn-post-commit.err @@ -153,14 +160,14 @@ fi #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-bad-prefix -run_fail "$TEST_KEY" rosie lookup --prefix=bar poetry +run_fail "$TEST_KEY" timeout 10 rosie lookup --prefix=bar poetry file_cmp "$TEST_KEY.out" "$TEST_KEY.out" foo-aa001/trunk@2 roses poetry Roses are Red,... @@ -321,7 +333,8 @@ __OUT__ file_cmp "$TEST_KEY.err" "$TEST_KEY.err" foo-aa001/trunk@2 roses poetry Roses are Red,... @@ -332,7 +345,8 @@ __OUT__ file_cmp "$TEST_KEY.err" "$TEST_KEY.err" . -#------------------------------------------------------------------------------- -. "$(dirname "$0")/test_header" - -if ! eslint --version 1>'/dev/null' 2>&1; then - skip_all '"eslint" command not available' -fi - -tests 3 - -TEST_KEY="${TEST_KEY_BASE}-docs" -run_pass "${TEST_KEY}" eslint --env browser --env jquery --env es6 \ - --parser-options=ecmaVersion:6 sphinx/_static/js -file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <'/dev/null' -file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <'/dev/null' - -exit diff --git a/t/syntax/test_header b/t/syntax/test_header deleted file mode 120000 index 90bd5a36f9..0000000000 --- a/t/syntax/test_header +++ /dev/null @@ -1 +0,0 @@ -../lib/bash/test_header \ No newline at end of file diff --git a/tox.ini b/tox.ini index 922db7b725..7ee23545dc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,6 @@ -[pycodestyle] -ignore= - ; module level import not at top of file - E402, - ; line break before binary operator - W503, - ; line break after binary operator - W504 +[flake8] exclude= .git, __pycache__, - .tox, ; purposely corrupt file t/rose-metadata-check/lib/custom_macro_corrupt.py diff --git a/usr/bin/rose b/usr/bin/rose index 6391202f5f..03ac6a4cdb 100755 --- a/usr/bin/rose +++ b/usr/bin/rose @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------- # Copyright (C) British Crown (Met Office) & Contributors. # diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000000..fb938cf97a --- /dev/null +++ b/yarn.lock @@ -0,0 +1,1200 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0": + version "7.12.11" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" + integrity sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw== + dependencies: + "@babel/highlight" "^7.10.4" + +"@babel/helper-validator-identifier@^7.10.4": + version "7.12.11" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" + integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== + +"@babel/highlight@^7.10.4": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.4.tgz#7d1bdfd65753538fabe6c38596cdb76d9ac60143" + integrity sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA== + dependencies: + "@babel/helper-validator-identifier" "^7.10.4" + chalk "^2.0.0" + js-tokens "^4.0.0" + +"@eslint/eslintrc@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.2.2.tgz#d01fc791e2fc33e88a29d6f3dc7e93d0cd784b76" + integrity sha512-EfB5OHNYp1F4px/LI/FEnGylop7nOqkQ1LRzCM0KccA2U8tvV8w01KBv37LbO7nW4H+YhKyo2LcJhRwjjV17QQ== + dependencies: + ajv "^6.12.4" + debug "^4.1.1" + espree "^7.3.0" + globals "^12.1.0" + ignore "^4.0.6" + import-fresh "^3.2.1" + js-yaml "^3.13.1" + lodash "^4.17.19" + minimatch "^3.0.4" + strip-json-comments "^3.1.1" + +"@types/json5@^0.0.29": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" + integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= + +acorn-jsx@^5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" + integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== + +acorn@^7.4.0: + version "7.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" + integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== + +ajv@^6.10.0, ajv@^6.12.4: + version "6.12.6" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== + dependencies: + fast-deep-equal "^3.1.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +ansi-colors@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" + integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== + +ansi-regex@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" + integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +ansi-styles@^4.0.0, ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +array-includes@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.2.tgz#a8db03e0b88c8c6aeddc49cb132f9bcab4ebf9c8" + integrity sha512-w2GspexNQpx+PutG3QpT437/BenZBj0M/MZGn5mzv/MofYqo0xmRHzn4lFsoDlWJ+THYsGJmFlW68WlDFx7VRw== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + es-abstract "^1.18.0-next.1" + get-intrinsic "^1.0.1" + is-string "^1.0.5" + +array.prototype.flat@^1.2.3: + version "1.2.4" + resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz#6ef638b43312bd401b4c6199fdec7e2dc9e9a123" + integrity sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + es-abstract "^1.18.0-next.1" + +astral-regex@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" + integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +call-bind@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.0.tgz#24127054bb3f9bdcb4b1fb82418186072f77b8ce" + integrity sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.0" + +callsites@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" + integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== + +chalk@^2.0.0: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +chalk@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" + integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +contains-path@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/contains-path/-/contains-path-0.1.0.tgz#fe8cf184ff6670b6baef01a9d4861a5cbec4120a" + integrity sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo= + +cross-spawn@^7.0.2: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + +debug@^2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +debug@^4.0.1, debug@^4.1.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" + integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + dependencies: + ms "2.1.2" + +deep-is@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" + integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= + +define-properties@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" + integrity sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ== + dependencies: + object-keys "^1.0.12" + +doctrine@1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-1.5.0.tgz#379dce730f6166f76cefa4e6707a159b02c5a6fa" + integrity sha1-N53Ocw9hZvds76TmcHoVmwLFpvo= + dependencies: + esutils "^2.0.2" + isarray "^1.0.0" + +doctrine@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" + integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== + dependencies: + esutils "^2.0.2" + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +enquirer@^2.3.5: + version "2.3.6" + resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" + integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== + dependencies: + ansi-colors "^4.1.1" + +error-ex@^1.2.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" + integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== + dependencies: + is-arrayish "^0.2.1" + +es-abstract@^1.18.0-next.1: + version "1.18.0-next.1" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.1.tgz#6e3a0a4bda717e5023ab3b8e90bec36108d22c68" + integrity sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA== + dependencies: + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.1" + is-callable "^1.2.2" + is-negative-zero "^2.0.0" + is-regex "^1.1.1" + object-inspect "^1.8.0" + object-keys "^1.1.1" + object.assign "^4.1.1" + string.prototype.trimend "^1.0.1" + string.prototype.trimstart "^1.0.1" + +es-to-primitive@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" + integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== + dependencies: + is-callable "^1.1.4" + is-date-object "^1.0.1" + is-symbol "^1.0.2" + +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + +eslint-config-standard@^14.1.1: + version "14.1.1" + resolved "https://registry.yarnpkg.com/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz#830a8e44e7aef7de67464979ad06b406026c56ea" + integrity sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg== + +eslint-import-resolver-node@^0.3.4: + version "0.3.4" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz#85ffa81942c25012d8231096ddf679c03042c717" + integrity sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA== + dependencies: + debug "^2.6.9" + resolve "^1.13.1" + +eslint-module-utils@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz#579ebd094f56af7797d19c9866c9c9486629bfa6" + integrity sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA== + dependencies: + debug "^2.6.9" + pkg-dir "^2.0.0" + +eslint-plugin-es@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz#75a7cdfdccddc0589934aeeb384175f221c57893" + integrity sha512-GUmAsJaN4Fc7Gbtl8uOBlayo2DqhwWvEzykMHSCZHU3XdJ+NSzzZcVhXh3VxX5icqQ+oQdIEawXX8xkR3mIFmQ== + dependencies: + eslint-utils "^2.0.0" + regexpp "^3.0.0" + +eslint-plugin-import@^2.22.1: + version "2.22.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.22.1.tgz#0896c7e6a0cf44109a2d97b95903c2bb689d7702" + integrity sha512-8K7JjINHOpH64ozkAhpT3sd+FswIZTfMZTjdx052pnWrgRCVfp8op9tbjpAk3DdUeI/Ba4C8OjdC0r90erHEOw== + dependencies: + array-includes "^3.1.1" + array.prototype.flat "^1.2.3" + contains-path "^0.1.0" + debug "^2.6.9" + doctrine "1.5.0" + eslint-import-resolver-node "^0.3.4" + eslint-module-utils "^2.6.0" + has "^1.0.3" + minimatch "^3.0.4" + object.values "^1.1.1" + read-pkg-up "^2.0.0" + resolve "^1.17.0" + tsconfig-paths "^3.9.0" + +eslint-plugin-node@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz#c95544416ee4ada26740a30474eefc5402dc671d" + integrity sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g== + dependencies: + eslint-plugin-es "^3.0.0" + eslint-utils "^2.0.0" + ignore "^5.1.1" + minimatch "^3.0.4" + resolve "^1.10.1" + semver "^6.1.0" + +eslint-plugin-promise@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz#845fd8b2260ad8f82564c1222fce44ad71d9418a" + integrity sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw== + +eslint-plugin-standard@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-standard/-/eslint-plugin-standard-5.0.0.tgz#c43f6925d669f177db46f095ea30be95476b1ee4" + integrity sha512-eSIXPc9wBM4BrniMzJRBm2uoVuXz2EPa+NXPk2+itrVt+r5SbKFERx/IgrK/HmfjddyKVz2f+j+7gBRvu19xLg== + +eslint-scope@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c" + integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== + dependencies: + esrecurse "^4.3.0" + estraverse "^4.1.1" + +eslint-utils@^2.0.0, eslint-utils@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.1.0.tgz#d2de5e03424e707dc10c74068ddedae708741b27" + integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg== + dependencies: + eslint-visitor-keys "^1.1.0" + +eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" + integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== + +eslint-visitor-keys@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8" + integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== + +eslint@^7.14.0: + version "7.16.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.16.0.tgz#a761605bf9a7b32d24bb7cde59aeb0fd76f06092" + integrity sha512-iVWPS785RuDA4dWuhhgXTNrGxHHK3a8HLSMBgbbU59ruJDubUraXN8N5rn7kb8tG6sjg74eE0RA3YWT51eusEw== + dependencies: + "@babel/code-frame" "^7.0.0" + "@eslint/eslintrc" "^0.2.2" + ajv "^6.10.0" + chalk "^4.0.0" + cross-spawn "^7.0.2" + debug "^4.0.1" + doctrine "^3.0.0" + enquirer "^2.3.5" + eslint-scope "^5.1.1" + eslint-utils "^2.1.0" + eslint-visitor-keys "^2.0.0" + espree "^7.3.1" + esquery "^1.2.0" + esutils "^2.0.2" + file-entry-cache "^6.0.0" + functional-red-black-tree "^1.0.1" + glob-parent "^5.0.0" + globals "^12.1.0" + ignore "^4.0.6" + import-fresh "^3.0.0" + imurmurhash "^0.1.4" + is-glob "^4.0.0" + js-yaml "^3.13.1" + json-stable-stringify-without-jsonify "^1.0.1" + levn "^0.4.1" + lodash "^4.17.19" + minimatch "^3.0.4" + natural-compare "^1.4.0" + optionator "^0.9.1" + progress "^2.0.0" + regexpp "^3.1.0" + semver "^7.2.1" + strip-ansi "^6.0.0" + strip-json-comments "^3.1.0" + table "^6.0.4" + text-table "^0.2.0" + v8-compile-cache "^2.0.3" + +espree@^7.3.0, espree@^7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-7.3.1.tgz#f2df330b752c6f55019f8bd89b7660039c1bbbb6" + integrity sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g== + dependencies: + acorn "^7.4.0" + acorn-jsx "^5.3.1" + eslint-visitor-keys "^1.3.0" + +esprima@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + +esquery@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.3.1.tgz#b78b5828aa8e214e29fb74c4d5b752e1c033da57" + integrity sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ== + dependencies: + estraverse "^5.1.0" + +esrecurse@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" + integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== + dependencies: + estraverse "^5.2.0" + +estraverse@^4.1.1: + version "4.3.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" + integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== + +estraverse@^5.1.0, estraverse@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" + integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== + +esutils@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + +fast-deep-equal@^3.1.1: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + +fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +fast-levenshtein@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" + integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= + +file-entry-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.0.tgz#7921a89c391c6d93efec2169ac6bf300c527ea0a" + integrity sha512-fqoO76jZ3ZnYrXLDRxBR1YvOvc0k844kcOg40bgsPrE25LAb/PDqTY+ho64Xh2c8ZXgIKldchCFHczG2UVRcWA== + dependencies: + flat-cache "^3.0.4" + +find-up@^2.0.0, find-up@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + integrity sha1-RdG35QbHF93UgndaK3eSCjwMV6c= + dependencies: + locate-path "^2.0.0" + +flat-cache@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" + integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== + dependencies: + flatted "^3.1.0" + rimraf "^3.0.2" + +flatted@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.0.tgz#a5d06b4a8b01e3a63771daa5cb7a1903e2e57067" + integrity sha512-tW+UkmtNg/jv9CSofAKvgVcO7c2URjhTdW1ZTkcAritblu8tajiYy7YisnIflEwtKssCtOxpnBRoCB7iap0/TA== + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + +functional-red-black-tree@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" + integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= + +get-intrinsic@^1.0.0, get-intrinsic@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.0.2.tgz#6820da226e50b24894e08859469dc68361545d49" + integrity sha512-aeX0vrFm21ILl3+JpFFRNe9aUvp6VFZb2/CTbgLb8j75kOhvoNYjt9d8KA/tJG4gSo8nzEDedRl0h7vDmBYRVg== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.1" + +glob-parent@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229" + integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ== + dependencies: + is-glob "^4.0.1" + +glob@^7.1.3: + version "7.1.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" + integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +globals@^12.1.0: + version "12.4.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-12.4.0.tgz#a18813576a41b00a24a97e7f815918c2e19925f8" + integrity sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg== + dependencies: + type-fest "^0.8.1" + +graceful-fs@^4.1.2: + version "4.2.4" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" + integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-symbols@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.1.tgz#9f5214758a44196c406d9bd76cebf81ec2dd31e8" + integrity sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg== + +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + +hosted-git-info@^2.1.4: + version "2.8.8" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" + integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg== + +ignore@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" + integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== + +ignore@^5.1.1: + version "5.1.8" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" + integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== + +import-fresh@^3.0.0, import-fresh@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== + dependencies: + parent-module "^1.0.0" + resolve-from "^4.0.0" + +imurmurhash@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +is-arrayish@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" + integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= + +is-callable@^1.1.4, is-callable@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.2.tgz#c7c6715cd22d4ddb48d3e19970223aceabb080d9" + integrity sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA== + +is-core-module@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a" + integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ== + dependencies: + has "^1.0.3" + +is-date-object@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e" + integrity sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g== + +is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= + +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + +is-glob@^4.0.0, is-glob@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" + integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== + dependencies: + is-extglob "^2.1.1" + +is-negative-zero@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" + integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w== + +is-regex@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.1.tgz#c6f98aacc546f6cec5468a07b7b153ab564a57b9" + integrity sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg== + dependencies: + has-symbols "^1.0.1" + +is-string@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6" + integrity sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ== + +is-symbol@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.3.tgz#38e1014b9e6329be0de9d24a414fd7441ec61937" + integrity sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ== + dependencies: + has-symbols "^1.0.1" + +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +js-yaml@^3.13.1: + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-stable-stringify-without-jsonify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" + integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= + +json5@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" + integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== + dependencies: + minimist "^1.2.0" + +levn@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== + dependencies: + prelude-ls "^1.2.1" + type-check "~0.4.0" + +load-json-file@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8" + integrity sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg= + dependencies: + graceful-fs "^4.1.2" + parse-json "^2.2.0" + pify "^2.0.0" + strip-bom "^3.0.0" + +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + integrity sha1-K1aLJl7slExtnA3pw9u7ygNUzY4= + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + +lodash@^4.17.19, lodash@^4.17.20: + version "4.17.20" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" + integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== + +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + +minimatch@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + +minimist@^1.2.0: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +natural-compare@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" + integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= + +normalize-package-data@^2.3.2: + version "2.5.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" + integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== + dependencies: + hosted-git-info "^2.1.4" + resolve "^1.10.0" + semver "2 || 3 || 4 || 5" + validate-npm-package-license "^3.0.1" + +object-inspect@^1.8.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.9.0.tgz#c90521d74e1127b67266ded3394ad6116986533a" + integrity sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw== + +object-keys@^1.0.12, object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + +object.assign@^4.1.1: + version "4.1.2" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" + integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + has-symbols "^1.0.1" + object-keys "^1.1.1" + +object.values@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.2.tgz#7a2015e06fcb0f546bd652486ce8583a4731c731" + integrity sha512-MYC0jvJopr8EK6dPBiO8Nb9mvjdypOachO5REGk6MXzujbBrAisKo3HmdEI6kZDL6fC31Mwee/5YbtMebixeag== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + es-abstract "^1.18.0-next.1" + has "^1.0.3" + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +optionator@^0.9.1: + version "0.9.1" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" + integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== + dependencies: + deep-is "^0.1.3" + fast-levenshtein "^2.0.6" + levn "^0.4.1" + prelude-ls "^1.2.1" + type-check "^0.4.0" + word-wrap "^1.2.3" + +p-limit@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" + integrity sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q== + dependencies: + p-try "^1.0.0" + +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + integrity sha1-IKAQOyIqcMj9OcwuWAaA893l7EM= + dependencies: + p-limit "^1.1.0" + +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + integrity sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M= + +parent-module@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" + integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== + dependencies: + callsites "^3.0.0" + +parse-json@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" + integrity sha1-9ID0BDTvgHQfhGkJn43qGPVaTck= + dependencies: + error-ex "^1.2.0" + +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + +path-parse@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" + integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + +path-type@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-2.0.0.tgz#f012ccb8415b7096fc2daa1054c3d72389594c73" + integrity sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM= + dependencies: + pify "^2.0.0" + +pify@^2.0.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" + integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= + +pkg-dir@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" + integrity sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s= + dependencies: + find-up "^2.1.0" + +prelude-ls@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== + +progress@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + +punycode@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +read-pkg-up@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-2.0.0.tgz#6b72a8048984e0c41e79510fd5e9fa99b3b549be" + integrity sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4= + dependencies: + find-up "^2.0.0" + read-pkg "^2.0.0" + +read-pkg@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-2.0.0.tgz#8ef1c0623c6a6db0dc6713c4bfac46332b2368f8" + integrity sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg= + dependencies: + load-json-file "^2.0.0" + normalize-package-data "^2.3.2" + path-type "^2.0.0" + +regexpp@^3.0.0, regexpp@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2" + integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q== + +resolve-from@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" + integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== + +resolve@^1.10.0, resolve@^1.10.1, resolve@^1.13.1, resolve@^1.17.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" + integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg== + dependencies: + is-core-module "^2.1.0" + path-parse "^1.0.6" + +rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + +"semver@2 || 3 || 4 || 5": + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + +semver@^6.1.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" + integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + +semver@^7.2.1: + version "7.3.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.4.tgz#27aaa7d2e4ca76452f98d3add093a72c943edc97" + integrity sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw== + dependencies: + lru-cache "^6.0.0" + +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + +slice-ansi@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" + integrity sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ== + dependencies: + ansi-styles "^4.0.0" + astral-regex "^2.0.0" + is-fullwidth-code-point "^3.0.0" + +spdx-correct@^3.0.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" + integrity sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w== + dependencies: + spdx-expression-parse "^3.0.0" + spdx-license-ids "^3.0.0" + +spdx-exceptions@^2.1.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz#3f28ce1a77a00372683eade4a433183527a2163d" + integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A== + +spdx-expression-parse@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz#cf70f50482eefdc98e3ce0a6833e4a53ceeba679" + integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q== + dependencies: + spdx-exceptions "^2.1.0" + spdx-license-ids "^3.0.0" + +spdx-license-ids@^3.0.0: + version "3.0.7" + resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.7.tgz#e9c18a410e5ed7e12442a549fbd8afa767038d65" + integrity sha512-U+MTEOO0AiDzxwFvoa4JVnMV6mZlJKk2sBLt90s7G0Gd0Mlknc7kxEn3nuDPNZRta7O2uy8oLcZLVT+4sqNZHQ== + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + +string-width@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" + integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.0" + +string.prototype.trimend@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.3.tgz#a22bd53cca5c7cf44d7c9d5c732118873d6cd18b" + integrity sha512-ayH0pB+uf0U28CtjlLvL7NaohvR1amUvVZk+y3DYb0Ey2PUV5zPkkKy9+U1ndVEIXO8hNg18eIv9Jntbii+dKw== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + +string.prototype.trimstart@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.3.tgz#9b4cb590e123bb36564401d59824298de50fd5aa" + integrity sha512-oBIBUy5lea5tt0ovtOFiEQaBkoBBkyJhZXzJYrSmDo5IUUqbOPvVezuRs/agBIdZ2p2Eo1FD6bD9USyBLfl3xg== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + +strip-ansi@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" + integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== + dependencies: + ansi-regex "^5.0.0" + +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM= + +strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +table@^6.0.4: + version "6.0.4" + resolved "https://registry.yarnpkg.com/table/-/table-6.0.4.tgz#c523dd182177e926c723eb20e1b341238188aa0d" + integrity sha512-sBT4xRLdALd+NFBvwOz8bw4b15htyythha+q+DVZqy2RS08PPC8O2sZFgJYEY7bJvbCFKccs+WIZ/cd+xxTWCw== + dependencies: + ajv "^6.12.4" + lodash "^4.17.20" + slice-ansi "^4.0.0" + string-width "^4.2.0" + +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= + +tsconfig-paths@^3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz#098547a6c4448807e8fcb8eae081064ee9a3c90b" + integrity sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw== + dependencies: + "@types/json5" "^0.0.29" + json5 "^1.0.1" + minimist "^1.2.0" + strip-bom "^3.0.0" + +type-check@^0.4.0, type-check@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== + dependencies: + prelude-ls "^1.2.1" + +type-fest@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" + integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== + +uri-js@^4.2.2: + version "4.4.0" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.0.tgz#aa714261de793e8a82347a7bcc9ce74e86f28602" + integrity sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g== + dependencies: + punycode "^2.1.0" + +v8-compile-cache@^2.0.3: + version "2.2.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132" + integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q== + +validate-npm-package-license@^3.0.1: + version "3.0.4" + resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" + integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== + dependencies: + spdx-correct "^3.0.0" + spdx-expression-parse "^3.0.0" + +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + +word-wrap@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" + integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==