diff --git a/.gitignore b/.gitignore index 02e56725..7a4f2ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,3 @@ python/soma_workflow.egg-info/ # Pycharm project# ################## .idea - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b10eee2..0f2cf6f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,24 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v4.0.1 hooks: + - id: check-added-large-files - id: check-case-conflict - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-shebang-scripts-are-executable - id: check-symlinks - id: check-xml - id: check-yaml - id: debug-statements + - id: end-of-file-fixer + - id: fix-byte-order-marker - id: fix-encoding-pragma - - id: check-added-large-files + #- id: trailing-whitespace # not ready yet - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 + rev: 3.9.2 hooks: - id: flake8 name: flake8 under python2 diff --git a/bin/soma_kill_servers b/bin/soma_kill_servers index b0a79626..c61f60b5 100755 --- a/bin/soma_kill_servers +++ b/bin/soma_kill_servers @@ -24,4 +24,3 @@ clear_db = options.clear connection.RemoteConnection.kill_remote_servers( resource_id, login=login, passwd=passwd, clear_db=clear_db) - diff --git a/bin/soma_restart_workflow b/bin/soma_restart_workflow index fde13d3b..5f608b48 100755 --- a/bin/soma_restart_workflow +++ b/bin/soma_restart_workflow @@ -33,5 +33,3 @@ if __name__ == '__main__': else: queue = sys.argv[3] wfc.restart_workflow(workflow_id, queue) - - diff --git a/bin/soma_stop_workflow b/bin/soma_stop_workflow index 3a1f5be4..b9175be3 100755 --- a/bin/soma_stop_workflow +++ b/bin/soma_stop_workflow @@ -28,6 +28,3 @@ if __name__ == '__main__': wfc = WorkflowController(sys.argv[2]) wfc.stop_workflow(workflow_id) - - - diff --git a/bin/soma_submit_workflow b/bin/soma_submit_workflow index ef1ef64e..02439017 100755 --- a/bin/soma_submit_workflow +++ b/bin/soma_submit_workflow @@ -41,6 +41,3 @@ if __name__ == '__main__': wfc = WorkflowController(args.resource) wfc.submit_workflow(workflow, name = args.name, queue = args.queue) - - - diff --git a/doc/source/changes_swf3.txt b/doc/source/changes_swf3.txt index 90bbed9e..58e954ea 100644 --- a/doc/source/changes_swf3.txt +++ b/doc/source/changes_swf3.txt @@ -97,4 +97,3 @@ Ex: light_mode = 1 scheduler_type = local_basic allowed_python_versions = 3 - diff --git a/doc/source/client_API.txt b/doc/source/client_API.txt index 72da4f5a..8567fcde 100644 --- a/doc/source/client_API.txt +++ b/doc/source/client_API.txt @@ -140,4 +140,3 @@ Helper .. automethod:: Helper.unserialize .. automethod:: Helper.convert_wf_file_for_p2_5 - diff --git a/doc/source/examples.txt b/doc/source/examples.txt index 89bc0f88..6884305c 100644 --- a/doc/source/examples.txt +++ b/doc/source/examples.txt @@ -439,9 +439,3 @@ the output files. The data directory "/tmp/soma_workflow_examples/" on your machine should now contains the file copy_of_myfile. - - - - - - diff --git a/doc/source/gui.txt b/doc/source/gui.txt index 8eb66816..622e1d33 100644 --- a/doc/source/gui.txt +++ b/doc/source/gui.txt @@ -181,5 +181,3 @@ Several plots can be displayed and refreshed at regular time interval: * the jobs function of the time, also showing the number or CPU cores used by each job (new in Soma-Workflow 3.0) * the number of jobs running function of the time. * the number of CPU cores running function of the time (new in Soma-Workflow 3.0) - - diff --git a/doc/source/index.txt b/doc/source/index.txt index 8288308d..645cce71 100644 --- a/doc/source/index.txt +++ b/doc/source/index.txt @@ -264,6 +264,3 @@ Authors and contributors .. :Release: |release| :Date: |today| - - - diff --git a/doc/source/install_config.txt b/doc/source/install_config.txt index 8f30197a..2d94fe2f 100644 --- a/doc/source/install_config.txt +++ b/doc/source/install_config.txt @@ -677,4 +677,3 @@ Ex: CPU_NB = 2 MAX_CPU_NB = 16 SCHEDULER_INTERVAL = 1 - diff --git a/doc/source/scheduler.txt b/doc/source/scheduler.txt index a3ded141..698086d9 100644 --- a/doc/source/scheduler.txt +++ b/doc/source/scheduler.txt @@ -68,4 +68,3 @@ MPI implementation .. automodule:: soma_workflow.schedulers.mpi_scheduler :members: - diff --git a/doc/source/status_list.txt b/doc/source/status_list.txt index f6755304..45749c4a 100644 --- a/doc/source/status_list.txt +++ b/doc/source/status_list.txt @@ -239,7 +239,3 @@ File Transfer status :height: 100 px :width: 100 px :scale: 30 - - - - diff --git a/doc/source/workflow_creation.txt b/doc/source/workflow_creation.txt index 77efc072..bb1d0814 100644 --- a/doc/source/workflow_creation.txt +++ b/doc/source/workflow_creation.txt @@ -477,4 +477,3 @@ A few engine execution jobs types have been defined and are available for workfl Such jobs are alternatives to :ref:`params_link_functions` described above. - diff --git a/doc/upload2github b/doc/upload2github deleted file mode 100644 index 50d6a160..00000000 --- a/doc/upload2github +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -e - -# -# This script is not needed any longer, the github push automatically triggers -# deploying docs on gh-pages (via travis-ci). -# - -# clean build directory -rm -rf ./build -mkdir ./build - -# clone the repo into a dir called build/html -git clone --branch gh-pages --no-checkout git@github.com:populse/soma-workflow.git ./build/html - -# create sphinx documentation in clear directory -make html - -# add configaration file to enable _ -cd ./build/html/ -touch .nojekyll - -# commit changes -git add . -git commit -a -m "Automatic update of soma-workflow documentation" - -# send commit to github server -git push origin gh-pages - -echo "You can now visit the soma-workflow page at http://neurospin.github.io/soma-workflow/" diff --git a/documents/memo_github.md b/documents/memo_github.md index 52a2b255..8c8b8951 100644 --- a/documents/memo_github.md +++ b/documents/memo_github.md @@ -16,4 +16,3 @@ $ git log --follow ./path/to/file ``` c.f http://git-scm.com/docs/git-log - diff --git a/ez_setup.py b/ez_setup.py index 6af0b717..3f6ba601 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Bootstrap setuptools installation diff --git a/pack2pypi.sh b/pack2pypi.sh index 22ed9ecf..051eaaa7 100755 --- a/pack2pypi.sh +++ b/pack2pypi.sh @@ -31,6 +31,3 @@ cat setup.py | sed -e "s/development version/$FULLVERSION/" > ./$PACKNAME/setup. VERSIONPATH=./$PACKNAME/python/soma_workflow echo fullVersion = \'$FULLVERSION\' > $VERSIONPATH/version.py echo shortVersion = \'$SHORTVERSION\' >> $VERSIONPATH/version.py - - - diff --git a/python/soma_workflow/clean_server.py b/python/soma_workflow/clean_server.py index 0cb6495e..e08063bc 100644 --- a/python/soma_workflow/clean_server.py +++ b/python/soma_workflow/clean_server.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import diff --git a/python/soma_workflow/configuration.py b/python/soma_workflow/configuration.py index b4d29dd3..735a1abc 100644 --- a/python/soma_workflow/configuration.py +++ b/python/soma_workflow/configuration.py @@ -1543,4 +1543,3 @@ def restore_soma_workflow_directory(): else: Configuration.search_config_path \ = staticmethod(Configuration._old_search_config_path) - diff --git a/python/soma_workflow/setup_client2server.py b/python/soma_workflow/setup_client2server.py index c95cc148..d581e66d 100644 --- a/python/soma_workflow/setup_client2server.py +++ b/python/soma_workflow/setup_client2server.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python # -*- coding: utf-8 -*- """ diff --git a/python/soma_workflow/setup_server.py b/python/soma_workflow/setup_server.py index 12b928de..54f2b016 100644 --- a/python/soma_workflow/setup_server.py +++ b/python/soma_workflow/setup_server.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python # -*- coding: utf-8 -*- """ diff --git a/python/soma_workflow/start_database_server.py b/python/soma_workflow/start_database_server.py index 53223b72..03609111 100644 --- a/python/soma_workflow/start_database_server.py +++ b/python/soma_workflow/start_database_server.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- ''' diff --git a/python/soma_workflow/test/data/PlatformTest/testlocal.stdin b/python/soma_workflow/test/data/PlatformTest/testlocal.stdin index 78ade449..3dbd4043 100644 --- a/python/soma_workflow/test/data/PlatformTest/testlocal.stdin +++ b/python/soma_workflow/test/data/PlatformTest/testlocal.stdin @@ -2,4 +2,3 @@ 0 1 0 - diff --git a/python/soma_workflow/test/data/PlatformTest/ubuntu.sh b/python/soma_workflow/test/data/PlatformTest/ubuntu.sh old mode 100644 new mode 100755 diff --git a/python/soma_workflow/test/data/TestDrmaaCTypes.py b/python/soma_workflow/test/data/TestDrmaaCTypes.py index c88671d2..42f9f65a 100644 --- a/python/soma_workflow/test/data/TestDrmaaCTypes.py +++ b/python/soma_workflow/test/data/TestDrmaaCTypes.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/python/soma_workflow/test/data/config_examples/soma-workflow-scheduler.cfg b/python/soma_workflow/test/data/config_examples/soma-workflow-scheduler.cfg index 2e9823ec..d74f4ac9 100644 --- a/python/soma_workflow/test/data/config_examples/soma-workflow-scheduler.cfg +++ b/python/soma_workflow/test/data/config_examples/soma-workflow-scheduler.cfg @@ -1,4 +1,4 @@ [local_scheduler] CPU_NB = 2 -PERIOD = 1 \ No newline at end of file +PERIOD = 1 diff --git a/python/soma_workflow/test/data/jobExamples/complete/file0 b/python/soma_workflow/test/data/jobExamples/complete/file0 old mode 100755 new mode 100644 index 580db7aa..0843220b --- a/python/soma_workflow/test/data/jobExamples/complete/file0 +++ b/python/soma_workflow/test/data/jobExamples/complete/file0 @@ -13,4 +13,3 @@ Directions 1. In a large bowl, sift together the flour, baking powder, salt and sugar. Make a well in the center and pour in the milk, egg and melted butter; mix until smooth. 2. Heat a lightly oiled griddle or frying pan over medium high heat. Pour or scoop the batter onto the griddle, using approximately 1/4 cup for each pancake. Brown on both sides and serve hot. - diff --git a/python/soma_workflow/test/data/jobExamples/complete/file1 b/python/soma_workflow/test/data/jobExamples/complete/file1 index b5476077..cbe647f0 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/file1 +++ b/python/soma_workflow/test/data/jobExamples/complete/file1 @@ -1,4 +1,3 @@ This is a Soma-Workflow example file used as input to job and workflow tests. - diff --git a/python/soma_workflow/test/data/jobExamples/complete/job5.py b/python/soma_workflow/test/data/jobExamples/complete/job5.py index 89aaa565..856154d2 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/job5.py +++ b/python/soma_workflow/test/data/jobExamples/complete/job5.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # calls job1 using params IO as json diff --git a/python/soma_workflow/test/data/jobExamples/complete/job6.py b/python/soma_workflow/test/data/jobExamples/complete/job6.py index b501c06a..2efdab33 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/job6.py +++ b/python/soma_workflow/test/data/jobExamples/complete/job6.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # calls job1 using params IO as json diff --git a/python/soma_workflow/test/data/jobExamples/complete/job7.py b/python/soma_workflow/test/data/jobExamples/complete/job7.py index 0ea2b98e..a907cae0 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/job7.py +++ b/python/soma_workflow/test/data/jobExamples/complete/job7.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # inputs: as env var SOMAWF_INPUT_PARAMS @@ -25,8 +24,8 @@ if not os.path.exists(out_dir): os.mkdir(out_dir) -filePathsOut = [os.path.join(out_dir, os.path.basename(f)) - for f in filePathsIn] +filePathsOut = [os.path.join(out_dir, os.path.basename(fp)) + for fp in filePathsIn] for fin, fout in zip(filePathsIn, filePathsOut): shutil.copy2(fin, fout) diff --git a/python/soma_workflow/test/data/jobExamples/complete/job8.py b/python/soma_workflow/test/data/jobExamples/complete/job8.py index f5bcb820..a44a6c7d 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/job8.py +++ b/python/soma_workflow/test/data/jobExamples/complete/job8.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # input: "input" param in json dict diff --git a/python/soma_workflow/test/data/jobExamples/complete/job9.py b/python/soma_workflow/test/data/jobExamples/complete/job9.py index 998642f6..08ab6be9 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/job9.py +++ b/python/soma_workflow/test/data/jobExamples/complete/job9.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # input: "inputs" param in json dict, list of files diff --git a/python/soma_workflow/test/data/jobExamples/complete/output_models/file13 b/python/soma_workflow/test/data/jobExamples/complete/output_models/file13 index 40d75144..d16c9eeb 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/output_models/file13 +++ b/python/soma_workflow/test/data/jobExamples/complete/output_models/file13 @@ -21,4 +21,3 @@ job8 output: This is a Soma-Workflow example file used as input to job and workflow tests. - diff --git a/python/soma_workflow/test/data/jobExamples/complete/output_models/file17 b/python/soma_workflow/test/data/jobExamples/complete/output_models/file17 index b763448f..a39da92d 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/output_models/file17 +++ b/python/soma_workflow/test/data/jobExamples/complete/output_models/file17 @@ -14,4 +14,3 @@ Directions 1. In a large bowl, sift together the flour, baking powder, salt and sugar. Make a well in the center and pour in the milk, egg and melted butter; mix until smooth. 2. Heat a lightly oiled griddle or frying pan over medium high heat. Pour or scoop the batter onto the griddle, using approximately 1/4 cup for each pancake. Brown on both sides and serve hot. - diff --git a/python/soma_workflow/test/data/jobExamples/complete/output_models/stdout_exception_job b/python/soma_workflow/test/data/jobExamples/complete/output_models/stdout_exception_job index a3f3a814..31ddc763 100644 --- a/python/soma_workflow/test/data/jobExamples/complete/output_models/stdout_exception_job +++ b/python/soma_workflow/test/data/jobExamples/complete/output_models/stdout_exception_job @@ -1 +1 @@ -This is a program which throws an exception. \ No newline at end of file +This is a program which throws an exception. diff --git a/python/soma_workflow/test/data/jobExamples/complete/stdin1 b/python/soma_workflow/test/data/jobExamples/complete/stdin1 old mode 100755 new mode 100644 diff --git a/python/soma_workflow/test/data/jobExamples/complete/stdin2 b/python/soma_workflow/test/data/jobExamples/complete/stdin2 old mode 100755 new mode 100644 diff --git a/python/soma_workflow/test/data/jobExamples/complete/stdin3 b/python/soma_workflow/test/data/jobExamples/complete/stdin3 old mode 100755 new mode 100644 diff --git a/python/soma_workflow/test/data/jobExamples/complete/stdin4 b/python/soma_workflow/test/data/jobExamples/complete/stdin4 old mode 100755 new mode 100644 diff --git a/python/soma_workflow/test/data/jobExamples/mpi/simple_mpi.sh b/python/soma_workflow/test/data/jobExamples/mpi/simple_mpi.sh index 8ec3a779..3ee965d5 100755 --- a/python/soma_workflow/test/data/jobExamples/mpi/simple_mpi.sh +++ b/python/soma_workflow/test/data/jobExamples/mpi/simple_mpi.sh @@ -1,4 +1,4 @@ #!/bin/bash echo "SOMA_JOB_MPI_BIN" $SOMA_JOB_MPI_BIN "SOMA_JOB_NODE_FILE" $SOMA_JOB_NODE_FILE -. $SOMA_JOB_MPI_BIN/mpirun -machinefile `eval echo $SOMA_JOB_NODE_FILE` -np $1 $2 \ No newline at end of file +. $SOMA_JOB_MPI_BIN/mpirun -machinefile `eval echo $SOMA_JOB_NODE_FILE` -np $1 $2 diff --git a/python/soma_workflow/test/data/jobExamples/special_transfers/example.hdr b/python/soma_workflow/test/data/jobExamples/special_transfers/example.hdr index 0dda084f..f0369527 100644 --- a/python/soma_workflow/test/data/jobExamples/special_transfers/example.hdr +++ b/python/soma_workflow/test/data/jobExamples/special_transfers/example.hdr @@ -1,3 +1,2 @@ File with extension .hdr - diff --git a/python/soma_workflow/test/data/jobExamples/special_transfers/example.img b/python/soma_workflow/test/data/jobExamples/special_transfers/example.img index 9c09ed7c..14a80be7 100644 --- a/python/soma_workflow/test/data/jobExamples/special_transfers/example.img +++ b/python/soma_workflow/test/data/jobExamples/special_transfers/example.img @@ -1,4 +1,3 @@ File with extension .img -