Merge pull request #711 from sardana-org/release-Jan18

Release Jan18

.bumpversion.cfg

@@ -3,7 +3,7 @@ commit = True
 message = Bump version {current_version} to {new_version}
 tag = False
 tag_name = {new_version}
-current_version = 2.3.2
+current_version = 2.4.0
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
 serialize = 
 	{major}.{minor}.{patch}-{release}

.flake8

@@ -0,0 +1,10 @@
+[flake8]
+# E129 is ignored to fulfill with PEP8 which allow the same indentation of
+# multi conditional lines ("if" statement) and the code nested inside of the
+# "if" statment. More details in:
+# https://github.com/sardana-org/sardana/pull/551#issuecomment-338572275
+#
+# The default configuration ignores: E121, E123, E126, E133, E226, E241,
+# E242, E704 and W503 so ignore them here as well. See more details in:
+# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+ignore = E121, E123, E126, E129, E133, E226, E241, E242, E704 and W503

.travis.yml

@@ -1,3 +1,5 @@
+group: deprecated-2017Q4
+
 language: python
 
 sudo: required
@@ -14,8 +16,10 @@ env:
 
 before_install:
   # install flake8 to perform python code style check in the script part
+  # install it using pip in order to get the newest version
   - if [ $TEST == "flake8" ]; then sudo apt-get update -qq ; fi
-  - if [ $TEST == "flake8" ]; then sudo apt-get install -qq python-flake8; fi
+  - if [ $TEST == "flake8" ]; then sudo apt-get install -qq python-pip; fi
+  - if [ $TEST == "flake8" ]; then sudo pip install flake8; fi
 
 install:
   # run reszelaz/sardana-test docker container (Debian8 with sardana-deps)

CHANGELOG.md

@@ -4,10 +4,130 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 This file follows the formats and conventions from [keepachangelog.com]
 
 
-## [Unreleased]
+## [2.4.0] 2018-03-14
 
 ### Added
-- ...
+- General hooks - hooks that can be configured with `defgh`, `udefgh` and `lsgh`
+  macros instead of attaching them programatically (#200, #646)
+- New API to `Stoppable` interface of pool controllers that allows synchronized
+  multiaxes stop/abort (#157, #592)
+- Macro execution logs can be saved to a file. Controlled with `logmacro` macro and
+  `LogMacroDir`, `LogMacroFormat` environment variables (#102, #480)
+- `addctrlib`, `relctrllib`, `relctrlcls` macros usefull when developing
+  controller classes (#541)
+- `meshct` macro - mesh composed from row scans executed in the continuous
+  way (#659)
+- Optional sending of logs to Logstash (www.elastic.co) configurable with
+  `LogstashHost` and `LogstashPort` Tango properties of Pool and MacroServer
+  (#699).
+- Relative continuous scans like `dscanct`, `d2scanct`, etc. (#605, #688)
+- Expose acquisition data in `ct` and `uct` macros via data macro property
+  (#471, #682, #684)
+- Notification to the user in case of a failed operation of stopping in spock
+  (#592)
+- Timeout/watchdog in continuous scans - especially usefull when
+  triggers may be missed e.g. not precise positioning (#136, #601)
+- Reintroduce intermediate events for counter/timer channels while
+  software acquisition is in progress (#625)
+- TaurusCounterTimerController - that can connect to different data
+  sources e.g. EPICS by using Taurus schemes (#628)
+- Allow deleting multiple measurement groups and multiple controllers
+  with udefmeas and udefctrl macros respectivelly (#361, #547)
+- Improve exception hangling in ascanct & co. (#600)
+- Allow to hide Taurus 4 related deprecation warnings
+  (`TAURUS_MAX_DEPRECATION_COUNTS` sardana custom setting) (#550)
+- Optional data extrapolation for the very first records in ascanct & co.
+  (`ApplyExtrapolation` environment variable) (#588)
+- Inform about an error when reading the sofware synchronized channel so
+  the record can be completed - value for the given trigger will not
+  arrive anymore (#581, #582)
+- `--file` option to sequencer - it allows to load a sequence file
+  directly on the application startup moment (#283, #551)
+- Report error line number when loading a sequence from a txt file
+  fails (#114, #552)
+- Present available pools at the macroserver creation moment in the
+  alphabetical order (#585, #586)
+- Present available doors at the spock profile creation moment in the
+  alphabetical order (#221, #558, #673)
+- `DiffractometerType` is stored in crystal file in HKL controller (#679)
+- Some backwards compatibility for element names in PQDN - recently
+  Taurus started using only FQDN (#625, #627)
+- Improve DumbRecorder (example of a custom file recorder) to write to
+  a file.
+- Data in scan Records can now be accessed via dict-like syntax (#644)
+- Example of a macro that uses other macros as hooks #649
+
+### Fixed
+- Spock waits until macro stopping is finished after Ctrl+C (#34. #596)
+- Limits of the underneeth motors are checked if we move a pseudo motor
+  (#36, #663, #704)
+- Limits of the underneeth motors are checked if we move a motor group
+  (#259, #560)
+- Eliminate a possibility of deadlock when aborting a macro (#693, #695,
+  #708)
+- Acquisition start sequence which in case of starting disabled channels
+  could unintentionally change the measurement group's configuration (#607,
+  #615)
+- Selection of the master timer/monitor for each of the acquisition
+  sub-actions (hardware and software) (#614)
+- Avoid "already involved in motion" errors due to wrong handling of
+  operation context and Tango state machine (#639)
+- Protect software synchronizer from errors in reading motor's position
+  (#694, #700)
+- Make the information about the element's instrument fully dynamic and
+  remove it from the serialized information (#122, #619)
+- uct macro (#319, #627)
+- Avoid measurement group start failures when to the disabled controller
+  is offline (#677, #681)
+- Allow to stop macro when it was previously paused (#348, #548)
+- Bug in theoretical motor position in ascanct & co. (#591)
+- Counter/timer TaurusValue widget when used with Taurus 4 - correctly show
+  the element's name (#617)
+- `relmaclib` reports the error message in case the macro has parameters
+  definition is incorrect (#377, #642)
+- Icons in macroexecution widgets when used with Taurus 4 (#578)
+- Spurious errors when reading RecordData attribute, normally triggered
+  on event subscription e.g. macrogui (#447, #598)
+- Possible too early acquisition triggering by trigger/gate due to the
+  wrong order ot starting trigger/gate and software synchronizer in the
+  synchronization action (#597)
+- Validation of motor positions agains their limits in ascanct & co. (#595)
+- Generation of theoretical timestamps in ascanct & co. (#602)
+- Maintain macrobutton's text when MacroServer is shut down (#293, #559)
+- Number of repetitions (always pass 1) passed to experimental channel
+  controllers in case software synchronization is in use (#594)
+- `Hookable.hooks` proprty setting - now it cleans the previous
+  configuration (#655)
+- `getData` of scan macros from the GSF (#683, #687)
+- Make PoolUtil thread-safe (#662, #655)
+- Dummy counter/timer now returns partial value when its acquisition was
+  aborted (#626)
+- Workaround for #427: make default values for repeat parameters of `wa`,
+  `pwa` and all list macros fully functional - also support execution with
+  `Macro.execMacro` (#654)
+- `getIntervalEstimation` method of the GSF for some scanning modes (#661)
+- Improved MacroServer creation wizard (#676)
+
+### Changed
+- FQDN are now used internally by sardana in its identifiers (#668, partially)
+- Make explicit file descriptor buffer synchronization (force effective write to
+  the file system) in SPEC and FIO recorders (#651)
+- Rename edctrl to edctrlcls macro (#541)
+- The way how the master timer/monitor for the acquisition actions is selected.
+  Previously the first one for the given synchronization was used, now it is
+  taken into account if it is enabled or disabled (next ones may be used then).
+  (#647, #648)
+- Macrobutton's text to from "<macro_name>" to "Run/Abort <macro_name>"
+  (#322, #554, #658)
+- Color policy in spock for IPython >= 5 from Linux to Neutral (#706 and #712)
+
+### Removed
+- `ElementList` attribute from the Door Tango device - `Element` attribute is
+  available on the MacroServer device (#556, #557, #653)
+- `raw_stop_all`, `raw_stop_one`, `_raw_stop_all`, `_raw_stop_one`, `stop_all`,
+  `stop_one`, `raw_abort_all`, `raw_abort_one`, `_raw_abort_all`, `_raw_abort_one`,
+  `abort_all`, `abort_one` methods of the `PoolController` class (#592)
+
 
 ## [2.3.2] - 2017-08-11
 For a full log of commits between versions run (in your git repo):
@@ -277,6 +397,7 @@ Main improvements since sardana 1.5.0 (aka Jan15):
 
 [keepachangelog.com]: http://keepachangelog.com
 [Unreleased]: https://github.com/sardana-org/sardana/compare/2.3.2...HEAD
+[2.4.0]: https://github.com/sardana-org/sardana/compare/2.3.2...2.4.0
 [2.3.2]: https://github.com/sardana-org/sardana/compare/2.3.1...2.3.2
 [2.3.1]: https://github.com/sardana-org/sardana/compare/2.3.0...2.3.1
 [2.3.0]: https://github.com/sardana-org/sardana/compare/2.2.3...2.3.0

doc/how_to_release.md

@@ -0,0 +1,140 @@
+# How to release (draft)
+
+This is a guide for sardana release managers: it details the steps for making
+an official release, including a checklist of stuff that should be manually
+tested.
+
+## The release process
+
+1. During all the development, use the Milestones to keep track of the intended
+   release for each issue.
+2. Previous to the release deadline, re-check the open issues/PR and update
+   the assignation issues/PR to the release milestone. Request feedback from
+   the devel community.
+3. Work to close all the PR/issues remaining open in the milestone. This can
+   be either done in develop or in a release branch called `release-XXX`
+   (where `XXX` is the milestone name, e.g. `Jul17`). If a release branch is
+   used, `develop` is freed to continue with integrations that may not be
+   suitable for this release. On the other hand, it adds a bit more work
+   because the release-related PRs (which are done against the `release-XXX`
+   branch), may need to be also merged to develop.
+   Note: the `release-XXX` branch *can* live in the sardana-org repo or on a
+   personal fork (in which case you should do step 4.iv **now** to allow other
+   integrators to push directly to it).
+4. Create the release branch if it was not done already in the previous step
+   and:
+    1. Review and update the CHANGELOG.md if necessary. See [this](http://keepachangelog.com).
+    2. Bump version using `bumpversion <major|minor|patch> && bumpversion release`
+       (use [semver](http://semver.org/) criteria to choose amongst `major`,
+       `minor` or `patch`. OPTIONAL: Sardana depends on Taurus, and the
+       required version of Taurus may need to be bumped as well. Taurus and
+       other dependencies are specified in: `setup.py` (requires list of
+       strings) and `src/sardana/requirements.py` (`__requires__` dictionary
+       and taurus.core value).
+    3. The version numbers used in the man pages of the Sardana scripts are
+       bumped (you may use `taurus/doc/makeman` script executing it from the
+       doc directory e.g. `sardana/doc`) and committing the changes.
+    4. Create a PR to merge the `release-XXX` against the **`master`** branch
+       of the sardana-org repo
+5. Request reviews in the PR from at least one integrator from each
+   participating institute. The master branch is protected, so the reviews need
+   to be cleared (or dismissed with an explanation) before the release can be
+   merged.
+6. Perform manual tests (see checklist below). You may use the CI artifacts
+   (e.g., from appveyor) and post the results in the comments of the PR.
+7. Once all reviews are cleared, update the date of the release in the
+   CHANGELOG.md, merge the PR and tag in master.
+8. Merge also the  `release-XXX` branch into develop, and bump the version of
+   develop with `bumpversion patch`
+
+## Manual test checklist
+
+This is a check-list of manual tests. It is just orientative. Expand it
+at will. This list assumes a clean environment with all Sardana dependencies
+already installed and access to a Tango system with the TangoTest DS running.
+
+Hint: this list can be used as a template to be copy-pasted on a release PR
+
+### Installation
+- [ ] Install Sardana from the tar.gz : `pip install <tarball_artifact_URL>`
+
+### Create testing environment and run testsuite
+- [ ] Start Pool demo2. In a console do `Pool demo2`.
+- [ ] Start MacroServer demo2 and connect to the Pool demo2.
+  In another console do: `MacroServer demo2`
+- [ ] Set MacroServer's MacroPath to point to the macro examples.
+  In another IPython console do:
+  `PyTango.DeviceProxy('macroserver/demo1/1').put_property({'MacroPath':'<path_to_sardana_installation_dir>/macroserver/macros/examples'})`
+- [ ] Restart MacroServer e.g. Ctrl+C in the MacroServer's console and
+  start it again.
+- [ ] Create spock profile demo2. In another console do `spock --profile=demo2`
+- [ ] In spock run `sar_demo` macro.
+- [ ] Edit `<path_to_sardana_installation_dir>/sardanacustomsettings.py`
+  to point to the demo2 door e.g. `UNITTEST_DOOR_NAME = "door/demo2/1"`
+- [ ] Run testsuite. In another console do `sardanatestsuite`
+
+### Test Sardana using Spock
+- [ ] Test interactive macros from spock e.g. `ask_for_moveable`, `ask_peak`
+- [ ] Execute `umvr` macro and verify that the position updates arrives.
+- [ ] Test `expconf`:
+  1. Configure scan files using expconf set ScanDir to: `/tmp/` and
+     ScanFile to: `demo1.h5, demo1.dat`.
+  2. Configure online plot to show counters.
+  3. Configure snapshot group: with a motor and the `sys/tg_test/1/ampli`
+     attribute.
+  4. Add the `sys/tg_test/1/double_scalar` attribute to the measurement
+     group.
+  5. Open online plot.
+  6. Set JsonRecorder to true. In spock do `senv JsonRecorder True`
+  7. Run step scan and verify if:
+     - Records appear in spock output.
+     - Records were stored in scan files.
+     - Records were plotted on the online plot
+- [ ] Run `showscan` and access to the last scan data.
+- [ ] Test `edmac`:
+  1. Modify existing macro: `ask_peak` and run it to verify that the change
+     was applied.
+  2. Create new macro in a new macro library:
+     `edmac my_macro <path_to_sardana_installation_dir>/macroserver/macros/examples/my_macros.py
+     and run it.
+
+### Test Sardana with TaurusGUI
+
+- [ ] Create the GUI using this [guide](https://sourceforge.net/p/sardana/wiki/Howto-GUI_creation)
+
+#### PMTV (PoolMotorTaurusValue)
+- [ ] Move motors from the slit panel in absolute and relative modes.
+- [ ] Show expert view.
+- [ ] Show compact mode.
+
+#### macroexecutor
+- [ ] Execute `ascan` macro
+- [ ] Pause it in the middel and resume
+- [ ] Abort it
+- [ ] Add it to favorites
+- [ ] Run `lsm` macro
+- [ ] Execute `ascan` from favorites
+- [ ] Run `lsmac` macro
+- [ ] Execute `ascan` from history
+- [ ] Edit `dscan` macro in spock yellow line and run it
+- [ ] Restart macroexecutor application
+- [ ] Run `lsm` from history
+- [ ] Run `ascan` from favorites
+
+#### sequencer
+- [ ] Add `ascan` macro to the sequence
+- [ ] Add `lsct` macro as a `post-acq` hook of `ascan`
+- [ ] Add `dscan` macro to the sequence
+- [ ] Run the sequence
+- [ ] Save sequence to a file
+- [ ] Start new sequence
+- [ ] Load sequence from a files
+- [ ] Run the loaded sequence
+
+#### sardanaeditor
+- [ ] Open sardanaeditor with macroserver name as argument.
+- [ ] Browse macro libraries and open an existing macro.
+- [ ] Edit existing macro and save & apply chaneges.
+- [ ] Execute macro to see if changes were aplied.
+- [ ] Create a new macro using template.
+- [ ] Execute the newly created macro.

doc/man/MacroServer.1

@@ -1,7 +1,7 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH MACROSERVER "1" "August 2017" "MacroServer 2.3.2" "User Commands"
+.TH MACROSERVER "1" "February 2018" "MacroServer 2.4.0" "User Commands"
 .SH NAME
-MacroServer \- manual page for MacroServer 2.3.2
+MacroServer \- manual page for MacroServer 2.4.0
 .SH SYNOPSIS
 .B usage:
 \fI\,MacroServer instance_name \/\fR[\fI\,options\/\fR]

doc/man/Pool.1

@@ -1,7 +1,7 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH POOL "1" "August 2017" "Pool 2.3.2" "User Commands"
+.TH POOL "1" "February 2018" "Pool 2.4.0" "User Commands"
 .SH NAME
-Pool \- manual page for Pool 2.3.2
+Pool \- manual page for Pool 2.4.0
 .SH SYNOPSIS
 .B usage:
 \fI\,Pool instance_name \/\fR[\fI\,options\/\fR]

doc/man/Sardana.1

@@ -1,7 +1,7 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH SARDANA "1" "August 2017" "Sardana 2.3.2" "User Commands"
+.TH SARDANA "1" "February 2018" "Sardana 2.4.0" "User Commands"
 .SH NAME
-Sardana \- manual page for Sardana 2.3.2
+Sardana \- manual page for Sardana 2.4.0
 .SH SYNOPSIS
 .B usage:
 \fI\,Sardana instance_name \/\fR[\fI\,options\/\fR]

doc/man/diffractometeralignment.1

@@ -1,7 +1,7 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH DIFFRACTOMETERALIGNMENT "1" "August 2017" "diffractometeralignment 2.3.2" "User Commands"
+.TH DIFFRACTOMETERALIGNMENT "1" "February 2018" "diffractometeralignment 2.4.0" "User Commands"
 .SH NAME
-diffractometeralignment \- manual page for diffractometeralignment 2.3.2
+diffractometeralignment \- manual page for diffractometeralignment 2.4.0
 .SH SYNOPSIS
 .B diffractometeralignment
 \fI\,<model> \/\fR[\fI\,door_name\/\fR]
@@ -38,3 +38,6 @@ e.g. tango://foo:1234)
 .TP
 \fB\-\-remote\-console\-port\fR=\fI\,PORT\/\fR
 enables remote debugging using the given port
+.TP
+\fB\-\-default\-formatter\fR=\fI\,FORMATTER\/\fR
+Override the default formatter

doc/man/hklscan.1

@@ -1,7 +1,7 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH HKLSCAN "1" "August 2017" "hklscan 2.3.2" "User Commands"
+.TH HKLSCAN "1" "February 2018" "hklscan 2.4.0" "User Commands"
 .SH NAME
-hklscan \- manual page for hklscan 2.3.2
+hklscan \- manual page for hklscan 2.4.0
 .SH SYNOPSIS
 .B hklscan
 \fI\,<model> \/\fR[\fI\,door_name\/\fR]
@@ -37,3 +37,6 @@ e.g. tango://foo:1234)
 .TP
 \fB\-\-remote\-console\-port\fR=\fI\,PORT\/\fR
 enables remote debugging using the given port
+.TP
+\fB\-\-default\-formatter\fR=\fI\,FORMATTER\/\fR
+Override the default formatter