Fixes to work with collection-2021* conda envs

.ci/bl-specific.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
-# cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/...
+# TODO: watch for https://github.com/NSLS-II-HXN/hxnfly/pull/8 to be
+# merged/released, then this install can be removed from here.
+python3 -m pip install --no-deps -vv git+https://github.com/NSLS-II-HXN/hxnfly@master
 
-conda install -y -c ${CONDA_CHANNEL_NAME} 03-id-hxn-collection
+sudo mkdir -v -p /home/xf03id/
+sudo chown -Rv $USER: /home/xf03id/
+touch /home/xf03id/benchmark.out

.gitignore

@@ -64,23 +64,28 @@ security
 profile_default
 
 *.sw*
-profile_collection/startup/20-detectors.py.bak.write-on-gpfs-on-merlin_matt_20200219
-profile_collection/startup/20-detectors.py.write-local-on-merlin-test_matt_20200219
+startup/20-detectors.py.bak.write-on-gpfs-on-merlin_matt_20200219
+startup/20-detectors.py.write-local-on-merlin-test_matt_20200219
 *~
-profile_collection/startup/#*.py#
-profile_collection/startup/eLog.pdf
-profile_collection/startup/eLog_info.obj
+startup/#*.py#
+startup/eLog.pdf
+startup/eLog_info.obj
 old/
 # Quick test:
-profile_collection/startup/20-detectors-eiger.py
+startup/20-detectors-eiger.py
+
+# Ignore temporary files
+startup/tmp_fig.pdf
+startup/tmp_img.png
+startup/eLog_info.obj
 
 # PostScript files to ignore (no .ps extension):
-profile_collection/startup/functools
-profile_collection/startup/gridspec
-profile_collection/startup/h5py
-profile_collection/startup/np
-profile_collection/startup/os
-profile_collection/startup/plt
-profile_collection/startup/shutil
-profile_collection/startup/sys
-profile_collection/startup/css-workspace-xf03id-ws1/
+startup/functools
+startup/gridspec
+startup/h5py
+startup/np
+startup/os
+startup/plt
+startup/shutil
+startup/sys
+startup/css-workspace-xf03id-ws1/

azure-pipelines.yml

@@ -7,12 +7,15 @@ resources:
       endpoint: github
 
 jobs:
-  - template: collection-2021-1.2.yml@templates
-    parameters:
-      beamline_acronym: HXN
   - template: nsls2-collection-2021-2.2.yml@templates
     parameters:
       beamline_acronym: HXN
   - template: nsls2-collection-2021-2.2-py39.yml@templates
     parameters:
       beamline_acronym: HXN
+  - template: nsls2-collection-2021-3.0-py37.yml@templates
+    parameters:
+      beamline_acronym: HXN
+  - template: nsls2-collection-2021-3.0-py39.yml@templates
+    parameters:
+      beamline_acronym: HXN

startup/00-startup.py

@@ -1,48 +1,85 @@
-import certifi
+import functools
+import os
+import time
+import uuid
 import warnings
+from collections import deque
+from datetime import datetime, timedelta, tzinfo
+
+# The following code allows to call Matplotlib API from threads (experimental)
+# Requires https://github.com/tacaswell/mpl-qtthread (not packaged yet)
+import matplotlib
+import matplotlib.backends.backend_qt5
+import mpl_qtthread
+# set up the teleporter
+mpl_qtthread.backend.initialize_qt_teleporter()
+# tell Matplotlib to use this backend
+matplotlib.use("module://mpl_qtthread.backend_agg")
+# suppress (now) spurious warnings for mpl3.3+
+mpl_qtthread.monkeypatch_pyplot()
+
+import certifi
+import ophyd
 import pandas as pd
+import pymongo
+import six
+from ophyd.signal import EpicsSignalBase
+
+import certifi
 import ophyd
+import pandas as pd
+import pymongo
+import six
+from ophyd.signal import EpicsSignalBase
+
+EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10)
 
 # Set up a Broker.
 # TODO clean this up
 from bluesky_kafka import Publisher
-from databroker import Broker
-from databroker.headersource.mongo import MDS
+from databroker.v0 import Broker
 from databroker.assets.mongo import Registry
-
 from databroker.headersource.core import doc_or_uid_to_uid
-
-from datetime import timedelta, datetime, tzinfo
-
-import pymongo
-from pymongo import MongoClient
-
-import uuid
+from databroker.headersource.mongo import MDS
+from databroker.v0 import Broker
 from jsonschema import validate as js_validate
-import six
-from collections import deque
+from pymongo import MongoClient
 
-import os
 os.environ["PPMAC_HOST"] = "xf03idc-ppmac1"
 
+bootstrap_servers = os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", None)
+if bootstrap_servers is None:
+    # https://github.com/NSLS-II/nslsii/blob/b332c34813adf798c38184292d21537ef4f653bb/nslsii/__init__.py#L710-L712
+    msg = ("The 'BLUESKY_KAFKA_BOOTSTRAP_SERVERS' environment variable must "
+           "be defined as a comma-delimited list of Kafka server addresses "
+           "or hostnames and ports as a string such as "
+           "``'kafka1:9092,kafka2:9092``")
+    raise RuntimeError(msg)
 
+kafka_password = os.getenv("BLUESKY_KAFKA_PASSWORD", None)
+if kafka_password is None:
+    msg = "The 'BLUESKY_KAFKA_PASSWORD' environment variable must be set."
+    raise RuntimeError(msg)
 
 kafka_publisher = Publisher(
         topic="hxn.bluesky.datum.documents",
-        bootstrap_servers=os.environ['BLUESKY_KAFKA_BOOTSTRAP_SERVERS'],
+        bootstrap_servers=bootstrap_servers,
         key=str(uuid.uuid4()),
         producer_config={
                 "acks": 1,
                 "message.timeout.ms": 3000,
                 "queue.buffering.max.kbytes": 10 * 1048576,
                 "compression.codec": "snappy",
                 "security.protocol": "SSL",
-                "ssl.ca.location": certifi.where()
-            },
+                "ssl.ca.location": certifi.where(),
+                "security.protocol": "SASL_SSL",
+                "sasl.mechanisms": "PLAIN",
+                "sasl.username": "beamline",
+                "sasl.password": kafka_password,
+                },
         flush_on_stop_doc=True,
     )
 
-
 # DB1
 db1_name = 'rs'
 db1_addr = 'mongodb://xf03id1-mdb01:27017,xf03id1-mdb02:27017,xf03id1-mdb03:27017'
@@ -257,6 +294,8 @@ def bulk_register_datum_table(self, resource_uid, dkwargs_table, validate=False)
 mds_db1 = MDS(_mds_config_db1, auth=False)
 db1 = Broker(mds_db1, CompositeRegistry(_fs_config_db1))
 
+
+# wrapper for two databases
 class CompositeBroker(Broker):
     """wrapper for two databases"""
 
@@ -326,16 +365,18 @@ def insert(self, name, doc):
 db = CompositeBroker(mds_db1, CompositeRegistry(_fs_config_db1))
 
 from hxntools.handlers import register as _hxn_register_handlers
+
 _hxn_register_handlers(db)
 del _hxn_register_handlers
 # do the rest of the standard configuration
 from IPython import get_ipython
 from nslsii import configure_base, configure_olog
 
-configure_base(get_ipython().user_ns, db, bec=False)
+configure_base(get_ipython().user_ns, db, bec=False, ipython_logging=False)
 # configure_olog(get_ipython().user_ns)
 
 from bluesky.callbacks.best_effort import BestEffortCallback
+
 bec = BestEffortCallback()
 
 # un import *
@@ -355,11 +396,11 @@ def insert(self, name, doc):
 RE.md['beamline_id'] = 'HXN'
 RE.verbose = True
 
-# set up some HXN specific callbacks
-from ophyd.callbacks import UidPublish
 from hxntools.scan_number import HxnScanNumberPrinter
 from hxntools.scan_status import HxnScanStatus
 from ophyd import EpicsSignal
+# set up some HXN specific callbacks
+from ophyd.callbacks import UidPublish
 
 uid_signal = EpicsSignal('XF:03IDC-ES{BS-Scan}UID-I', name='uid_signal')
 uid_broadcaster = UidPublish(uid_signal)
@@ -372,7 +413,7 @@ def flush_on_stop_doc(name, doc):
         kafka_publisher.flush()
 
 # This is needed to prevent the local buffer from filling.
-RE.subscribe('stop', flush_on_stop_doc)
+RE.subscribe(flush_on_stop_doc, 'stop')
 
 # Pass on only start/stop documents to a few subscriptions
 for _event in ('start', 'stop'):
@@ -389,6 +430,7 @@ def ensure_proposal_id(md):
 
 # be nice on segfaults
 import faulthandler
+
 faulthandler.enable()
 
 # set up logging framework
@@ -417,8 +459,8 @@ def ensure_proposal_id(md):
 pd.options.display.max_rows = None
 pd.options.display.max_columns = 10
 
+from bluesky.plan_stubs import mov
 
-from bluesky.plan_stubs import  mov
 # from bluesky.utils import register_transform
 
 def register_transform(RE, *, prefix='<'):
@@ -435,6 +477,7 @@ def register_transform(RE, *, prefix='<'):
         valid python syntax or an existing transform you are on your own.
     '''
     import IPython
+
     # from IPython.core.inputtransformer2 import StatelessInputTransformer
 
  #   @StatelessInputTransformer.wrap
@@ -522,7 +565,17 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs):
     if as_string is None:
         as_string = self._string
 
-    with self._lock:
+    ###########################################
+    # Usedf only for old ophyd 1.3.3 and older.
+    from distutils.version import LooseVersion
+
+    import ophyd
+
+    if ophyd.__version__ < LooseVersion('1.4'):
+        self._metadata_lock = self._lock
+    ###########################################
+
+    with self._metadata_lock:
         if not self._read_pv.connected:
             if not self._read_pv.wait_for_connection(connection_timeout):
                 raise TimeoutError('Failed to connect to %s' %
@@ -557,11 +610,9 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs):
     return ret
 
 
-from ophyd import EpicsSignal
-from ophyd import EpicsSignalRO
+from ophyd import EpicsSignal, EpicsSignalRO
 from ophyd.areadetector import EpicsSignalWithRBV
 
 EpicsSignal.get = _epicssignal_get
 EpicsSignalRO.get = _epicssignal_get
 EpicsSignalWithRBV.get = _epicssignal_get
-

startup/13-mll.py

@@ -32,9 +32,9 @@ class HxnMLLSample(NamedDevice):
     sz = Cpt(EpicsMotor, 'XF:03IDC-ES{ANC350:3-Ax:2}Mtr', doc='coarse z')
 
     kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1}KillAll-Cmd.PROC',
-               doc='kill all piezos')
+               doc='kill all piezos', kind='omitted')
     zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1}KillZero-Cmd.PROC',
-               doc='zero all piezos')
+               doc='zero all piezos', kind='omitted')
     # sz1 = Cpt(EpicsMotor, 'XF:03IDC-ES{ANC350:3-Ax:3}Mtr', doc='coarse z1')
     # sz1 was replaced with vz when controller 2 died
 
@@ -69,12 +69,12 @@ class HxnMLLDiffractionSample(NamedDevice):
     #dssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssy}Mtr', doc='fine_y')
     #dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssx}Mtr', doc='fine_z')
     # swap back dssx and dssz (08/07/19, X. Huang)
-    dssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssx}Mtr', doc='fine_x')
-    dssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssy}Mtr', doc='fine_y')
-    dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssz}Mtr', doc='fine_z')
+    dssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssx}Mtr', doc='fine_x', timeout = 120)
+    dssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssy}Mtr', doc='fine_y', timeout = 120)
+    dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssz}Mtr', doc='fine_z', timeout = 120)
 
     kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-Diff}Kill-Cmd.PROC',
-               doc='kill all piezos')
+               doc='kill all piezos', kind='omitted')
 
 
 smlld = HxnMLLDiffractionSample('', name='smlld')

startup/15-zp.py

@@ -41,9 +41,9 @@ class SmarpodRotationAxis(SmarpodBase):
 class HxnZPSample(NamedDevice):
     # Zoneplate module fine sample stage axes (closed on cap
     # sensors/interferometer)
-    zpssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssx}Mtr', doc='fine x')
-    zpssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssy}Mtr', doc='fine y')
-    zpssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssz}Mtr', doc='fine z')
+    zpssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssx}Mtr', doc='fine x', timeout = 120)
+    zpssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssy}Mtr', doc='fine y', timeout = 120)
+    zpssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssz}Mtr', doc='fine z', timeout = 120)
 
     # rotary underneath sample
     zpsth = Cpt(EpicsMotor, 'XF:03IDC-ES{SC210:1-Ax:1}Mtr', doc='theta')
@@ -58,8 +58,8 @@ class HxnZPSample(NamedDevice):
     smarthy = Cpt(SmarpodRotationAxis, axis=3, doc='smarpod theta around y')
     smarthz = Cpt(SmarpodRotationAxis, axis=1, doc='smarpod theta around z')
 
-    kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Kill-Cmd.PROC')
-    zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Zero-Cmd.PROC')
+    kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Kill-Cmd.PROC', kind='omitted')
+    zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Zero-Cmd.PROC', kind='omitted')
     mode = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Mode-I')
 
 

startup/21-xspress3.py

@@ -1,7 +1,7 @@
 from ophyd.device import (Component as Cpt)
 
-from hxntools.detectors.xspress3 import (Xspress3FileStore,
-                                         Xspress3Channel)
+from nslsii.detectors.xspress3 import (Xspress3FileStore,
+                                       Xspress3Channel)
 from hxntools.detectors.hxn_xspress3 import HxnXspress3DetectorBase
 import threading
 from ophyd import DeviceStatus
@@ -139,7 +139,12 @@ def unstage(self, *args, **kwargs):
 
 
 # Create directories on the xspress3 server, otherwise scans can fail:
-xspress3.make_directories.put(True)
+#====================================================================
+# Note from DAMA: commented it out during the 2020-2 deployment visit.
+# The corresponding code is in
+# https://github.com/NSLS-II/nslsii/blob/master/nslsii/detectors/xspress3.py
+#====================================================================
+# xspress3.make_directories.put(True)
 
 
 elem_K_list = np.array(['Na','Mg','Al','Si','P','S','Cl','Ar','K','Ca','Sc','Ti','V','Cr','Mn','Fe','Co','Ni','Cu','Zn','Ga','Ge','As','Se','Br','Kr','Rb','Sr','Y','Zr','Nb','Mo','Tc','Ru','Rh','Pd','Ag','Cd','In','Sn','Sb','Te','I','Xe','Cs','Ba','La','Hf','Ta','W','Re','Os','Ir','Pt','Au','Hg','Tl','Pb','Bi','Po','At','Rn','Fr','Ra','Ac','Ce','Pr','Nd','Pm','Sm','Eu','Gd','Tb','Dy','Ho','Er','Tm','Yb','Lu','Th','Pa','U','Np','Pu','Am','Cm','Bk','Cf'])
@@ -156,7 +161,7 @@ def unstage(self, *args, **kwargs):
 
 
 def xspress3_roi_setup():
-    elem_list = np.array(['Mg','Si','Ge','Cl','Ga','Ti','Fe','Zn','Mn','Cu','Ni','Cr','Co','Pt_L','Hf_L','W_L'])
+    elem_list = np.array(['V','Si','S','P','Ca','K','Fe','Zn','Al','Ti','Ni','Cr','Co','Pt_M','Au_L','Mn'])
     num_elem = np.size(elem_list)
     if num_elem > 16:
         num_elem = 16
@@ -173,6 +178,10 @@ def xspress3_roi_setup():
                 break
             channel.set_roi(i+1, energy-150, energy+150, name=elem_list[i])
 
+            # Uncomment the following lines for debugging liveplots without beam.
+            # if elem_list[i] == "Au_L":
+            #     channel.set_roi(i+1, 0, 40000, name=elem_list[i])
+
 
 '''
 def xspress3_roi_setup():
@@ -267,4 +276,3 @@ def configure_xspress3(sclr):
                 attr.kind = 'config'
 
 configure_xspress3(xspress3)
-