From 60034c80ac7eb1bd2991bbed7d0a86a10a47df3f Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 21 Jan 2021 17:56:49 -0500 Subject: [PATCH 01/16] Add a missing file to fix CI --- .ci/bl-specific.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/bl-specific.sh b/.ci/bl-specific.sh index 7aece2c..b77b4ec 100644 --- a/.ci/bl-specific.sh +++ b/.ci/bl-specific.sh @@ -3,3 +3,7 @@ # cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/... conda install -y -c ${CONDA_CHANNEL_NAME} 03-id-hxn-collection + +mkdir -v -p /home/xf03id/ + +touch /home/xf03id/benchmark.out From 9f1071ff0161c5e03d16cc9d740c93db31d5c869 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 21 Jan 2021 18:11:37 -0500 Subject: [PATCH 02/16] mkdir with sudo --- .ci/bl-specific.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/bl-specific.sh b/.ci/bl-specific.sh index b77b4ec..b8c6aed 100644 --- a/.ci/bl-specific.sh +++ b/.ci/bl-specific.sh @@ -4,6 +4,6 @@ conda install -y -c ${CONDA_CHANNEL_NAME} 03-id-hxn-collection -mkdir -v -p /home/xf03id/ - +sudo mkdir -v -p /home/xf03id/ +sudo chown -Rv $USER: /home/xf03id/ touch /home/xf03id/benchmark.out From 69f0d404f0bb2d61087271cf44ce4a4a4bf03ab3 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 14:31:36 -0400 Subject: [PATCH 03/16] Use default value for `BLUESKY_KAFKA_BOOTSTRAP_SERVERS` --- startup/00-startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 8405497..575c776 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -29,7 +29,7 @@ kafka_publisher = Publisher( topic="hxn.bluesky.datum.documents", - bootstrap_servers=os.environ['BLUESKY_KAFKA_BOOTSTRAP_SERVERS'], + bootstrap_servers=os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", "") key=str(uuid.uuid4()), producer_config={ "acks": 1, From 2414f7c2cb929e4b02ac4ff3d409ada240d16218 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 14:37:54 -0400 Subject: [PATCH 04/16] CI: do not install bl-specific packages --- .ci/bl-specific.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/bl-specific.sh b/.ci/bl-specific.sh index b8c6aed..d0b1cb8 100644 --- a/.ci/bl-specific.sh +++ b/.ci/bl-specific.sh @@ -1,9 +1,5 @@ #!/bin/bash -# cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/... - -conda install -y -c ${CONDA_CHANNEL_NAME} 03-id-hxn-collection - sudo mkdir -v -p /home/xf03id/ sudo chown -Rv $USER: /home/xf03id/ touch /home/xf03id/benchmark.out From 22c87d25c1f42f095310c652c328fb5011782dda Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 14:42:31 -0400 Subject: [PATCH 05/16] Fix missing comma --- startup/00-startup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 575c776..4950559 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -26,10 +26,9 @@ os.environ["PPMAC_HOST"] = "xf03idc-ppmac1" - kafka_publisher = Publisher( topic="hxn.bluesky.datum.documents", - bootstrap_servers=os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", "") + bootstrap_servers=os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", ""), key=str(uuid.uuid4()), producer_config={ "acks": 1, From 8502195f567677f154be3c88b580a014202df383 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 14 Oct 2020 16:26:00 -0400 Subject: [PATCH 06/16] WIP: Fixes to work with collection-2020-2.0rc8 conda env --- startup/00-startup.py | 25 +++++++-- startup/13-mll.py | 6 +- startup/15-zp.py | 4 +- startup/21-xspress3.py | 11 +++- startup/70-users_plans.py | 114 ++++++++++++++++++++------------------ 5 files changed, 95 insertions(+), 65 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 4950559..b356df9 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -6,7 +6,7 @@ # Set up a Broker. # TODO clean this up from bluesky_kafka import Publisher -from databroker import Broker +from databroker.v0 import Broker from databroker.headersource.mongo import MDS from databroker.assets.mongo import Registry @@ -254,8 +254,17 @@ def bulk_register_datum_table(self, resource_uid, dkwargs_table, validate=False) mds_db1 = MDS(_mds_config_db1, auth=False) -db1 = Broker(mds_db1, CompositeRegistry(_fs_config_db1)) +cr1 = CompositeRegistry(_fs_config_db1) +db1 = Broker(mds_db1, cr1) +# Broker 2 + +mds_db2 = MDS(_mds_config_db2, auth=False) +cr2 = CompositeRegistry(_fs_config_db2) +db2 = Broker(mds_db2, cr2) + + +# wrapper for two databases class CompositeBroker(Broker): """wrapper for two databases""" @@ -521,7 +530,15 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs): if as_string is None: as_string = self._string - with self._lock: + ########################################### + # Usedf only for old ophyd 1.3.3 and older. + from distutils.version import LooseVersion + import ophyd + if ophyd.__version__ < LooseVersion('1.4'): + self._metadata_lock = self._lock + ########################################### + + with self._metadata_lock: if not self._read_pv.connected: if not self._read_pv.wait_for_connection(connection_timeout): raise TimeoutError('Failed to connect to %s' % @@ -560,7 +577,7 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs): from ophyd import EpicsSignalRO from ophyd.areadetector import EpicsSignalWithRBV + EpicsSignal.get = _epicssignal_get EpicsSignalRO.get = _epicssignal_get EpicsSignalWithRBV.get = _epicssignal_get - diff --git a/startup/13-mll.py b/startup/13-mll.py index 37c32c8..2138e6f 100755 --- a/startup/13-mll.py +++ b/startup/13-mll.py @@ -32,9 +32,9 @@ class HxnMLLSample(NamedDevice): sz = Cpt(EpicsMotor, 'XF:03IDC-ES{ANC350:3-Ax:2}Mtr', doc='coarse z') kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1}KillAll-Cmd.PROC', - doc='kill all piezos') + doc='kill all piezos', kind='omitted') zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1}KillZero-Cmd.PROC', - doc='zero all piezos') + doc='zero all piezos', kind='omitted') # sz1 = Cpt(EpicsMotor, 'XF:03IDC-ES{ANC350:3-Ax:3}Mtr', doc='coarse z1') # sz1 was replaced with vz when controller 2 died @@ -74,7 +74,7 @@ class HxnMLLDiffractionSample(NamedDevice): dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssz}Mtr', doc='fine_z') kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-Diff}Kill-Cmd.PROC', - doc='kill all piezos') + doc='kill all piezos', kind='omitted') smlld = HxnMLLDiffractionSample('', name='smlld') diff --git a/startup/15-zp.py b/startup/15-zp.py index e3fe377..7233cb3 100755 --- a/startup/15-zp.py +++ b/startup/15-zp.py @@ -58,8 +58,8 @@ class HxnZPSample(NamedDevice): smarthy = Cpt(SmarpodRotationAxis, axis=3, doc='smarpod theta around y') smarthz = Cpt(SmarpodRotationAxis, axis=1, doc='smarpod theta around z') - kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Kill-Cmd.PROC') - zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Zero-Cmd.PROC') + kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Kill-Cmd.PROC', kind='omitted') + zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Zero-Cmd.PROC', kind='omitted') mode = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Mode-I') diff --git a/startup/21-xspress3.py b/startup/21-xspress3.py index 35cdbd2..6887b13 100755 --- a/startup/21-xspress3.py +++ b/startup/21-xspress3.py @@ -1,7 +1,7 @@ from ophyd.device import (Component as Cpt) -from hxntools.detectors.xspress3 import (Xspress3FileStore, - Xspress3Channel) +from nslsii.detectors.xspress3 import (Xspress3FileStore, + Xspress3Channel) from hxntools.detectors.hxn_xspress3 import HxnXspress3DetectorBase import threading from ophyd import DeviceStatus @@ -139,7 +139,12 @@ def unstage(self, *args, **kwargs): # Create directories on the xspress3 server, otherwise scans can fail: -xspress3.make_directories.put(True) +#==================================================================== +# Note from DAMA: commented it out during the 2020-2 deployment visit. +# The corresponding code is in +# https://github.com/NSLS-II/nslsii/blob/master/nslsii/detectors/xspress3.py +#==================================================================== +# xspress3.make_directories.put(True) elem_K_list = np.array(['Na','Mg','Al','Si','P','S','Cl','Ar','K','Ca','Sc','Ti','V','Cr','Mn','Fe','Co','Ni','Cu','Zn','Ga','Ge','As','Se','Br','Kr','Rb','Sr','Y','Zr','Nb','Mo','Tc','Ru','Rh','Pd','Ag','Cd','In','Sn','Sb','Te','I','Xe','Cs','Ba','La','Hf','Ta','W','Re','Os','Ir','Pt','Au','Hg','Tl','Pb','Bi','Po','At','Rn','Fr','Ra','Ac','Ce','Pr','Nd','Pm','Sm','Eu','Gd','Tb','Dy','Ho','Er','Tm','Yb','Lu','Th','Pa','U','Np','Pu','Am','Cm','Bk','Cf']) diff --git a/startup/70-users_plans.py b/startup/70-users_plans.py index 3d254d8..8fadd5e 100755 --- a/startup/70-users_plans.py +++ b/startup/70-users_plans.py @@ -111,7 +111,12 @@ def mll_z_fly2d(z_start, z_end, z_num, mot1, start1, end1, num1, mot2, start2, e yield from bps.mov(smlld.sbz, init_sz) save_page() + def zp_z_2dscan(z_start, z_end, z_num, mot1, start1, end1, num1, mot2, start2, end2, num2, acq_time, elem='Co'): + """This function hasn't been used for a while. + + TODO: convert to use 'yield from ...'. + """ z_step = (z_end - z_start)/z_num init_sz = zps.zpsz.position movr(zps.zpsz, z_start/1000) @@ -122,6 +127,7 @@ def zp_z_2dscan(z_start, z_end, z_num, mot1, start1, end1, num1, mot2, start2, e movr(zps.zpsz, z_step/1000) mov(zps.zpsz, init_sz) + def go_det(det): if det == 'merlin': @@ -242,7 +248,7 @@ def mll_mosaic_scan(x_start, x_end, x_num, x_block, y_start, y_end, y_num, y_blo #start mosaic scan for i in range(y_block): for j in range(x_block): - smll_sync_piezos() + yield from smll_sync_piezos() RE(fly2d(smll.ssx, x_start, x_end, x_num, smll.ssy, y_start, y_end, y_num, acq_time, return_speed=40)) dx = x_block_size movr_sx(dx) @@ -2154,11 +2160,13 @@ def mono_m1(pf_start, pf_end, pf_num, b_start, b_end, b_num): def smll_kill_piezos(): - smll.kill.put(1) + # smll.kill.put(1) + yield from bps.mv(smll.kill, 1) yield from bps.sleep(5) def smll_zero_piezos(): - smll.zero.put(1) + # smll.zero.put(1) + yield from bps.mv(smll.zero, 1) yield from bps.sleep(3) def smll_sync_piezos(): @@ -2177,7 +2185,7 @@ def movr_sx(dist): print('Current ssy = %.3f' % c_ssy) print('Current ssz = %.3f' % c_ssz) - smll_kill_piezos() + yield from smll_kill_piezos() t_ssx = c_ssx + dist @@ -2186,20 +2194,20 @@ def movr_sx(dist): dx, dz = sample_to_lab(dxp, dzp, alpha) - movr(sx, dx) - movr(sz, dz) + yield from bps.movr(sx, dx) + yield from bps.movr(sz, dz) dy = c_ssy -smll.ssy.position - movr(sy, dy) + yield from bps.movr(sy, dy) - sleep(5) + yield from bps.sleep(5) - smll_sync_piezos() + yield from smll_sync_piezos() - mov(ssx, t_ssx) - mov(ssy, c_ssy) - mov(ssz, c_ssz) + yield from bps.mov(ssx, t_ssx) + yield from bps.mov(ssy, c_ssy) + yield from bps.mov(ssz, c_ssz) print('Post-move x = %.3f' % smll.ssx.position) print('Post-move y = %.3f' % smll.ssy.position) @@ -2215,7 +2223,7 @@ def mov_sx(t_pos): print('Current ssy = %.3f' % c_ssy) print('Current ssz = %.3f' % c_ssz) - smll_kill_piezos() + yield from smll_kill_piezos() t_ssx = t_pos @@ -2224,19 +2232,19 @@ def mov_sx(t_pos): dx, dz = sample_to_lab(dxp, dzp, alpha) - movr(sx, dx) - movr(sz, dz) + yield from bps.movr(sx, dx) + yield from bps.movr(sz, dz) dy = c_ssy - smll.ssy.position - movr(sy, dy) + yield from bps.movr(sy, dy) - sleep(5) + yield from bps.sleep(5) - smll_sync_piezos() - mov(ssx, t_ssx) - mov(ssy, c_ssy) - mov(ssz, c_ssz) + yield from smll_sync_piezos() + yield from bps.mov(ssx, t_ssx) + yield from bps.mov(ssy, c_ssy) + yield from bps.mov(ssz, c_ssz) print('Post-move x = %.3f' % (smll.ssx.position)) print('Post-move y = %.3f' % (smll.ssy.position)) @@ -2253,26 +2261,26 @@ def movr_sy(dist): print('Current ssy = %.3f' % c_ssy) print('Current ssz = %.3f' % c_ssz) - smll_kill_piezos() + yield from smll_kill_piezos() t_ssy = c_ssy + dist dy = t_ssy - smll.ssy.position - movr(sy, dy) + yield from bps.movr(sy, dy) dxp = c_ssx - smll.ssx.position dzp = c_ssz - smll.ssz.position dx, dz = sample_to_lab(dxp, dzp, alpha) - movr(sx, dx) - movr(sz, dz) + yield from bps.movr(sx, dx) + yield from bps.movr(sz, dz) - sleep(5) + yield from bps.sleep(5) - smll_sync_piezos() - mov(ssx, c_ssx) - mov(ssy, t_ssy) - mov(ssz, c_ssz) + yield from smll_sync_piezos() + yield from bps.mov(ssy, t_ssy) + yield from bps.mov(ssz, c_ssz) + yield from bps.mov(ssx, c_ssx) print('Post-move x = %.3f' % smll.ssx.position) print('Post-move y = %.3f' % smll.ssy.position) @@ -2288,26 +2296,26 @@ def mov_sy(t_pos): print('Current ssy = %.3f' % c_ssy) print('Current ssz = %.3f' % c_ssz) - smll_kill_piezos() + yield from smll_kill_piezos() t_ssy = t_pos dy = t_ssy - smll.ssy.position - movr(sy, dy) + yield from bps.movr(sy, dy) dxp = c_ssx - smll.ssx.position dzp = c_ssz - smll.ssz.position dx, dz = sample_to_lab(dxp, dzp, alpha) - movr(sbx, dx) - movr(sbz, dz) + yield from bps.movr(sbx, dx) + yield from bps.movr(sbz, dz) - sleep(5) + yield from bps.sleep(5) - smll_sync_piezos() - mov(ssx, c_ssx) - mov(ssy, t_ssy) - mov(ssz, c_ssz) + yield from smll_sync_piezos() + yield from bps.mov(ssx, c_ssx) + yield from bps.mov(ssy, t_ssy) + yield from bps.mov(ssz, c_ssz) print('Post-move x = %.3f' % (smll.ssx.position)) print('Post-move y = %.3f' % (smll.ssy.position)) @@ -2324,20 +2332,20 @@ def movr_sz(dist): print('Current ssy = %.3f' % c_ssy) print('Current ssz = %.3f' % c_ssz) - smll_kill_piezos() + yield from smll_kill_piezos() t_ssz = c_ssz + dist*np.cos(alpha) dz = t_ssz - smll.dssz.position dy = c_ssy - smll.dssy.position - movr(sbz, dz) - movr(dsy, dy/1000.0) + yield from bps.movr(sbz, dz) + yield from bps.movr(dsy, dy/1000.0) - sleep(5) + yield from bps.sleep(5) - smll_sync_piezos() - mov(dssy, c_ssy) - mov(dssz, t_ssz) + yield from smll_sync_piezos() + yield from bps.mov(dssy, c_ssy) + yield from bps.mov(dssz, t_ssz) print('post-move x = %.3f' % smll.dssx.position) print('Post-move y = %.3f' % smll.dssy.position) @@ -2354,20 +2362,20 @@ def mov_sz(t_pos): print('Current ssy = %.3f' % c_ssy) print('Current ssz = %.3f' % c_ssz) - smll_kill_piezos() + yield from smll_kill_piezos() t_ssz = t_pos dz = (t_ssz - smll.ssz.position)/np.cos(alpha) dy = c_ssy - smll.ssy.position - movr(sz, dz) - movr(sy, dy) + yield from bps.movr(sz, dz) + yield from bps.movr(sy, dy) - sleep(5) + yield from bps.sleep(5) - smll_sync_piezos() - mov(ssy, c_ssy) - mov(ssz, t_ssz) + yield from smll_sync_piezos() + yield from bps.mov(ssy, c_ssy) + yield from bps.mov(ssz, t_ssz) print('Post-move x = %.3f' % smll.ssx.position) print('Post-move y = %.3f' % smll.ssy.position) From 9a05425c4095d788f86c72813d67dc35282fcf4d Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 15:14:36 -0400 Subject: [PATCH 07/16] Remove `db2` --- startup/00-startup.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index b356df9..56bdc5c 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -257,12 +257,6 @@ def bulk_register_datum_table(self, resource_uid, dkwargs_table, validate=False) cr1 = CompositeRegistry(_fs_config_db1) db1 = Broker(mds_db1, cr1) -# Broker 2 - -mds_db2 = MDS(_mds_config_db2, auth=False) -cr2 = CompositeRegistry(_fs_config_db2) -db2 = Broker(mds_db2, cr2) - # wrapper for two databases class CompositeBroker(Broker): From 34819c86b8be2cf9075c2bc4df2abe04bea6235a Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 15:20:56 -0400 Subject: [PATCH 08/16] Sync changes with master (manually) --- startup/00-startup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 56bdc5c..09e81e7 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -254,11 +254,9 @@ def bulk_register_datum_table(self, resource_uid, dkwargs_table, validate=False) mds_db1 = MDS(_mds_config_db1, auth=False) -cr1 = CompositeRegistry(_fs_config_db1) -db1 = Broker(mds_db1, cr1) +db1 = Broker(mds_db1, CompositeRegistry(_fs_config_db1)) -# wrapper for two databases class CompositeBroker(Broker): """wrapper for two databases""" From 4344afa94121272f2ce422f14da02067b4b1ac85 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 15:22:48 -0400 Subject: [PATCH 09/16] Sync more changes with master (manually) --- startup/00-startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 09e81e7..7d36fd0 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -6,7 +6,7 @@ # Set up a Broker. # TODO clean this up from bluesky_kafka import Publisher -from databroker.v0 import Broker +from databroker import Broker from databroker.headersource.mongo import MDS from databroker.assets.mongo import Registry From 06ef527f026ada3931c7882237fa8b9011d88165 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 16:43:59 -0400 Subject: [PATCH 10/16] Use databroker.v0 for `Broker` --- startup/00-startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 7d36fd0..09e81e7 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -6,7 +6,7 @@ # Set up a Broker. # TODO clean this up from bluesky_kafka import Publisher -from databroker import Broker +from databroker.v0 import Broker from databroker.headersource.mongo import MDS from databroker.assets.mongo import Registry From 4a65b1848ead9d79b783e5c3f82141a0baa6194e Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 18:08:30 -0400 Subject: [PATCH 11/16] Add ophyd timeout of 10 seconds --- startup/00-startup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/startup/00-startup.py b/startup/00-startup.py index 09e81e7..3413ee0 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -3,6 +3,9 @@ import pandas as pd import ophyd +from ophyd.signal import EpicsSignalBase +EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10) + # Set up a Broker. # TODO clean this up from bluesky_kafka import Publisher From 5e50e68a72a6d390047a2375d859b5cd8b05c476 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 18:34:54 -0400 Subject: [PATCH 12/16] CI: install `hxnfly` from the branch with the fixed imports --- .ci/bl-specific.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/bl-specific.sh b/.ci/bl-specific.sh index d0b1cb8..cd9091c 100644 --- a/.ci/bl-specific.sh +++ b/.ci/bl-specific.sh @@ -1,5 +1,9 @@ #!/bin/bash +# TODO: watch for https://github.com/NSLS-II-HXN/hxnfly/pull/8 to be +# merged/released, then this install can be removed from here. +python3 -m pip install --no-deps -vv git+https://github.com/NSLS-II-HXN/hxnfly@change-xspress3-imports + sudo mkdir -v -p /home/xf03id/ sudo chown -Rv $USER: /home/xf03id/ touch /home/xf03id/benchmark.out From 023150d77332c4bb44f281ddd62ce70d5e0865e9 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 18:35:56 -0400 Subject: [PATCH 13/16] Add missing imports; sort imports with VScode feature --- startup/00-startup.py | 47 +++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 3413ee0..02047a3 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -1,31 +1,30 @@ -import certifi +import functools +import os +import time +import uuid import warnings -import pandas as pd -import ophyd +from collections import deque +from datetime import datetime, timedelta, tzinfo +import certifi +import ophyd +import pandas as pd +import pymongo +import six from ophyd.signal import EpicsSignalBase + EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10) # Set up a Broker. # TODO clean this up from bluesky_kafka import Publisher -from databroker.v0 import Broker -from databroker.headersource.mongo import MDS from databroker.assets.mongo import Registry - from databroker.headersource.core import doc_or_uid_to_uid - -from datetime import timedelta, datetime, tzinfo - -import pymongo -from pymongo import MongoClient - -import uuid +from databroker.headersource.mongo import MDS +from databroker.v0 import Broker from jsonschema import validate as js_validate -import six -from collections import deque +from pymongo import MongoClient -import os os.environ["PPMAC_HOST"] = "xf03idc-ppmac1" @@ -329,6 +328,7 @@ def insert(self, name, doc): db = CompositeBroker(mds_db1, CompositeRegistry(_fs_config_db1)) from hxntools.handlers import register as _hxn_register_handlers + _hxn_register_handlers(db) del _hxn_register_handlers # do the rest of the standard configuration @@ -339,6 +339,7 @@ def insert(self, name, doc): # configure_olog(get_ipython().user_ns) from bluesky.callbacks.best_effort import BestEffortCallback + bec = BestEffortCallback() # un import * @@ -358,11 +359,11 @@ def insert(self, name, doc): RE.md['beamline_id'] = 'HXN' RE.verbose = True -# set up some HXN specific callbacks -from ophyd.callbacks import UidPublish from hxntools.scan_number import HxnScanNumberPrinter from hxntools.scan_status import HxnScanStatus from ophyd import EpicsSignal +# set up some HXN specific callbacks +from ophyd.callbacks import UidPublish uid_signal = EpicsSignal('XF:03IDC-ES{BS-Scan}UID-I', name='uid_signal') uid_broadcaster = UidPublish(uid_signal) @@ -392,6 +393,7 @@ def ensure_proposal_id(md): # be nice on segfaults import faulthandler + faulthandler.enable() # set up logging framework @@ -421,7 +423,8 @@ def ensure_proposal_id(md): pd.options.display.max_columns = 10 -from bluesky.plan_stubs import mov +from bluesky.plan_stubs import mov + # from bluesky.utils import register_transform def register_transform(RE, *, prefix='<'): @@ -438,6 +441,7 @@ def register_transform(RE, *, prefix='<'): valid python syntax or an existing transform you are on your own. ''' import IPython + # from IPython.core.inputtransformer2 import StatelessInputTransformer # @StatelessInputTransformer.wrap @@ -528,6 +532,7 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs): ########################################### # Usedf only for old ophyd 1.3.3 and older. from distutils.version import LooseVersion + import ophyd if ophyd.__version__ < LooseVersion('1.4'): self._metadata_lock = self._lock @@ -568,11 +573,9 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs): return ret -from ophyd import EpicsSignal -from ophyd import EpicsSignalRO +from ophyd import EpicsSignal, EpicsSignalRO from ophyd.areadetector import EpicsSignalWithRBV - EpicsSignal.get = _epicssignal_get EpicsSignalRO.get = _epicssignal_get EpicsSignalWithRBV.get = _epicssignal_get From e035e8546c0447987d8f30f2d579da08fd6a78b3 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 19:06:53 -0400 Subject: [PATCH 14/16] Make the `BLUESKY_KAFKA_BOOTSTRAP_SERVERS` env var required and better instructions on setting it --- startup/00-startup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 02047a3..ee3bf7e 100755 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -28,9 +28,18 @@ os.environ["PPMAC_HOST"] = "xf03idc-ppmac1" +bootstrap_servers = os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", None) +if bootstrap_servers is None: + # https://github.com/NSLS-II/nslsii/blob/b332c34813adf798c38184292d21537ef4f653bb/nslsii/__init__.py#L710-L712 + msg = ("The 'BLUESKY_KAFKA_BOOTSTRAP_SERVERS' environment variable must " + "be defined as a comma-delimited list of Kafka server addresses " + "or hostnames and ports as a string such as " + "``'kafka1:9092,kafka2:9092``") + raise RuntimeError(msg) + kafka_publisher = Publisher( topic="hxn.bluesky.datum.documents", - bootstrap_servers=os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", ""), + bootstrap_servers=bootstrap_servers, key=str(uuid.uuid4()), producer_config={ "acks": 1, From 73e45ddef9c4e91b1f35931d712985cb98a5c3be Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 19:09:10 -0400 Subject: [PATCH 15/16] CI: change to the PR branch `add-kafka-vars` of profile-collection-ci repo --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7b5feeb..8e02202 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,7 +3,8 @@ resources: - repository: templates type: github name: NSLS-II/profile-collection-ci - ref: refs/heads/main + # ref: refs/heads/main + ref: refs/heads/add-kafka-vars endpoint: github jobs: From 1afdfcfe0c52262389251f69b3eb870880231d79 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 27 Oct 2021 20:32:16 -0400 Subject: [PATCH 16/16] CI: use `main` branch of profile-collection-ci repo --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8e02202..7b5feeb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,8 +3,7 @@ resources: - repository: templates type: github name: NSLS-II/profile-collection-ci - # ref: refs/heads/main - ref: refs/heads/add-kafka-vars + ref: refs/heads/main endpoint: github jobs: