From 9724d199768df26fcba439b4009df686342885f5 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 21 Feb 2024 17:32:40 -0500 Subject: [PATCH 01/12] WIP: test with string IOC --- pyproject.toml | 3 +- src/srx_caproto_iocs/example/__init__.py | 1 + src/srx_caproto_iocs/example/caproto_ioc.py | 60 ++++++++++++++ src/srx_caproto_iocs/example/ophyd.py | 15 ++++ tests/conftest.py | 88 ++++++++++++++------- tests/test_string_ioc.py | 52 ++++++++++++ 6 files changed, 191 insertions(+), 28 deletions(-) create mode 100644 src/srx_caproto_iocs/example/__init__.py create mode 100644 src/srx_caproto_iocs/example/caproto_ioc.py create mode 100644 src/srx_caproto_iocs/example/ophyd.py create mode 100644 tests/test_string_ioc.py diff --git a/pyproject.toml b/pyproject.toml index d81d23d..ac54ad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ markers = [ "hardware: marks tests as requiring the hardware IOC to be available/running (deselect with '-m \"not hardware\"')", "tiled: marks tests as requiring tiled", "cloud_friendly: marks tests to be able to execute in the CI in the cloud", + "needs_epics_core: marks tests as requiring epics-core executables such as caget, caput, etc." ] [tool.coverage] @@ -135,7 +136,7 @@ extend-select = [ "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib "RET", # flake8-return - "RUF", # Ruff-specific + # "RUF", # Ruff-specific "SIM", # flake8-simplify # "T20", # flake8-print "UP", # pyupgrade diff --git a/src/srx_caproto_iocs/example/__init__.py b/src/srx_caproto_iocs/example/__init__.py new file mode 100644 index 0000000..5baeb5f --- /dev/null +++ b/src/srx_caproto_iocs/example/__init__.py @@ -0,0 +1 @@ +""""Example Caproto IOC code.""" diff --git a/src/srx_caproto_iocs/example/caproto_ioc.py b/src/srx_caproto_iocs/example/caproto_ioc.py new file mode 100644 index 0000000..d4b38a9 --- /dev/null +++ b/src/srx_caproto_iocs/example/caproto_ioc.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import textwrap + +from caproto import ChannelType +from caproto.server import PVGroup, pvproperty, run, template_arg_parser + +from ..base import check_args + + +class CaprotoStringIOC(PVGroup): + """Test channel types for strings.""" + + common_kwargs = {"max_length": 255, "string_encoding": "utf-8"} + + bare_string = pvproperty( + value="bare_string", doc="A test for a bare string", **common_kwargs + ) + implicit_string_type = pvproperty( + value="implicit_string_type", + doc="A test for an implicit string type", + report_as_string=True, + **common_kwargs, + ) + string_type = pvproperty( + value="string_type", + doc="A test for a string type", + dtype=str, + report_as_string=True, + **common_kwargs, + ) + string_type_enum = pvproperty( + value="string_type_enum", + doc="A test for a string type", + dtype=ChannelType.STRING, + **common_kwargs, + ) + char_type_as_string = pvproperty( + value="char_type_as_string", + doc="A test for a char type reported as string", + report_as_string=True, + dtype=ChannelType.CHAR, + **common_kwargs, + ) + char_type = pvproperty( + value="char_type", + doc="A test for a char type not reported as string", + dtype=ChannelType.CHAR, + **common_kwargs, + ) + + +if __name__ == "__main__": + parser, split_args = template_arg_parser( + default_prefix="", desc=textwrap.dedent(CaprotoStringIOC.__doc__) + ) + ioc_options, run_options = check_args(parser, split_args) + ioc = CaprotoStringIOC(**ioc_options) + + run(ioc.pvdb, **run_options) diff --git a/src/srx_caproto_iocs/example/ophyd.py b/src/srx_caproto_iocs/example/ophyd.py new file mode 100644 index 0000000..68c20c5 --- /dev/null +++ b/src/srx_caproto_iocs/example/ophyd.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from ophyd import Component as Cpt +from ophyd import Device, EpicsSignal + + +class OphydChannelTypes(Device): + """An ophyd Device which works with the CaprotoIOCChannelTypes caproto IOC.""" + + bare_string = Cpt(EpicsSignal, "bare_string", string=True) + implicit_string_type = Cpt(EpicsSignal, "implicit_string_type", string=True) + string_type = Cpt(EpicsSignal, "string_type", string=True) + string_type_enum = Cpt(EpicsSignal, "string_type_enum", string=True) + char_type_as_string = Cpt(EpicsSignal, "char_type_as_string", string=True) + char_type = Cpt(EpicsSignal, "char_type", string=True) diff --git a/tests/conftest.py b/tests/conftest.py index 5137c09..0512b94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,62 +5,91 @@ import subprocess import sys import time as ttime -from pprint import pformat -import netifaces import pytest from srx_caproto_iocs.base import OphydDeviceWithCaprotoIOC +from srx_caproto_iocs.example.ophyd import OphydChannelTypes CAPROTO_PV_PREFIX = "BASE:{{Dev:Save1}}:" OPHYD_PV_PREFIX = CAPROTO_PV_PREFIX.replace("{{", "{").replace("}}", "}") -@pytest.fixture() -def base_ophyd_device(): - dev = OphydDeviceWithCaprotoIOC( - OPHYD_PV_PREFIX, name="ophyd_device_with_caproto_ioc" - ) - yield dev - dev.ioc_stage.put("unstaged") - - -@pytest.fixture(scope="session") -def base_caproto_ioc(wait=5): +def get_epics_env(): first_three = ".".join(socket.gethostbyname(socket.gethostname()).split(".")[:3]) broadcast = f"{first_three}.255" print(f"{broadcast = }") - env = { + # from pprint import pformat + # import netifaces + # interfaces = netifaces.interfaces() + # print(f"{interfaces = }") + # for interface in interfaces: + # addrs = netifaces.ifaddresses(interface) + # try: + # print(f"{interface = }: {pformat(addrs[netifaces.AF_INET])}") + # except Exception as e: + # print(f"{interface = }: exception:\n {e}") + + return { "EPICS_CAS_BEACON_ADDR_LIST": os.getenv("EPICS_CA_ADDR_LIST", broadcast), "EPICS_CAS_AUTO_BEACON_ADDR_LIST": "no", } - print(f"Updating env with:\n\n{pformat(env)}\n") - os.environ.update(env) - interfaces = netifaces.interfaces() - print(f"{interfaces = }") - for interface in interfaces: - addrs = netifaces.ifaddresses(interface) - try: - print(f"{interface = }: {pformat(addrs[netifaces.AF_INET])}") - except Exception as e: - print(f"{interface = }: exception:\n {e}") +def start_ioc_subprocess(ioc_name="srx_caproto_iocs.base", pv_prefix=CAPROTO_PV_PREFIX): + env = get_epics_env() - command = f"{sys.executable} -m srx_caproto_iocs.base --prefix={CAPROTO_PV_PREFIX} --list-pvs" + command = f"{sys.executable} -m {ioc_name} --prefix={pv_prefix} --list-pvs" print( f"\nStarting caproto IOC in via a fixture using the following command:\n\n {command}\n" ) - p = subprocess.Popen( + return subprocess.Popen( command.split(), start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, - env=os.environ, + env=env, + ) + + +@pytest.fixture(scope="session") +def base_caproto_ioc(wait=5): + p = start_ioc_subprocess( + ioc_name="srx_caproto_iocs.base", pv_prefix=CAPROTO_PV_PREFIX + ) + + print(f"Wait for {wait} seconds...") + ttime.sleep(wait) + + yield p + + p.terminate() + + std_out, std_err = p.communicate() + std_out = std_out.decode() + sep = "=" * 80 + print(f"STDOUT:\n{sep}\n{std_out}") + print(f"STDERR:\n{sep}\n{std_err}") + + +@pytest.fixture() +def base_ophyd_device(): + dev = OphydDeviceWithCaprotoIOC( + OPHYD_PV_PREFIX, name="ophyd_device_with_caproto_ioc" ) + yield dev + dev.ioc_stage.put("unstaged") + + +@pytest.fixture(scope="session") +def caproto_ioc_channel_types(wait=5): + p = start_ioc_subprocess( + ioc_name="srx_caproto_iocs.example.caproto_ioc", pv_prefix=CAPROTO_PV_PREFIX + ) + print(f"Wait for {wait} seconds...") ttime.sleep(wait) @@ -73,3 +102,8 @@ def base_caproto_ioc(wait=5): sep = "=" * 80 print(f"STDOUT:\n{sep}\n{std_out}") print(f"STDERR:\n{sep}\n{std_err}") + + +@pytest.fixture() +def ophyd_channel_types(): + return OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") diff --git a/tests/test_string_ioc.py b/tests/test_string_ioc.py new file mode 100644 index 0000000..d91edec --- /dev/null +++ b/tests/test_string_ioc.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import string +import subprocess + +import pytest + +LIMIT = 40 - 1 +STRING_39 = string.ascii_letters[:LIMIT] +STRING_LONGER = string.ascii_letters + + +@pytest.mark.cloud_friendly() +@pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) +def test_strings( + # caproto_ioc_channel_types, + ophyd_channel_types, + value, +): + ophyd_channel_types.bare_string.put(value) + + if len(value) <= LIMIT: + ophyd_channel_types.implicit_string_type.put(value) + else: + with pytest.raises(ValueError): + ophyd_channel_types.implicit_string_type.put(value) + + if len(value) <= LIMIT: + ophyd_channel_types.string_type.put(value) + else: + with pytest.raises(ValueError): + ophyd_channel_types.string_type.put(value) + + if len(value) <= LIMIT: + ophyd_channel_types.char_type_as_string.put(value) + else: + with pytest.raises(ValueError): + ophyd_channel_types.char_type_as_string.put(value) + + ophyd_channel_types.char_type.put(value) + + +@pytest.mark.cloud_friendly() +@pytest.mark.needs_epics_core() +@pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) +def test_with_epics_core(ophyd_channel_types, value): + for cpt in ophyd_channel_types.component_names: + ret = subprocess.run( + ["caput", "-S", getattr(ophyd_channel_types, cpt).pvname, value], + check=False, + ) + print(f"{cpt=}: {ret.returncode=}\n") From dc98043b2ca396d718cd5d82c685ff7112a5984b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 21 Feb 2024 19:38:47 -0500 Subject: [PATCH 02/12] STY: fix linting errors --- tests/test_string_ioc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_string_ioc.py b/tests/test_string_ioc.py index d91edec..31af7ac 100644 --- a/tests/test_string_ioc.py +++ b/tests/test_string_ioc.py @@ -5,7 +5,7 @@ import pytest -LIMIT = 40 - 1 +LIMIT = 39 STRING_39 = string.ascii_letters[:LIMIT] STRING_LONGER = string.ascii_letters @@ -22,19 +22,19 @@ def test_strings( if len(value) <= LIMIT: ophyd_channel_types.implicit_string_type.put(value) else: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.implicit_string_type.put(value) if len(value) <= LIMIT: ophyd_channel_types.string_type.put(value) else: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.string_type.put(value) if len(value) <= LIMIT: ophyd_channel_types.char_type_as_string.put(value) else: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.char_type_as_string.put(value) ophyd_channel_types.char_type.put(value) From cfe47e2db45b619cd2849c98e1d04a72151155dc Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 21 Feb 2024 19:39:14 -0500 Subject: [PATCH 03/12] Parametrize the script to start different caproto IOCs --- scripts/run-caproto-ioc.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/run-caproto-ioc.sh b/scripts/run-caproto-ioc.sh index 76c8257..9cfaefe 100644 --- a/scripts/run-caproto-ioc.sh +++ b/scripts/run-caproto-ioc.sh @@ -2,6 +2,8 @@ set -vxeuo pipefail +CAPROTO_IOC="${1:-srx_caproto_iocs.base}" + # shellcheck source=/dev/null if [ -f "/etc/profile.d/epics.sh" ]; then . /etc/profile.d/epics.sh @@ -10,4 +12,4 @@ fi export EPICS_CAS_AUTO_BEACON_ADDR_LIST="no" export EPICS_CAS_BEACON_ADDR_LIST="${EPICS_CA_ADDR_LIST:-127.0.0.255}" -python -m srx_caproto_iocs.base --prefix="BASE:{{Dev:Save1}}:" --list-pvs +python -m "${CAPROTO_IOC}" --prefix="BASE:{{Dev:Save1}}:" --list-pvs From 9f1f5c68ec99c7e7f1f364c8ddb03b0857e07c81 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 21 Feb 2024 22:57:40 -0500 Subject: [PATCH 04/12] TST: `cainfo` and `caput` tests for string caproto IOC --- tests/conftest.py | 7 ++++- tests/test_string_ioc.py | 68 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0512b94..f4e30ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import os import socket +import string import subprocess import sys import time as ttime @@ -106,4 +107,8 @@ def caproto_ioc_channel_types(wait=5): @pytest.fixture() def ophyd_channel_types(): - return OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") + dev = OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") + letters = iter(list(string.ascii_letters)) + for cpt in sorted(dev.component_names): + getattr(dev, cpt).put(next(letters)) + return dev diff --git a/tests/test_string_ioc.py b/tests/test_string_ioc.py index 31af7ac..af2e46c 100644 --- a/tests/test_string_ioc.py +++ b/tests/test_string_ioc.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import string import subprocess @@ -13,7 +14,7 @@ @pytest.mark.cloud_friendly() @pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) def test_strings( - # caproto_ioc_channel_types, + caproto_ioc_channel_types, ophyd_channel_types, value, ): @@ -31,6 +32,12 @@ def test_strings( with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.string_type.put(value) + if len(value) <= LIMIT: + ophyd_channel_types.string_type_enum.put(value) + else: + with pytest.raises(ValueError, match="byte string too long"): + ophyd_channel_types.string_type_enum.put(value) + if len(value) <= LIMIT: ophyd_channel_types.char_type_as_string.put(value) else: @@ -40,13 +47,64 @@ def test_strings( ophyd_channel_types.char_type.put(value) +@pytest.mark.cloud_friendly() +@pytest.mark.needs_epics_core() +def test_cainfo(caproto_ioc_channel_types, ophyd_channel_types): + for cpt in sorted(ophyd_channel_types.component_names): + command = ["cainfo", getattr(ophyd_channel_types, cpt).pvname] + command_str = " ".join(command) + ret = subprocess.run( + command, + check=False, + capture_output=True, + ) + stdout = ret.stdout.decode() + print( + f"command: {command_str}\n {ret.returncode=}\n STDOUT:\n{ret.stdout.decode()}\n STDERR:\n{ret.stderr.decode()}\n" + ) + assert ret.returncode == 0 + if cpt in [ + "char_type_as_string", + "implicit_string_type", + "string_type", + "string_type_enum", + ]: + assert "Native data type: DBF_STRING" in stdout + else: + assert "Native data type: DBF_CHAR" in stdout + + @pytest.mark.cloud_friendly() @pytest.mark.needs_epics_core() @pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) -def test_with_epics_core(ophyd_channel_types, value): - for cpt in ophyd_channel_types.component_names: +def test_caput(caproto_ioc_channel_types, ophyd_channel_types, value): + option = "" + for cpt in sorted(ophyd_channel_types.component_names): + if cpt in [ + "char_type_as_string", + "implicit_string_type", + "string_type", + "string_type_enum", + ]: + option = "-s" + would_trim = True + else: + option = "-S" + would_trim = False + command = ["caput", option, getattr(ophyd_channel_types, cpt).pvname, value] + command_str = " ".join(command) ret = subprocess.run( - ["caput", "-S", getattr(ophyd_channel_types, cpt).pvname, value], + command, check=False, + capture_output=True, + ) + stdout = ret.stdout.decode() + print( + f"command: {command_str}\n {ret.returncode=}\n STDOUT:\n{stdout}\n STDERR:\n{ret.stderr.decode()}\n" ) - print(f"{cpt=}: {ret.returncode=}\n") + assert ret.returncode == 0 + actual = re.search("New : (.*)", stdout).group(1).split()[-1].rstrip() + if not would_trim or len(value) == LIMIT: + assert actual == value + else: + assert len(actual) < len(value) From 2d05e7646e4215ec2a7d9aecf0f4fd68a9104afa Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 00:33:17 -0500 Subject: [PATCH 05/12] TST: fix the blocking issue in the tests --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index f4e30ea..9cf01ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,13 +46,14 @@ def start_ioc_subprocess(ioc_name="srx_caproto_iocs.base", pv_prefix=CAPROTO_PV_ print( f"\nStarting caproto IOC in via a fixture using the following command:\n\n {command}\n" ) + os.environ.update(env) return subprocess.Popen( command.split(), start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, - env=env, + env=os.environ, ) From 9c9f53988c06fe338fa16c26ff54f45136d03e4f Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 02:30:41 -0500 Subject: [PATCH 06/12] CI: install `epics-base` package from conda-forge (for `caget`, `caput`, `cainfo`, etc.) --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++++----- pyproject.toml | 1 - 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35705f2..1e227f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,16 +53,15 @@ jobs: env: TZ: America/New_York + defaults: + run: + shell: bash -l {0} + steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Set env vars run: | @@ -75,9 +74,33 @@ jobs: export DATETIME_STRING=$(date +%Y%m%d%H%M%S) echo "DATETIME_STRING=${DATETIME_STRING}" >> $GITHUB_ENV + # - uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # allow-prereleases: true + + - name: Set up Python ${{ matrix.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: + ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} + python-version: ${{ matrix.python-version }} + mamba-version: "*" + miniforge-version: latest + channels: conda-forge + use-mamba: true + + - name: Install epics-base packages from CF + run: | + set -vxeuo pipefail + mamba install -c conda-forge -y epics-base "setuptools<67" + conda env list + conda list + - name: Install package run: | set -vxeuo pipefail + which caput python -m pip install .[test] - name: Test package diff --git a/pyproject.toml b/pyproject.toml index ac54ad8..580497f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "netifaces", "pytest >=6", "pytest-cov >=3", ] From 590e5543de21db6ff86a8370b3627f1ee402f8d8 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 02:52:49 -0500 Subject: [PATCH 07/12] Minor tweaks and addressing review comments from PR#2 --- .github/workflows/ci.yml | 2 +- scripts/run-act.sh | 4 +++- scripts/test-file-saving.sh | 5 +---- src/srx_caproto_iocs/example/ophyd.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e227f6..3088332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: - name: Set env vars run: | - + set -x export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV diff --git a/scripts/run-act.sh b/scripts/run-act.sh index 212dd68..768ad28 100644 --- a/scripts/run-act.sh +++ b/scripts/run-act.sh @@ -1,3 +1,5 @@ #!/bin/bash -act -W .github/workflows/ci.yml -j checks --matrix python-version:3.11 +PYTHON_VERSION="${1:-3.11}" + +act -W .github/workflows/ci.yml -j checks --matrix python-version:"${PYTHON_VERSION}" diff --git a/scripts/test-file-saving.sh b/scripts/test-file-saving.sh index 423d569..2159796 100644 --- a/scripts/test-file-saving.sh +++ b/scripts/test-file-saving.sh @@ -10,10 +10,7 @@ fi num="${1:-50}" data_dir="/tmp/test/$(date +%Y/%m/%d)" - -if [ ! -d "${data_dir}" ]; then - mkdir -v -p "${data_dir}" -fi +mkdir -v -p "${data_dir}" caput "BASE:{Dev:Save1}:write_dir" "${data_dir}" caput "BASE:{Dev:Save1}:file_name" "saveme_{num:06d}_{uid}.h5" diff --git a/src/srx_caproto_iocs/example/ophyd.py b/src/srx_caproto_iocs/example/ophyd.py index 68c20c5..a44ac08 100644 --- a/src/srx_caproto_iocs/example/ophyd.py +++ b/src/srx_caproto_iocs/example/ophyd.py @@ -5,7 +5,7 @@ class OphydChannelTypes(Device): - """An ophyd Device which works with the CaprotoIOCChannelTypes caproto IOC.""" + """An ophyd Device which works with the CaprotoStringIOC caproto IOC.""" bare_string = Cpt(EpicsSignal, "bare_string", string=True) implicit_string_type = Cpt(EpicsSignal, "implicit_string_type", string=True) From ab77f7ff51676409de017f407eaf2027bd64acb2 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 03:27:32 -0500 Subject: [PATCH 08/12] CI: use micromamba for the python env --- .github/workflows/ci.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3088332..6bdf544 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,22 +80,12 @@ jobs: # allow-prereleases: true - name: Set up Python ${{ matrix.python-version }} with conda - uses: conda-incubator/setup-miniconda@v3 + uses: mamba-org/setup-micromamba@v1 with: - activate-environment: - ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} - python-version: ${{ matrix.python-version }} - mamba-version: "*" - miniforge-version: latest - channels: conda-forge - use-mamba: true - - - name: Install epics-base packages from CF - run: | - set -vxeuo pipefail - mamba install -c conda-forge -y epics-base "setuptools<67" - conda env list - conda list + init-shell: bash + environment-name: ${{env.REPOSITORY_NAME}}-py${{matrix.python-version}} + create-args: >- + python=${{ matrix.python-version }} epics-base setuptools<67 - name: Install package run: | From def5ded510af630193b8cbc18901a8686d608aa7 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 03:56:00 -0500 Subject: [PATCH 09/12] Change `write_dir` and `file_name` PV types to CHAR and fix the relevant test --- src/srx_caproto_iocs/base.py | 4 ++-- tests/test_base_ophyd.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 2ad314f..9fb1faf 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -39,14 +39,14 @@ class CaprotoSaveIOC(PVGroup): value="/tmp", doc="The directory to write data to. It support datetime formatting, e.g. '/tmp/det/%Y/%m/%d/'", string_encoding="utf-8", - report_as_string=True, + dtype=ChannelType.CHAR, max_length=255, ) file_name = pvproperty( value="test.h5", doc="The file name of the file to write to. It support .format() based formatting, e.g. 'scan_{num:06d}.h5'", string_encoding="utf-8", - report_as_string=True, + dtype=ChannelType.CHAR, max_length=255, ) full_file_path = pvproperty( diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index f5ecade..d052bb1 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -22,7 +22,12 @@ def test_base_ophyd_templates( date = now(as_object=True) write_dir_root = Path(tmpdirname) dir_template = f"{write_dir_root}/{date_template}" - write_dir = Path(date.strftime(dir_template)) + + # We pre-create the test directory in advance as the IOC is not supposed to create one. + # The assumption for the IOC is that the directory will exist before saving a file to that. + # We need to replace the blank spaces below for it to work, as the IOC will replace + # any blank spaces in `full_file_path` before returning the value. + write_dir = Path(date.strftime(dir_template).replace(" ", "_")) write_dir.mkdir(parents=True, exist_ok=True) file_template = "scan_{num:06d}_{uid}.hdf5" From ccca4d3c63669aeffc3e4c1868df6d564bfd1989 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 03:59:53 -0500 Subject: [PATCH 10/12] Minor improvements --- scripts/run-act.sh | 2 ++ tests/conftest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/run-act.sh b/scripts/run-act.sh index 768ad28..074a8c6 100644 --- a/scripts/run-act.sh +++ b/scripts/run-act.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -vxeuo pipefail + PYTHON_VERSION="${1:-3.11}" act -W .github/workflows/ci.yml -j checks --matrix python-version:"${PYTHON_VERSION}" diff --git a/tests/conftest.py b/tests/conftest.py index 9cf01ec..a605f72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,7 +109,7 @@ def caproto_ioc_channel_types(wait=5): @pytest.fixture() def ophyd_channel_types(): dev = OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") - letters = iter(list(string.ascii_letters)) + letters = iter(string.ascii_letters) for cpt in sorted(dev.component_names): getattr(dev, cpt).put(next(letters)) return dev From 7d0e104cb63c737509efe214a3d7e486f429712c Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 23 Feb 2024 04:06:24 -0500 Subject: [PATCH 11/12] sanitize file path better * replaces unsupport characters with underscores _ * github artifact upload does not support the special chars: - Double quote " - Colon : - Less than < - Greater than > - Vertical bar | - Asterisk * - Question mark ? - Carriage return \r - Line feed \n * note that \r, \n are included in whitespace regex symbol \s --- src/srx_caproto_iocs/base.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 9fb1faf..570794d 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import textwrap import threading import time as ttime @@ -116,12 +117,15 @@ async def stage(self, instance, value): if value == StageStates.STAGED.value: # Steps: - # 1. Render 'write_dir' with datetime lib and replace any blank spaces with underscores. - # 2. Render 'file_name' with .format(). - # 3. Replace blank spaces with underscores. + # 1. Render 'write_dir' with datetime lib + # 2. Replace unsupported characters with underscores (sanitize). + # 3. Check if sanitized 'write_dir' exists + # 4. Render 'file_name' with .format(). + # 5. Replace unsupported characters with underscores. + sanitizer = re.compile(pattern=r"[\":<>|\*\?\s]") date = now(as_object=True) - write_dir = Path(date.strftime(self.write_dir.value).replace(" ", "_")) + write_dir = Path(sanitizer.sub("_", date.strftime(self.write_dir.value))) if not write_dir.exists(): msg = f"Path '{write_dir}' does not exist." print(msg) @@ -136,8 +140,7 @@ async def stage(self, instance, value): full_file_path = write_dir / file_name.format( num=self.frame_num.value, uid=uid, suid=uid[:8] ) - full_file_path = str(full_file_path) - full_file_path.replace(" ", "_") + full_file_path = sanitizer.sub("_", str(full_file_path)) print(f"{now()}: {full_file_path = }") From 571539ed508b78a71afd8fb122d96cc6d7027877 Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 23 Feb 2024 09:18:51 -0500 Subject: [PATCH 12/12] update `test_base_ophyd.py` to use the correct path sanitizer --- tests/test_base_ophyd.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index d052bb1..7e514bd 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import shutil import time as ttime import uuid @@ -25,9 +26,10 @@ def test_base_ophyd_templates( # We pre-create the test directory in advance as the IOC is not supposed to create one. # The assumption for the IOC is that the directory will exist before saving a file to that. - # We need to replace the blank spaces below for it to work, as the IOC will replace - # any blank spaces in `full_file_path` before returning the value. - write_dir = Path(date.strftime(dir_template).replace(" ", "_")) + # We need to substitute the unsupported characters below for it to work, as the IOC will do + # the same in `full_file_path` before returning the value. + sanitizer = re.compile(pattern=r"[\":<>|\*\?\s]") + write_dir = Path(sanitizer.sub("_", date.strftime(dir_template))) write_dir.mkdir(parents=True, exist_ok=True) file_template = "scan_{num:06d}_{uid}.hdf5"