From e62bc004ea1e0a0a20ec09403c05dca3e3468ffd Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 1 Aug 2023 17:06:58 +0200 Subject: [PATCH 01/14] Bump mypy.ini to Python 3.8 Previously python_requires was bumped to 3.8, but mypy.ini wasn't updated with it. --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index cf05bf408..bbe6845e2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,4 @@ [mypy] ignore_missing_imports = 1 files = src/spead2, examples, tests -python_version = 3.7 +python_version = 3.8 From de87b39d615b058204025964b6afa5831a59dfcf Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 1 Aug 2023 17:06:14 +0200 Subject: [PATCH 02/14] Add pre-commit hooks --- .pre-commit-config.yaml | 38 ++++++++++++++++++++++++++++++++ doc/license.rst | 1 - include/spead2/portable_endian.h | 20 ++++++++--------- include/spead2/py_common.h | 2 +- src/common_memory_pool.cpp | 2 +- 5 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..1535e48c9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-executables-have-shebangs + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.4.1 + hooks: + - id: mypy + # Passing filenames to mypy can do odd things. See + # https://github.com/pre-commit/mirrors-mypy/issues/33. + # mypy.ini determines the set of files that will actually be checked. + pass_filenames: false + # The pre-commit hook passes some options, but we set options in mypy.ini. + args: [] + # The pre-commit hook only has python, not pyi. + types: [] + types_or: [python, pyi] + additional_dependencies: [ + 'numpy==1.24.4' + ] + - repo: https://github.com/jazzband/pip-tools + rev: 7.1.0 + hooks: + - id: pip-compile + args: [requirements.in] + language_version: python3.8 + files: '^requirements\.(?:in|txt)$' diff --git a/doc/license.rst b/doc/license.rst index 7b7214316..6d7f88658 100644 --- a/doc/license.rst +++ b/doc/license.rst @@ -13,4 +13,3 @@ details. You should have received a copy of the GNU Lesser General Public License along with this program. If not, see http://www.gnu.org/licenses/. - diff --git a/include/spead2/portable_endian.h b/include/spead2/portable_endian.h index 076bbe95b..155c930de 100644 --- a/include/spead2/portable_endian.h +++ b/include/spead2/portable_endian.h @@ -25,12 +25,12 @@ # define htole16(x) (x) # define be16toh(x) __builtin_bswap16(x) # define le16toh(x) (x) - + # define htobe32(x) __builtin_bswap32(x) # define htole32(x) (x) # define be32toh(x) __builtin_bswap32(x) # define le32toh(x) (x) - + # define htobe64(x) __builtin_bswap64(x) # define htole64(x) (x) # define be64toh(x) __builtin_bswap64(x) @@ -42,12 +42,12 @@ # define htole16(x) __builtin_bswap16(x) # define be16toh(x) (x) # define le16toh(x) __builtin_bswap16(x) - + # define htobe32(x) (x) # define htole32(x) __builtin_bswap32(x) # define be32toh(x) (x) # define le32toh(x) __builtin_bswap32(x) - + # define htobe64(x) (x) # define htole64(x) __builtin_bswap64(x) # define be64toh(x) (x) @@ -65,12 +65,12 @@ # define htole16(x) OSSwapHostToLittleInt16(x) # define be16toh(x) OSSwapBigToHostInt16(x) # define le16toh(x) OSSwapLittleToHostInt16(x) - + # define htobe32(x) OSSwapHostToBigInt32(x) # define htole32(x) OSSwapHostToLittleInt32(x) # define be32toh(x) OSSwapBigToHostInt32(x) # define le32toh(x) OSSwapLittleToHostInt32(x) - + # define htobe64(x) OSSwapHostToBigInt64(x) # define htole64(x) OSSwapHostToLittleInt64(x) # define be64toh(x) OSSwapBigToHostInt64(x) @@ -109,12 +109,12 @@ # define htole16(x) (x) # define be16toh(x) ntohs(x) # define le16toh(x) (x) - + # define htobe32(x) htonl(x) # define htole32(x) (x) # define be32toh(x) ntohl(x) # define le32toh(x) (x) - + # define htobe64(x) htonll(x) # define htole64(x) (x) # define be64toh(x) ntohll(x) @@ -127,12 +127,12 @@ # define htole16(x) __builtin_bswap16(x) # define be16toh(x) (x) # define le16toh(x) __builtin_bswap16(x) - + # define htobe32(x) (x) # define htole32(x) __builtin_bswap32(x) # define be32toh(x) (x) # define le32toh(x) __builtin_bswap32(x) - + # define htobe64(x) (x) # define htole64(x) __builtin_bswap64(x) # define be64toh(x) (x) diff --git a/include/spead2/py_common.h b/include/spead2/py_common.h index 399bf1eb9..6fc0aecb9 100644 --- a/include/spead2/py_common.h +++ b/include/spead2/py_common.h @@ -357,7 +357,7 @@ namespace detail // Convert a base class pointer to member function to the derived class template -static inline auto up(Result (Base::*func)(Args...)) -> Result (Derived::*)(Args...) +static inline auto up(Result (Base::*func)(Args...)) -> Result (Derived::*)(Args...) { return static_cast(func); } diff --git a/src/common_memory_pool.cpp b/src/common_memory_pool.cpp index 81f9f0541..334d5a0ff 100644 --- a/src/common_memory_pool.cpp +++ b/src/common_memory_pool.cpp @@ -84,7 +84,7 @@ class memory_pool_deleter state->allocator.reset(); } - memory_allocator::deleter &get_base_deleter() { return state->base_deleter; } + memory_allocator::deleter &get_base_deleter() { return state->base_deleter; } const memory_allocator::deleter &get_base_deleter() const { return state->base_deleter; } }; From 249b8c7fded01b514a10ec64f2e6e18882a932ee Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 1 Aug 2023 17:12:05 +0200 Subject: [PATCH 03/14] Make get_version.py executable To keep the pre-commit hooks happy. --- gen/get_version.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 gen/get_version.py diff --git a/gen/get_version.py b/gen/get_version.py old mode 100644 new mode 100755 From a51bb76777c90d34abc5491a36560e048434821c Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 1 Aug 2023 17:13:21 +0200 Subject: [PATCH 04/14] Remove flake8 from requirements.in It's now handled via pre-commit. --- requirements.in | 1 - requirements.txt | 8 -------- 2 files changed, 9 deletions(-) diff --git a/requirements.in b/requirements.in index b046e1ae1..366a7a476 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,3 @@ -flake8 jinja2 netifaces numba diff --git a/requirements.txt b/requirements.txt index 2320d8396..97e51f52a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,6 @@ # exceptiongroup==1.1.2 # via pytest -flake8==6.0.0 - # via -r requirements.in importlib-metadata==6.8.0 # via numba iniconfig==2.0.0 @@ -18,8 +16,6 @@ llvmlite==0.40.1 # via numba markupsafe==2.1.3 # via jinja2 -mccabe==0.7.0 - # via flake8 netifaces==0.11.0 # via -r requirements.in numba==0.57.1 @@ -35,12 +31,8 @@ packaging==23.1 # pytest pluggy==1.2.0 # via pytest -pycodestyle==2.10.0 - # via flake8 pycparser==2.21 # via -r requirements.in -pyflakes==3.0.1 - # via flake8 pytest==7.4.0 # via # -r requirements.in From 744a11ad9e84eb567e1b7466e6c5e3ac2ffa9994 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 1 Aug 2023 17:14:39 +0200 Subject: [PATCH 05/14] Run pre-commit instead of flake8 in Actions --- .github/workflows/test.yml | 2 +- requirements.in | 1 + requirements.txt | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eeb9681ae..f4ad3e5dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -111,7 +111,7 @@ jobs: - run: pip install -v . - name: Run tests run: ./.ci/py-tests.sh - - run: flake8 + - run: pre-commit run --all-files - name: Show ccache stats run: ccache -s diff --git a/requirements.in b/requirements.in index 366a7a476..34391a0aa 100644 --- a/requirements.in +++ b/requirements.in @@ -3,6 +3,7 @@ netifaces numba numpy packaging +pre-commit pycparser pytest pytest-asyncio diff --git a/requirements.txt b/requirements.txt index 97e51f52a..5cee8e182 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,16 @@ # # pip-compile requirements.in # +cfgv==3.3.1 + # via pre-commit +distlib==0.3.7 + # via virtualenv exceptiongroup==1.1.2 # via pytest +filelock==3.12.2 + # via virtualenv +identify==2.5.26 + # via pre-commit importlib-metadata==6.8.0 # via numba iniconfig==2.0.0 @@ -18,6 +26,8 @@ markupsafe==2.1.3 # via jinja2 netifaces==0.11.0 # via -r requirements.in +nodeenv==1.8.0 + # via pre-commit numba==0.57.1 # via -r requirements.in numpy==1.24.4 @@ -29,8 +39,12 @@ packaging==23.1 # via # -r requirements.in # pytest +platformdirs==3.10.0 + # via virtualenv pluggy==1.2.0 # via pytest +pre-commit==3.3.3 + # via -r requirements.in pycparser==2.21 # via -r requirements.in pytest==7.4.0 @@ -42,9 +56,16 @@ pytest-asyncio==0.21.0 # via -r requirements.in pytest-timeout==2.1.0 # via -r requirements.in +pyyaml==6.0.1 + # via pre-commit scipy==1.10.1 # via -r requirements.in tomli==2.0.1 # via pytest +virtualenv==20.24.2 + # via pre-commit zipp==3.16.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From bfb75bf96485fdef378110b5a01bb3d7f52c88f9 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 1 Aug 2023 17:25:27 +0200 Subject: [PATCH 06/14] Skip pip-compile pre-commit hook in Actions It was failing because a number of matrix items didn't have Python 3.8 installed. --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4ad3e5dd..3cc39fa2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -111,7 +111,8 @@ jobs: - run: pip install -v . - name: Run tests run: ./.ci/py-tests.sh - - run: pre-commit run --all-files + # Skip pip-compile because not all configurations have a python3.8 binary + - run: SKIP=pip-compile pre-commit run --all-files - name: Show ccache stats run: ccache -s From f5d00332882d2e7e137e5feabc974f94d2c7c355 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 09:22:52 +0200 Subject: [PATCH 07/14] Run isort over the code Also add it to pre-commit config. --- .pre-commit-config.yaml | 5 +++ examples/recv_chunk_group_example.py | 10 ++--- examples/recv_chunk_ring_example.py | 10 ++--- examples/test_recv.py | 3 +- examples/test_send.py | 6 ++- examples/test_send_asyncio.py | 3 +- gen/gen_loader.py | 3 +- gen/get_version.py | 2 +- pyproject.toml | 6 +++ setup.py | 2 +- src/spead2/__init__.py | 54 +++++++++++++++------------ src/spead2/__init__.pyi | 16 +++++++- src/spead2/recv/__init__.py | 10 +++-- src/spead2/recv/__init__.pyi | 14 ++++++- src/spead2/recv/asyncio.py | 2 +- src/spead2/recv/numba.py | 1 - src/spead2/recv/numba.pyi | 1 - src/spead2/send/__init__.py | 20 ++++++++-- src/spead2/send/__init__.pyi | 15 ++++++-- src/spead2/send/asyncio.py | 9 +++-- src/spead2/send/asyncio.pyi | 1 - src/spead2/tools/bench_asyncio.py | 10 ++--- src/spead2/tools/cmdline.py | 1 - src/spead2/tools/recv_asyncio.py | 5 ++- src/spead2/tools/send_asyncio.py | 9 +++-- tests/shutdown.py | 2 +- tests/test_passthrough.py | 4 +- tests/test_passthrough_asyncio.py | 4 +- tests/test_recv.py | 2 +- tests/test_recv_asyncio.py | 3 +- tests/test_recv_chunk_stream.py | 9 ++--- tests/test_recv_chunk_stream_group.py | 8 ++-- tests/test_send.py | 2 +- tests/test_send_asyncio.py | 2 +- 34 files changed, 160 insertions(+), 94 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1535e48c9..5793d2c2b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,9 @@ repos: + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/examples/recv_chunk_group_example.py b/examples/recv_chunk_group_example.py index 27fb4b598..afc234f9e 100755 --- a/examples/recv_chunk_group_example.py +++ b/examples/recv_chunk_group_example.py @@ -19,14 +19,14 @@ # ringbuffers. To test it, run # spead2_send localhost:8888 localhost:8889 --heaps 1000 --heap-size 65536 --rate 10 -from spead2.numba import intp_to_voidptr -import spead2.recv -from spead2.recv.numba import chunk_place_data - import numba -from numba import types import numpy as np import scipy +from numba import types + +import spead2.recv +from spead2.numba import intp_to_voidptr +from spead2.recv.numba import chunk_place_data HEAP_PAYLOAD_SIZE = 65536 HEAPS_PER_CHUNK = 64 diff --git a/examples/recv_chunk_ring_example.py b/examples/recv_chunk_ring_example.py index 2cadc164d..6de136688 100755 --- a/examples/recv_chunk_ring_example.py +++ b/examples/recv_chunk_ring_example.py @@ -19,14 +19,14 @@ # test it, run # spead2_send localhost:8888 --heaps 1000 --heap-size 65536 --rate 10 -from spead2.numba import intp_to_voidptr -import spead2.recv -from spead2.recv.numba import chunk_place_data - import numba -from numba import types import numpy as np import scipy +from numba import types + +import spead2.recv +from spead2.numba import intp_to_voidptr +from spead2.recv.numba import chunk_place_data HEAP_PAYLOAD_SIZE = 65536 HEAPS_PER_CHUNK = 64 diff --git a/examples/test_recv.py b/examples/test_recv.py index 3748fea82..f526708f0 100755 --- a/examples/test_recv.py +++ b/examples/test_recv.py @@ -15,9 +15,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import logging + import spead2 import spead2.recv -import logging logging.basicConfig(level=logging.INFO) diff --git a/examples/test_send.py b/examples/test_send.py index 8cf33a641..be329b2ae 100755 --- a/examples/test_send.py +++ b/examples/test_send.py @@ -15,11 +15,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -import spead2 -import spead2.send import logging + import numpy as np +import spead2 +import spead2.send + logging.basicConfig(level=logging.INFO) thread_pool = spead2.ThreadPool() diff --git a/examples/test_send_asyncio.py b/examples/test_send_asyncio.py index f66ab574f..b3b5a4799 100755 --- a/examples/test_send_asyncio.py +++ b/examples/test_send_asyncio.py @@ -15,8 +15,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -import logging import asyncio +import logging import numpy as np @@ -24,7 +24,6 @@ import spead2.send import spead2.send.asyncio - logging.basicConfig(level=logging.INFO) thread_pool = spead2.ThreadPool() diff --git a/gen/gen_loader.py b/gen/gen_loader.py index f26f55777..51a650921 100755 --- a/gen/gen_loader.py +++ b/gen/gen_loader.py @@ -22,9 +22,8 @@ import jinja2 from pycparser import c_ast -from pycparser.c_parser import CParser from pycparser.c_generator import CGenerator - +from pycparser.c_parser import CParser # The typedefs are arbitrary and just used to allow pycparser to parse the # code. Note that some functions in infiniband/verbs.h are implemented as diff --git a/gen/get_version.py b/gen/get_version.py index aab07050f..c37e265c9 100755 --- a/gen/get_version.py +++ b/gen/get_version.py @@ -18,8 +18,8 @@ import argparse import re import sys -from packaging.version import Version +from packaging.version import Version parser = argparse.ArgumentParser() parser.add_argument('mode', choices=('major', 'minor', 'patch', 'full')) diff --git a/pyproject.toml b/pyproject.toml index fe05c5f17..1bb39e3c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,9 @@ archs = ["x86_64"] CC = "ccache gcc" CFLAGS = "-flto" LDFLAGS = "-flto" + +[tool.isort] +profile = "black" +combine_as_imports = true +line_length = 100 +py_version = 38 diff --git a/setup.py b/setup.py index ff5b950e4..ba452a24f 100755 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ import os.path import subprocess -from setuptools import setup from pybind11.setup_helpers import ParallelCompile, Pybind11Extension, build_ext +from setuptools import setup class BuildExt(build_ext): diff --git a/src/spead2/__init__.py b/src/spead2/__init__.py index 63aa2da5e..bd8ed59ce 100644 --- a/src/spead2/__init__.py +++ b/src/spead2/__init__.py @@ -13,45 +13,53 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -import numbers as _numbers import logging +import numbers as _numbers import struct import numpy as _np import spead2._spead2 -from spead2._spead2 import ( # noqa: F401 - Flavour, ThreadPool, Stopped, Empty, Full, - MemoryAllocator, MmapAllocator, MemoryPool, InprocQueue, +from spead2._spead2 import ( # noqa: F401 BUG_COMPAT_DESCRIPTOR_WIDTHS, + BUG_COMPAT_PYSPEAD_0_5_2, BUG_COMPAT_SHAPE_BIT_1, BUG_COMPAT_SWAP_ENDIAN, - BUG_COMPAT_PYSPEAD_0_5_2, - NULL_ID, - HEAP_CNT_ID, - HEAP_LENGTH_ID, - PAYLOAD_OFFSET_ID, - PAYLOAD_LENGTH_ID, - DESCRIPTOR_ID, - STREAM_CTRL_ID, - DESCRIPTOR_NAME_ID, + CTRL_DESCRIPTOR_REISSUE, + CTRL_DESCRIPTOR_UPDATE, + CTRL_STREAM_START, + CTRL_STREAM_STOP, DESCRIPTOR_DESCRIPTION_ID, - DESCRIPTOR_SHAPE_ID, + DESCRIPTOR_DTYPE_ID, DESCRIPTOR_FORMAT_ID, + DESCRIPTOR_ID, DESCRIPTOR_ID_ID, - DESCRIPTOR_DTYPE_ID, - CTRL_STREAM_START, - CTRL_DESCRIPTOR_REISSUE, - CTRL_STREAM_STOP, - CTRL_DESCRIPTOR_UPDATE, + DESCRIPTOR_NAME_ID, + DESCRIPTOR_SHAPE_ID, + HEAP_CNT_ID, + HEAP_LENGTH_ID, + MEMCPY_NONTEMPORAL, MEMCPY_STD, - MEMCPY_NONTEMPORAL) + NULL_ID, + PAYLOAD_LENGTH_ID, + PAYLOAD_OFFSET_ID, + STREAM_CTRL_ID, + Empty, + Flavour, + Full, + InprocQueue, + MemoryAllocator, + MemoryPool, + MmapAllocator, + Stopped, + ThreadPool, +) + try: - from spead2._spead2 import IbvContext # noqa: F401 + from spead2._spead2 import IbvContext # noqa: F401 except ImportError: pass -from spead2._version import __version__ # noqa: F401 - +from spead2._version import __version__ # noqa: F401 _logger = logging.getLogger(__name__) _UNRESERVED_ID = 0x1000 #: First ID that can be auto-allocated diff --git a/src/spead2/__init__.pyi b/src/spead2/__init__.pyi index 331add088..8624494c6 100644 --- a/src/spead2/__init__.pyi +++ b/src/spead2/__init__.pyi @@ -13,10 +13,22 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -from typing import (List, Sequence, Optional, Tuple, Any, Union, - Dict, KeysView, ValuesView, Text, overload) +from typing import ( + Any, + Dict, + KeysView, + List, + Optional, + Sequence, + Text, + Tuple, + Union, + ValuesView, + overload, +) import numpy as np + try: from numpy.typing import DTypeLike as _DTypeLike except ImportError: diff --git a/src/spead2/recv/__init__.py b/src/spead2/recv/__init__.py index c72ecd1e8..b26d0d057 100644 --- a/src/spead2/recv/__init__.py +++ b/src/spead2/recv/__init__.py @@ -56,11 +56,11 @@ from collections.abc import Sequence as _Sequence -from spead2._spead2.recv import ( # noqa: F401 +from spead2._spead2.recv import ( # noqa: F401 Chunk, + ChunkRingbuffer, ChunkRingPair, ChunkRingStream, - ChunkRingbuffer, ChunkStreamConfig, ChunkStreamGroupConfig, ChunkStreamGroupMember, @@ -73,9 +73,11 @@ StreamStatConfig, StreamStats, ) -from . import stream_stat_indices # noqa: F401 + +from . import stream_stat_indices # noqa: F401 + try: - from spead2._spead2.recv import UdpIbvConfig # noqa: F401 + from spead2._spead2.recv import UdpIbvConfig # noqa: F401 except ImportError: pass diff --git a/src/spead2/recv/__init__.pyi b/src/spead2/recv/__init__.pyi index e1e258ac3..9ff02850d 100644 --- a/src/spead2/recv/__init__.pyi +++ b/src/spead2/recv/__init__.pyi @@ -16,7 +16,19 @@ import collections.abc import enum import socket -from typing import Iterator, Iterable, Any, List, Tuple, Sequence, Union, Text, Optional, ClassVar, overload +from typing import ( + Any, + ClassVar, + Iterable, + Iterator, + List, + Optional, + Sequence, + Text, + Tuple, + Union, + overload, +) import spead2 from spead2 import _EndpointList diff --git a/src/spead2/recv/asyncio.py b/src/spead2/recv/asyncio.py index 707d9b88c..b1de603eb 100644 --- a/src/spead2/recv/asyncio.py +++ b/src/spead2/recv/asyncio.py @@ -16,9 +16,9 @@ """ Integration between spead2.recv and asyncio """ +import asyncio import collections import functools -import asyncio import spead2.recv diff --git a/src/spead2/recv/numba.py b/src/spead2/recv/numba.py index d704b70fc..7fb7adf8d 100644 --- a/src/spead2/recv/numba.py +++ b/src/spead2/recv/numba.py @@ -17,7 +17,6 @@ from numba import types - try: _size_t = types.size_t except AttributeError: diff --git a/src/spead2/recv/numba.pyi b/src/spead2/recv/numba.pyi index 0ba36deba..8267a0143 100644 --- a/src/spead2/recv/numba.pyi +++ b/src/spead2/recv/numba.pyi @@ -15,5 +15,4 @@ import numba.types - chunk_place_data: numba.types.Record diff --git a/src/spead2/send/__init__.py b/src/spead2/send/__init__.py index 7c8a1a0a6..6e628b3f3 100644 --- a/src/spead2/send/__init__.py +++ b/src/spead2/send/__init__.py @@ -18,11 +18,23 @@ import weakref import spead2 as _spead2 -from spead2._spead2.send import ( # noqa: F401 - RateMethod, StreamConfig, GroupMode, Heap, HeapReference, HeapReferenceList, - PacketGenerator, Stream, BytesStream, UdpStream, TcpStream, InprocStream) +from spead2._spead2.send import ( # noqa: F401 + BytesStream, + GroupMode, + Heap, + HeapReference, + HeapReferenceList, + InprocStream, + PacketGenerator, + RateMethod, + Stream, + StreamConfig, + TcpStream, + UdpStream, +) + try: - from spead2._spead2.send import UdpIbvStream, UdpIbvConfig # noqa: F401 + from spead2._spead2.send import UdpIbvConfig, UdpIbvStream # noqa: F401 except ImportError: pass diff --git a/src/spead2/send/__init__.pyi b/src/spead2/send/__init__.pyi index 8171426d7..49dcc36c0 100644 --- a/src/spead2/send/__init__.pyi +++ b/src/spead2/send/__init__.pyi @@ -13,11 +13,20 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -from typing import ( - Text, Union, List, Tuple, Iterator, Iterable, Optional, Sequence, ClassVar, overload -) import enum import socket +from typing import ( + ClassVar, + Iterable, + Iterator, + List, + Optional, + Sequence, + Text, + Tuple, + Union, + overload, +) import spead2 from spead2 import _EndpointList diff --git a/src/spead2/send/asyncio.py b/src/spead2/send/asyncio.py index 413131fa9..d4ef570fe 100644 --- a/src/spead2/send/asyncio.py +++ b/src/spead2/send/asyncio.py @@ -19,10 +19,11 @@ import asyncio - -from spead2._spead2.send import UdpStreamAsyncio as _UdpStreamAsyncio -from spead2._spead2.send import TcpStreamAsyncio as _TcpStreamAsyncio -from spead2._spead2.send import InprocStreamAsyncio as _InprocStreamAsyncio +from spead2._spead2.send import ( + InprocStreamAsyncio as _InprocStreamAsyncio, + TcpStreamAsyncio as _TcpStreamAsyncio, + UdpStreamAsyncio as _UdpStreamAsyncio, +) def _set_result(future, result): diff --git a/src/spead2/send/asyncio.pyi b/src/spead2/send/asyncio.pyi index 9527ad56f..10d5e4de9 100644 --- a/src/spead2/send/asyncio.pyi +++ b/src/spead2/send/asyncio.pyi @@ -19,7 +19,6 @@ from typing import List, Optional, Union, overload import spead2 import spead2.send - from spead2 import _EndpointList class AsyncStream(spead2.send.Stream): diff --git a/src/spead2/tools/bench_asyncio.py b/src/spead2/tools/bench_asyncio.py index b353b2061..9df1d5ad3 100644 --- a/src/spead2/tools/bench_asyncio.py +++ b/src/spead2/tools/bench_asyncio.py @@ -24,14 +24,14 @@ the master end. """ -import sys import argparse -import json +import asyncio import collections +import json import logging -import traceback +import sys import timeit -import asyncio +import traceback import numpy as np @@ -40,8 +40,8 @@ import spead2.recv.asyncio import spead2.send import spead2.send.asyncio -from . import cmdline +from . import cmdline ARG_ENUMS = { "rate_method": spead2.send.RateMethod diff --git a/src/spead2/tools/cmdline.py b/src/spead2/tools/cmdline.py index 226560a98..70f2f5bd6 100644 --- a/src/spead2/tools/cmdline.py +++ b/src/spead2/tools/cmdline.py @@ -20,7 +20,6 @@ import spead2.recv import spead2.send - _HAVE_IBV = hasattr(spead2.recv.Stream, 'add_udp_ibv_reader') diff --git a/src/spead2/tools/recv_asyncio.py b/src/spead2/tools/recv_asyncio.py index b2d5bdb0b..13af50c7e 100644 --- a/src/spead2/tools/recv_asyncio.py +++ b/src/spead2/tools/recv_asyncio.py @@ -20,14 +20,15 @@ necessary, to allow multiple code-paths to be exercised. """ -import logging import argparse -import signal import asyncio +import logging +import signal import spead2 import spead2.recv import spead2.recv.asyncio + from . import cmdline diff --git a/src/spead2/tools/send_asyncio.py b/src/spead2/tools/send_asyncio.py index e7318dbaf..9eca7ae0a 100644 --- a/src/spead2/tools/send_asyncio.py +++ b/src/spead2/tools/send_asyncio.py @@ -18,19 +18,20 @@ This is mainly a benchmark application, but also demonstrates the API. """ +import argparse +import asyncio +import collections +import itertools import logging import sys import time -import itertools -import argparse -import collections -import asyncio import numpy as np import spead2 import spead2.send import spead2.send.asyncio + from . import cmdline diff --git a/tests/shutdown.py b/tests/shutdown.py index 702e9d551..a7ac61a54 100644 --- a/tests/shutdown.py +++ b/tests/shutdown.py @@ -27,9 +27,9 @@ from numba import types import spead2 +import spead2._spead2 import spead2.recv import spead2.send -import spead2._spead2 from spead2.numba import intp_to_voidptr from spead2.recv.numba import chunk_place_data diff --git a/tests/test_passthrough.py b/tests/test_passthrough.py index a8a6217b2..c2fb29412 100644 --- a/tests/test_passthrough.py +++ b/tests/test_passthrough.py @@ -19,13 +19,13 @@ import socket import sys -import numpy as np import netifaces +import numpy as np import pytest import spead2 -import spead2.send import spead2.recv +import spead2.send def _assert_items_equal(item1, item2): diff --git a/tests/test_passthrough_asyncio.py b/tests/test_passthrough_asyncio.py index 5d1935956..c34b96cdb 100644 --- a/tests/test_passthrough_asyncio.py +++ b/tests/test_passthrough_asyncio.py @@ -15,14 +15,14 @@ """Test that data can be passed through the various async transports.""" -import socket import asyncio +import socket import pytest import spead2 -import spead2.send import spead2.recv.asyncio +import spead2.send import spead2.send.asyncio from . import test_passthrough diff --git a/tests/test_recv.py b/tests/test_recv.py index 05214a70c..a32e8bd3e 100644 --- a/tests/test_recv.py +++ b/tests/test_recv.py @@ -13,10 +13,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import platform import socket import struct import sys -import platform import time import numpy as np diff --git a/tests/test_recv_asyncio.py b/tests/test_recv_asyncio.py index 365ff2a61..3f04613c6 100644 --- a/tests/test_recv_asyncio.py +++ b/tests/test_recv_asyncio.py @@ -18,9 +18,8 @@ import pytest import spead2 -import spead2.send import spead2.recv.asyncio - +import spead2.send pytestmark = [pytest.mark.asyncio] diff --git a/tests/test_recv_chunk_stream.py b/tests/test_recv_chunk_stream.py index 48d4dbf37..9291d2e14 100644 --- a/tests/test_recv_chunk_stream.py +++ b/tests/test_recv_chunk_stream.py @@ -21,17 +21,16 @@ import weakref import numba -from numba import types import numpy as np -import scipy import pytest +import scipy +from numba import types import spead2 -from spead2.numba import intp_to_voidptr import spead2.recv as recv -from spead2.recv.numba import chunk_place_data import spead2.send as send - +from spead2.numba import intp_to_voidptr +from spead2.recv.numba import chunk_place_data HEAP_PAYLOAD_SIZE = 1024 HEAPS_PER_CHUNK = 10 diff --git a/tests/test_recv_chunk_stream_group.py b/tests/test_recv_chunk_stream_group.py index 39a852d00..af574d469 100644 --- a/tests/test_recv_chunk_stream_group.py +++ b/tests/test_recv_chunk_stream_group.py @@ -21,19 +21,21 @@ import weakref import numba -from numba import types import numpy as np import pytest import scipy +from numba import types import spead2 import spead2.recv as recv import spead2.send as send from spead2.numba import intp_to_voidptr from spead2.recv.numba import chunk_place_data - from tests.test_recv_chunk_stream import ( - CHUNK_PAYLOAD_SIZE, HEAP_PAYLOAD_SIZE, HEAPS_PER_CHUNK, place_plain_llc + CHUNK_PAYLOAD_SIZE, + HEAP_PAYLOAD_SIZE, + HEAPS_PER_CHUNK, + place_plain_llc, ) STREAMS = 4 diff --git a/tests/test_send.py b/tests/test_send.py index 8742dcdf2..e3b36105b 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -18,8 +18,8 @@ import math import platform import struct -import time import threading +import time import weakref import numpy as np diff --git a/tests/test_send_asyncio.py b/tests/test_send_asyncio.py index d8de79928..c9790cbb8 100644 --- a/tests/test_send_asyncio.py +++ b/tests/test_send_asyncio.py @@ -14,8 +14,8 @@ # along with this program. If not, see . import asyncio -import logging import gc +import logging import weakref import numpy as np From dadf9ad3257aaf5ebac7ce39030f77112fcdcd56 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 09:53:23 +0200 Subject: [PATCH 08/14] Add black to pre-commit --- .pre-commit-config.yaml | 4 ++++ pyproject.toml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5793d2c2b..a30c4e090 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,10 @@ repos: hooks: - id: isort name: isort + - repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 1bb39e3c0..4b33c1a41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,7 @@ profile = "black" combine_as_imports = true line_length = 100 py_version = 38 + +[tool.black] +line-length = 100 +target-versions = ["py38", "py39", "py310", "py311", "py312"] From a958c627a5a453670fbafbaa4edac16084f16cba Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 10:32:26 +0200 Subject: [PATCH 09/14] Add black formatting to the codebase --- doc/conf.py | 54 +- examples/recv_chunk_group_example.py | 25 +- examples/recv_chunk_ring_example.py | 26 +- examples/test_recv.py | 4 +- examples/test_send.py | 5 +- examples/test_send_asyncio.py | 10 +- gen/gen_loader.py | 96 +-- gen/get_version.py | 20 +- setup.py | 91 +-- src/spead2/__init__.py | 237 +++---- src/spead2/__init__.pyi | 71 ++- src/spead2/numba.py | 1 + src/spead2/recv/__init__.pyi | 154 +++-- src/spead2/recv/asyncio.py | 2 +- src/spead2/recv/numba.py | 26 +- src/spead2/send/__init__.py | 29 +- src/spead2/send/__init__.pyi | 245 +++++--- src/spead2/send/asyncio.py | 28 +- src/spead2/send/asyncio.pyi | 47 +- src/spead2/tools/bench_asyncio.py | 202 +++--- src/spead2/tools/cmdline.py | 198 +++--- src/spead2/tools/recv_asyncio.py | 30 +- src/spead2/tools/send_asyncio.py | 85 +-- tests/shutdown.py | 22 +- tests/test_common.py | 92 +-- tests/test_passthrough.py | 372 +++++++---- tests/test_passthrough_asyncio.py | 79 ++- tests/test_recv.py | 661 +++++++++++--------- tests/test_recv_asyncio.py | 4 +- tests/test_recv_chunk_stream.py | 125 ++-- tests/test_recv_chunk_stream_group.py | 34 +- tests/test_send.py | 857 ++++++++++++++------------ tests/test_send_asyncio.py | 20 +- 33 files changed, 2294 insertions(+), 1658 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 02dc09213..e3a8e7fc4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -19,32 +19,34 @@ # # import os # import sys -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath("..")) # Build doxygen first on readthedocs -rtd = os.environ.get('READTHEDOCS') == 'True' +rtd = os.environ.get("READTHEDOCS") == "True" if rtd: - subprocess.check_call(['doxygen']) + subprocess.check_call(["doxygen"]) # ReadTheDocs can't build the extension (no Boost), so we have to mock it # See http://docs.readthedocs.io/en/latest/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules - autodoc_mock_imports = ['spead2._spead2'] + autodoc_mock_imports = ["spead2._spead2"] # -- Project information ----------------------------------------------------- -project = 'spead2' -copyright = '2015–2021, National Research Foundation (SARAO)' -author = 'National Research Foundation (SARAO)' +project = "spead2" +copyright = "2015–2021, National Research Foundation (SARAO)" +author = "National Research Foundation (SARAO)" + def get_version(): globals_ = {} root = os.path.dirname(os.path.dirname(__file__)) - with open(os.path.join(root, 'src', 'spead2', '_version.py')) as f: + with open(os.path.join(root, "src", "spead2", "_version.py")) as f: code = f.read() exec(code, globals_) - release = globals_['__version__'] - match = re.match('^(\d+)\.(\d+)', release) + release = globals_["__version__"] + match = re.match("^(\d+)\.(\d+)", release) return match.group(0), release + version, release = get_version() # -- General configuration --------------------------------------------------- @@ -53,30 +55,30 @@ def get_version(): # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.mathjax', - 'sphinx.ext.napoleon', - 'sphinx_rtd_theme', - 'breathe' + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx_rtd_theme", + "breathe", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -breathe_projects = {'spead2': './doxygen/xml'} -breathe_default_project = 'spead2' +breathe_projects = {"spead2": "./doxygen/xml"} +breathe_default_project = "spead2" intersphinx_mapping = { - 'python': ('https://docs.python.org/3/', None), - 'numpy': ('https://numpy.org/doc/stable/', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/', None), - 'numba': ('https://numba.readthedocs.io/en/latest/', None) + "python": ("https://docs.python.org/3/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "scipy": ("https://docs.scipy.org/doc/scipy/", None), + "numba": ("https://numba.readthedocs.io/en/latest/", None), } # -- Options for HTML output ------------------------------------------------- @@ -84,9 +86,9 @@ def get_version(): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] diff --git a/examples/recv_chunk_group_example.py b/examples/recv_chunk_group_example.py index afc234f9e..282212465 100755 --- a/examples/recv_chunk_group_example.py +++ b/examples/recv_chunk_group_example.py @@ -49,39 +49,30 @@ def chunk_place(data_ptr, data_size): def main(): NUM_STREAMS = 2 MAX_CHUNKS = 4 - place_callback = scipy.LowLevelCallable( - chunk_place.ctypes, - signature='void (void *, size_t)' - ) + place_callback = scipy.LowLevelCallable(chunk_place.ctypes, signature="void (void *, size_t)") chunk_config = spead2.recv.ChunkStreamConfig( items=[spead2.HEAP_CNT_ID, spead2.HEAP_LENGTH_ID], max_chunks=MAX_CHUNKS, - place=place_callback) + place=place_callback, + ) group_config = spead2.recv.ChunkStreamGroupConfig(max_chunks=MAX_CHUNKS) data_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS) free_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS) group = spead2.recv.ChunkStreamRingGroup(group_config, data_ring, free_ring) for _ in range(NUM_STREAMS): - group.emplace_back( - spead2.ThreadPool(), - spead2.recv.StreamConfig(), - chunk_config - ) + group.emplace_back(spead2.ThreadPool(), spead2.recv.StreamConfig(), chunk_config) for _ in range(MAX_CHUNKS): chunk = spead2.recv.Chunk( - present=np.empty(HEAPS_PER_CHUNK, np.uint8), - data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8) + present=np.empty(HEAPS_PER_CHUNK, np.uint8), data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8) ) group.add_free_chunk(chunk) for i in range(NUM_STREAMS): - group[i].add_udp_reader(8888 + i, buffer_size=1024 * 1024, bind_hostname='127.0.0.1') + group[i].add_udp_reader(8888 + i, buffer_size=1024 * 1024, bind_hostname="127.0.0.1") for chunk in data_ring: n_present = np.sum(chunk.present) - print( - f"Received chunk {chunk.chunk_id} with " - f"{n_present} / {HEAPS_PER_CHUNK} heaps") + print(f"Received chunk {chunk.chunk_id} with {n_present} / {HEAPS_PER_CHUNK} heaps") group.add_free_chunk(chunk) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/examples/recv_chunk_ring_example.py b/examples/recv_chunk_ring_example.py index 6de136688..921f6c69a 100755 --- a/examples/recv_chunk_ring_example.py +++ b/examples/recv_chunk_ring_example.py @@ -48,36 +48,28 @@ def chunk_place(data_ptr, data_size): def main(): MAX_CHUNKS = 4 - place_callback = scipy.LowLevelCallable( - chunk_place.ctypes, - signature='void (void *, size_t)' - ) + place_callback = scipy.LowLevelCallable(chunk_place.ctypes, signature="void (void *, size_t)") chunk_config = spead2.recv.ChunkStreamConfig( items=[spead2.HEAP_CNT_ID, spead2.HEAP_LENGTH_ID], max_chunks=MAX_CHUNKS, - place=place_callback) + place=place_callback, + ) data_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS) free_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS) stream = spead2.recv.ChunkRingStream( - spead2.ThreadPool(), - spead2.recv.StreamConfig(), - chunk_config, - data_ring, - free_ring) + spead2.ThreadPool(), spead2.recv.StreamConfig(), chunk_config, data_ring, free_ring + ) for i in range(MAX_CHUNKS): chunk = spead2.recv.Chunk( - present=np.empty(HEAPS_PER_CHUNK, np.uint8), - data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8) + present=np.empty(HEAPS_PER_CHUNK, np.uint8), data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8) ) stream.add_free_chunk(chunk) - stream.add_udp_reader(8888, buffer_size=1024 * 1024, bind_hostname='127.0.0.1') + stream.add_udp_reader(8888, buffer_size=1024 * 1024, bind_hostname="127.0.0.1") for chunk in data_ring: n_present = np.sum(chunk.present) - print( - f"Received chunk {chunk.chunk_id} with " - f"{n_present} / {HEAPS_PER_CHUNK} heaps") + print(f"Received chunk {chunk.chunk_id} with {n_present} / {HEAPS_PER_CHUNK} heaps") stream.add_free_chunk(chunk) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/examples/test_recv.py b/examples/test_recv.py index f526708f0..9e753ddd6 100755 --- a/examples/test_recv.py +++ b/examples/test_recv.py @@ -25,11 +25,11 @@ thread_pool = spead2.ThreadPool() stream = spead2.recv.Stream( thread_pool, - spead2.recv.StreamConfig(memory_allocator=spead2.MemoryPool(16384, 26214400, 12, 8)) + spead2.recv.StreamConfig(memory_allocator=spead2.MemoryPool(16384, 26214400, 12, 8)), ) del thread_pool if 0: - with open('junkspeadfile', 'rb') as f: + with open("junkspeadfile", "rb") as f: text = f.read() stream.add_buffer_reader(text) else: diff --git a/examples/test_send.py b/examples/test_send.py index be329b2ae..e834e11b9 100755 --- a/examples/test_send.py +++ b/examples/test_send.py @@ -26,12 +26,13 @@ thread_pool = spead2.ThreadPool() stream = spead2.send.UdpStream( - thread_pool, [("127.0.0.1", 8888)], spead2.send.StreamConfig(rate=1e7)) + thread_pool, [("127.0.0.1", 8888)], spead2.send.StreamConfig(rate=1e7) +) del thread_pool shape = (40, 50) ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48, 0)) -item = ig.add_item(0x1234, 'foo', 'a foo item', shape=shape, dtype=np.int32) +item = ig.add_item(0x1234, "foo", "a foo item", shape=shape, dtype=np.int32) item.value = np.zeros(shape, np.int32) stream.send_heap(ig.get_heap()) stream.send_heap(ig.get_end()) diff --git a/examples/test_send_asyncio.py b/examples/test_send_asyncio.py index b3b5a4799..83443bc6f 100755 --- a/examples/test_send_asyncio.py +++ b/examples/test_send_asyncio.py @@ -28,17 +28,15 @@ thread_pool = spead2.ThreadPool() stream = spead2.send.asyncio.UdpStream( - thread_pool, [("127.0.0.1", 8888)], spead2.send.StreamConfig(rate=1e7)) + thread_pool, [("127.0.0.1", 8888)], spead2.send.StreamConfig(rate=1e7) +) del thread_pool # Make sure this doesn't crash anything shape = (40, 50) ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48, 0)) -item = ig.add_item(0x1234, 'foo', 'a foo item', shape=shape, dtype=np.int32) +item = ig.add_item(0x1234, "foo", "a foo item", shape=shape, dtype=np.int32) item.value = np.zeros(shape, np.int32) -futures = [ - stream.async_send_heap(ig.get_heap()), - stream.async_send_heap(ig.get_end()) -] +futures = [stream.async_send_heap(ig.get_heap()), stream.async_send_heap(ig.get_end())] # Delete things to check that there are no refcounting bugs del ig del stream diff --git a/gen/gen_loader.py b/gen/gen_loader.py index 51a650921..5d679204e 100755 --- a/gen/gen_loader.py +++ b/gen/gen_loader.py @@ -28,7 +28,7 @@ # The typedefs are arbitrary and just used to allow pycparser to parse the # code. Note that some functions in infiniband/verbs.h are implemented as # static inline functions, and so do not get listed here. -IBV_DECLS = ''' +IBV_DECLS = """ typedef unsigned long size_t; typedef unsigned long uint64_t; @@ -80,9 +80,9 @@ struct ibv_mr *ibv_reg_mr_iova2(struct ibv_pd *pd, void *addr, size_t length, uint64_t iova, unsigned int access); -''' +""" -RDMACM_DECLS = ''' +RDMACM_DECLS = """ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr); struct rdma_event_channel *rdma_create_event_channel(void); @@ -94,9 +94,9 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel); int rdma_destroy_id(struct rdma_cm_id *id); -''' +""" -MLX5DV_DECLS = ''' +MLX5DV_DECLS = """ typedef int bool; typedef unsigned long uint64_t; @@ -110,7 +110,7 @@ struct mlx5dv_wq_init_attr *mlx5_wq_attr); int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type); -''' +""" class RenameVisitor(c_ast.NodeVisitor): @@ -198,8 +198,18 @@ class Library: @c has_. """ - def __init__(self, name, headers, soname, guard, decls, wrappers=(), optional=(), - *, fail_log_level='warning'): + def __init__( + self, + name, + headers, + soname, + guard, + decls, + wrappers=(), + optional=(), + *, + fail_log_level="warning" + ): self.name = name self.headers = list(headers) self.soname = soname @@ -212,51 +222,63 @@ def __init__(self, name, headers, soname, guard, decls, wrappers=(), optional=() self.environment = jinja2.Environment( autoescape=False, trim_blocks=True, - loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), ) - self.environment.filters['gen'] = gen_node - self.environment.filters['rename'] = rename_func - self.environment.filters['ptr'] = make_func_ptr - self.environment.filters['args'] = func_args - self.environment.globals['name'] = self.name - self.environment.globals['headers'] = self.headers - self.environment.globals['soname'] = self.soname - self.environment.globals['guard'] = self.guard - self.environment.globals['nodes'] = self.nodes - self.environment.globals['wrappers'] = set(wrappers) - self.environment.globals['optional'] = set(optional) - self.environment.globals['fail_log_level'] = fail_log_level + self.environment.filters["gen"] = gen_node + self.environment.filters["rename"] = rename_func + self.environment.filters["ptr"] = make_func_ptr + self.environment.filters["args"] = func_args + self.environment.globals["name"] = self.name + self.environment.globals["headers"] = self.headers + self.environment.globals["soname"] = self.soname + self.environment.globals["guard"] = self.guard + self.environment.globals["nodes"] = self.nodes + self.environment.globals["wrappers"] = set(wrappers) + self.environment.globals["optional"] = set(optional) + self.environment.globals["fail_log_level"] = fail_log_level def header(self): - return self.environment.get_template('template.h').render() + return self.environment.get_template("template.h").render() def cxx(self): - return self.environment.get_template('template.cpp').render() + return self.environment.get_template("template.cpp").render() def main(argv): parser = argparse.ArgumentParser() - parser.add_argument('type', choices=['header', 'cxx']) - parser.add_argument('library', choices=['rdmacm', 'ibv', 'mlx5dv']) + parser.add_argument("type", choices=["header", "cxx"]) + parser.add_argument("library", choices=["rdmacm", "ibv", "mlx5dv"]) args = parser.parse_args() - if args.library == 'rdmacm': - lib = Library('rdmacm', ['rdma/rdma_cma.h'], 'librdmacm.so.1', 'SPEAD2_USE_IBV', - RDMACM_DECLS) - elif args.library == 'ibv': - lib = Library('ibv', ['infiniband/verbs.h'], 'libibverbs.so.1', 'SPEAD2_USE_IBV', - IBV_DECLS, - ['ibv_create_qp', 'ibv_query_device'], - ['ibv_reg_mr_iova2']) + if args.library == "rdmacm": + lib = Library( + "rdmacm", ["rdma/rdma_cma.h"], "librdmacm.so.1", "SPEAD2_USE_IBV", RDMACM_DECLS + ) + elif args.library == "ibv": + lib = Library( + "ibv", + ["infiniband/verbs.h"], + "libibverbs.so.1", + "SPEAD2_USE_IBV", + IBV_DECLS, + ["ibv_create_qp", "ibv_query_device"], + ["ibv_reg_mr_iova2"], + ) else: - lib = Library('mlx5dv', ['infiniband/mlx5dv.h'], 'libmlx5.so.1', 'SPEAD2_USE_MLX5DV', - MLX5DV_DECLS, fail_log_level='debug') + lib = Library( + "mlx5dv", + ["infiniband/mlx5dv.h"], + "libmlx5.so.1", + "SPEAD2_USE_MLX5DV", + MLX5DV_DECLS, + fail_log_level="debug", + ) - if args.type == 'header': + if args.type == "header": print(lib.header()) else: print(lib.cxx()) -if __name__ == '__main__': +if __name__ == "__main__": main(sys.argv) diff --git a/gen/get_version.py b/gen/get_version.py index c37e265c9..f47e3410e 100755 --- a/gen/get_version.py +++ b/gen/get_version.py @@ -22,24 +22,24 @@ from packaging.version import Version parser = argparse.ArgumentParser() -parser.add_argument('mode', choices=('major', 'minor', 'patch', 'full')) +parser.add_argument("mode", choices=("major", "minor", "patch", "full")) args = parser.parse_args() -with open('src/spead2/_version.py') as version_file: +with open("src/spead2/_version.py") as version_file: line = version_file.readline().strip() match = re.fullmatch(r'__version__ = "(.+)"', line) if not match: - print('src/spead2/_version.py does not match the expected format', file=sys.stderr) + print("src/spead2/_version.py does not match the expected format", file=sys.stderr) sys.exit(1) version_str = match.group(1) version = Version(version_str) mode = args.mode -if mode == 'major': - print(version.major, end='') -elif mode == 'minor': - print(version.minor, end='') -elif mode == 'patch': - print(version.micro, end='') +if mode == "major": + print(version.major, end="") +elif mode == "minor": + print(version.minor, end="") +elif mode == "patch": + print(version.micro, end="") else: - print(version_str, end='') + print(version_str, end="") diff --git a/setup.py b/setup.py index ba452a24f..e3492eaa4 100755 --- a/setup.py +++ b/setup.py @@ -27,42 +27,40 @@ class BuildExt(build_ext): user_options = build_ext.user_options + [ - ('coverage', None, - 'build with GCC --coverage option'), - ('split-debug=', None, - 'write debug symbols to a separate directory') + ("coverage", None, "build with GCC --coverage option"), + ("split-debug=", None, "write debug symbols to a separate directory"), ] - boolean_options = build_ext.boolean_options + ['coverage'] + boolean_options = build_ext.boolean_options + ["coverage"] def initialize_options(self): build_ext.initialize_options(self) # setuptools bug causes these to be lost during reinitialization by # ./setup.py develop - if not hasattr(self, 'coverage'): + if not hasattr(self, "coverage"): self.coverage = None - if not hasattr(self, 'split_debug'): + if not hasattr(self, "split_debug"): self.split_debug = None def run(self): self.mkpath(self.build_temp) - subprocess.check_call(os.path.abspath('configure'), cwd=self.build_temp) + subprocess.check_call(os.path.abspath("configure"), cwd=self.build_temp) config = configparser.ConfigParser() - config.read(os.path.join(self.build_temp, 'python-build.cfg')) + config.read(os.path.join(self.build_temp, "python-build.cfg")) for extension in self.extensions: - extension.extra_compile_args.extend(config['compiler']['CFLAGS'].split()) - extension.extra_link_args.extend(config['compiler']['LIBS'].split()) + extension.extra_compile_args.extend(config["compiler"]["CFLAGS"].split()) + extension.extra_link_args.extend(config["compiler"]["LIBS"].split()) if self.coverage: - extension.extra_compile_args.extend(['-g', '-O0', '--coverage']) - extension.libraries.extend(['gcov']) + extension.extra_compile_args.extend(["-g", "-O0", "--coverage"]) + extension.libraries.extend(["gcov"]) if self.split_debug: - extension.extra_compile_args.extend(['-g']) - extension.include_dirs.insert(0, os.path.join(self.build_temp, 'include')) + extension.extra_compile_args.extend(["-g"]) + extension.include_dirs.insert(0, os.path.join(self.build_temp, "include")) super().run() def build_extensions(self): # Stop GCC complaining about -Wstrict-prototypes in C++ code try: - self.compiler.compiler_so.remove('-Wstrict-prototypes') + self.compiler.compiler_so.remove("-Wstrict-prototypes") except ValueError: pass super().build_extensions() @@ -82,57 +80,64 @@ def build_extension(self, ext): if self.split_debug: os.makedirs(self.split_debug, exist_ok=True) basename = os.path.basename(ext_path) - debug_path = os.path.join(self.split_debug, basename + '.debug') - self.spawn(['objcopy', '--only-keep-debug', '--', ext_path, debug_path]) - self.spawn(['strip', '--strip-debug', '--strip-unneeded', '--', ext_path]) + debug_path = os.path.join(self.split_debug, basename + ".debug") + self.spawn(["objcopy", "--only-keep-debug", "--", ext_path, debug_path]) + self.spawn(["strip", "--strip-debug", "--strip-unneeded", "--", ext_path]) old_cwd = os.getcwd() # See the documentation for --add-gnu-debuglink for why it needs to be # run from the directory containing the file. ext_path_abs = os.path.abspath(ext_path) try: os.chdir(self.split_debug) - self.spawn(['objcopy', '--add-gnu-debuglink=' + os.path.basename(debug_path), - '--', ext_path_abs]) + self.spawn( + [ + "objcopy", + "--add-gnu-debuglink=" + os.path.basename(debug_path), + "--", + ext_path_abs, + ] + ) finally: os.chdir(old_cwd) - self.spawn(['chmod', '-x', '--', debug_path]) + self.spawn(["chmod", "-x", "--", debug_path]) # Can't actually install on readthedocs.org because we can't compile, # but we need setup.py to still be successful to make the doc build work. -rtd = os.environ.get('READTHEDOCS') == 'True' +rtd = os.environ.get("READTHEDOCS") == "True" if not rtd: - if not os.path.exists(os.path.join(os.path.dirname(__file__), 'configure')): - raise SystemExit("configure not found. Either download a release " + - "from https://pypi.org/project/spead2 or run " + - "./bootstrap.sh if not using a release.") + if not os.path.exists(os.path.join(os.path.dirname(__file__), "configure")): + raise SystemExit( + "configure not found. Either download a release " + + "from https://pypi.org/project/spead2 or run " + + "./bootstrap.sh if not using a release." + ) extensions = [ Pybind11Extension( - '_spead2', - sources=(glob.glob('src/py_*.cpp') + - glob.glob('src/common_*.cpp') + - glob.glob('src/recv_*.cpp') + - glob.glob('src/send_*.cpp')), - depends=glob.glob('include/spead2/*.h'), - language='c++', - include_dirs=['include'], + "_spead2", + sources=( + glob.glob("src/py_*.cpp") + + glob.glob("src/common_*.cpp") + + glob.glob("src/recv_*.cpp") + + glob.glob("src/send_*.cpp") + ), + depends=glob.glob("include/spead2/*.h"), + language="c++", + include_dirs=["include"], # We don't need to pass boost objects across shared library # boundaries. These macros makes -fvisibility=hidden do its job. # The first is asio-specific, while the latter is only used in # Boost 1.81+. define_macros=[ - ('BOOST_ASIO_DISABLE_VISIBILITY', None), - ('BOOST_DISABLE_EXPLICIT_SYMBOL_VISIBILITY', None) - ]) + ("BOOST_ASIO_DISABLE_VISIBILITY", None), + ("BOOST_DISABLE_EXPLICIT_SYMBOL_VISIBILITY", None), + ], + ) ] else: extensions = [] ParallelCompile("SPEAD2_NUM_BUILD_JOBS").install() -setup( - ext_package='spead2', - ext_modules=extensions, - cmdclass={'build_ext': BuildExt} -) +setup(ext_package="spead2", ext_modules=extensions, cmdclass={"build_ext": BuildExt}) diff --git a/src/spead2/__init__.py b/src/spead2/__init__.py index bd8ed59ce..cb5bd4542 100644 --- a/src/spead2/__init__.py +++ b/src/spead2/__init__.py @@ -62,7 +62,7 @@ from spead2._version import __version__ # noqa: F401 _logger = logging.getLogger(__name__) -_UNRESERVED_ID = 0x1000 #: First ID that can be auto-allocated +_UNRESERVED_ID = 0x1000 #: First ID that can be auto-allocated _FASTPATH_NONE = 0 _FASTPATH_IMMEDIATE = 1 @@ -82,10 +82,10 @@ def parse_range_list(ranges): """ if not ranges: return [] - parts = ranges.split(',') + parts = ranges.split(",") out = [] for part in parts: - fields = part.split('-', 1) + fields = part.split("-", 1) if len(fields) == 2: start = int(fields[0]) end = int(fields[1]) @@ -127,30 +127,30 @@ class Descriptor: is a tuple of field code and bit length. """ - def __init__(self, id, name, description, shape, dtype=None, order='C', format=None): + def __init__(self, id, name, description, shape, dtype=None, order="C", format=None): shape = tuple(shape) unknowns = sum([x is None for x in shape]) if unknowns > 1: - raise ValueError('Cannot have multiple unknown dimensions') + raise ValueError("Cannot have multiple unknown dimensions") if dtype is not None: dtype = _np.dtype(dtype) if dtype.hasobject: - raise ValueError('Cannot use dtype that has reference-counted objects') + raise ValueError("Cannot use dtype that has reference-counted objects") if format is not None: - raise ValueError('Only one of dtype and format can be specified') + raise ValueError("Only one of dtype and format can be specified") if dtype.itemsize == 0: - raise ValueError('Cannot use zero-sized dtype') + raise ValueError("Cannot use zero-sized dtype") if unknowns > 0: - raise ValueError('Cannot have unknown dimensions when using numpy descriptor') + raise ValueError("Cannot have unknown dimensions when using numpy descriptor") self._internal_dtype = dtype else: if format is None: - raise ValueError('One of dtype and format must be specified') - if order != 'C': + raise ValueError("One of dtype and format must be specified") + if order != "C": raise ValueError("When specifying format, order must be 'C'") self._internal_dtype = self._parse_format(format) - if order not in ['C', 'F']: + if order not in ["C", "F"]: raise ValueError("Order must be 'C' or 'F'") self.id = id self.name = name @@ -161,11 +161,13 @@ def __init__(self, id, name, description, shape, dtype=None, order='C', format=N self.format = format if not self._internal_dtype.hasobject: self._fastpath = _FASTPATH_NUMPY - elif (not shape and - dtype is None and - len(format) == 1 and - format[0][0] in ('u', 'i') and - format[0][1] < 64): + elif ( + not shape + and dtype is None + and len(format) == 1 + and format[0][0] in ("u", "i") + and format[0][1] < 64 + ): self._fastpath = _FASTPATH_IMMEDIATE else: self._fastpath = _FASTPATH_NONE @@ -182,30 +184,31 @@ def _parse_numpy_header(cls, header): raise ValueError(msg % d) keys = list(d.keys()) keys.sort() - if keys != ['descr', 'fortran_order', 'shape']: + if keys != ["descr", "fortran_order", "shape"]: msg = "Descriptor does not contain the correct keys: %r" raise ValueError(msg % (keys,)) # Sanity-check the values. - if (not isinstance(d['shape'], tuple) or - not all([isinstance(x, _numbers.Integral) and x >= 0 for x in d['shape']])): + if not isinstance(d["shape"], tuple) or not all( + [isinstance(x, _numbers.Integral) and x >= 0 for x in d["shape"]] + ): msg = "shape is not valid: %r" - raise ValueError(msg % (d['shape'],)) - if not isinstance(d['fortran_order'], bool): + raise ValueError(msg % (d["shape"],)) + if not isinstance(d["fortran_order"], bool): msg = "fortran_order is not a valid bool: %r" - raise ValueError(msg % (d['fortran_order'],)) + raise ValueError(msg % (d["fortran_order"],)) try: - dtype = _np.dtype(d['descr']) + dtype = _np.dtype(d["descr"]) except TypeError as e: msg = "descr is not a valid dtype descriptor: %r" - raise ValueError(msg % (d['descr'],)) from e - order = 'F' if d['fortran_order'] else 'C' - return d['shape'], order, dtype + raise ValueError(msg % (d["descr"],)) from e + order = "F" if d["fortran_order"] else "C" + return d["shape"], order, dtype @classmethod def _make_numpy_header(self, shape, dtype, order): return "{{'descr': {!r}, 'fortran_order': {!r}, 'shape': {!r}}}".format( - _np.lib.format.dtype_to_descr(dtype), order == 'F', - tuple(shape)) + _np.lib.format.dtype_to_descr(dtype), order == "F", tuple(shape) + ) @classmethod def _parse_format(cls, fmt): @@ -219,25 +222,26 @@ def _parse_format(cls, fmt): """ fields = [] if not fmt: - raise ValueError('empty format') + raise ValueError("empty format") for code, length in fmt: if length <= 0: if length == 0: - raise ValueError('zero-length field (bug_compat mismatch?)') + raise ValueError("zero-length field (bug_compat mismatch?)") else: - raise ValueError('negative-length field') - if ((code in ('u', 'i') and length in (8, 16, 32, 64)) or - (code == 'f' and length in (32, 64))): - fields.append('>' + code + str(length // 8)) - elif code == 'b' and length == 8: - fields.append('?') - elif code == 'c' and length == 8: - fields.append('S1') + raise ValueError("negative-length field") + if (code in ("u", "i") and length in (8, 16, 32, 64)) or ( + code == "f" and length in (32, 64) + ): + fields.append(">" + code + str(length // 8)) + elif code == "b" and length == 8: + fields.append("?") + elif code == "c" and length == 8: + fields.append("S1") else: - if code not in ['u', 'i', 'b']: - raise ValueError(f'illegal format ({code}, {length})') - fields.append('O') - return _np.dtype(','.join(fields)) + if code not in ["u", "i", "b"]: + raise ValueError(f"illegal format ({code}, {length})") + fields.append("O") + return _np.dtype(",".join(fields)) @property def itemsize_bits(self): @@ -262,7 +266,8 @@ def allow_immediate(self): is best not to send them at all. """ return not self.is_variable_size() and ( - self.dtype is not None or self.itemsize_bits % 8 == 0) + self.dtype is not None or self.itemsize_bits % 8 == 0 + ) def dynamic_shape(self, max_elements): """Determine the dynamic shape, given incoming data that is big enough @@ -274,7 +279,7 @@ def dynamic_shape(self, max_elements): if x is not None: known *= x else: - assert unknown_pos == -1, 'Shape has multiple unknown dimensions' + assert unknown_pos == -1, "Shape has multiple unknown dimensions" unknown_pos = i if unknown_pos == -1: return self.shape @@ -301,33 +306,38 @@ def from_raw(cls, raw_descriptor, flavour): dtype = None format = None if raw_descriptor.numpy_header: - header = raw_descriptor.numpy_header.decode('ascii') + header = raw_descriptor.numpy_header.decode("ascii") shape, order, dtype = cls._parse_numpy_header(header) if flavour.bug_compat & BUG_COMPAT_SWAP_ENDIAN: dtype = dtype.newbyteorder() else: shape = raw_descriptor.shape - order = 'C' + order = "C" format = raw_descriptor.format return cls( raw_descriptor.id, - raw_descriptor.name.decode('ascii'), - raw_descriptor.description.decode('ascii'), - shape, dtype, order, format) + raw_descriptor.name.decode("ascii"), + raw_descriptor.description.decode("ascii"), + shape, + dtype, + order, + format, + ) def to_raw(self, flavour): raw = spead2._spead2.RawDescriptor() raw.id = self.id - raw.name = self.name.encode('ascii') - raw.description = self.description.encode('ascii') + raw.name = self.name.encode("ascii") + raw.description = self.description.encode("ascii") raw.shape = self.shape if self.dtype is not None: if flavour.bug_compat & BUG_COMPAT_SWAP_ENDIAN: dtype = self.dtype.newbyteorder() else: dtype = self.dtype - raw.numpy_header = self._make_numpy_header( - self.shape, dtype, self.order).encode('ascii') + raw.numpy_header = self._make_numpy_header(self.shape, dtype, self.order).encode( + "ascii" + ) else: raw.format = self.format return raw @@ -343,10 +353,10 @@ class Item(Descriptor): """ def __init__(self, *args, **kw): - value = kw.pop('value', None) + value = kw.pop("value", None) super().__init__(*args, **kw) self._value = value - self.version = 1 #: Version number + self.version = 1 #: Version number @property def value(self): @@ -398,13 +408,13 @@ def _write_bits(cls, array): thereafter call `send((value, bits))` to add that many bits into the array. You must call `close()` to flush any partial bytes.""" pos = 0 - current = 0 # bits not yet written into array + current = 0 # bits not yet written into array current_bits = 0 try: while True: (value, bits) = yield if value < 0 or value >= (1 << bits): - raise ValueError('Value is out of range for number of bits') + raise ValueError("Value is out of range for number of bits") current = (current << bits) | value current_bits += bits while current_bits >= 8: @@ -414,7 +424,7 @@ def _write_bits(cls, array): pos += 1 except GeneratorExit: if current_bits > 0: - current <<= (8 - current_bits) + current <<= 8 - current_bits array[pos] = current def _load_recursive(self, shape, gen): @@ -430,26 +440,26 @@ def _load_recursive(self, shape, gen): for code, length in self.format: field = None raw = gen.send(length) - if code == 'u': + if code == "u": field = raw - elif code == 'i': + elif code == "i": field = raw # Interpret as 2's complement if field >= 1 << (length - 1): field -= 1 << length - elif code == 'b': + elif code == "b": field = bool(raw) - elif code == 'c': - field = struct.pack('B', raw) - elif code == 'f': + elif code == "c": + field = struct.pack("B", raw) + elif code == "f": if length == 32: field = _np.uint32(raw).view(_np.float32) elif length == 64: field = _np.uint64(raw).view(_np.float64) else: - raise ValueError('unhandled float length {}'.format((code, length))) + raise ValueError("unhandled float length {}".format((code, length))) else: - raise ValueError('unhandled format {}'.format((code, length))) + raise ValueError("unhandled format {}".format((code, length))) fields.append(field) if len(fields) == 1: ans = fields[0] @@ -466,42 +476,45 @@ def _store_recursive(self, dims, value, gen): value = (value,) for (code, length), field in zip(self.format, value): raw = None - if code == 'u': + if code == "u": raw = int(field) if raw < 0 or raw >= (1 << length): - raise ValueError(f'{raw} is out of range for u{length}') - elif code == 'i': + raise ValueError(f"{raw} is out of range for u{length}") + elif code == "i": top_bit = 1 << (length - 1) raw = int(field) if raw < -top_bit or raw >= top_bit: - raise ValueError(f'{field} is out of range for i{length}') + raise ValueError(f"{field} is out of range for i{length}") # convert to 2's complement if raw < 0: raw += 2 * top_bit - elif code == 'b': + elif code == "b": raw = 1 if field else 0 - elif code == 'c': + elif code == "c": raw = ord(field) - elif code == 'f': + elif code == "f": if length == 32: raw = _np.float32(field).view(_np.uint32) elif length == 64: raw = _np.float64(field).view(_np.uint64) else: - raise ValueError('unhandled float length {}'.format((code, length))) + raise ValueError("unhandled float length {}".format((code, length))) else: - raise ValueError('unhandled format {}'.format((code, length))) + raise ValueError("unhandled format {}".format((code, length))) gen.send((raw, length)) - def set_from_raw(self, raw_item, new_order='='): + def set_from_raw(self, raw_item, new_order="="): raw_value = _np.array(raw_item, _np.uint8, copy=False) if self._fastpath == _FASTPATH_NUMPY: max_elements = raw_value.shape[0] // self._internal_dtype.itemsize shape = self.dynamic_shape(max_elements) elements = _shape_elements(shape) if elements > max_elements: - raise ValueError('Item {} has too few elements for shape ({} < {})'.format( - self.name, max_elements, elements)) + raise ValueError( + "Item {} has too few elements for shape ({} < {})".format( + self.name, max_elements, elements + ) + ) size_bytes = elements * self._internal_dtype.itemsize if raw_item.is_immediate: # Immediates get head padding instead of tail padding @@ -511,14 +524,17 @@ def set_from_raw(self, raw_item, new_order='='): array1d = raw_value[:size_bytes] array1d = array1d.view(dtype=self._internal_dtype) # Force the byte order if requested - array1d = array1d.astype(self._internal_dtype.newbyteorder(new_order), - casting='equiv', copy=False) + array1d = array1d.astype( + self._internal_dtype.newbyteorder(new_order), casting="equiv", copy=False + ) value = _np.reshape(array1d, shape, self.order) - elif (self._fastpath == _FASTPATH_IMMEDIATE and - raw_item.is_immediate and - raw_value.shape[0] * 8 == self.format[0][1]): + elif ( + self._fastpath == _FASTPATH_IMMEDIATE + and raw_item.is_immediate + and raw_value.shape[0] * 8 == self.format[0][1] + ): value = raw_item.immediate_value - if self.format[0][0] == 'i': + if self.format[0][0] == "i": top = 1 << (self.format[0][1] - 1) if value >= top: value -= 2 * top @@ -529,23 +545,26 @@ def set_from_raw(self, raw_item, new_order='='): elements = _shape_elements(shape) bits = elements * itemsize_bits if elements > max_elements: - raise ValueError('Item {} has too few elements for shape ({} < {})'.format( - self.name, max_elements, elements)) + raise ValueError( + "Item {} has too few elements for shape ({} < {})".format( + self.name, max_elements, elements + ) + ) if raw_item.is_immediate: # Immediates get head padding instead of tail padding size_bytes = (bits + 7) // 8 raw_value = raw_value[-size_bytes:] gen = self._read_bits(raw_value) - gen.send(None) # Initialisation of the generator + gen.send(None) # Initialisation of the generator value = _np.array(self._load_recursive(shape, gen), self._internal_dtype) if len(self.shape) == 0 and isinstance(value, _np.ndarray): # Convert zero-dimensional array to scalar value = value[()] - elif len(self.shape) == 1 and self.format == [('c', 8)]: + elif len(self.shape) == 1 and self.format == [("c", 8)]: # Convert array of characters to a string - value = b''.join(value).decode('ascii') + value = b"".join(value).decode("ascii") self.value = value def _num_elements(self): @@ -556,7 +575,7 @@ def _num_elements(self): for size in self.shape: ans *= len(cur) if ans == 0: - return ans # Prevents IndexError below + return ans # Prevents IndexError below cur = cur[0] return ans @@ -585,24 +604,24 @@ def _transform_value(self): """ value = self.value if value is None: - raise ValueError('Cannot send a value of None') + raise ValueError("Cannot send a value of None") if isinstance(value, (bytes, str)) and len(self.shape) == 1: # This is complicated by Python 3 not providing a simple way to # turn a bytes object into a list of one-byte objects, the way # list(str) does. value = [self.value[i : i + 1] for i in range(len(self.value))] if self._fastpath == _FASTPATH_IMMEDIATE and self.itemsize_bits % 8 == 0: - value = _np.array(value, dtype='>u8', order=self.order, copy=False) + value = _np.array(value, dtype=">u8", order=self.order, copy=False) if not self.compatible_shape(value.shape): - raise ValueError(f'Value has shape {value.shape}, expected {self.shape}') + raise ValueError(f"Value has shape {value.shape}, expected {self.shape}") # Truncate to just the low-order bytes that are needed. Note: this # doesn't check for overflows, because the value might only get # filled in later if it is referenced rather than copied. - value = value[_np.newaxis].view(_np.uint8)[-(self.itemsize_bits // 8):] + value = value[_np.newaxis].view(_np.uint8)[-(self.itemsize_bits // 8) :] else: value = _np.array(value, dtype=self._internal_dtype, order=self.order, copy=False) if not self.compatible_shape(value.shape): - raise ValueError(f'Value has shape {value.shape}, expected {self.shape}') + raise ValueError(f"Value has shape {value.shape}, expected {self.shape}") return value def to_buffer(self): @@ -626,7 +645,7 @@ def to_buffer(self): gen.close() return out else: - if self.order == 'F': + if self.order == "F": # numpy doesn't allow buffer protocol to be used on arrays that # aren't C-contiguous, but transposition just fiddles the # strides of the view without creating a new array. @@ -670,13 +689,15 @@ def _add_item(self, item): old_by_name = None # Check if this is just the same thing - if (old is not None and - old.name == item.name and - old.description == item.description and - old.shape == item.shape and - old.dtype == item.dtype and - old.order == item.order and - old.format == item.format): + if ( + old is not None + and old.name == item.name + and old.description == item.description + and old.shape == item.shape + and old.dtype == item.dtype + and old.order == item.order + and old.format == item.format + ): # Descriptor is the same, so just transfer the value. If the value # is None, then we've only been given a descriptor to add. if item.value is not None: @@ -684,7 +705,7 @@ def _add_item(self, item): return if old is not None or old_by_name is not None: - _logger.info('Descriptor replacement for ID %#x, name %s', item.id, item.name) + _logger.info("Descriptor replacement for ID %#x, name %s", item.id, item.name) # Ensure the version number is seen to increment, regardless of # whether accessed by name or ID. new_version = item.version @@ -756,7 +777,7 @@ def __len__(self): """Number of items""" return len(self._by_name) - def update(self, heap, new_order='='): + def update(self, heap, new_order="="): """Update the item descriptors and items from an incoming heap. Parameters @@ -780,11 +801,11 @@ def update(self, heap, new_order='='): updated_items = {} for raw_item in heap.get_items(): if raw_item.id <= STREAM_CTRL_ID: - continue # Special fields, not real items + continue # Special fields, not real items try: item = self._by_id[raw_item.id] except KeyError: - _logger.warning('Item with ID %#x received but there is no descriptor', raw_item.id) + _logger.warning("Item with ID %#x received but there is no descriptor", raw_item.id) else: item.set_from_raw(raw_item, new_order=new_order) item.version += 1 diff --git a/src/spead2/__init__.pyi b/src/spead2/__init__.pyi index 8624494c6..3c5b88da6 100644 --- a/src/spead2/__init__.pyi +++ b/src/spead2/__init__.pyi @@ -74,8 +74,9 @@ class Empty(RuntimeError): class Flavour: @overload - def __init__(self, version: int, item_pointer_bits : int, - heap_address_bits: int, bug_compat: int = ...) -> None: ... + def __init__( + self, version: int, item_pointer_bits: int, heap_address_bits: int, bug_compat: int = ... + ) -> None: ... @overload def __init__(self) -> None: ... def __eq__(self, o: object) -> bool: ... @@ -105,12 +106,25 @@ class MemoryPool(MemoryAllocator): warn_on_empty: bool @overload - def __init__(self, lower: int, upper: int, max_free: int, initial: int, - allocator: Optional[MemoryAllocator] = None) -> None: ... + def __init__( + self, + lower: int, + upper: int, + max_free: int, + initial: int, + allocator: Optional[MemoryAllocator] = None, + ) -> None: ... @overload - def __init__(self, thread_pool: ThreadPool, lower: int, upper: int, max_free: int, initial: int, - low_water: int, allocator: MemoryAllocator) -> None: ... - + def __init__( + self, + thread_pool: ThreadPool, + lower: int, + upper: int, + max_free: int, + initial: int, + low_water: int, + allocator: MemoryAllocator, + ) -> None: ... class InprocQueue: def __init__(self) -> None: ... @@ -144,9 +158,16 @@ class Descriptor: order: str format: Optional[List[Tuple[str, int]]] - def __init__(self, id: int, name: str, description: str, - shape: Sequence[Optional[int]], dtype: Optional[_DTypeLike] = None, - order: str = ..., format: Optional[List[Tuple[str, int]]] = None) -> None: ... + def __init__( + self, + id: int, + name: str, + description: str, + shape: Sequence[Optional[int]], + dtype: Optional[_DTypeLike] = None, + order: str = ..., + format: Optional[List[Tuple[str, int]]] = None, + ) -> None: ... @property def itemsize_bits(self) -> int: ... @property @@ -162,10 +183,17 @@ class Descriptor: class Item(Descriptor): version: int - def __init__(self, id: int, name: str, description: str, - shape: Sequence[Optional[int]], dtype: Optional[_DTypeLike] = None, - order: str = ..., format: Optional[List[Tuple[str, int]]] = None, - value: Any = None) -> None: ... + def __init__( + self, + id: int, + name: str, + description: str, + shape: Sequence[Optional[int]], + dtype: Optional[_DTypeLike] = None, + order: str = ..., + format: Optional[List[Tuple[str, int]]] = None, + value: Any = None, + ) -> None: ... @property def value(self) -> Any: ... @value.setter @@ -176,10 +204,17 @@ class Item(Descriptor): class ItemGroup: def __init__(self) -> None: ... - def add_item(self, id: Optional[int], name: str, description: str, - shape: Sequence[Optional[int]], dtype: Optional[_DTypeLike] = None, - order: str = 'C', format: Optional[List[Tuple[str, int]]] = None, - value: Any = None) -> Item: ... + def add_item( + self, + id: Optional[int], + name: str, + description: str, + shape: Sequence[Optional[int]], + dtype: Optional[_DTypeLike] = None, + order: str = "C", + format: Optional[List[Tuple[str, int]]] = None, + value: Any = None, + ) -> Item: ... def __getitem__(self, key: Union[int, str]) -> Item: ... def __contains__(self, key: Union[int, str]) -> bool: ... def keys(self) -> KeysView[str]: ... diff --git a/src/spead2/numba.py b/src/spead2/numba.py index a0ee47d83..980222028 100644 --- a/src/spead2/numba.py +++ b/src/spead2/numba.py @@ -37,6 +37,7 @@ def codegen(context, builder, signature, args): rtype = signature.return_type llrtype = context.get_value_type(rtype) return builder.inttoptr(src, llrtype) + return sig, codegen diff --git a/src/spead2/recv/__init__.pyi b/src/spead2/recv/__init__.pyi index 9ff02850d..b99326b29 100644 --- a/src/spead2/recv/__init__.pyi +++ b/src/spead2/recv/__init__.pyi @@ -65,7 +65,6 @@ class StreamStatConfig: class Mode(enum.Enum): COUNTER: int = ... MAXIMUM: int = ... - def __init__(self, name: str, mode: StreamStatConfig.Mode = ...) -> None: ... @property def name(self) -> str: ... @@ -86,7 +85,6 @@ class StreamStats: search_dist: int @property def config(self) -> List[StreamStatConfig]: ... - @overload def __getitem__(self, index: int) -> int: ... @overload @@ -118,10 +116,19 @@ class StreamConfig: stream_id: int @property def stats(self) -> List[StreamStatConfig]: ... - def __init__(self, *, max_heaps: int = ..., substreams: int = ..., bug_compat: int = ..., - memcpy: int = ..., memory_allocator: spead2.MemoryAllocator = ..., - stop_on_stop_item: bool = ..., allow_unsized_heaps: bool = ..., - allow_out_of_order: bool = ..., stream_id: int = ...) -> None: ... + def __init__( + self, + *, + max_heaps: int = ..., + substreams: int = ..., + bug_compat: int = ..., + memcpy: int = ..., + memory_allocator: spead2.MemoryAllocator = ..., + stop_on_stop_item: bool = ..., + allow_unsized_heaps: bool = ..., + allow_out_of_order: bool = ..., + stream_id: int = ... + ) -> None: ... def add_stat(self, name: str, mode: StreamStatConfig.Mode = ...) -> int: ... def get_stat_index(self, name: str) -> int: ... def next_stat_index(self) -> int: ... @@ -131,8 +138,13 @@ class RingStreamConfig: heaps: int contiguous_only: bool incomplete_keep_payload_ranges: bool - def __init__(self, *, heaps: int = ..., contiguous_only: bool = ..., - incomplete_keep_payload_ranges: bool = ...) -> None: ... + def __init__( + self, + *, + heaps: int = ..., + contiguous_only: bool = ..., + incomplete_keep_payload_ranges: bool = ... + ) -> None: ... class UdpIbvConfig: DEFAULT_BUFFER_SIZE: ClassVar[int] @@ -146,9 +158,16 @@ class UdpIbvConfig: comp_vector: int max_poll: int - def __init__(self, *, endpoints: _EndpointList = ..., interface_address: str = ..., - buffer_size: int = ..., max_size: int = ..., comp_vector: int = ..., - max_poll: int = ...) -> None: ... + def __init__( + self, + *, + endpoints: _EndpointList = ..., + interface_address: str = ..., + buffer_size: int = ..., + max_size: int = ..., + comp_vector: int = ..., + max_poll: int = ... + ) -> None: ... class Ringbuffer: def size(self) -> int: ... @@ -165,31 +184,56 @@ class _Stream: def add_buffer_reader(self, buffer: Any) -> None: ... @overload - def add_udp_reader(self, port: int, max_size: int = ..., buffer_size: int = ..., - bind_hostname: str = ...) -> None: ... + def add_udp_reader( + self, port: int, max_size: int = ..., buffer_size: int = ..., bind_hostname: str = ... + ) -> None: ... @overload def add_udp_reader(self, socket: socket.socket, max_size: int = ...) -> None: ... @overload - def add_udp_reader(self, multicast_group: str, port: int, max_size: int = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... + def add_udp_reader( + self, + multicast_group: str, + port: int, + max_size: int = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... @overload - def add_udp_reader(self, multicast_group: str, port: int, max_size: int = ..., - buffer_size: int = ..., interface_index: int = ...) -> None: ... + def add_udp_reader( + self, + multicast_group: str, + port: int, + max_size: int = ..., + buffer_size: int = ..., + interface_index: int = ..., + ) -> None: ... @overload - def add_tcp_reader(self, port: int, max_size: int = ..., buffer_size: int = ..., - bind_hostname: str = ...) -> None: ... + def add_tcp_reader( + self, port: int, max_size: int = ..., buffer_size: int = ..., bind_hostname: str = ... + ) -> None: ... @overload def add_tcp_reader(self, acceptor: socket.socket, max_size: int = ...) -> None: ... @overload - def add_udp_ibv_reader(self, multicast_group: str, port: int, - interface_address: str, - max_size: int = ..., buffer_size: int = ..., - comp_vector: int = ..., max_poll: int = ...) -> None: ... + def add_udp_ibv_reader( + self, + multicast_group: str, + port: int, + interface_address: str, + max_size: int = ..., + buffer_size: int = ..., + comp_vector: int = ..., + max_poll: int = ..., + ) -> None: ... @overload - def add_udp_ibv_reader(self, endpoints: Sequence[Tuple[str, int]], - interface_address: str, - max_size: int = ..., buffer_size: int = ..., - comp_vector: int = ..., max_poll: int = ...) -> None: ... + def add_udp_ibv_reader( + self, + endpoints: Sequence[Tuple[str, int]], + interface_address: str, + max_size: int = ..., + buffer_size: int = ..., + comp_vector: int = ..., + max_poll: int = ..., + ) -> None: ... @overload def add_udp_ibv_reader(self, config: UdpIbvConfig) -> None: ... def add_udp_pcap_file_reader(self, filename: str, filter: str = ...) -> None: ... @@ -203,8 +247,12 @@ class _Stream: # We make a dummy _RingStream base class because mypy objects to the async stream # type overloading get as a coroutine. class _RingStream(_Stream): - def __init__(self, thread_pool: spead2.ThreadPool, config: StreamConfig = ..., - ring_config: RingStreamConfig = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + config: StreamConfig = ..., + ring_config: RingStreamConfig = ..., + ) -> None: ... def __iter__(self) -> Iterator[Heap]: ... def get_nowait(self) -> Heap: ... @property @@ -228,21 +276,31 @@ class ChunkStreamConfig: def disable_packet_presence(self) -> None: ... @property def packet_presence_payload_size(self) -> int: ... - def __init__( - self, *, items: List[int] = ..., max_chunks: int = ..., - place: Optional[tuple] = ..., max_heap_extra: int = ...) -> None: ... + self, + *, + items: List[int] = ..., + max_chunks: int = ..., + place: Optional[tuple] = ..., + max_heap_extra: int = ... + ) -> None: ... class Chunk: chunk_id: int stream_id: int present: object # optional buffer protocol - data: object # optional buffer protocol - extra: object # optional buffer protocol + data: object # optional buffer protocol + extra: object # optional buffer protocol def __init__( - self, *, chunk_id: int = ..., stream_id: int = ..., - present: object = ..., data: object = ..., extra: object = ...) -> None: ... + self, + *, + chunk_id: int = ..., + stream_id: int = ..., + present: object = ..., + data: object = ..., + extra: object = ... + ) -> None: ... # Dummy base class because the async ChunkRingbuffer.get has a different # signature to the synchronous version. @@ -277,32 +335,40 @@ class ChunkRingPair: class ChunkRingStream(_Stream, ChunkRingPair): def __init__( - self, thread_pool: spead2.ThreadPool, config: StreamConfig, + self, + thread_pool: spead2.ThreadPool, + config: StreamConfig, chunk_stream_config: ChunkStreamConfig, - data_ringbuffer: _ChunkRingbuffer, free_ringbuffer: _ChunkRingbuffer) -> None: ... + data_ringbuffer: _ChunkRingbuffer, + free_ringbuffer: _ChunkRingbuffer, + ) -> None: ... class ChunkStreamGroupConfig: class EvictionMode(enum.Enum): LOSSY = ... LOSSLESS = ... - DEFAULT_MAX_CHUNKS: ClassVar[int] @property def max_chunks(self) -> int: ... @property def eviction_mode(self) -> ChunkStreamGroupConfig.EvictionMode: ... - def __init__(self, *, max_chunks=..., eviction_mode=...) -> None: ... class ChunkStreamRingGroup(ChunkRingPair, collections.abc.Sequence[ChunkStreamGroupMember]): def __init__( - self, config: ChunkStreamGroupConfig, data_ringbuffer: _ChunkRingbuffer, - free_ringbuffer: _ChunkRingbuffer) -> None: ... + self, + config: ChunkStreamGroupConfig, + data_ringbuffer: _ChunkRingbuffer, + free_ringbuffer: _ChunkRingbuffer, + ) -> None: ... @property def config(self) -> ChunkStreamGroupConfig: ... def emplace_back( - self, thread_pool: spead2.ThreadPool, config: spead2.recv.StreamConfig, - chunk_stream_config: spead2.recv.ChunkStreamConfig) -> ChunkStreamGroupMember: ... + self, + thread_pool: spead2.ThreadPool, + config: spead2.recv.StreamConfig, + chunk_stream_config: spead2.recv.ChunkStreamConfig, + ) -> ChunkStreamGroupMember: ... def stop(self) -> None: ... # These are marked abstract in Sequence, so need to be implemented here @overload diff --git a/src/spead2/recv/asyncio.py b/src/spead2/recv/asyncio.py index b1de603eb..4ba4241f9 100644 --- a/src/spead2/recv/asyncio.py +++ b/src/spead2/recv/asyncio.py @@ -24,7 +24,7 @@ class _Waiter: - __slots__ = ['callback', 'future'] + __slots__ = ["callback", "future"] def __init__(self, callback): self.callback = callback diff --git a/src/spead2/recv/numba.py b/src/spead2/recv/numba.py index 7fb7adf8d..42d7ce0fb 100644 --- a/src/spead2/recv/numba.py +++ b/src/spead2/recv/numba.py @@ -23,18 +23,20 @@ # Older versions of numba.types doesn't have a size_t, so assume it is the same as uintptr_t _size_t = types.uintp -chunk_place_data = types.Record.make_c_struct([ - ('packet', types.intp), # uint8_t * - ('packet_size', _size_t), - ('items', types.intp), # s_item_pointer_t * - ('chunk_id', types.int64), - ('heap_index', _size_t), - ('heap_offset', _size_t), - ('batch_stats', types.intp), # uint64_t * - ('extra', types.intp), # uint8_t * - ('extra_offset', _size_t), - ('extra_size', _size_t) -]) +chunk_place_data = types.Record.make_c_struct( + [ + ("packet", types.intp), # uint8_t * + ("packet_size", _size_t), + ("items", types.intp), # s_item_pointer_t * + ("chunk_id", types.int64), + ("heap_index", _size_t), + ("heap_offset", _size_t), + ("batch_stats", types.intp), # uint64_t * + ("extra", types.intp), # uint8_t * + ("extra_offset", _size_t), + ("extra_size", _size_t), + ] +) """Numba record type representing the C structure used in the chunk placement callback. Numba doesn't (as of 0.54) support pointers in records, so the pointer fields diff --git a/src/spead2/send/__init__.py b/src/spead2/send/__init__.py index 6e628b3f3..7642a4e81 100644 --- a/src/spead2/send/__init__.py +++ b/src/spead2/send/__init__.py @@ -61,11 +61,12 @@ class HeapGenerator: The SPEAD protocol flavour used for heaps generated by :py:meth:`get_heap` and :py:meth:`get_end`. """ + def __init__(self, item_group, descriptor_frequency=None, flavour=_spead2.Flavour()): # Workaround to avoid creating a self-reference when it's bundled into # the same class. self._item_group = item_group if item_group is not self else None - self._info = {} # Maps ID to _ItemInfo + self._info = {} # Maps ID to _ItemInfo self._descriptor_frequency = descriptor_frequency # Counter for calls to add_to_heap. This is independent of the # protocol-level heap count. @@ -81,8 +82,10 @@ def _descriptor_stale(self, item, info): if info.descriptor_cnt is None: # Never been sent before return True - if self._descriptor_frequency is not None \ - and self._descriptor_cnt - info.descriptor_cnt >= self._descriptor_frequency: + if ( + self._descriptor_frequency is not None + and self._descriptor_cnt - info.descriptor_cnt >= self._descriptor_frequency + ): # This descriptor is due for a resend return True # Check for complete replacement of the item @@ -93,7 +96,7 @@ def _descriptor_stale(self, item, info): return True return False - def add_to_heap(self, heap, descriptors='stale', data='stale'): + def add_to_heap(self, heap, descriptors="stale", data="stale"): """Update a heap to contains all the new items and item descriptors since the last call. @@ -119,19 +122,20 @@ def add_to_heap(self, heap, descriptors='stale', data='stale'): ValueError if `descriptors` or `data` is not one of the legal values """ - if descriptors not in ['stale', 'all', 'none']: + if descriptors not in ["stale", "all", "none"]: raise ValueError("descriptors must be one of 'stale', 'all', 'none'") - if data not in ['stale', 'all', 'none']: + if data not in ["stale", "all", "none"]: raise ValueError("data must be one of 'stale', 'all', 'none'") item_group = self._item_group if self._item_group is not None else self for item in item_group.values(): info = self._get_info(item) - if (descriptors == 'all') or (descriptors == 'stale' - and self._descriptor_stale(item, info)): + if (descriptors == "all") or ( + descriptors == "stale" and self._descriptor_stale(item, info) + ): heap.add_descriptor(item) info.descriptor_cnt = self._descriptor_cnt if item.value is not None: - if (data == 'all') or (data == 'stale' and info.version != item.version): + if (data == "all") or (data == "stale" and info.version != item.version): heap.add_item(item) info.version = item.version self._descriptor_cnt += 1 @@ -147,15 +151,13 @@ def get_heap(self, *args, **kwargs): return heap def get_start(self): - """Return a heap that contains only a start-of-stream marker. - """ + """Return a heap that contains only a start-of-stream marker.""" heap = Heap(self._flavour) heap.add_start() return heap def get_end(self): - """Return a heap that contains only an end-of-stream marker. - """ + """Return a heap that contains only an end-of-stream marker.""" heap = Heap(self._flavour) heap.add_end() return heap @@ -163,6 +165,7 @@ def get_end(self): class ItemGroup(_spead2.ItemGroup, HeapGenerator): """Bundles an ItemGroup and HeapGenerator into a single class""" + def __init__(self, *args, **kwargs): _spead2.ItemGroup.__init__(self) HeapGenerator.__init__(self, self, *args, **kwargs) diff --git a/src/spead2/send/__init__.pyi b/src/spead2/send/__init__.pyi index 49dcc36c0..ef7b03af7 100644 --- a/src/spead2/send/__init__.pyi +++ b/src/spead2/send/__init__.pyi @@ -78,11 +78,16 @@ class StreamConfig: burst_rate_ratio: float rate_method: RateMethod - def __init__(self, *, max_packet_size: int = ..., rate: float = ..., - burst_size: int = ..., max_heaps: int = ..., - burst_rate_ratio: float = ..., - rate_method: RateMethod = ...) -> None: ... - + def __init__( + self, + *, + max_packet_size: int = ..., + rate: float = ..., + burst_size: int = ..., + max_heaps: int = ..., + burst_rate_ratio: float = ..., + rate_method: RateMethod = ... + ) -> None: ... @property def burst_rate(self) -> float: ... @@ -96,65 +101,109 @@ class Stream: def num_substreams(self) -> int: ... class SyncStream(Stream): - def send_heap(self, heap: Heap, cnt: int = ..., substream_index = ...) -> None: ... - def send_heaps(self, heaps: Union[List[HeapReference], HeapReferenceList], mode: GroupMode) -> None: ... + def send_heap(self, heap: Heap, cnt: int = ..., substream_index=...) -> None: ... + def send_heaps( + self, heaps: Union[List[HeapReference], HeapReferenceList], mode: GroupMode + ) -> None: ... class _UdpStream: DEFAULT_BUFFER_SIZE: ClassVar[int] @overload - def __init__(self, thread_pool: spead2.ThreadPool, - hostname: str, port: int, - config: StreamConfig = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + hostname: str, + port: int, + config: StreamConfig = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - hostname: str, port: int, - config: StreamConfig, - buffer_size: int, - ttl: int) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + hostname: str, + port: int, + config: StreamConfig, + buffer_size: int, + ttl: int, + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - multicast_group: str, port: int, - config: StreamConfig, - ttl: int, interface_address: str) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + multicast_group: str, + port: int, + config: StreamConfig, + ttl: int, + interface_address: str, + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - multicast_group: str, port: int, - config: StreamConfig, - ttl: int, interface_index: int) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + multicast_group: str, + port: int, + config: StreamConfig, + ttl: int, + interface_index: int, + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - socket: socket.socket, hostname: str, port: int, - config: StreamConfig = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + socket: socket.socket, + hostname: str, + port: int, + config: StreamConfig = ..., + ) -> None: ... # Endpoint list variants @overload - def __init__(self, thread_pool: spead2.ThreadPool, - endpoints: _EndpointList, - config: StreamConfig = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + endpoints: _EndpointList, + config: StreamConfig = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - endpoints: _EndpointList, - config: StreamConfig, - buffer_size: int, - ttl: int) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + endpoints: _EndpointList, + config: StreamConfig, + buffer_size: int, + ttl: int, + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - endpoints: _EndpointList, - config: StreamConfig, - ttl: int, interface_address: str) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + endpoints: _EndpointList, + config: StreamConfig, + ttl: int, + interface_address: str, + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - endpoints: _EndpointList, - config: StreamConfig, - ttl: int, interface_index: int) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + endpoints: _EndpointList, + config: StreamConfig, + ttl: int, + interface_index: int, + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - socket: socket.socket, - endpoints: _EndpointList, - config: StreamConfig = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + socket: socket.socket, + endpoints: _EndpointList, + config: StreamConfig = ..., + ) -> None: ... class UdpStream(_UdpStream, SyncStream): pass @@ -171,28 +220,39 @@ class UdpIbvConfig: max_poll: int memory_regions: list - def __init__(self, *, endpoints: _EndpointList = ..., interface_address: str = ..., - buffer_size: int = ..., ttl: int = ..., comp_vector: int = ..., - max_poll: int = ..., memory_regions: list = ...) -> None: ... - + def __init__( + self, + *, + endpoints: _EndpointList = ..., + interface_address: str = ..., + buffer_size: int = ..., + ttl: int = ..., + comp_vector: int = ..., + max_poll: int = ..., + memory_regions: list = ... + ) -> None: ... class _UdpIbvStream: DEFAULT_BUFFER_SIZE: ClassVar[int] DEFAULT_MAX_POLL: ClassVar[int] @overload - def __init__(self, thread_pool: spead2.ThreadPool, - multicast_group: str, port: int, - config: StreamConfig, - interface_address: str, - buffer_size: int = ..., ttl: int = ..., - comp_vector: int = ..., max_pool: int = ...) -> None: ... - + def __init__( + self, + thread_pool: spead2.ThreadPool, + multicast_group: str, + port: int, + config: StreamConfig, + interface_address: str, + buffer_size: int = ..., + ttl: int = ..., + comp_vector: int = ..., + max_pool: int = ..., + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - config: StreamConfig, - udp_ibv_config: UdpIbvConfig) -> None: ... - + def __init__( + self, thread_pool: spead2.ThreadPool, config: StreamConfig, udp_ibv_config: UdpIbvConfig + ) -> None: ... class UdpIbvStream(_UdpIbvStream, SyncStream): pass @@ -202,23 +262,32 @@ class _TcpStream: class TcpStream(_TcpStream, SyncStream): @overload - def __init__(self, thread_pool: spead2.ThreadPool, socket: socket.socket, - config: StreamConfig = ...) -> None: ... + def __init__( + self, thread_pool: spead2.ThreadPool, socket: socket.socket, config: StreamConfig = ... + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - hostname: str, port: int, - config: StreamConfig = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + hostname: str, + port: int, + config: StreamConfig = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, - endpoints: _EndpointList, - config: StreamConfig = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + endpoints: _EndpointList, + config: StreamConfig = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... class BytesStream(SyncStream): def getvalue(self) -> bytes: ... - def __init__(self, thread_pool: spead2.ThreadPool, - config: StreamConfig = ...) -> None: ... + def __init__(self, thread_pool: spead2.ThreadPool, config: StreamConfig = ...) -> None: ... class _InprocStream: @property @@ -226,23 +295,33 @@ class _InprocStream: @property def queues(self) -> Sequence[spead2.InprocQueue]: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, queue: spead2.InprocQueue, - config: StreamConfig = ...) -> None: ... + def __init__( + self, thread_pool: spead2.ThreadPool, queue: spead2.InprocQueue, config: StreamConfig = ... + ) -> None: ... @overload - def __init__(self, thread_pool: spead2.ThreadPool, queues: List[spead2.InprocQueue], - config: StreamConfig = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + queues: List[spead2.InprocQueue], + config: StreamConfig = ..., + ) -> None: ... class InprocStream(_InprocStream, SyncStream): pass class HeapGenerator: - def __init__(self, item_group: spead2.ItemGroup, descriptor_frequency: Optional[int] = None, - flavour: spead2.Flavour = spead2.Flavour()) -> None: ... + def __init__( + self, + item_group: spead2.ItemGroup, + descriptor_frequency: Optional[int] = None, + flavour: spead2.Flavour = spead2.Flavour(), + ) -> None: ... def add_to_heap(self, heap: Heap, descriptors: str = ..., data: str = ...) -> Heap: ... def get_heap(self, descriptors: str = ..., data: str = ...) -> Heap: ... def get_start(self) -> Heap: ... def get_end(self) -> Heap: ... class ItemGroup(spead2.ItemGroup, HeapGenerator): - def __init__(self, descriptor_frequency: Optional[int] = None, - flavour: spead2.Flavour = ...) -> None: ... + def __init__( + self, descriptor_frequency: Optional[int] = None, flavour: spead2.Flavour = ... + ) -> None: ... diff --git a/src/spead2/send/asyncio.py b/src/spead2/send/asyncio.py index d4ef570fe..71c5f9adf 100644 --- a/src/spead2/send/asyncio.py +++ b/src/spead2/send/asyncio.py @@ -56,6 +56,7 @@ def callback(exc, bytes_transferred): if self._active == 0: loop.remove_reader(self.fd) self._last_queued_future = None # Purely to free the memory + queued = call(callback) if self._active == 0: loop.add_reader(self.fd, self.process_callbacks) @@ -79,13 +80,11 @@ def async_send_heap(self, heap, cnt=-1, substream_index=0): Substream on which to send the heap """ meth = super().async_send_heap - return self._async_send( - lambda callback: meth(heap, callback, cnt, substream_index)) + return self._async_send(lambda callback: meth(heap, callback, cnt, substream_index)) def async_send_heaps(self, heaps, mode): meth = super().async_send_heaps - return self._async_send( - lambda callback: meth(heaps, callback, mode)) + return self._async_send(lambda callback: meth(heaps, callback, mode)) async def async_flush(self): """Asynchronously wait for all enqueued heaps to be sent. Note that @@ -99,10 +98,11 @@ async def async_flush(self): return Wrapped -UdpStream = _wrap_class('UdpStream', _UdpStreamAsyncio) -UdpStream.__doc__ = \ - """SPEAD over UDP with asynchronous sends. The other constructors - defined for :py:class:`spead2.send.UdpStream` are also applicable here. +UdpStream = _wrap_class("UdpStream", _UdpStreamAsyncio) +UdpStream.__doc__ = """SPEAD over UDP with asynchronous sends. + + The other constructors defined for :py:class:`spead2.send.UdpStream` are + also applicable here. Parameters ---------- @@ -117,7 +117,7 @@ async def async_flush(self): to OS limits. """ -_TcpStreamBase = _wrap_class('TcpStream', _TcpStreamAsyncio) +_TcpStreamBase = _wrap_class("TcpStream", _TcpStreamAsyncio) class TcpStream(_TcpStreamBase): @@ -158,9 +158,8 @@ def callback(arg): return stream -InprocStream = _wrap_class('InprocStream', _InprocStreamAsyncio) -InprocStream.__doc__ = \ - """SPEAD over reliable in-process transport. +InprocStream = _wrap_class("InprocStream", _InprocStreamAsyncio) +InprocStream.__doc__ = """SPEAD over reliable in-process transport. .. note:: @@ -182,9 +181,8 @@ def callback(arg): try: from spead2._spead2.send import UdpIbvStreamAsyncio as _UdpIbvStreamAsyncio - UdpIbvStream = _wrap_class('UdpIbvStream', _UdpIbvStreamAsyncio) - UdpIbvStream.__doc__ = \ - """Like :class:`UdpStream`, but using the Infiniband Verbs API. + UdpIbvStream = _wrap_class("UdpIbvStream", _UdpIbvStreamAsyncio) + UdpIbvStream.__doc__ = """Like :class:`UdpStream`, but using the Infiniband Verbs API. Parameters ---------- diff --git a/src/spead2/send/asyncio.pyi b/src/spead2/send/asyncio.pyi index 10d5e4de9..a21d56af1 100644 --- a/src/spead2/send/asyncio.pyi +++ b/src/spead2/send/asyncio.pyi @@ -25,11 +25,14 @@ class AsyncStream(spead2.send.Stream): @property def fd(self) -> int: ... def flush(self) -> None: ... - def async_send_heap(self, heap: spead2.send.Heap, cnt: int = ..., - substream_index: int = ...) -> asyncio.Future[int]: ... - def async_send_heaps(self, - heaps: Union[List[spead2.send.HeapReference], spead2.send.HeapReferenceList], - mode: spead2.send.GroupMode) -> asyncio.Future[int]: ... + def async_send_heap( + self, heap: spead2.send.Heap, cnt: int = ..., substream_index: int = ... + ) -> asyncio.Future[int]: ... + def async_send_heaps( + self, + heaps: Union[List[spead2.send.HeapReference], spead2.send.HeapReferenceList], + mode: spead2.send.GroupMode, + ) -> asyncio.Future[int]: ... async def async_flush(self) -> None: ... class UdpStream(spead2.send._UdpStream, AsyncStream): @@ -39,21 +42,33 @@ class UdpIbvStream(spead2.send._UdpIbvStream, AsyncStream): pass class TcpStream(spead2.send._TcpStream, AsyncStream): - def __init__(self, thread_pool: spead2.ThreadPool, socket: socket.socket, - config: spead2.send.StreamConfig = ...) -> None: ... + def __init__( + self, + thread_pool: spead2.ThreadPool, + socket: socket.socket, + config: spead2.send.StreamConfig = ..., + ) -> None: ... @overload @classmethod - async def connect(self, thread_pool: spead2.ThreadPool, - hostname: str, port: int, - config: spead2.send.StreamConfig = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... + async def connect( + self, + thread_pool: spead2.ThreadPool, + hostname: str, + port: int, + config: spead2.send.StreamConfig = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... @overload @classmethod - async def connect(self, thread_pool: spead2.ThreadPool, - endpoints: _EndpointList, - config: spead2.send.StreamConfig = ..., - buffer_size: int = ..., interface_address: str = ...) -> None: ... - + async def connect( + self, + thread_pool: spead2.ThreadPool, + endpoints: _EndpointList, + config: spead2.send.StreamConfig = ..., + buffer_size: int = ..., + interface_address: str = ..., + ) -> None: ... class InprocStream(spead2.send._InprocStream, AsyncStream): pass diff --git a/src/spead2/tools/bench_asyncio.py b/src/spead2/tools/bench_asyncio.py index 9df1d5ad3..cf277f36c 100644 --- a/src/spead2/tools/bench_asyncio.py +++ b/src/spead2/tools/bench_asyncio.py @@ -43,9 +43,7 @@ from . import cmdline -ARG_ENUMS = { - "rate_method": spead2.send.RateMethod -} +ARG_ENUMS = {"rate_method": spead2.send.RateMethod} def serialize_args(args): @@ -81,52 +79,52 @@ async def run_stream(self, stream): return num_heaps def _write(self, s): - self.writer.write(s.encode('ascii')) + self.writer.write(s.encode("ascii")) async def run_control(self): try: stream_task = None while True: command = await self.reader.readline() - command = command.decode('ascii') + command = command.decode("ascii") logging.debug("command = %s", command) if not command: break command = json.loads(command) - if command['cmd'] == 'start': + if command["cmd"] == "start": if stream_task is not None: logging.warning("Start received while already running: %s", command) continue - args = deserialize_args(command['args']) + args = deserialize_args(command["args"]) protocol = cmdline.ProtocolOptions() receiver = cmdline.ReceiverOptions(protocol) - for (key, value) in command['protocol'].items(): + for key, value in command["protocol"].items(): setattr(protocol, key, value) - for (key, value) in command['receiver'].items(): + for key, value in command["receiver"].items(): setattr(receiver, key, value) stream = spead2.recv.asyncio.Stream( receiver.make_thread_pool(), receiver.make_stream_config(), - receiver.make_ring_stream_config() + receiver.make_ring_stream_config(), ) - if getattr(receiver, 'ibv') and not hasattr(stream, 'add_udp_ibv_reader'): - logging.error('--recv-ibv passed but agent does not support ibv') + if getattr(receiver, "ibv") and not hasattr(stream, "add_udp_ibv_reader"): + logging.error("--recv-ibv passed but agent does not support ibv") sys.exit(1) endpoint = args.multicast or args.endpoint receiver.add_readers(stream, [endpoint]) stream_task = asyncio.ensure_future(self.run_stream(stream)) - self._write('ready\n') - elif command['cmd'] == 'stop': + self._write("ready\n") + elif command["cmd"] == "stop": if stream_task is None: logging.warning("Stop received when already stopped") continue stream.stop() received_heaps = await stream_task - self._write(json.dumps({'received_heaps': received_heaps}) + '\n') + self._write(json.dumps({"received_heaps": received_heaps}) + "\n") stream_task = None stream = None - elif command['cmd'] == 'exit': + elif command["cmd"] == "exit": break else: logging.warning("Bad command: %s", command) @@ -161,7 +159,7 @@ async def send_stream(item_group, stream, num_heaps, args): if i == num_heaps: heap = item_group.get_end() else: - heap = item_group.get_heap(data='all') + heap = item_group.get_heap(data="all") task = asyncio.ensure_future(stream.async_send_heap(heap)) tasks.append(task) for task in tasks: @@ -169,39 +167,55 @@ async def send_stream(item_group, stream, num_heaps, args): return transferred -async def measure_connection_once(args, protocol, sender, receiver, - rate, num_heaps, required_heaps): +async def measure_connection_once( + args, protocol, sender, receiver, rate, num_heaps, required_heaps +): def write(s): - writer.write(s.encode('ascii')) + writer.write(s.encode("ascii")) host, port = cmdline.parse_endpoint(args.endpoint) reader, writer = await asyncio.open_connection(host, port) - write(json.dumps( - { - 'cmd': 'start', - 'args': serialize_args(args), - 'protocol': {key: value for (key, value) in protocol.__dict__.items() - if not key.startswith('_')}, - 'receiver': {key: value for (key, value) in receiver.__dict__.items() - if not key.startswith('_')} - }) + '\n') + write( + json.dumps( + { + "cmd": "start", + "args": serialize_args(args), + "protocol": { + key: value + for (key, value) in protocol.__dict__.items() + if not key.startswith("_") + }, + "receiver": { + key: value + for (key, value) in receiver.__dict__.items() + if not key.startswith("_") + }, + } + ) + + "\n" + ) # Wait for "ready" response response = await reader.readline() - assert response == b'ready\n' + assert response == b"ready\n" item_group = spead2.send.ItemGroup(flavour=sender.make_flavour()) - item_group.add_item(id=None, name='Test item', - description='A test item with arbitrary value', - shape=(args.heap_size,), dtype=np.uint8, - value=np.zeros((args.heap_size,), dtype=np.uint8)) - - sender.rate = rate * 8e-9 # Convert to Gb/s + item_group.add_item( + id=None, + name="Test item", + description="A test item with arbitrary value", + shape=(args.heap_size,), + dtype=np.uint8, + value=np.zeros((args.heap_size,), dtype=np.uint8), + ) + + sender.rate = rate * 8e-9 # Convert to Gb/s endpoint = args.endpoint if args.multicast is not None: endpoint = args.multicast memory_regions = [item.value for item in item_group.values()] stream = await sender.make_stream( - sender.make_thread_pool(), [cmdline.parse_endpoint(endpoint)], memory_regions) + sender.make_thread_pool(), [cmdline.parse_endpoint(endpoint)], memory_regions + ) start = timeit.default_timer() transferred = await send_stream(item_group, stream, num_heaps, args) @@ -210,16 +224,21 @@ def write(s): actual_rate = transferred / elapsed # Give receiver time to catch up with any queue await asyncio.sleep(0.1) - write(json.dumps({'cmd': 'stop'}) + '\n') + write(json.dumps({"cmd": "stop"}) + "\n") # Read number of heaps received response = await reader.readline() - response = json.loads(response.decode('ascii')) - received_heaps = response['received_heaps'] + response = json.loads(response.decode("ascii")) + received_heaps = response["received_heaps"] await asyncio.sleep(0.5) await writer.drain() writer.close() - logging.debug("Received %d/%d heaps in %f seconds, rate %.3f Gbps", - received_heaps, num_heaps, elapsed, actual_rate * 8e-9) + logging.debug( + "Received %d/%d heaps in %f seconds, rate %.3f Gbps", + received_heaps, + num_heaps, + elapsed, + actual_rate * 8e-9, + ) return received_heaps >= required_heaps, actual_rate @@ -228,8 +247,9 @@ async def measure_connection(args, protocol, sender, receiver, rate, num_heaps, rate_sum = 0.0 passes = 5 for i in range(passes): - status, actual_rate = await measure_connection_once(args, protocol, sender, receiver, - rate, num_heaps, required_heaps) + status, actual_rate = await measure_connection_once( + args, protocol, sender, receiver, rate, num_heaps, required_heaps + ) good = good and status rate_sum += actual_rate return good, rate_sum / passes @@ -242,8 +262,9 @@ async def run_master(args, protocol, sender, receiver): # does not matter. Also do a warmup run first to warm up the receiver. num_heaps = int(1e9 / args.heap_size) + 2 await measure_connection_once(args, protocol, sender, receiver, 0.0, num_heaps, 0) # warmup - good, actual_rate = await measure_connection(args, protocol, sender, receiver, - 0.0, num_heaps, num_heaps - 1) + good, actual_rate = await measure_connection( + args, protocol, sender, receiver, 0.0, num_heaps, num_heaps - 1 + ) if good: if not args.quiet: print("Limited by send speed") @@ -258,10 +279,14 @@ async def run_master(args, protocol, sender, receiver): rate = (low + high) * 0.5 num_heaps = int(max(1e9, rate) / args.heap_size) + 2 good, actual_rate = await measure_connection( - args, protocol, sender, receiver, rate, num_heaps, num_heaps - 1) + args, protocol, sender, receiver, rate, num_heaps, num_heaps - 1 + ) if not args.quiet: - print("Rate: {:.3f} Gbps ({:.3f} actual): {}".format( - rate * 8e-9, actual_rate * 8e-9, "GOOD" if good else "BAD")) + print( + "Rate: {:.3f} Gbps ({:.3f} actual): {}".format( + rate * 8e-9, actual_rate * 8e-9, "GOOD" if good else "BAD" + ) + ) if good: low = rate best_actual = actual_rate @@ -276,54 +301,61 @@ async def run_master(args, protocol, sender, receiver): def main(): parser = argparse.ArgumentParser() - parser.add_argument('--log', metavar='LEVEL', default='INFO', help='Log level [%(default)s]') - subparsers = parser.add_subparsers(title='subcommands', required=True) + parser.add_argument("--log", metavar="LEVEL", default="INFO", help="Log level [%(default)s]") + subparsers = parser.add_subparsers(title="subcommands", required=True) - master = subparsers.add_parser('master') + master = subparsers.add_parser("master") sender_map = { - 'buffer': 'send_buffer', - 'packet': None, # Use receiver's packet size - 'rate': None, # Controlled by test - 'bind': 'send_bind', - 'ibv': 'send_ibv', - 'ibv_vector': 'send_ibv_vector', - 'ibv_max_poll': 'send_ibv_max_poll', - 'affinity': 'send-affinity', - 'threads': 'send-threads' + "buffer": "send_buffer", + "packet": None, # Use receiver's packet size + "rate": None, # Controlled by test + "bind": "send_bind", + "ibv": "send_ibv", + "ibv_vector": "send_ibv_vector", + "ibv_max_poll": "send_ibv_max_poll", + "affinity": "send-affinity", + "threads": "send-threads", } receiver_map = { - 'buffer': 'recv_buffer', - 'bind': 'recv_bind', - 'ibv': 'recv_ibv', - 'ibv_vector': 'recv_ibv_vector', - 'ibv_max_poll': 'recv_ibv_max_poll', - 'affinity': 'recv-affinity', - 'threads': 'recv-threads', - 'mem_pool': None, - 'mem_lower': None, - 'mem_upper': None + "buffer": "recv_buffer", + "bind": "recv_bind", + "ibv": "recv_ibv", + "ibv_vector": "recv_ibv_vector", + "ibv_max_poll": "recv_ibv_max_poll", + "affinity": "recv-affinity", + "threads": "recv-threads", + "mem_pool": None, + "mem_lower": None, + "mem_upper": None, } - protocol = cmdline.ProtocolOptions(name_map={'tcp': None}) + protocol = cmdline.ProtocolOptions(name_map={"tcp": None}) sender = cmdline.SenderOptions(protocol, name_map=sender_map) receiver = cmdline.ReceiverOptions(protocol, name_map=receiver_map) - master.add_argument('--quiet', action='store_true', default=False, - help='Print only the final result') - master.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, - help='Payload size for heap [%(default)s]') - master.add_argument('--multicast', metavar='ADDRESS', type=str, - help='Send via multicast group [unicast]') + master.add_argument( + "--quiet", action="store_true", default=False, help="Print only the final result" + ) + master.add_argument( + "--heap-size", + metavar="BYTES", + type=int, + default=4194304, + help="Payload size for heap [%(default)s]", + ) + master.add_argument( + "--multicast", metavar="ADDRESS", type=str, help="Send via multicast group [unicast]" + ) protocol.add_arguments(master) - sender.add_arguments(master.add_argument_group('sender options')) - receiver.add_arguments(master.add_argument_group('receiver options')) - master.add_argument('endpoint', metavar='HOST:PORT') - agent = subparsers.add_parser('agent') - agent.add_argument('port', type=int) + sender.add_arguments(master.add_argument_group("sender options")) + receiver.add_arguments(master.add_argument_group("receiver options")) + master.add_argument("endpoint", metavar="HOST:PORT") + agent = subparsers.add_parser("agent") + agent.add_argument("port", type=int) args = parser.parse_args() logging.basicConfig(level=getattr(logging, args.log.upper())) - if 'endpoint' in args: + if "endpoint" in args: if args.send_ibv and not args.multicast: - parser.error('--send-ibv requires --multicast') + parser.error("--send-ibv requires --multicast") receiver.mem_pool = True receiver.mem_lower = args.heap_size receiver.mem_upper = args.heap_size + 1024 # more than enough for overheads diff --git a/src/spead2/tools/cmdline.py b/src/spead2/tools/cmdline.py index 70f2f5bd6..e55fd9774 100644 --- a/src/spead2/tools/cmdline.py +++ b/src/spead2/tools/cmdline.py @@ -20,15 +20,15 @@ import spead2.recv import spead2.send -_HAVE_IBV = hasattr(spead2.recv.Stream, 'add_udp_ibv_reader') +_HAVE_IBV = hasattr(spead2.recv.Stream, "add_udp_ibv_reader") def parse_endpoint(endpoint): - if ':' in endpoint: - host, port = endpoint.rsplit(':', 1) + if ":" in endpoint: + host, port = endpoint.rsplit(":", 1) port = int(port) else: - host = '' + host = "" port = int(endpoint) return host, port @@ -66,10 +66,10 @@ def _add_argument(self, parser, name, *args, **kwargs): - No `default` should be provided. The value currently stored in the object is used as the default. """ - assert 'dest' not in kwargs + assert "dest" not in kwargs new_name = self._name_map.get(name, name) if new_name is not None: - flag = '--' + new_name.replace('_', '-') + flag = "--" + new_name.replace("_", "-") parser.add_argument(flag, *args, dest=new_name, default=getattr(self, name), **kwargs) def _extract_args(self, namespace): @@ -79,7 +79,7 @@ def _extract_args(self, namespace): the arguments into the object. """ for name in self.__dict__: - if name.startswith('_'): + if name.startswith("_"): continue mapped_name = self._name_map.get(name, name) if mapped_name is not None: @@ -95,10 +95,10 @@ def __init__(self, name_map=None) -> None: self.pyspead = False def add_arguments(self, parser): - self._add_argument(parser, 'tcp', action='store_true', - help='Use TCP instead of UDP') - self._add_argument(parser, 'pyspead', action='store_true', - help='Be bug-compatible with PySPEAD') + self._add_argument(parser, "tcp", action="store_true", help="Use TCP instead of UDP") + self._add_argument( + parser, "pyspead", action="store_true", help="Be bug-compatible with PySPEAD" + ) def notify(self, parser, namespace): self._extract_args(namespace) @@ -114,7 +114,7 @@ class SharedOptions(_Options): def __init__(self, protocol, name_map=None) -> None: super().__init__(name_map) self._protocol = protocol - self.buffer = None # Default depends on protocol + self.buffer = None # Default depends on protocol self.bind = None if _HAVE_IBV: # These must be set conditionally, because _extract_args requires @@ -125,18 +125,32 @@ def __init__(self, protocol, name_map=None) -> None: self.affinity = None def add_arguments(self, parser): - self._add_argument(parser, 'buffer', type=int, help='Socket buffer size') - self._add_argument(parser, 'bind', type=str, help='Interface address') + self._add_argument(parser, "buffer", type=int, help="Socket buffer size") + self._add_argument(parser, "bind", type=str, help="Interface address") if _HAVE_IBV: - self._add_argument(parser, 'ibv', action='store_true', help='Use ibverbs [no]') - self._add_argument(parser, 'ibv_vector', type=int, metavar='N', - help='Completion vector, or -1 to use polling [%(default)s]') - self._add_argument(parser, 'ibv_max_poll', type=int, - help='Maximum number of times to poll in a row [%(default)s]') - self._add_argument(parser, 'affinity', type=spead2.parse_range_list, - help='List of CPUs to pin threads to [no affinity]') - self._add_argument(parser, 'threads', type=int, - help='Number of worker threads [%(default)s]') + self._add_argument(parser, "ibv", action="store_true", help="Use ibverbs [no]") + self._add_argument( + parser, + "ibv_vector", + type=int, + metavar="N", + help="Completion vector, or -1 to use polling [%(default)s]", + ) + self._add_argument( + parser, + "ibv_max_poll", + type=int, + help="Maximum number of times to poll in a row [%(default)s]", + ) + self._add_argument( + parser, + "affinity", + type=spead2.parse_range_list, + help="List of CPUs to pin threads to [no affinity]", + ) + self._add_argument( + parser, "threads", type=int, help="Number of worker threads [%(default)s]" + ) def make_thread_pool(self): if self.affinity: @@ -165,33 +179,48 @@ def __init__(self, protocol, name_map=None) -> None: self.ibv_max_poll = spead2.recv.UdpIbvConfig.DEFAULT_MAX_POLL def add_arguments(self, parser): - self._add_argument(parser, 'memcpy_nt', action='store_true', - help='Use non-temporal memcpy') - self._add_argument(parser, 'concurrent_heaps', type=int, - help='Maximum number of in-flight heaps per substream [%(default)s]') - self._add_argument(parser, 'substreams', type=int, - help='Number of parallel substreams [%(default)s]') - self._add_argument(parser, 'ring_heaps', type=int, - help='Ring buffer capacity in heaps [%(default)s]') - self._add_argument(parser, 'mem_pool', action='store_true', help='Use a memory pool') - self._add_argument(parser, 'mem_lower', type=int, - help='Minimum allocation which will use the memory pool [%(default)s]') - self._add_argument(parser, 'mem_upper', type=int, - help='Maximum allocation which will use the memory pool [%(default)s]') - self._add_argument(parser, 'mem_max_free', type=int, - help='Maximum free memory buffers [%(default)s]') - self._add_argument(parser, 'mem_initial', type=int, - help='Initial free memory buffers [%(default)s]') - self._add_argument(parser, 'packet', type=int, help='Maximum packet size to accept') + self._add_argument(parser, "memcpy_nt", action="store_true", help="Use non-temporal memcpy") + self._add_argument( + parser, + "concurrent_heaps", + type=int, + help="Maximum number of in-flight heaps per substream [%(default)s]", + ) + self._add_argument( + parser, "substreams", type=int, help="Number of parallel substreams [%(default)s]" + ) + self._add_argument( + parser, "ring_heaps", type=int, help="Ring buffer capacity in heaps [%(default)s]" + ) + self._add_argument(parser, "mem_pool", action="store_true", help="Use a memory pool") + self._add_argument( + parser, + "mem_lower", + type=int, + help="Minimum allocation which will use the memory pool [%(default)s]", + ) + self._add_argument( + parser, + "mem_upper", + type=int, + help="Maximum allocation which will use the memory pool [%(default)s]", + ) + self._add_argument( + parser, "mem_max_free", type=int, help="Maximum free memory buffers [%(default)s]" + ) + self._add_argument( + parser, "mem_initial", type=int, help="Initial free memory buffers [%(default)s]" + ) + self._add_argument(parser, "packet", type=int, help="Maximum packet size to accept") super().add_arguments(parser) def notify(self, parser, namespace): self._extract_args(namespace) if _HAVE_IBV: if self.ibv and not self.bind: - parser.error('--ibv requires --bind') + parser.error("--ibv requires --bind") if self._protocol.tcp and self.ibv: - parser.error('--ibv and --tcp are incompatible') + parser.error("--ibv and --tcp are incompatible") if self.buffer is None: if self._protocol.tcp: @@ -215,8 +244,9 @@ def make_stream_config(self): config.substreams = self.substreams config.bug_compat = spead2.BUG_COMPAT_PYSPEAD_0_5_2 if self._protocol.pyspead else 0 if self.mem_pool: - config.memory_allocator = spead2.MemoryPool(self.mem_lower, self.mem_upper, - self.mem_max_free, self.mem_initial) + config.memory_allocator = spead2.MemoryPool( + self.mem_lower, self.mem_upper, self.mem_max_free, self.mem_initial + ) if self.memcpy_nt: config.memcpy = spead2.MEMCPY_NONTEMPORAL return config @@ -235,7 +265,7 @@ def add_readers(self, stream, endpoints, *, allow_pcap=False): try: stream.add_udp_pcap_file_reader(source) except AttributeError: - raise RuntimeError('spead2 was compiled without pcap support') from None + raise RuntimeError("spead2 was compiled without pcap support") from None else: if self._protocol.tcp: stream.add_tcp_reader(port, self.packet, self.buffer, host) @@ -249,11 +279,13 @@ def add_readers(self, stream, endpoints, *, allow_pcap=False): stream.add_udp_ibv_reader( spead2.recv.UdpIbvConfig( endpoints=ibv_endpoints, - interface_address=self.bind or '', + interface_address=self.bind or "", max_size=self.packet, buffer_size=self.buffer, comp_vector=self.ibv_vector, - max_poll=self.ibv_max_poll)) + max_poll=self.ibv_max_poll, + ) + ) class SenderOptions(SharedOptions): @@ -281,30 +313,47 @@ def parse_rate_method(value): rate_method_names = list(spead2.send.RateMethod.__members__) - self._add_argument(parser, 'addr_bits', type=int, - help='Heap address bits [%(default)s]') - self._add_argument(parser, 'packet', type=int, - help='Maximum packet size to send [%(default)s]') - self._add_argument(parser, 'burst', metavar='BYTES', type=int, - help='Burst size [%(default)s]') - self._add_argument(parser, 'burst_rate_ratio', metavar='RATIO', type=float, - help='Hard rate limit, relative to --rate [%(default)s]') - self._add_argument(parser, 'max_heaps', metavar='HEAPS', type=int, - help='Maximum heaps in flight [%(default)s]') - self._add_argument(parser, 'rate_method', metavar='METHOD', type=parse_rate_method, - help=f'Method for rate limiting ({"/".join(rate_method_names)})') - self._add_argument(parser, 'rate', metavar='Gb/s', type=float, - help='Transmission rate bound [no limit]') - self._add_argument(parser, 'ttl', type=int, help='TTL for multicast target') + self._add_argument(parser, "addr_bits", type=int, help="Heap address bits [%(default)s]") + self._add_argument( + parser, "packet", type=int, help="Maximum packet size to send [%(default)s]" + ) + self._add_argument( + parser, "burst", metavar="BYTES", type=int, help="Burst size [%(default)s]" + ) + self._add_argument( + parser, + "burst_rate_ratio", + metavar="RATIO", + type=float, + help="Hard rate limit, relative to --rate [%(default)s]", + ) + self._add_argument( + parser, + "max_heaps", + metavar="HEAPS", + type=int, + help="Maximum heaps in flight [%(default)s]", + ) + self._add_argument( + parser, + "rate_method", + metavar="METHOD", + type=parse_rate_method, + help=f'Method for rate limiting ({"/".join(rate_method_names)})', + ) + self._add_argument( + parser, "rate", metavar="Gb/s", type=float, help="Transmission rate bound [no limit]" + ) + self._add_argument(parser, "ttl", type=int, help="TTL for multicast target") super().add_arguments(parser) def notify(self, parser, namespace): self._extract_args(namespace) if _HAVE_IBV: if self.ibv and not self.bind: - parser.error('--ibv requires --bind') + parser.error("--ibv requires --bind") if self._protocol.tcp and self.ibv: - parser.error('--ibv and --tcp are incompatible') + parser.error("--ibv and --tcp are incompatible") if self.buffer is None: if self._protocol.tcp: self.buffer = spead2.send.asyncio.TcpStream.DEFAULT_BUFFER_SIZE @@ -324,32 +373,35 @@ def make_stream_config(self): burst_size=self.burst, burst_rate_ratio=self.burst_rate_ratio, max_heaps=self.max_heaps, - rate_method=self.rate_method) + rate_method=self.rate_method, + ) async def make_stream(self, thread_pool, endpoints, memory_regions): config = self.make_stream_config() if self._protocol.tcp: return await spead2.send.asyncio.TcpStream.connect( - thread_pool, endpoints, config, self.buffer, self.bind or '') + thread_pool, endpoints, config, self.buffer, self.bind or "" + ) elif _HAVE_IBV and self.ibv: return spead2.send.asyncio.UdpIbvStream( thread_pool, config, spead2.send.UdpIbvConfig( endpoints=endpoints, - interface_address=self.bind or '', + interface_address=self.bind or "", buffer_size=self.buffer, ttl=self.ttl or 1, comp_vector=self.ibv_vector, max_poll=self.ibv_max_poll, - memory_regions=memory_regions - ) + memory_regions=memory_regions, + ), ) else: kwargs = {} if self.ttl is not None: - kwargs['ttl'] = self.ttl + kwargs["ttl"] = self.ttl if self.bind: - kwargs['interface_address'] = self.bind + kwargs["interface_address"] = self.bind return spead2.send.asyncio.UdpStream( - thread_pool, endpoints, config, self.buffer, **kwargs) + thread_pool, endpoints, config, self.buffer, **kwargs + ) diff --git a/src/spead2/tools/recv_asyncio.py b/src/spead2/tools/recv_asyncio.py index 13af50c7e..3c5edbd0a 100644 --- a/src/spead2/tools/recv_asyncio.py +++ b/src/spead2/tools/recv_asyncio.py @@ -34,18 +34,18 @@ def get_args(): parser = argparse.ArgumentParser() - parser.add_argument('source', nargs='+', help='Sources (filename, host:port or port)') + parser.add_argument("source", nargs="+", help="Sources (filename, host:port or port)") - group = parser.add_argument_group('Output options') - group.add_argument('--log', metavar='LEVEL', default='INFO', help='Log level [%(default)s]') - group.add_argument('--values', action='store_true', help='Show heap values') - group.add_argument('--descriptors', action='store_true', help='Show descriptors') + group = parser.add_argument_group("Output options") + group.add_argument("--log", metavar="LEVEL", default="INFO", help="Log level [%(default)s]") + group.add_argument("--values", action="store_true", help="Show heap values") + group.add_argument("--descriptors", action="store_true", help="Show descriptors") - group = parser.add_argument_group('Input options') - group.add_argument('--max-heaps', type=int, help='Stop receiving after this many heaps') - group.add_argument('--joint', action='store_true', help='Treat all sources as a single stream') + group = parser.add_argument_group("Input options") + group.add_argument("--max-heaps", type=int, help="Stop receiving after this many heaps") + group.add_argument("--joint", action="store_true", help="Treat all sources as a single stream") - group = parser.add_argument_group('Protocol and performance options') + group = parser.add_argument_group("Protocol and performance options") protocol = cmdline.ProtocolOptions() receiver = cmdline.ReceiverOptions(protocol) protocol.add_arguments(group) @@ -72,16 +72,20 @@ async def run_stream(stream, name, args): if args.descriptors: for raw_descriptor in heap.get_descriptors(): descriptor = spead2.Descriptor.from_raw(raw_descriptor, heap.flavour) - print('''\ + print( + """\ Descriptor for {0.name} ({0.id:#x}) description: {0.description} format: {0.format} dtype: {0.dtype} - shape: {0.shape}'''.format(descriptor)) + shape: {0.shape}""".format( + descriptor + ) + ) changed = item_group.update(heap) - for (key, item) in changed.items(): + for key, item in changed.items(): if args.values: - print(key, '=', item.value) + print(key, "=", item.value) else: print(key) except ValueError as e: diff --git a/src/spead2/tools/send_asyncio.py b/src/spead2/tools/send_asyncio.py index 9eca7ae0a..1f6d9b3cf 100644 --- a/src/spead2/tools/send_asyncio.py +++ b/src/spead2/tools/send_asyncio.py @@ -36,42 +36,46 @@ def parse_endpoint(endpoint): - if ':' not in endpoint: - raise ValueError('destination must have the form :') + if ":" not in endpoint: + raise ValueError("destination must have the form :") return cmdline.parse_endpoint(endpoint) def get_args(): parser = argparse.ArgumentParser() - parser.add_argument('destination', type=parse_endpoint, nargs='+', metavar='HOST:PORT') - - group = parser.add_argument_group('Data options') - group.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, - help='Payload size for heap [%(default)s]') - group.add_argument('--items', type=int, default=1, - help='Number of items per heap [%(default)s]') - group.add_argument('--dtype', type=str, default=' 1: - parser.error('only one destination is supported with TCP') + parser.error("only one destination is supported with TCP") return args, sender @@ -85,8 +89,7 @@ async def run(item_group, stream, args): rep = itertools.repeat(False) else: rep = itertools.chain( - itertools.repeat(False, args.heaps), - itertools.repeat(True, len(args.destination)) + itertools.repeat(False, args.heaps), itertools.repeat(True, len(args.destination)) ) n_substreams = stream.num_substreams for i, is_end in enumerate(rep): @@ -100,8 +103,8 @@ async def run(item_group, stream, args): heap = item_group.get_end() else: heap = item_group.get_heap( - descriptors='all' if i < n_substreams else 'stale', - data='all') + descriptors="all" if i < n_substreams else "stale", data="all" + ) task = asyncio.ensure_future(stream.async_send_heap(heap, substream_index=i % n_substreams)) tasks.append(task) while len(tasks) > 0: @@ -112,9 +115,12 @@ async def run(item_group, stream, args): last_error = error elapsed = time.time() - start_time if last_error is not None: - logging.warn('%d errors, last one: %s', n_errors, last_error) - print('Sent {} bytes in {:.6f}s, {:.6f} Gb/s'.format( - n_bytes, elapsed, n_bytes * 8 / elapsed / 1e9)) + logging.warn("%d errors, last one: %s", n_errors, last_error) + print( + "Sent {} bytes in {:.6f}s, {:.6f} Gb/s".format( + n_bytes, elapsed, n_bytes * 8 / elapsed / 1e9 + ) + ) async def async_main(): @@ -125,16 +131,21 @@ async def async_main(): elements = args.heap_size // (args.items * dtype.itemsize) heap_size = elements * args.items * dtype.itemsize if heap_size != args.heap_size: - logging.warn('Heap size is not an exact multiple: using %d instead of %d', - heap_size, args.heap_size) + logging.warn( + "Heap size is not an exact multiple: using %d instead of %d", heap_size, args.heap_size + ) item_group = spead2.send.ItemGroup( - descriptor_frequency=args.descriptors, - flavour=sender.make_flavour()) + descriptor_frequency=args.descriptors, flavour=sender.make_flavour() + ) for i in range(args.items): - item_group.add_item(id=None, name=f'Test item {i}', - description='A test item with arbitrary value', - shape=(elements,), dtype=dtype, - value=np.zeros((elements,), dtype=dtype)) + item_group.add_item( + id=None, + name=f"Test item {i}", + description="A test item with arbitrary value", + shape=(elements,), + dtype=dtype, + value=np.zeros((elements,), dtype=dtype), + ) thread_pool = sender.make_thread_pool() memory_regions = [item.value for item in item_group.values()] stream = await sender.make_stream(thread_pool, args.destination, memory_regions) diff --git a/tests/shutdown.py b/tests/shutdown.py index a7ac61a54..d57e61f5a 100644 --- a/tests/shutdown.py +++ b/tests/shutdown.py @@ -42,7 +42,7 @@ def test_logging_shutdown(): # Set a log level that won't actually display the messages. logging.basicConfig(level=logging.ERROR) for i in range(20000): - spead2._spead2.log_info(f'Test message {i}') + spead2._spead2.log_info(f"Test message {i}") def test_running_thread_pool(): @@ -55,10 +55,11 @@ def test_running_stream(): logging.basicConfig(level=logging.ERROR) stream = spead2.recv.Stream(spead2.ThreadPool()) stream.add_udp_reader(7148) - sender = spead2.send.UdpStream(spead2.ThreadPool(), 'localhost', 7148) + sender = spead2.send.UdpStream(spead2.ThreadPool(), "localhost", 7148) ig = spead2.send.ItemGroup() - ig.add_item(id=None, name='test', description='test', - shape=(), format=[('u', 32)], value=0xdeadbeef) + ig.add_item( + id=None, name="test", description="test", shape=(), format=[("u", 32)], value=0xDEADBEEF + ) heap = ig.get_heap() for i in range(5): sender.send_heap(heap) @@ -79,25 +80,22 @@ def place(data_ptr, data_size): global group group = spead2.recv.ChunkStreamRingGroup( spead2.recv.ChunkStreamGroupConfig( - max_chunks=2, - eviction_mode=spead2.recv.ChunkStreamGroupConfig.EvictionMode.LOSSLESS + max_chunks=2, eviction_mode=spead2.recv.ChunkStreamGroupConfig.EvictionMode.LOSSLESS ), spead2.recv.ChunkRingbuffer(4), - spead2.recv.ChunkRingbuffer(4) + spead2.recv.ChunkRingbuffer(4), ) for _ in range(group.free_ringbuffer.maxsize): chunk = spead2.recv.Chunk(data=np.zeros(1024, np.uint8), present=np.zeros(1, np.uint8)) group.add_free_chunk(chunk) - place_llc = scipy.LowLevelCallable(place.ctypes, signature='void (void *, size_t)') + place_llc = scipy.LowLevelCallable(place.ctypes, signature="void (void *, size_t)") for _ in range(2): group.emplace_back( spead2.ThreadPool(), spead2.recv.StreamConfig(), spead2.recv.ChunkStreamConfig( - items=[spead2.HEAP_CNT_ID, spead2.HEAP_LENGTH_ID], - max_chunks=2, - place=place_llc - ) + items=[spead2.HEAP_CNT_ID, spead2.HEAP_LENGTH_ID], max_chunks=2, place=place_llc + ), ) queues = [spead2.InprocQueue() for _ in group] for queue, stream in zip(queues, group): diff --git a/tests/test_common.py b/tests/test_common.py index 2a9610d73..ce7f6357a 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -23,19 +23,20 @@ class TestParseRangeList: def test_empty(self): - assert spead2.parse_range_list('') == [] + assert spead2.parse_range_list("") == [] def test_simple(self): - assert spead2.parse_range_list('1,2,5') == [1, 2, 5] + assert spead2.parse_range_list("1,2,5") == [1, 2, 5] def test_ranges(self): - assert spead2.parse_range_list('100,4-6,8,10-10,12-13') == [100, 4, 5, 6, 8, 10, 12, 13] + assert spead2.parse_range_list("100,4-6,8,10-10,12-13") == [100, 4, 5, 6, 8, 10, 12, 13] class TestThreadPool: """Smoke tests for :py:class:`spead2.ThreadPool`. These are very simple tests, because it is not actually possible to check things like the thread affinity.""" + def test_simple(self): spead2.ThreadPool() spead2.ThreadPool(4) @@ -98,10 +99,10 @@ class TestItem: def test_nonascii_value(self): """Using a non-ASCII unicode character raises a :py:exc:`UnicodeEncodeError`.""" - item1 = spead2.Item(0x1000, 'name1', 'description', - (None,), format=[('c', 8)], value='\u0200') - item2 = spead2.Item(0x1001, 'name2', 'description2', (), - dtype='S5', value='\u0201') + item1 = spead2.Item( + 0x1000, "name1", "description", (None,), format=[("c", 8)], value="\u0200" + ) + item2 = spead2.Item(0x1001, "name2", "description2", (), dtype="S5", value="\u0201") with pytest.raises(UnicodeEncodeError): item1.to_buffer() with pytest.raises(UnicodeEncodeError): @@ -110,55 +111,54 @@ def test_nonascii_value(self): def test_format_and_dtype(self): """Specifying both a format and dtype raises :py:exc:`ValueError`.""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', - (1, 2), format=[('c', 8)], dtype='S1') + spead2.Item(0x1000, "name", "description", (1, 2), format=[("c", 8)], dtype="S1") def test_no_format_or_dtype(self): """At least one of format and dtype must be specified.""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', (1, 2), format=None) + spead2.Item(0x1000, "name", "description", (1, 2), format=None) def test_invalid_order(self): """The `order` parameter must be either 'C' or 'F'.""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', (1, 2), np.int32, order='K') + spead2.Item(0x1000, "name", "description", (1, 2), np.int32, order="K") def test_fortran_fallback(self): """The `order` parameter must be 'C' for legacy formats.""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', (1, 2), format=[('u', 32)], order='F') + spead2.Item(0x1000, "name", "description", (1, 2), format=[("u", 32)], order="F") def test_empty_format(self): """Format must not be empty""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', (1, 2), format=[]) + spead2.Item(0x1000, "name", "description", (1, 2), format=[]) def test_assign_none(self): """Changing a value back to `None` raises :py:exc:`ValueError`.""" - item = spead2.Item(0x1000, 'name', 'description', (), np.int32) + item = spead2.Item(0x1000, "name", "description", (), np.int32) with pytest.raises(ValueError): item.value = None def test_multiple_unknown(self): """Multiple unknown dimensions are not allowed.""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', (5, None, 3, None), format=[('u', 32)]) + spead2.Item(0x1000, "name", "description", (5, None, 3, None), format=[("u", 32)]) def test_numpy_unknown(self): """Unknown dimensions are not permitted when using a numpy descriptor""" with pytest.raises(ValueError): - spead2.Item(0x1000, 'name', 'description', (5, None), np.int32) + spead2.Item(0x1000, "name", "description", (5, None), np.int32) def test_nonascii_name(self): """Name with non-ASCII characters must fail""" with pytest.raises(UnicodeEncodeError): - item = spead2.Item(0x1000, '\u0200', 'description', (), np.int32) + item = spead2.Item(0x1000, "\u0200", "description", (), np.int32) item.to_raw(spead2.Flavour()) def test_nonascii_description(self): """Description with non-ASCII characters must fail""" with pytest.raises(UnicodeEncodeError): - item = spead2.Item(0x1000, 'name', '\u0200', (), np.int32) + item = spead2.Item(0x1000, "name", "\u0200", (), np.int32) item.to_raw(spead2.Flavour()) @@ -168,48 +168,48 @@ class TestItemGroup: def test_allocate_id(self): """Automatic allocation of IDs must skip over already-allocated IDs""" ig = spead2.ItemGroup() - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32) - ig.add_item(0x1003, 'item 2', 'item 2', (), np.int32) - ig.add_item(None, 'item 3', 'item 3', (), np.int32) - ig.add_item(None, 'item 4', 'item 4', (), np.int32) - ig.add_item(None, 'item 5', 'item 5', (), np.int32) - assert ig[0x1001].name == 'item 3' - assert ig[0x1002].name == 'item 4' - assert ig[0x1004].name == 'item 5' + ig.add_item(0x1000, "item 1", "item 1", (), np.int32) + ig.add_item(0x1003, "item 2", "item 2", (), np.int32) + ig.add_item(None, "item 3", "item 3", (), np.int32) + ig.add_item(None, "item 4", "item 4", (), np.int32) + ig.add_item(None, "item 5", "item 5", (), np.int32) + assert ig[0x1001].name == "item 3" + assert ig[0x1002].name == "item 4" + assert ig[0x1004].name == "item 5" def test_replace_rename(self): """When a new item is added with a known ID but a different name, the old item must cease to exist.""" ig = spead2.ItemGroup() - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32) - ig.add_item(0x1000, 'renamed', 'renamed', (), np.int32) + ig.add_item(0x1000, "item 1", "item 1", (), np.int32) + ig.add_item(0x1000, "renamed", "renamed", (), np.int32) assert list(ig.ids()) == [0x1000] - assert list(ig.keys()) == ['renamed'] + assert list(ig.keys()) == ["renamed"] def test_replace_clobber_name(self): """When a new item is added that collides with an existing name, that other item is deleted.""" ig = spead2.ItemGroup() - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32) - ig.add_item(0x1001, 'item 1', 'clobbered', (), np.int32) + ig.add_item(0x1000, "item 1", "item 1", (), np.int32) + ig.add_item(0x1001, "item 1", "clobbered", (), np.int32) assert list(ig.ids()) == [0x1001] - assert list(ig.keys()) == ['item 1'] - assert ig['item 1'].description == 'clobbered' - assert ig['item 1'] is ig[0x1001] + assert list(ig.keys()) == ["item 1"] + assert ig["item 1"].description == "clobbered" + assert ig["item 1"] is ig[0x1001] def test_replace_reset_value(self): """When a descriptor is replaced with an identical one but a new value, the new value must take effect and the version must be incremented.""" ig = spead2.ItemGroup() - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32, value=np.int32(4)) - ig.add_item(0x1001, 'item 2', 'item 2', (), np.int32, value=np.int32(5)) + ig.add_item(0x1000, "item 1", "item 1", (), np.int32, value=np.int32(4)) + ig.add_item(0x1001, "item 2", "item 2", (), np.int32, value=np.int32(5)) item1 = ig[0x1000] item2 = ig[0x1001] item1_version = item1.version item2_version = item2.version - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32, value=np.int32(6)) - ig.add_item(0x1001, 'item 2', 'item 2', (), np.int32) + ig.add_item(0x1000, "item 1", "item 1", (), np.int32, value=np.int32(6)) + ig.add_item(0x1001, "item 2", "item 2", (), np.int32) assert item1 is ig[0x1000] assert item2 is ig[0x1001] assert item1.value == np.int32(6) @@ -221,10 +221,10 @@ def test_replace_change_shape(self): """When a descriptor changes the shape, the old item must be discarded and ``None`` used in its place. The version must be bumped.""" ig = spead2.ItemGroup() - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32, value=np.int32(4)) + ig.add_item(0x1000, "item 1", "item 1", (), np.int32, value=np.int32(4)) item1 = ig[0x1000] item1_version = item1.version - ig.add_item(0x1000, 'item 1', 'bigger', (3, 4), np.int32) + ig.add_item(0x1000, "item 1", "bigger", (3, 4), np.int32) assert item1 is not ig[0x1000] assert ig[0x1000].value is None assert ig[0x1000].version > item1_version @@ -233,10 +233,10 @@ def test_replace_clobber_both(self): """Adding a new item that collides with one item on name and another on ID causes both to be dropped.""" ig = spead2.ItemGroup() - ig.add_item(0x1000, 'item 1', 'item 1', (), np.int32) - ig.add_item(0x1001, 'item 2', 'item 2', (), np.int32) - ig.add_item(0x1000, 'item 2', 'clobber', (), np.int32) + ig.add_item(0x1000, "item 1", "item 1", (), np.int32) + ig.add_item(0x1001, "item 2", "item 2", (), np.int32) + ig.add_item(0x1000, "item 2", "clobber", (), np.int32) assert list(ig.ids()) == [0x1000] - assert list(ig.keys()) == ['item 2'] - assert ig[0x1000] is ig['item 2'] - assert ig[0x1000].description == 'clobber' + assert list(ig.keys()) == ["item 2"] + assert ig[0x1000] is ig["item 2"] + assert ig[0x1000].description == "clobber" diff --git a/tests/test_passthrough.py b/tests/test_passthrough.py index c2fb29412..d3b10d5c6 100644 --- a/tests/test_passthrough.py +++ b/tests/test_passthrough.py @@ -36,16 +36,16 @@ def _assert_items_equal(item1, item2): assert item1.format == item2.format # Byte order need not match, provided that values are received correctly if item1.dtype is not None and item2.dtype is not None: - assert item1.dtype.newbyteorder('<') == item2.dtype.newbyteorder('<') + assert item1.dtype.newbyteorder("<") == item2.dtype.newbyteorder("<") else: assert item1.dtype == item2.dtype assert item1.order == item2.order # Comparing arrays has many issues. Convert them to lists where appropriate value1 = item1.value value2 = item2.value - if hasattr(value1, 'tolist'): + if hasattr(value1, "tolist"): value1 = value1.tolist() - if hasattr(value2, 'tolist'): + if hasattr(value2, "tolist"): value2 = value2.tolist() assert value1 == value2 @@ -67,13 +67,13 @@ class BaseTestPassthrough: @classmethod def check_ipv6(cls): if not socket.has_ipv6: - pytest.skip('platform does not support IPv6') + pytest.skip("platform does not support IPv6") with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as sock: # Travis build systems fail to bind to an IPv6 address try: sock.bind(("::1", 8888)) except OSError: - pytest.skip('platform cannot bind IPv6 localhost address') + pytest.skip("platform cannot bind IPv6 localhost address") if cls.requires_ipv6_multicast: with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as sock: # Github Actions on MacOS doesn't have routes to multicast @@ -81,15 +81,24 @@ def check_ipv6(cls): sock.connect(("ff14::1234", 8888)) sock.send(b"test") except OSError: - pytest.skip('platform cannot transmit to an IPv6 multicast address') - - def _test_item_groups(self, item_groups, *, - memcpy=spead2.MEMCPY_STD, allocator=None, - new_order='=', group_mode=None): + pytest.skip("platform cannot transmit to an IPv6 multicast address") + + def _test_item_groups( + self, + item_groups, + *, + memcpy=spead2.MEMCPY_STD, + allocator=None, + new_order="=", + group_mode=None, + ): received_item_groups = self.transmit_item_groups( item_groups, - memcpy=memcpy, allocator=allocator, - new_order=new_order, group_mode=group_mode) + memcpy=memcpy, + allocator=allocator, + new_order=new_order, + group_mode=group_mode, + ) assert len(received_item_groups) == len(item_groups) for received_item_group, item_group in zip(received_item_groups, item_groups): assert_item_groups_equal(item_group, received_item_group) @@ -97,32 +106,51 @@ def _test_item_groups(self, item_groups, *, if item.dtype is not None: assert item.value.dtype == item.value.dtype.newbyteorder(new_order) - def _test_item_group(self, item_group, *, - memcpy=spead2.MEMCPY_STD, allocator=None, - new_order='=', group_mode=None): + def _test_item_group( + self, + item_group, + *, + memcpy=spead2.MEMCPY_STD, + allocator=None, + new_order="=", + group_mode=None, + ): self._test_item_groups( [item_group], memcpy=memcpy, allocator=allocator, new_order=new_order, - group_mode=group_mode) + group_mode=group_mode, + ) def test_numpy_simple(self): """A basic array with numpy encoding""" ig = spead2.send.ItemGroup() data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16) - ig.add_item(id=0x2345, name='name', description='description', - shape=data.shape, dtype=data.dtype, value=data) + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=data.shape, + dtype=data.dtype, + value=data, + ) self._test_item_group(ig) def test_numpy_byteorder(self): """A numpy array in non-native byte order""" ig = spead2.send.ItemGroup() data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.dtype(np.uint16).newbyteorder()) - ig.add_item(id=0x2345, name='name', description='description', - shape=data.shape, dtype=data.dtype, value=data) + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=data.shape, + dtype=data.dtype, + value=data, + ) self._test_item_group(ig) - self._test_item_group(ig, new_order='|') + self._test_item_group(ig, new_order="|") def test_numpy_large(self): """A numpy style array split across several packets. It also @@ -130,12 +158,18 @@ def test_numpy_large(self): to test that those all work.""" # macOS doesn't have a big enough socket buffer to reliably transmit # the whole thing over UDP. - if self.is_lossy and sys.platform == 'darwin': + if self.is_lossy and sys.platform == "darwin": pytest.skip("macOS can't reliably handle large heaps over UDP") ig = spead2.send.ItemGroup() data = np.random.randn(100, 200) - ig.add_item(id=0x2345, name='name', description='description', - shape=data.shape, dtype=data.dtype, value=data) + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=data.shape, + dtype=data.dtype, + value=data, + ) allocator = spead2.MmapAllocator() pool = spead2.MemoryPool(1, 4096, 4, 4, allocator) self._test_item_group(ig, memcpy=spead2.MEMCPY_NONTEMPORAL, allocator=pool) @@ -144,19 +178,26 @@ def test_fallback_struct_whole_bytes(self): """A structure with non-byte-aligned elements, but which is byte-aligned overall.""" ig = spead2.send.ItemGroup() - format = [('u', 4), ('f', 64), ('i', 4)] + format = [("u", 4), ("f", 64), ("i", 4)] data = (12, 1.5, -3) - ig.add_item(id=0x2345, name='name', description='description', - shape=(), format=format, value=data) + ig.add_item( + id=0x2345, name="name", description="description", shape=(), format=format, value=data + ) self._test_item_group(ig) def test_string(self): """Byte string is converted to array of characters and back.""" ig = spead2.send.ItemGroup() - format = [('c', 8)] - data = 'Hello world' - ig.add_item(id=0x2345, name='name', description='description', - shape=(None,), format=format, value=data) + format = [("c", 8)] + data = "Hello world" + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=(None,), + format=format, + value=data, + ) self._test_item_group(ig) def test_fallback_array_partial_bytes_small(self): @@ -164,46 +205,61 @@ def test_fallback_array_partial_bytes_small(self): and is small enough to encode in an immediate. """ ig = spead2.send.ItemGroup() - format = [('u', 7)] + format = [("u", 7)] data = [127, 12, 123] - ig.add_item(id=0x2345, name='name', description='description', - shape=(len(data),), format=format, value=data) + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=(len(data),), + format=format, + value=data, + ) self._test_item_group(ig) def test_fallback_types(self): """An array structure using a mix of types.""" ig = spead2.send.ItemGroup() - format = [('b', 1), ('i', 7), ('c', 8), ('f', 32)] - data = [(True, 17, b'y', 1.0), (False, -23, b'n', -1.0)] - ig.add_item(id=0x2345, name='name', description='description', - shape=(2,), format=format, value=data) + format = [("b", 1), ("i", 7), ("c", 8), ("f", 32)] + data = [(True, 17, b"y", 1.0), (False, -23, b"n", -1.0)] + ig.add_item( + id=0x2345, name="name", description="description", shape=(2,), format=format, value=data + ) self._test_item_group(ig) def test_numpy_fallback_struct(self): """A structure specified using a format, but which is encodable using numpy.""" ig = spead2.send.ItemGroup() - format = [('u', 8), ('f', 32)] + format = [("u", 8), ("f", 32)] data = (12, 1.5) - ig.add_item(id=0x2345, name='name', description='description', - shape=(), format=format, value=data) + ig.add_item( + id=0x2345, name="name", description="description", shape=(), format=format, value=data + ) self._test_item_group(ig) def test_fallback_struct_partial_bytes(self): """A structure which takes a fractional number of bytes per element.""" ig = spead2.send.ItemGroup() - format = [('u', 4), ('f', 64)] + format = [("u", 4), ("f", 64)] data = (12, 1.5) - ig.add_item(id=0x2345, name='name', description='description', - shape=(), format=format, value=data) + ig.add_item( + id=0x2345, name="name", description="description", shape=(), format=format, value=data + ) self._test_item_group(ig) def test_fallback_scalar(self): """Send a scalar using fallback format descriptor.""" ig = spead2.send.ItemGroup() - format = [('f', 64)] + format = [("f", 64)] data = 1.5 - ig.add_item(id=0x2345, name='scalar name', description='scalar description', - shape=(), format=format, value=data) + ig.add_item( + id=0x2345, + name="scalar name", + description="scalar description", + shape=(), + format=format, + value=data, + ) self._test_item_group(ig) def test_many_items(self): @@ -214,13 +270,20 @@ def test_many_items(self): """ ig = spead2.send.ItemGroup() for i in range(50): - name = f'test item {i}' - ig.add_item(id=0x2345 + i, name=name, description=name, - shape=(), format=[('u', 40)], value=0x12345 * i) + name = f"test item {i}" + ig.add_item( + id=0x2345 + i, + name=name, + description=name, + shape=(), + format=[("u", 40)], + value=0x12345 * i, + ) self._test_item_group(ig) - def transmit_item_groups(self, item_groups, *, - memcpy, allocator, new_order='=', group_mode=None): + def transmit_item_groups( + self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None + ): """Transmit `item_groups` over the chosen transport. Return the item groups received at the other end. Each item group will @@ -233,15 +296,11 @@ def transmit_item_groups(self, item_groups, *, if allocator is not None: recv_config.memory_allocator = allocator receivers = [ - spead2.recv.Stream(spead2.ThreadPool(), recv_config) - for i in range(len(item_groups)) + spead2.recv.Stream(spead2.ThreadPool(), recv_config) for i in range(len(item_groups)) ] self.prepare_receivers(receivers) sender = self.prepare_senders(spead2.ThreadPool(), len(item_groups)) - gens = [ - spead2.send.HeapGenerator(item_group) - for item_group in item_groups - ] + gens = [spead2.send.HeapGenerator(item_group) for item_group in item_groups] if len(item_groups) != 1: # Use reversed order so that if everything is actually going # through the same transport it will get picked up. @@ -251,7 +310,7 @@ def transmit_item_groups(self, item_groups, *, spead2.send.HeapReference(gen.get_heap(), substream_index=i) for i, gen in reversed(list(enumerate(gens))) ], - group_mode + group_mode, ) # For the stop heaps, use a HeapReferenceList to test it. hrl = spead2.send.HeapReferenceList( @@ -311,14 +370,21 @@ def test_substreams(self): item_groups = [] for i in range(4): ig = spead2.ItemGroup() - ig.add_item(id=0x2345, name='int', description='an integer', - shape=(), format=[('i', 32)], value=i) + ig.add_item( + id=0x2345, + name="int", + description="an integer", + shape=(), + format=[("i", 32)], + value=i, + ) item_groups.append(ig) self._test_item_groups(item_groups) - @pytest.mark.parametrize('size', [10, 20000]) - @pytest.mark.parametrize('group_mode', [spead2.send.GroupMode.ROUND_ROBIN, - spead2.send.GroupMode.SERIAL]) + @pytest.mark.parametrize("size", [10, 20000]) + @pytest.mark.parametrize( + "group_mode", [spead2.send.GroupMode.ROUND_ROBIN, spead2.send.GroupMode.SERIAL] + ) def test_group_modes(self, size, group_mode): # The interleaving and substream features are independent, but the # test framework is set up for one item group per substream. @@ -326,21 +392,34 @@ def test_group_modes(self, size, group_mode): for i in range(4): value = np.random.randint(0, 256, size=size).astype(np.uint8) ig = spead2.ItemGroup() - ig.add_item(id=0x2345, name='arr', description='a random array', - shape=(size,), dtype='u8', value=value) + ig.add_item( + id=0x2345, + name="arr", + description="a random array", + shape=(size,), + dtype="u8", + value=value, + ) item_groups.append(ig) self._test_item_groups(item_groups, group_mode=group_mode) - @pytest.mark.parametrize('group_mode', [spead2.send.GroupMode.ROUND_ROBIN, - spead2.send.GroupMode.SERIAL]) + @pytest.mark.parametrize( + "group_mode", [spead2.send.GroupMode.ROUND_ROBIN, spead2.send.GroupMode.SERIAL] + ) def test_group_modes_mixed_sizes(self, group_mode): sizes = [20000, 2000, 40000, 30000] item_groups = [] for size in sizes: value = np.random.randint(0, 256, size=size).astype(np.uint8) ig = spead2.ItemGroup() - ig.add_item(id=0x2345, name='arr', description='a random array', - shape=(size,), dtype='u8', value=value) + ig.add_item( + id=0x2345, + name="arr", + description="a random array", + shape=(size,), + dtype="u8", + value=value, + ) item_groups.append(ig) self._test_item_groups(item_groups, group_mode=group_mode) @@ -362,27 +441,31 @@ def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): return spead2.send.UdpStream( - thread_pool, "localhost", 8888, + thread_pool, + "localhost", + 8888, spead2.send.StreamConfig(rate=1e7), - buffer_size=0) + buffer_size=0, + ) else: return spead2.send.UdpStream( thread_pool, [("localhost", 8888 + i) for i in range(n)], spead2.send.StreamConfig(rate=1e7), - buffer_size=0) + buffer_size=0, + ) def test_empty_endpoints(self): with pytest.raises(ValueError): - spead2.send.UdpStream( - spead2.ThreadPool(), [], spead2.send.StreamConfig(rate=1e7)) + spead2.send.UdpStream(spead2.ThreadPool(), [], spead2.send.StreamConfig(rate=1e7)) def test_mixed_protocols(self): with pytest.raises(ValueError): spead2.send.UdpStream( spead2.ThreadPool(), - [('127.0.0.1', 8888), ('::1', 8888)], - spead2.send.StreamConfig(rate=1e7)) + [("127.0.0.1", 8888), ("::1", 8888)], + spead2.send.StreamConfig(rate=1e7), + ) class TestPassthroughUdp6(BaseTestPassthroughSubstreams): @@ -397,15 +480,15 @@ def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): return spead2.send.UdpStream( - thread_pool, "::1", 8888, - spead2.send.StreamConfig(rate=1e7), - buffer_size=0) + thread_pool, "::1", 8888, spead2.send.StreamConfig(rate=1e7), buffer_size=0 + ) else: return spead2.send.UdpStream( thread_pool, [("::1", 8888 + i) for i in range(n)], spead2.send.StreamConfig(rate=1e7), - buffer_size=0) + buffer_size=0, + ) class TestPassthroughUdpCustomSocket(BaseTestPassthroughSubstreams): @@ -418,7 +501,7 @@ def prepare_receivers(self, receivers): recv_sock.bind(("127.0.0.1", 0)) self._ports.append(recv_sock.getsockname()[1]) receiver.add_udp_reader(socket=recv_sock) - recv_sock.close() # spead2 duplicates the socket + recv_sock.close() # spead2 duplicates the socket def prepare_senders(self, thread_pool, n): assert len(self._ports) == n @@ -426,56 +509,71 @@ def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): sender = spead2.send.UdpStream( - thread_pool, send_sock, "127.0.0.1", self._ports[0], - spead2.send.StreamConfig(rate=1e7)) + thread_pool, + send_sock, + "127.0.0.1", + self._ports[0], + spead2.send.StreamConfig(rate=1e7), + ) else: sender = spead2.send.UdpStream( - thread_pool, send_sock, + thread_pool, + send_sock, [("127.0.0.1", port) for port in self._ports], - spead2.send.StreamConfig(rate=1e7)) - send_sock.close() # spead2 duplicates the socket + spead2.send.StreamConfig(rate=1e7), + ) + send_sock.close() # spead2 duplicates the socket return sender class TestPassthroughUdpMulticast(BaseTestPassthroughSubstreams): is_lossy = True - MCAST_GROUP = '239.255.88.88' - INTERFACE_ADDRESS = '127.0.0.1' + MCAST_GROUP = "239.255.88.88" + INTERFACE_ADDRESS = "127.0.0.1" def prepare_receivers(self, receivers): for i, receiver in enumerate(receivers): receiver.add_udp_reader( - self.MCAST_GROUP, 8887 - i, interface_address=self.INTERFACE_ADDRESS) + self.MCAST_GROUP, 8887 - i, interface_address=self.INTERFACE_ADDRESS + ) def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): return spead2.send.UdpStream( - thread_pool, self.MCAST_GROUP, 8887, + thread_pool, + self.MCAST_GROUP, + 8887, spead2.send.StreamConfig(rate=1e7), - buffer_size=0, ttl=1, interface_address=self.INTERFACE_ADDRESS) + buffer_size=0, + ttl=1, + interface_address=self.INTERFACE_ADDRESS, + ) else: return spead2.send.UdpStream( thread_pool, [(self.MCAST_GROUP, 8887 - i) for i in range(n)], spead2.send.StreamConfig(rate=1e7), - buffer_size=0, ttl=1, interface_address=self.INTERFACE_ADDRESS) + buffer_size=0, + ttl=1, + interface_address=self.INTERFACE_ADDRESS, + ) class TestPassthroughUdp6Multicast(TestPassthroughUdp6): requires_ipv6_multicast = True - MCAST_GROUP = 'ff14::1234' + MCAST_GROUP = "ff14::1234" @classmethod def get_interface_index(cls): - if not hasattr(socket, 'if_nametoindex'): - pytest.skip('socket.if_nametoindex does not exist') + if not hasattr(socket, "if_nametoindex"): + pytest.skip("socket.if_nametoindex does not exist") for iface in netifaces.interfaces(): addrs = netifaces.ifaddresses(iface).get(netifaces.AF_INET6, []) for addr in addrs: - if addr['addr'] != '::1': + if addr["addr"] != "::1": return socket.if_nametoindex(iface) - pytest.skip('could not find suitable interface for test') + pytest.skip("could not find suitable interface for test") def prepare_receivers(self, receivers): interface_index = self.get_interface_index() @@ -487,33 +585,41 @@ def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): return spead2.send.UdpStream( - thread_pool, self.MCAST_GROUP, 8887, + thread_pool, + self.MCAST_GROUP, + 8887, spead2.send.StreamConfig(rate=1e7), - buffer_size=0, ttl=0, interface_index=interface_index) + buffer_size=0, + ttl=0, + interface_index=interface_index, + ) else: return spead2.send.UdpStream( thread_pool, [(self.MCAST_GROUP, 8887 - i) for i in range(n)], spead2.send.StreamConfig(rate=1e7), - buffer_size=0, ttl=0, interface_index=interface_index) + buffer_size=0, + ttl=0, + interface_index=interface_index, + ) class TestPassthroughUdpIbv(BaseTestPassthroughSubstreams): is_lossy = True - MCAST_GROUP = '239.255.88.88' + MCAST_GROUP = "239.255.88.88" def _interface_address(self): - ifaddr = os.getenv('SPEAD2_TEST_IBV_INTERFACE_ADDRESS') + ifaddr = os.getenv("SPEAD2_TEST_IBV_INTERFACE_ADDRESS") if not ifaddr: - pytest.skip('Envar SPEAD2_TEST_IBV_INTERFACE_ADDRESS not set') + pytest.skip("Envar SPEAD2_TEST_IBV_INTERFACE_ADDRESS not set") return ifaddr def setup_method(self): # mlx5 drivers only enable multicast loopback if there are multiple # device contexts. The sender and receiver end up sharing one, so we # need to explicitly create another. - if not hasattr(spead2, 'IbvContext'): - pytest.skip('IBV support not compiled in') + if not hasattr(spead2, "IbvContext"): + pytest.skip("IBV support not compiled in") self._extra_context = spead2.IbvContext(self._interface_address()) def teardown_method(self): @@ -524,7 +630,9 @@ def prepare_receivers(self, receivers): receiver.add_udp_ibv_reader( spead2.recv.UdpIbvConfig( endpoints=[(self.MCAST_GROUP, 8876 + i)], - interface_address=self._interface_address())) + interface_address=self._interface_address(), + ) + ) def prepare_senders(self, thread_pool, n): # The buffer size is deliberately reduced so that we test the @@ -532,10 +640,13 @@ def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): return spead2.send.UdpIbvStream( - thread_pool, self.MCAST_GROUP, 8876, + thread_pool, + self.MCAST_GROUP, + 8876, spead2.send.StreamConfig(rate=1e7), self._interface_address(), - buffer_size=64 * 1024) + buffer_size=64 * 1024, + ) else: return spead2.send.UdpIbvStream( thread_pool, @@ -543,31 +654,38 @@ def prepare_senders(self, thread_pool, n): spead2.send.UdpIbvConfig( endpoints=[(self.MCAST_GROUP, 8876 + i) for i in range(n)], interface_address=self._interface_address(), - buffer_size=64 * 1024 - ) + buffer_size=64 * 1024, + ), ) - @pytest.mark.parametrize('num_items', [0, 1, 3, 4, 10]) + @pytest.mark.parametrize("num_items", [0, 1, 3, 4, 10]) def test_memory_regions(self, num_items): receiver = spead2.recv.Stream(spead2.ThreadPool(), spead2.recv.StreamConfig()) receiver.add_udp_ibv_reader( spead2.recv.UdpIbvConfig( - endpoints=[(self.MCAST_GROUP, 8876)], - interface_address=self._interface_address())) + endpoints=[(self.MCAST_GROUP, 8876)], interface_address=self._interface_address() + ) + ) ig = spead2.send.ItemGroup() data = [np.random.randn(50) for i in range(num_items)] for i in range(num_items): - ig.add_item(id=0x2345 + i, name=f'name {i}', description=f'description {i}', - shape=data[i].shape, dtype=data[i].dtype, value=data[i]) + ig.add_item( + id=0x2345 + i, + name=f"name {i}", + description=f"description {i}", + shape=data[i].shape, + dtype=data[i].dtype, + value=data[i], + ) sender = spead2.send.UdpIbvStream( spead2.ThreadPool(), spead2.send.StreamConfig(rate=1e7), spead2.send.UdpIbvConfig( endpoints=[(self.MCAST_GROUP, 8876)], interface_address=self._interface_address(), - memory_regions=data - ) + memory_regions=data, + ), ) sender.send_heap(ig.get_heap()) sender.send_heap(ig.get_end()) @@ -612,8 +730,9 @@ def prepare_sender(self, thread_pool): class TestPassthroughMem(BaseTestPassthrough): - def transmit_item_groups(self, item_groups, *, - memcpy, allocator, new_order='=', group_mode=None): + def transmit_item_groups( + self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None + ): assert len(item_groups) == 1 assert group_mode is None thread_pool = spead2.ThreadPool(2) @@ -646,12 +765,17 @@ def prepare_senders(self, thread_pool, n): else: return spead2.send.InprocStream(thread_pool, self._queues) - def transmit_item_groups(self, item_groups, *, - memcpy, allocator, new_order='=', group_mode=None): + def transmit_item_groups( + self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None + ): self._queues = [spead2.InprocQueue() for ig in item_groups] ret = super().transmit_item_groups( - item_groups, memcpy=memcpy, allocator=allocator, - new_order=new_order, group_mode=group_mode) + item_groups, + memcpy=memcpy, + allocator=allocator, + new_order=new_order, + group_mode=group_mode, + ) for queue in self._queues: queue.stop() return ret diff --git a/tests/test_passthrough_asyncio.py b/tests/test_passthrough_asyncio.py index c34b96cdb..2c159a098 100644 --- a/tests/test_passthrough_asyncio.py +++ b/tests/test_passthrough_asyncio.py @@ -29,20 +29,26 @@ class BaseTestPassthroughAsync(test_passthrough.BaseTestPassthrough): - def transmit_item_groups(self, item_groups, *, - memcpy, allocator, new_order='=', group_mode=None): + def transmit_item_groups( + self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None + ): loop = asyncio.new_event_loop() try: return loop.run_until_complete( self.transmit_item_groups_async( item_groups, - memcpy=memcpy, allocator=allocator, - new_order=new_order, group_mode=group_mode)) + memcpy=memcpy, + allocator=allocator, + new_order=new_order, + group_mode=group_mode, + ) + ) finally: loop.close() - async def transmit_item_groups_async(self, item_groups, *, - memcpy, allocator, new_order='=', group_mode=None): + async def transmit_item_groups_async( + self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None + ): if self.requires_ipv6: self.check_ipv6() recv_config = spead2.recv.StreamConfig(memcpy=memcpy) @@ -54,10 +60,7 @@ async def transmit_item_groups_async(self, item_groups, *, ] await self.prepare_receivers(receivers) sender = await self.prepare_senders(spead2.ThreadPool(), len(item_groups)) - gens = [ - spead2.send.HeapGenerator(item_group) - for item_group in item_groups - ] + gens = [spead2.send.HeapGenerator(item_group) for item_group in item_groups] if len(item_groups) != 1: # Use reversed order so that if everything is actually going # through the same transport it will get picked up. @@ -66,7 +69,8 @@ async def transmit_item_groups_async(self, item_groups, *, [ spead2.send.HeapReference(gen.get_heap(), substream_index=i) for i, gen in reversed(list(enumerate(gens))) - ], group_mode + ], + group_mode, ) # Use a HeapReferenceList to test it hrl = spead2.send.HeapReferenceList( @@ -107,7 +111,8 @@ async def prepare_senders(self, thread_pool, n): class BaseTestPassthroughSubstreamsAsync( - test_passthrough.BaseTestPassthroughSubstreams, BaseTestPassthroughAsync): + test_passthrough.BaseTestPassthroughSubstreams, BaseTestPassthroughAsync +): async def prepare_receivers(self, receivers): raise NotImplementedError() @@ -124,15 +129,19 @@ async def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): return spead2.send.asyncio.UdpStream( - thread_pool, "localhost", 8888, + thread_pool, + "localhost", + 8888, spead2.send.StreamConfig(rate=1e7), - buffer_size=0) + buffer_size=0, + ) else: return spead2.send.asyncio.UdpStream( thread_pool, [("localhost", 8888 + i) for i in range(n)], spead2.send.StreamConfig(rate=1e7), - buffer_size=0) + buffer_size=0, + ) class TestPassthroughUdpCustomSocket(BaseTestPassthroughSubstreamsAsync): @@ -140,7 +149,7 @@ async def prepare_receivers(self, receivers): self._ports = [] for receiver in receivers: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.bind(('127.0.0.1', 0)) + sock.bind(("127.0.0.1", 0)) self._ports.append(sock.getsockname()[1]) receiver.add_udp_reader(sock) sock.close() @@ -150,13 +159,19 @@ async def prepare_senders(self, thread_pool, n): if n == 1: with pytest.deprecated_call(): stream = spead2.send.asyncio.UdpStream( - thread_pool, sock, '127.0.0.1', self._ports[0], - spead2.send.StreamConfig(rate=1e7)) + thread_pool, + sock, + "127.0.0.1", + self._ports[0], + spead2.send.StreamConfig(rate=1e7), + ) else: stream = spead2.send.asyncio.UdpStream( - thread_pool, sock, - [('127.0.0.1', port) for port in self._ports], - spead2.send.StreamConfig(rate=1e7)) + thread_pool, + sock, + [("127.0.0.1", port) for port in self._ports], + spead2.send.StreamConfig(rate=1e7), + ) sock.close() return stream @@ -166,8 +181,7 @@ async def prepare_receiver(self, receiver): receiver.add_tcp_reader(8888, bind_hostname="127.0.0.1") async def prepare_sender(self, thread_pool): - sender = await spead2.send.asyncio.TcpStream.connect( - thread_pool, [("127.0.0.1", 8888)]) + sender = await spead2.send.asyncio.TcpStream.connect(thread_pool, [("127.0.0.1", 8888)]) return sender @@ -175,7 +189,7 @@ class TestPassthroughTcpCustomSocket(BaseTestPassthroughAsync): async def prepare_receiver(self, receiver): sock = socket.socket() # Prevent second iteration of the test from failing - sock.bind(('127.0.0.1', 0)) + sock.bind(("127.0.0.1", 0)) self._port = sock.getsockname()[1] sock.listen(1) receiver.add_tcp_reader(sock) @@ -184,9 +198,8 @@ async def prepare_receiver(self, receiver): async def prepare_sender(self, thread_pool): sock = socket.socket() sock.setblocking(False) - await asyncio.get_event_loop().sock_connect(sock, ('127.0.0.1', self._port)) - sender = spead2.send.asyncio.TcpStream( - thread_pool, sock) + await asyncio.get_event_loop().sock_connect(sock, ("127.0.0.1", self._port)) + sender = spead2.send.asyncio.TcpStream(thread_pool, sock) sock.close() return sender @@ -205,13 +218,17 @@ async def prepare_senders(self, thread_pool, n): else: return spead2.send.asyncio.InprocStream(thread_pool, self._queues) - async def transmit_item_groups_async(self, item_groups, *, - memcpy, allocator, new_order='=', group_mode=None): + async def transmit_item_groups_async( + self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None + ): self._queues = [spead2.InprocQueue() for ig in item_groups] ret = await super().transmit_item_groups_async( item_groups, - memcpy=memcpy, allocator=allocator, - new_order=new_order, group_mode=group_mode) + memcpy=memcpy, + allocator=allocator, + new_order=new_order, + group_mode=group_mode, + ) for queue in self._queues: queue.stop() return ret diff --git a/tests/test_recv.py b/tests/test_recv.py index a32e8bd3e..349df8f20 100644 --- a/tests/test_recv.py +++ b/tests/test_recv.py @@ -36,9 +36,9 @@ def __init__(self, id, value, immediate=False, offset=0): def encode(self, heap_address_bits): if self.immediate: - return struct.pack('>Q', (1 << 63) | (self.id << heap_address_bits) | self.value) + return struct.pack(">Q", (1 << 63) | (self.id << heap_address_bits) | self.value) else: - return struct.pack('>Q', (self.id << heap_address_bits) | self.offset) + return struct.pack(">Q", (self.id << heap_address_bits) | self.offset) class Flavour: @@ -49,22 +49,29 @@ def __init__(self, heap_address_bits, bug_compat=0): def _encode_be(self, size, value): """Encodes `value` as big-endian in `size` bytes""" assert size <= 8 - packed = struct.pack('>Q', value) - return packed[8 - size:] + packed = struct.pack(">Q", value) + return packed[8 - size :] def make_packet(self, items, payload): """Generate data for a packet at a low level. The value of non-immediate items are taken from the payload; the values in the objects are ignored. """ data = [] - data.append(struct.pack('>BBBBHH', 0x53, 0x4, - (64 - self.heap_address_bits) // 8, - self.heap_address_bits // 8, 0, - len(items))) + data.append( + struct.pack( + ">BBBBHH", + 0x53, + 0x4, + (64 - self.heap_address_bits) // 8, + self.heap_address_bits // 8, + 0, + len(items), + ) + ) for item in items: data.append(item.encode(self.heap_address_bits)) data.append(bytes(payload)) - return b''.join(data) + return b"".join(data) def make_format(self, format): if self.bug_compat & spead2.BUG_COMPAT_DESCRIPTOR_WIDTHS: @@ -74,9 +81,9 @@ def make_format(self, format): data = [] for field in format: assert len(field[0]) == 1 - data.append(field[0].encode('ascii')) + data.append(field[0].encode("ascii")) data.append(self._encode_be(field_size, field[1])) - return b''.join(data) + return b"".join(data) def make_shape(self, shape): if self.bug_compat & spead2.BUG_COMPAT_DESCRIPTOR_WIDTHS: @@ -91,13 +98,13 @@ def make_shape(self, shape): data = [] for value in shape: if value is None: - data.append(struct.pack('>B', variable_marker)) + data.append(struct.pack(">B", variable_marker)) data.append(self._encode_be(field_size - 1, 0)) elif value >= 0: data.append(self._encode_be(field_size, value)) else: - raise ValueError('Shape must contain non-negative values and None') - return b''.join(data) + raise ValueError("Shape must contain non-negative values and None") + return b"".join(data) def make_packet_heap(self, heap_cnt, items, *, duplicate_item_pointers=False, packets=None): """Construct all the packets in a heap. @@ -143,11 +150,12 @@ def make_packet_heap(self, heap_cnt, items, *, duplicate_item_pointers=False, pa Item(spead2.HEAP_CNT_ID, heap_cnt, True), Item(spead2.PAYLOAD_OFFSET_ID, start, True), Item(spead2.PAYLOAD_LENGTH_ID, stop - start, True), - Item(spead2.HEAP_LENGTH_ID, payload_size, True)] + Item(spead2.HEAP_LENGTH_ID, payload_size, True), + ] packet_items.extend(item_ptrs) if not duplicate_item_pointers: item_ptrs.clear() # Only put them in the first packet - out.append(self.make_packet(packet_items, payload[start : stop])) + out.append(self.make_packet(packet_items, payload[start:stop])) if packets is None: return out[0] else: @@ -155,41 +163,51 @@ def make_packet_heap(self, heap_cnt, items, *, duplicate_item_pointers=False, pa def make_plain_descriptor(self, id, name, description, format, shape): if not isinstance(name, bytes): - name = name.encode('ascii') + name = name.encode("ascii") if not isinstance(description, bytes): - description = description.encode('ascii') - return Item(spead2.DESCRIPTOR_ID, self.make_packet_heap( - 1, - [ - Item(spead2.DESCRIPTOR_ID_ID, id, True), - Item(spead2.DESCRIPTOR_NAME_ID, name), - Item(spead2.DESCRIPTOR_DESCRIPTION_ID, description), - Item(spead2.DESCRIPTOR_FORMAT_ID, self.make_format(format)), - Item(spead2.DESCRIPTOR_SHAPE_ID, self.make_shape(shape)) - ])) + description = description.encode("ascii") + return Item( + spead2.DESCRIPTOR_ID, + self.make_packet_heap( + 1, + [ + Item(spead2.DESCRIPTOR_ID_ID, id, True), + Item(spead2.DESCRIPTOR_NAME_ID, name), + Item(spead2.DESCRIPTOR_DESCRIPTION_ID, description), + Item(spead2.DESCRIPTOR_FORMAT_ID, self.make_format(format)), + Item(spead2.DESCRIPTOR_SHAPE_ID, self.make_shape(shape)), + ], + ), + ) def make_numpy_descriptor_raw(self, id, name, description, header): if not isinstance(name, bytes): - name = name.encode('ascii') + name = name.encode("ascii") if not isinstance(description, bytes): - description = description.encode('ascii') + description = description.encode("ascii") if not isinstance(header, bytes): - header = header.encode('ascii') - return Item(spead2.DESCRIPTOR_ID, self.make_packet_heap( - 1, - [ - Item(spead2.DESCRIPTOR_ID_ID, id, True), - Item(spead2.DESCRIPTOR_NAME_ID, name), - Item(spead2.DESCRIPTOR_DESCRIPTION_ID, description), - Item(spead2.DESCRIPTOR_DTYPE_ID, header) - ])) + header = header.encode("ascii") + return Item( + spead2.DESCRIPTOR_ID, + self.make_packet_heap( + 1, + [ + Item(spead2.DESCRIPTOR_ID_ID, id, True), + Item(spead2.DESCRIPTOR_NAME_ID, name), + Item(spead2.DESCRIPTOR_DESCRIPTION_ID, description), + Item(spead2.DESCRIPTOR_DTYPE_ID, header), + ], + ), + ) def make_numpy_descriptor(self, id, name, description, dtype, shape, fortran_order=False): - header = str({ - 'descr': np.lib.format.dtype_to_descr(np.dtype(dtype)), - 'fortran_order': bool(fortran_order), - 'shape': tuple(shape) - }) + header = str( + { + "descr": np.lib.format.dtype_to_descr(np.dtype(dtype)), + "fortran_order": bool(fortran_order), + "shape": tuple(shape), + } + ) return self.make_numpy_descriptor_raw(id, name, description, header) def make_numpy_descriptor_from(self, id, name, description, array): @@ -198,9 +216,10 @@ def make_numpy_descriptor_from(self, id, name, description, array): elif array.flags.f_contiguous: fortran_order = True else: - raise ValueError('Array must be C or Fortran-order contiguous') + raise ValueError("Array must be C or Fortran-order contiguous") return self.make_numpy_descriptor( - id, name, description, array.dtype, array.shape, fortran_order) + id, name, description, array.dtype, array.shape, fortran_order + ) FLAVOUR = Flavour(48) @@ -220,15 +239,20 @@ def data_to_heaps(self, data, **kwargs): thread_pool = spead2.ThreadPool() config = recv.StreamConfig(bug_compat=self.flavour.bug_compat) ring_config = recv.RingStreamConfig() - for key in {'stop_on_stop_item', 'allow_unsized_heaps', 'allow_out_of_order', - 'max_heaps', 'substreams'}: + for key in { + "stop_on_stop_item", + "allow_unsized_heaps", + "allow_out_of_order", + "max_heaps", + "substreams", + }: if key in kwargs: setattr(config, key, kwargs.pop(key)) - for key in {'contiguous_only', 'incomplete_keep_payload_ranges'}: + for key in {"contiguous_only", "incomplete_keep_payload_ranges"}: if key in kwargs: setattr(ring_config, key, kwargs.pop(key)) if kwargs: - raise ValueError(f'Unexpected keyword arguments ({kwargs})') + raise ValueError(f"Unexpected keyword arguments ({kwargs})") stream = recv.Stream(thread_pool, config, ring_config) stream.add_buffer_reader(data) return list(stream) @@ -259,9 +283,11 @@ def test_scalar_int(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_scalar_int', 'a scalar integer', [('i', 32)], []), - Item(0x1234, struct.pack('>i', -123456789)) - ]) + 0x1234, "test_scalar_int", "a scalar integer", [("i", 32)], [] + ), + Item(0x1234, struct.pack(">i", -123456789)), + ], + ) item = self.data_to_item(packet, 0x1234) assert isinstance(item.value, np.int32) assert item.value == -123456789 @@ -271,9 +297,11 @@ def test_scalar_int_immediate(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_scalar_int', 'a scalar integer', [('u', 32)], []), - Item(0x1234, 0x12345678, True) - ]) + 0x1234, "test_scalar_int", "a scalar integer", [("u", 32)], [] + ), + Item(0x1234, 0x12345678, True), + ], + ) item = self.data_to_item(packet, 0x1234) assert isinstance(item.value, np.uint32) assert item.value == 0x12345678 @@ -283,15 +311,19 @@ def test_scalar_int_immediate_fastpath(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_scalar_uint', 'a scalar integer', [('u', 48)], []), + 0x1234, "test_scalar_uint", "a scalar integer", [("u", 48)], [] + ), self.flavour.make_plain_descriptor( - 0x1235, 'test_scalar_positive', 'a positive scalar integer', [('i', 48)], []), + 0x1235, "test_scalar_positive", "a positive scalar integer", [("i", 48)], [] + ), self.flavour.make_plain_descriptor( - 0x1236, 'test_scalar_negative', 'a negative scalar integer', [('i', 48)], []), + 0x1236, "test_scalar_negative", "a negative scalar integer", [("i", 48)], [] + ), Item(0x1234, 0x9234567890AB, True), Item(0x1235, 0x1234567890AB, True), - Item(0x1236, 0x9234567890AB, True) - ]) + Item(0x1236, 0x9234567890AB, True), + ], + ) ig = self.data_to_ig(packet) assert len(ig) == 3 assert ig[0x1234].value == 0x9234567890AB @@ -303,20 +335,24 @@ def test_string(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_string', 'a byte string', [('c', 8)], [None]), - Item(0x1234, b'Hello world') - ]) + 0x1234, "test_string", "a byte string", [("c", 8)], [None] + ), + Item(0x1234, b"Hello world"), + ], + ) item = self.data_to_item(packet, 0x1234) - assert item.value == 'Hello world' + assert item.value == "Hello world" def test_array(self): packet = self.flavour.make_packet_heap( 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_array', 'an array of floats', [('f', 32)], (3, 2)), - Item(0x1234, struct.pack('>6f', *np.arange(1.5, 7.5))) - ]) + 0x1234, "test_array", "an array of floats", [("f", 32)], (3, 2) + ), + Item(0x1234, struct.pack(">6f", *np.arange(1.5, 7.5))), + ], + ) item = self.data_to_item(packet, 0x1234) expected = np.array([[1.5, 2.5], [3.5, 4.5], [5.5, 6.5]], dtype=np.float32) np.testing.assert_equal(expected, item.value) @@ -326,11 +362,13 @@ def test_array_fields(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_array', 'an array of floats', [('f', 32), ('i', 8)], (3,)), - Item(0x1234, struct.pack('>fbfbfb', 1.5, 1, 2.5, 2, 4.5, -4)) - ]) + 0x1234, "test_array", "an array of floats", [("f", 32), ("i", 8)], (3,) + ), + Item(0x1234, struct.pack(">fbfbfb", 1.5, 1, 2.5, 2, 4.5, -4)), + ], + ) item = self.data_to_item(packet, 0x1234) - dtype = np.dtype('=f4,i1') + dtype = np.dtype("=f4,i1") assert item.value.dtype == dtype expected = np.array([(1.5, 1), (2.5, 2), (4.5, -4)], dtype=dtype) np.testing.assert_equal(expected, item.value) @@ -346,9 +384,11 @@ def test_array_numpy(self): 1, [ self.flavour.make_numpy_descriptor_from( - 0x1234, 'test_array_numpy', 'an array of floats', expected), - Item(0x1234, send.data) - ]) + 0x1234, "test_array_numpy", "an array of floats", expected + ), + Item(0x1234, send.data), + ], + ) item = self.data_to_item(packet, 0x1234) assert item.value.dtype == expected.dtype np.testing.assert_equal(expected, item.value) @@ -365,9 +405,11 @@ def test_array_numpy_fortran(self): 1, [ self.flavour.make_numpy_descriptor_from( - 0x1234, 'test_array_numpy_fortran', 'an array of floats', expected), - Item(0x1234, np.ravel(send, 'K').data) - ]) + 0x1234, "test_array_numpy_fortran", "an array of floats", expected + ), + Item(0x1234, np.ravel(send, "K").data), + ], + ) item = self.data_to_item(packet, 0x1234) assert item.value.dtype == expected.dtype np.testing.assert_equal(expected, item.value) @@ -378,9 +420,11 @@ def test_fallback_uint(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_fallback_uint', 'an array of 12-bit uints', [('u', 12)], (3,)), - Item(0x1234, b'\xAB\xCD\xEF\x12\x30') - ]) + 0x1234, "test_fallback_uint", "an array of 12-bit uints", [("u", 12)], (3,) + ), + Item(0x1234, b"\xAB\xCD\xEF\x12\x30"), + ], + ) item = self.data_to_item(packet, 0x1234) np.testing.assert_equal(expected, item.value) @@ -390,22 +434,29 @@ def test_fallback_int(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_fallback_uint', 'an array of 12-bit ints', [('i', 12)], (3,)), - Item(0x1234, b'\xAB\xCD\xEF\x12\x30') - ]) + 0x1234, "test_fallback_uint", "an array of 12-bit ints", [("i", 12)], (3,) + ), + Item(0x1234, b"\xAB\xCD\xEF\x12\x30"), + ], + ) item = self.data_to_item(packet, 0x1234) np.testing.assert_equal(expected, item.value) def test_fallback_types(self): - expected = np.array([(True, 17, 'y', 1.0), (False, -23, 'n', -1.0)], dtype='O,O,S1,>f4') + expected = np.array([(True, 17, "y", 1.0), (False, -23, "n", -1.0)], dtype="O,O,S1,>f4") packet = self.flavour.make_packet_heap( 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_fallback_uint', 'an array with bools, int, chars and floats', - [('b', 1), ('i', 7), ('c', 8), ('f', 32)], (2,)), - Item(0x1234, b'\x91y\x3F\x80\x00\x00' + b'\x69n\xBF\x80\x00\x00') - ]) + 0x1234, + "test_fallback_uint", + "an array with bools, int, chars and floats", + [("b", 1), ("i", 7), ("c", 8), ("f", 32)], + (2,), + ), + Item(0x1234, b"\x91y\x3F\x80\x00\x00" + b"\x69n\xBF\x80\x00\x00"), + ], + ) item = self.data_to_item(packet, 0x1234) np.testing.assert_equal(expected, item.value) @@ -415,21 +466,21 @@ def test_fallback_scalar(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_fallback_scalar', 'a scalar with unusual type', - [('u', 48)], ()), - Item(0x1234, b'\x12\x34\x56\x78\x90\xAB') - ]) + 0x1234, "test_fallback_scalar", "a scalar with unusual type", [("u", 48)], () + ), + Item(0x1234, b"\x12\x34\x56\x78\x90\xAB"), + ], + ) item = self.data_to_item(packet, 0x1234) assert item.value == expected def test_duplicate_item_pointers(self): payload = bytearray(64) payload[:] = range(64) - packets = self.flavour.make_packet_heap(1, [ - Item(0x1600, 12345, True), - Item(0x5000, payload, False)], - packets=2) - heaps = self.data_to_heaps(b''.join(packets)) + packets = self.flavour.make_packet_heap( + 1, [Item(0x1600, 12345, True), Item(0x5000, payload, False)], packets=2 + ) + heaps = self.data_to_heaps(b"".join(packets)) assert len(heaps) == 1 items = heaps[0].get_items() assert len(items) == 2 @@ -444,11 +495,12 @@ def test_duplicate_item_pointers(self): def test_out_of_order_packets(self): payload = bytearray(64) payload[:] = range(64) - packets = self.flavour.make_packet_heap(1, [ - Item(0x1600, 12345, True), - Item(0x5000, payload, False)], - packets=[(32, 64), (0, 32)]) - heaps = self.data_to_heaps(b''.join(packets), allow_out_of_order=True) + packets = self.flavour.make_packet_heap( + 1, + [Item(0x1600, 12345, True), Item(0x5000, payload, False)], + packets=[(32, 64), (0, 32)], + ) + heaps = self.data_to_heaps(b"".join(packets), allow_out_of_order=True) assert len(heaps) == 1 items = heaps[0].get_items() assert len(items) == 2 @@ -463,13 +515,14 @@ def test_out_of_order_packets(self): def test_out_of_order_disallowed(self): payload = bytearray(64) payload[:] = range(64) - packets = self.flavour.make_packet_heap(1, [ - Item(0x1600, 12345, True), - Item(0x5000, payload, False)], - packets=[(32, 64), (0, 32)]) - heaps = self.data_to_heaps(b''.join(packets), - contiguous_only=False, - incomplete_keep_payload_ranges=True) + packets = self.flavour.make_packet_heap( + 1, + [Item(0x1600, 12345, True), Item(0x5000, payload, False)], + packets=[(32, 64), (0, 32)], + ) + heaps = self.data_to_heaps( + b"".join(packets), contiguous_only=False, incomplete_keep_payload_ranges=True + ) assert len(heaps) == 1 items = heaps[0].get_items() assert items == [] @@ -480,20 +533,20 @@ def test_out_of_order_disallowed(self): def test_substreams(self): payload = bytearray(64) payload[:] = range(64) - stream0_packets = self.flavour.make_packet_heap(2, [ - Item(0x1600, 12345, True), - Item(0x5000, payload, False)], - packets=2) + stream0_packets = self.flavour.make_packet_heap( + 2, [Item(0x1600, 12345, True), Item(0x5000, payload, False)], packets=2 + ) stream1_packets = [] for i in range(3, 20, 2): - stream1_packets.extend(self.flavour.make_packet_heap(i, [ - Item(0x1600, 54321, True), - Item(0x5001, payload, False)], - packets=2)) + stream1_packets.extend( + self.flavour.make_packet_heap( + i, [Item(0x1600, 54321, True), Item(0x5001, payload, False)], packets=2 + ) + ) # Interleave the packets from stream 1 into the middle of the substream # 0 packet. packets = stream0_packets[0:1] + stream1_packets + stream0_packets[1:] - heaps = self.data_to_heaps(b''.join(packets), substreams=2) + heaps = self.data_to_heaps(b"".join(packets), substreams=2) assert len(heaps) == 10 assert heaps[-1].cnt == 2 items = heaps[-1].get_items() @@ -505,18 +558,21 @@ def test_substreams(self): def test_incomplete_heaps(self): payload = bytearray(96) payload[:] = range(96) - packets = self.flavour.make_packet_heap(1, [ - Item(0x1600, 12345, True), - Item(0x5000, payload, False)], - packets=[(5, 12), (32, 64)]) - heaps = self.data_to_heaps(b''.join(packets), - contiguous_only=False, - incomplete_keep_payload_ranges=True, - allow_out_of_order=True) + packets = self.flavour.make_packet_heap( + 1, + [Item(0x1600, 12345, True), Item(0x5000, payload, False)], + packets=[(5, 12), (32, 64)], + ) + heaps = self.data_to_heaps( + b"".join(packets), + contiguous_only=False, + incomplete_keep_payload_ranges=True, + allow_out_of_order=True, + ) assert len(heaps) == 1 assert isinstance(heaps[0], recv.IncompleteHeap) items = heaps[0].get_items() - assert len(items) == 1 # Addressed item must be excluded + assert len(items) == 1 # Addressed item must be excluded assert items[0].id == 0x1600 assert heaps[0].heap_length == 96 assert heaps[0].received_length == 39 @@ -524,51 +580,55 @@ def test_incomplete_heaps(self): def test_is_start_of_stream(self): packet = self.flavour.make_packet_heap( - 1, - [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START, immediate=True)]) + 1, [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START, immediate=True)] + ) heaps = self.data_to_heaps(packet) assert heaps[0].is_start_of_stream() is True packet = self.flavour.make_packet_heap( - 1, - [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_DESCRIPTOR_REISSUE, immediate=True)]) + 1, [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_DESCRIPTOR_REISSUE, immediate=True)] + ) heaps = self.data_to_heaps(packet) assert heaps[0].is_start_of_stream() is False def test_is_end_of_stream(self): packet = self.flavour.make_packet_heap( - 1, - [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_STOP, immediate=True)]) + 1, [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_STOP, immediate=True)] + ) heaps = self.data_to_heaps(packet, stop_on_stop_item=False) assert heaps[0].is_end_of_stream() is True assert heaps[0].is_start_of_stream() is False def test_no_stop_on_stop_item(self): packet1 = self.flavour.make_packet_heap( - 1, - [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_STOP, immediate=True)]) + 1, [Item(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_STOP, immediate=True)] + ) packet2 = self.flavour.make_packet_heap( 2, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_string', 'a byte string', [('c', 8)], [None]), - Item(0x1234, b'Hello world') - ]) + 0x1234, "test_string", "a byte string", [("c", 8)], [None] + ), + Item(0x1234, b"Hello world"), + ], + ) heaps = self.data_to_heaps(packet1 + packet2, stop_on_stop_item=False) assert len(heaps) == 2 ig = spead2.ItemGroup() ig.update(heaps[0]) ig.update(heaps[1]) - assert ig['test_string'].value == 'Hello world' + assert ig["test_string"].value == "Hello world" def test_size_mismatch(self): packet = self.flavour.make_packet_heap( 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'bad', 'an item with insufficient data', [('u', 32)], (5, 5)), - Item(0x1234, b'\0' * 99) - ]) + 0x1234, "bad", "an item with insufficient data", [("u", 32)], (5, 5) + ), + Item(0x1234, b"\0" * 99), + ], + ) heaps = self.data_to_heaps(packet) assert len(heaps) == 1 ig = spead2.ItemGroup() @@ -579,14 +639,16 @@ def test_numpy_object(self): """numpy dtypes can contain Python objects (by pointer). These can't be used for SPEAD. """ - dtype = np.dtype('f4,O') + dtype = np.dtype("f4,O") packet = self.flavour.make_packet_heap( 1, [ self.flavour.make_numpy_descriptor( - 0x1234, 'object', 'an item with object pointers', dtype, (5,)), - Item(0x1234, b'?' * 100) - ]) + 0x1234, "object", "an item with object pointers", dtype, (5,) + ), + Item(0x1234, b"?" * 100), + ], + ) heaps = self.data_to_heaps(packet) assert len(heaps) == 1 ig = spead2.ItemGroup() @@ -600,9 +662,11 @@ def test_numpy_zero_size(self): 1, [ self.flavour.make_numpy_descriptor( - 0x1234, 'empty', 'an item with zero-byte dtype', dtype, (5,)), - Item(0x1234, b'') - ]) + 0x1234, "empty", "an item with zero-byte dtype", dtype, (5,) + ), + Item(0x1234, b""), + ], + ) heaps = self.data_to_heaps(packet) assert len(heaps) == 1 ig = spead2.ItemGroup() @@ -611,27 +675,26 @@ def test_numpy_zero_size(self): def test_numpy_malformed(self): """Malformed numpy header must raise :py:exc:`ValueError`.""" + def helper(header): packet = self.flavour.make_packet_heap( - 1, - [ - self.flavour.make_numpy_descriptor_raw( - 0x1234, 'name', 'description', header) - ]) + 1, [self.flavour.make_numpy_descriptor_raw(0x1234, "name", "description", header)] + ) heaps = self.data_to_heaps(packet) assert len(heaps) == 1 ig = spead2.ItemGroup() with pytest.raises(ValueError): ig.update(heaps[0]) - helper("{'descr': 'S1'") # Syntax error: no closing brace - helper("123") # Not a dictionary - helper("import os") # Security check + + helper("{'descr': 'S1'") # Syntax error: no closing brace + helper("123") # Not a dictionary + helper("import os") # Security check helper("{'descr': 'S1'}") # Missing keys helper("{'descr': 'S1', 'fortran_order': False, 'shape': (), 'foo': 'bar'}") # Extra keys - helper("{'descr': 'S1', 'fortran_order': False, 'shape': (-1,)}") # Bad shape - helper("{'descr': 1, 'fortran_order': False, 'shape': ()}") # Bad descriptor - helper("{'descr': '+-', 'fortran_order': False, 'shape': ()}") # Bad descriptor - helper("{'descr': 'S1', 'fortran_order': 0, 'shape': ()}") # Bad fortran_order + helper("{'descr': 'S1', 'fortran_order': False, 'shape': (-1,)}") # Bad shape + helper("{'descr': 1, 'fortran_order': False, 'shape': ()}") # Bad descriptor + helper("{'descr': '+-', 'fortran_order': False, 'shape': ()}") # Bad descriptor + helper("{'descr': 'S1', 'fortran_order': 0, 'shape': ()}") # Bad fortran_order helper("{'descr': 'S1', 'fortran_order': False, 'shape': (None,)}") # Bad shape def test_nonascii_value(self): @@ -641,9 +704,11 @@ def test_nonascii_value(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, 'test_string', 'a byte string', [('c', 8)], [None]), - Item(0x1234, '\u0200'.encode()) - ]) + 0x1234, "test_string", "a byte string", [("c", 8)], [None] + ), + Item(0x1234, "\u0200".encode()), + ], + ) heaps = self.data_to_heaps(packet) ig = spead2.ItemGroup() with pytest.raises(UnicodeDecodeError): @@ -656,8 +721,10 @@ def test_nonascii_name(self): 1, [ self.flavour.make_plain_descriptor( - 0x1234, b'\xEF', 'a byte string', [('c', 8)], [None]) - ]) + 0x1234, b"\xEF", "a byte string", [("c", 8)], [None] + ) + ], + ) heaps = self.data_to_heaps(packet) ig = spead2.ItemGroup() with pytest.raises(UnicodeDecodeError): @@ -667,11 +734,8 @@ def test_nonascii_description(self): """Receiving non-ASCII characters in an item description must raise :py:exc:`UnicodeDecodeError`.""" packet = self.flavour.make_packet_heap( - 1, - [ - self.flavour.make_plain_descriptor( - 0x1234, 'name', b'\xEF', [('c', 8)], [None]) - ]) + 1, [self.flavour.make_plain_descriptor(0x1234, "name", b"\xEF", [("c", 8)], [None])] + ) heaps = self.data_to_heaps(packet) ig = spead2.ItemGroup() with pytest.raises(UnicodeDecodeError): @@ -688,14 +752,18 @@ def test_no_heap_size(self): Item(spead2.HEAP_CNT_ID, 1, True), Item(0x1000, None, False, offset=0), Item(spead2.PAYLOAD_OFFSET_ID, 0, True), - Item(spead2.PAYLOAD_LENGTH_ID, 64, True) - ], payload1) + Item(spead2.PAYLOAD_LENGTH_ID, 64, True), + ], + payload1, + ) packet2 = self.flavour.make_packet( [ Item(spead2.HEAP_CNT_ID, 1, True), Item(spead2.PAYLOAD_OFFSET_ID, 64, True), - Item(spead2.PAYLOAD_LENGTH_ID, 32, True) - ], payload2) + Item(spead2.PAYLOAD_LENGTH_ID, 32, True), + ], + payload2, + ) heaps = self.data_to_heaps(packet1 + packet2) assert len(heaps) == 1 raw_items = heaps[0].get_items() @@ -709,13 +777,15 @@ def test_disallow_unsized_heaps(self, caplog): Item(spead2.HEAP_CNT_ID, 1, True), Item(0x1000, None, False, offset=0), Item(spead2.PAYLOAD_OFFSET_ID, 0, True), - Item(spead2.PAYLOAD_LENGTH_ID, 64, True) - ], bytes(np.arange(0, 64, dtype=np.uint8).data)) - with caplog.at_level('INFO', 'spead2'): + Item(spead2.PAYLOAD_LENGTH_ID, 64, True), + ], + bytes(np.arange(0, 64, dtype=np.uint8).data), + ) + with caplog.at_level("INFO", "spead2"): heaps = self.data_to_heaps(packet, allow_unsized_heaps=False) # Logging is asynchronous, so we have to give it a bit of time time.sleep(0.1) - assert caplog.messages == ['packet rejected because it has no HEAP_LEN'] + assert caplog.messages == ["packet rejected because it has no HEAP_LEN"] assert len(heaps) == 0 def test_bad_offset(self): @@ -726,8 +796,10 @@ def test_bad_offset(self): Item(spead2.HEAP_LENGTH_ID, 64, True), Item(spead2.PAYLOAD_OFFSET_ID, 0, True), Item(spead2.PAYLOAD_LENGTH_ID, 64, True), - Item(0x1000, None, False, offset=65) - ], b'\0' * 64) + Item(0x1000, None, False, offset=65), + ], + b"\0" * 64, + ) heaps = self.data_to_heaps(packet) assert len(heaps) == 0 @@ -736,15 +808,15 @@ class TestStreamConfig: """Tests for :class:`spead2.recv.StreamConfig`.""" expected_stats = [ - recv.StreamStatConfig('heaps'), - recv.StreamStatConfig('incomplete_heaps_evicted'), - recv.StreamStatConfig('incomplete_heaps_flushed'), - recv.StreamStatConfig('packets'), - recv.StreamStatConfig('batches'), - recv.StreamStatConfig('max_batch', recv.StreamStatConfig.Mode.MAXIMUM), - recv.StreamStatConfig('single_packet_heaps'), - recv.StreamStatConfig('search_dist'), - recv.StreamStatConfig('worker_blocked') + recv.StreamStatConfig("heaps"), + recv.StreamStatConfig("incomplete_heaps_evicted"), + recv.StreamStatConfig("incomplete_heaps_flushed"), + recv.StreamStatConfig("packets"), + recv.StreamStatConfig("batches"), + recv.StreamStatConfig("max_batch", recv.StreamStatConfig.Mode.MAXIMUM), + recv.StreamStatConfig("single_packet_heaps"), + recv.StreamStatConfig("search_dist"), + recv.StreamStatConfig("worker_blocked"), ] def test_default_construct(self): @@ -786,7 +858,7 @@ def test_kwargs_construct(self): stop_on_stop_item=False, allow_unsized_heaps=False, allow_out_of_order=True, - stream_id=123 + stream_id=123, ) assert config.max_heaps == 5 assert config.bug_compat == spead2.BUG_COMPAT_PYSPEAD_0_5_2 @@ -803,7 +875,7 @@ def test_max_heaps_zero(self): def test_bad_bug_compat(self): with pytest.raises(ValueError): - recv.StreamConfig(bug_compat=0xff) + recv.StreamConfig(bug_compat=0xFF) def test_bad_kwarg(self): with pytest.raises(TypeError): @@ -812,22 +884,22 @@ def test_bad_kwarg(self): def test_stats(self): config = recv.StreamConfig() base_index = config.next_stat_index() - counter_index = config.add_stat('counter') - maximum_index = config.add_stat('maximum', recv.StreamStatConfig.Mode.MAXIMUM) + counter_index = config.add_stat("counter") + maximum_index = config.add_stat("maximum", recv.StreamStatConfig.Mode.MAXIMUM) assert config.stats == self.expected_stats + [ - recv.StreamStatConfig('counter', recv.StreamStatConfig.Mode.COUNTER), - recv.StreamStatConfig('maximum', recv.StreamStatConfig.Mode.MAXIMUM) + recv.StreamStatConfig("counter", recv.StreamStatConfig.Mode.COUNTER), + recv.StreamStatConfig("maximum", recv.StreamStatConfig.Mode.MAXIMUM), ] assert counter_index == base_index - assert config.get_stat_index('counter') == counter_index - assert config.get_stat_index('maximum') == maximum_index + assert config.get_stat_index("counter") == counter_index + assert config.get_stat_index("maximum") == maximum_index with pytest.raises(IndexError): - config.get_stat_index('does_not_exist') + config.get_stat_index("does_not_exist") # Can't add duplicates with pytest.raises(ValueError): - config.add_stat('heaps') + config.add_stat("heaps") with pytest.raises(ValueError): - config.add_stat('counter') + config.add_stat("counter") class TestRingStreamConfig: @@ -854,12 +926,12 @@ def test_heaps_zero(self): recv.RingStreamConfig(heaps=0) -@pytest.mark.skipif(not hasattr(spead2, 'IbvContext'), reason='IBV support not compiled in') +@pytest.mark.skipif(not hasattr(spead2, "IbvContext"), reason="IBV support not compiled in") class TestUdpIbvConfig: def test_default_construct(self): config = recv.UdpIbvConfig() assert config.endpoints == [] - assert config.interface_address == '' + assert config.interface_address == "" assert config.buffer_size == recv.UdpIbvConfig.DEFAULT_BUFFER_SIZE assert config.max_size == recv.UdpIbvConfig.DEFAULT_MAX_SIZE assert config.comp_vector == 0 @@ -867,14 +939,15 @@ def test_default_construct(self): def test_kwargs_construct(self): config = recv.UdpIbvConfig( - endpoints=[('hello', 1234), ('goodbye', 2345)], - interface_address='1.2.3.4', + endpoints=[("hello", 1234), ("goodbye", 2345)], + interface_address="1.2.3.4", buffer_size=100, max_size=4321, comp_vector=-1, - max_poll=1000) - assert config.endpoints == [('hello', 1234), ('goodbye', 2345)] - assert config.interface_address == '1.2.3.4' + max_poll=1000, + ) + assert config.endpoints == [("hello", 1234), ("goodbye", 2345)] + assert config.interface_address == "1.2.3.4" assert config.buffer_size == 100 assert config.max_size == 4321 assert config.comp_vector == -1 @@ -898,50 +971,41 @@ def test_bad_max_size(self): config.max_size = 0 def test_no_endpoints(self): - config = recv.UdpIbvConfig(interface_address='10.0.0.1') - stream = recv.Stream( - spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) - with pytest.raises(ValueError, match='endpoints is empty'): + config = recv.UdpIbvConfig(interface_address="10.0.0.1") + stream = recv.Stream(spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) + with pytest.raises(ValueError, match="endpoints is empty"): stream.add_udp_ibv_reader(config) def test_ipv6_endpoints(self): - config = recv.UdpIbvConfig( - endpoints=[('::1', 8888)], - interface_address='10.0.0.1') - stream = recv.Stream( - spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) - with pytest.raises(ValueError, match='endpoint is not an IPv4 address'): + config = recv.UdpIbvConfig(endpoints=[("::1", 8888)], interface_address="10.0.0.1") + stream = recv.Stream(spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) + with pytest.raises(ValueError, match="endpoint is not an IPv4 address"): stream.add_udp_ibv_reader(config) def test_no_interface_address(self): - config = recv.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)]) - stream = recv.Stream( - spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) - with pytest.raises(ValueError, match='interface address'): + config = recv.UdpIbvConfig(endpoints=[("239.255.88.88", 8888)]) + stream = recv.Stream(spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) + with pytest.raises(ValueError, match="interface address"): stream.add_udp_ibv_reader(config) def test_bad_interface_address(self): config = recv.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)], - interface_address='this is not an interface address') - stream = recv.Stream( - spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) - with pytest.raises(RuntimeError, match='Host not found'): + endpoints=[("239.255.88.88", 8888)], + interface_address="this is not an interface address", + ) + stream = recv.Stream(spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) + with pytest.raises(RuntimeError, match="Host not found"): stream.add_udp_ibv_reader(config) def test_ipv6_interface_address(self): - config = recv.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)], - interface_address='::1') - stream = recv.Stream( - spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) - with pytest.raises(ValueError, match='interface address'): + config = recv.UdpIbvConfig(endpoints=[("239.255.88.88", 8888)], interface_address="::1") + stream = recv.Stream(spead2.ThreadPool(), recv.StreamConfig(), recv.RingStreamConfig()) + with pytest.raises(ValueError, match="interface address"): stream.add_udp_ibv_reader(config) @pytest.mark.skipif( - platform.python_implementation() == 'PyPy', - reason='Deprecations not being report on PyPy due to pybind/pybind11#3110' + platform.python_implementation() == "PyPy", + reason="Deprecations not being report on PyPy due to pybind/pybind11#3110", ) def test_deprecated_constants(self): with pytest.deprecated_call(): @@ -965,11 +1029,17 @@ def test_full_stop(self): sender = send.BytesStream(thread_pool) ig = send.ItemGroup() data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16) - ig.add_item(id=0x2345, name='name', description='description', - shape=data.shape, dtype=data.dtype, value=data) + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=data.shape, + dtype=data.dtype, + value=data, + ) gen = send.HeapGenerator(ig) for i in range(10): - sender.send_heap(gen.get_heap(data='all')) + sender.send_heap(gen.get_heap(data="all")) recv_ring_config = recv.RingStreamConfig(heaps=4) receiver = recv.Stream(thread_pool, ring_config=recv_ring_config) receiver.add_buffer_reader(sender.getvalue()) @@ -981,7 +1051,7 @@ def test_full_stop(self): assert receiver.ringbuffer.capacity() == 4 assert receiver.ringbuffer.size() == 4 - receiver.stop() # This unblocks all remaining heaps + receiver.stop() # This unblocks all remaining heaps stats = receiver.stats assert stats.heaps == 10 assert stats.packets == 10 @@ -997,10 +1067,16 @@ def test_no_stop_heap(self): sender = send.BytesStream(thread_pool) ig = send.ItemGroup() data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16) - ig.add_item(id=0x2345, name='name', description='description', - shape=data.shape, dtype=data.dtype, value=data) + ig.add_item( + id=0x2345, + name="name", + description="description", + shape=data.shape, + dtype=data.dtype, + value=data, + ) gen = send.HeapGenerator(ig) - sender.send_heap(gen.get_heap(data='all')) + sender.send_heap(gen.get_heap(data="all")) sender.send_heap(gen.get_end()) receiver = recv.Stream(thread_pool) receiver.add_buffer_reader(sender.getvalue()) @@ -1037,14 +1113,14 @@ def stats(self): def custom_stats_fixtures(self): stream_config = recv.StreamConfig() indices = {} - indices['counter'] = stream_config.add_stat('counter') - indices['maximum'] = stream_config.add_stat('maximum', recv.StreamStatConfig.Mode.MAXIMUM) + indices["counter"] = stream_config.add_stat("counter") + indices["maximum"] = stream_config.add_stat("maximum", recv.StreamStatConfig.Mode.MAXIMUM) receiver = recv.Stream(spead2.ThreadPool(), stream_config) stats = receiver.stats - stats['heaps'] = 10 - stats['max_batch'] = 15 - stats['counter'] = 123 - stats['maximum'] = 456 + stats["heaps"] = 10 + stats["max_batch"] = 15 + stats["counter"] = 123 + stats["maximum"] = 456 return stats, indices @pytest.fixture @@ -1058,14 +1134,14 @@ def custom_indices(self, custom_stats_fixtures): @pytest.fixture def custom_stats2(self): stream_config = recv.StreamConfig() - stream_config.add_stat('counter') - stream_config.add_stat('maximum', recv.StreamStatConfig.Mode.MAXIMUM) + stream_config.add_stat("counter") + stream_config.add_stat("maximum", recv.StreamStatConfig.Mode.MAXIMUM) receiver = recv.Stream(spead2.ThreadPool(), stream_config) stats = receiver.stats - stats['heaps'] = 20 - stats['max_batch'] = 17 - stats['counter'] = 300 - stats['maximum'] = 400 + stats["heaps"] = 20 + stats["max_batch"] = 17 + stats["counter"] = 300 + stats["maximum"] = 400 return stats def test_properties(self, stats): @@ -1081,19 +1157,19 @@ def test_properties(self, stats): assert stats.config == TestStreamConfig.expected_stats def test_getitem_name(self, stats): - assert stats['heaps'] == 10 - assert stats['incomplete_heaps_evicted'] == 20 - assert stats['incomplete_heaps_flushed'] == 30 - assert stats['packets'] == 40 - assert stats['batches'] == 50 - assert stats['max_batch'] == 60 - assert stats['single_packet_heaps'] == 70 - assert stats['search_dist'] == 80 - assert stats['worker_blocked'] == 90 + assert stats["heaps"] == 10 + assert stats["incomplete_heaps_evicted"] == 20 + assert stats["incomplete_heaps_flushed"] == 30 + assert stats["packets"] == 40 + assert stats["batches"] == 50 + assert stats["max_batch"] == 60 + assert stats["single_packet_heaps"] == 70 + assert stats["search_dist"] == 80 + assert stats["worker_blocked"] == 90 def test_getitem_name_missing(self, stats): with pytest.raises(KeyError): - stats['not_a_stat'] + stats["not_a_stat"] def test_getitem_index(self, stats): assert stats[recv.stream_stat_indices.HEAPS] == 10 @@ -1104,34 +1180,34 @@ def test_getitem_index_out_of_range(self, stats): stats[1000] def test_contains(self, stats): - assert 'heaps' in stats - assert 'not_a_stat' not in stats + assert "heaps" in stats + assert "not_a_stat" not in stats def test_len(self, stats): assert len(stats) == len(TestStreamConfig.expected_stats) def test_custom(self, stats, custom_stats, custom_indices): - assert custom_stats['counter'] == 123 - assert custom_stats['maximum'] == 456 - assert custom_stats[custom_indices['counter']] == 123 - assert custom_stats[custom_indices['maximum']] == 456 - assert 'counter' in custom_stats + assert custom_stats["counter"] == 123 + assert custom_stats["maximum"] == 456 + assert custom_stats[custom_indices["counter"]] == 123 + assert custom_stats[custom_indices["maximum"]] == 456 + assert "counter" in custom_stats assert len(custom_stats) == len(stats) + 2 def test_add(self, custom_stats, custom_stats2): total = custom_stats + custom_stats2 - assert total['heaps'] == custom_stats['heaps'] + custom_stats2['heaps'] - assert total['max_batch'] == max(custom_stats['max_batch'], custom_stats2['max_batch']) - assert total['counter'] == custom_stats['counter'] + custom_stats2['counter'] - assert total['maximum'] == max(custom_stats['maximum'], custom_stats2['maximum']) + assert total["heaps"] == custom_stats["heaps"] + custom_stats2["heaps"] + assert total["max_batch"] == max(custom_stats["max_batch"], custom_stats2["max_batch"]) + assert total["counter"] == custom_stats["counter"] + custom_stats2["counter"] + assert total["maximum"] == max(custom_stats["maximum"], custom_stats2["maximum"]) def test_add_assign(self, custom_stats, custom_stats2): total = custom_stats + custom_stats2 # Verified by the previous test custom_stats += custom_stats2 - assert custom_stats['heaps'] == total['heaps'] - assert custom_stats['max_batch'] == total['max_batch'] - assert custom_stats['counter'] == total['counter'] - assert custom_stats['maximum'] == total['maximum'] + assert custom_stats["heaps"] == total["heaps"] + assert custom_stats["max_batch"] == total["max_batch"] + assert custom_stats["counter"] == total["counter"] + assert custom_stats["maximum"] == total["maximum"] def test_add_mismatched(self, stats, custom_stats): with pytest.raises(ValueError): @@ -1145,9 +1221,7 @@ def test_iterate(self, custom_stats): assert list(custom_stats.items()) == [ (s.name, custom_stats[s.name]) for s in custom_stats.config ] - assert list(custom_stats.values()) == [ - custom_stats[s.name] for s in custom_stats.config - ] + assert list(custom_stats.values()) == [custom_stats[s.name] for s in custom_stats.config] class TestUdpReader: @@ -1157,8 +1231,7 @@ def test_out_of_range_udp_port(self): receiver.add_udp_reader(100000) @pytest.mark.skipif( - sys.platform == 'darwin', - reason='Test does not work with macos on Github Actions' + sys.platform == "darwin", reason="Test does not work with macos on Github Actions" ) def test_illegal_udp_port(self): receiver = recv.Stream(spead2.ThreadPool()) @@ -1222,13 +1295,15 @@ def simple_packet(self): 1, [ FLAVOUR.make_plain_descriptor( - 0x1234, 'test_scalar_int', 'a scalar integer', [('u', 32)], []), - Item(0x1234, 0x12345678, True) - ]) + 0x1234, "test_scalar_int", "a scalar integer", [("u", 32)], [] + ), + Item(0x1234, 0x12345678, True), + ], + ) def test_bad_header(self): """A nonsense header followed by a normal packet""" - data = b'deadbeef' + self.simple_packet() + data = b"deadbeef" + self.simple_packet() item = self.data_to_item(data, 0x1234) assert isinstance(item.value, np.uint32) assert item.value == 0x12345678 @@ -1240,9 +1315,11 @@ def test_packet_too_big(self): 1, [ FLAVOUR.make_numpy_descriptor_from( - 0x2345, 'test_big_array', 'over-sized packet', zeros), - Item(0x2345, zeros.data) - ]) + 0x2345, "test_big_array", "over-sized packet", zeros + ), + Item(0x2345, zeros.data), + ], + ) data = packet1 + self.simple_packet() item = self.data_to_item(data, 0x1234) assert isinstance(item.value, np.uint32) diff --git a/tests/test_recv_asyncio.py b/tests/test_recv_asyncio.py index 3f04613c6..56652038a 100644 --- a/tests/test_recv_asyncio.py +++ b/tests/test_recv_asyncio.py @@ -48,8 +48,8 @@ class MyChunk(spead2.recv.Chunk): """Subclasses Chunk to carry extra metadata.""" def __init__(self, label, **kwargs): - kwargs.setdefault('data', bytearray(10)) - kwargs.setdefault('present', bytearray(1)) + kwargs.setdefault("data", bytearray(10)) + kwargs.setdefault("present", bytearray(1)) super().__init__(**kwargs) self.label = label diff --git a/tests/test_recv_chunk_stream.py b/tests/test_recv_chunk_stream.py index 9291d2e14..602e2fd24 100644 --- a/tests/test_recv_chunk_stream.py +++ b/tests/test_recv_chunk_stream.py @@ -56,10 +56,12 @@ def check_refcount(objlist): assert weak() is None -user_data_type = types.Record.make_c_struct([ - ('scale', types.int_), # A scale applied to heap_index - ('placed_heaps_index', types.uintp) # Index at which to update stats -]) +user_data_type = types.Record.make_c_struct( + [ + ("scale", types.int_), # A scale applied to heap_index + ("placed_heaps_index", types.uintp), # Index at which to update stats + ] +) @numba.cfunc(types.void(types.CPointer(chunk_place_data), types.uintp), nopython=True) @@ -76,7 +78,8 @@ def place_plain(data_ptr, data_size): @numba.cfunc( types.void(types.CPointer(chunk_place_data), types.uintp, types.CPointer(user_data_type)), - nopython=True) + nopython=True, +) def place_bind(data_ptr, data_size, user_data_ptr): # Takes a np.int_ in via user_data to scale the heap index. data = numba.carray(data_ptr, 1) @@ -90,14 +93,18 @@ def place_bind(data_ptr, data_size, user_data_ptr): heap_index = heap_cnt % HEAPS_PER_CHUNK data[0].heap_index = heap_index * user_data[0].scale data[0].heap_offset = heap_index * HEAP_PAYLOAD_SIZE - batch_stats = numba.carray(intp_to_voidptr(data[0].batch_stats), - user_data[0].placed_heaps_index + 1, dtype=np.uint64) + batch_stats = numba.carray( + intp_to_voidptr(data[0].batch_stats), + user_data[0].placed_heaps_index + 1, + dtype=np.uint64, + ) batch_stats[user_data[0].placed_heaps_index] += 1 @numba.cfunc( types.void(types.CPointer(chunk_place_data), types.uintp, types.CPointer(user_data_type)), - nopython=True) + nopython=True, +) def place_extra(data_ptr, data_size, user_data_ptr): # Writes the 'extra' SPEAD item (items[3]) to the extra output array data = numba.carray(data_ptr, 1) @@ -116,10 +123,11 @@ def place_extra(data_ptr, data_size, user_data_ptr): # ctypes doesn't distinguish equivalent integer types, so we have to # specify the signature explicitly. -place_plain_llc = scipy.LowLevelCallable(place_plain.ctypes, signature='void (void *, size_t)') +place_plain_llc = scipy.LowLevelCallable(place_plain.ctypes, signature="void (void *, size_t)") place_bind_llc = scipy.LowLevelCallable( - place_bind.ctypes, signature='void (void *, size_t, void *)') -place_extra_llc = scipy.LowLevelCallable(place_extra.ctypes, signature='void (void *, size_t)') + place_bind.ctypes, signature="void (void *, size_t, void *)" +) +place_extra_llc = scipy.LowLevelCallable(place_extra.ctypes, signature="void (void *, size_t)") class TestChunkStreamConfig: @@ -152,7 +160,7 @@ def test_set_place_non_capsule(self): recv.ChunkStreamConfig(place=(1,)) def test_set_place_bad_signature(self): - place = scipy.LowLevelCallable(place_plain.ctypes, signature='void (void)') + place = scipy.LowLevelCallable(place_plain.ctypes, signature="void (void)") # One might expect TypeError, but ValueError is what scipy uses for # invalid signatures. with pytest.raises(ValueError): @@ -237,7 +245,7 @@ def test_qsize(self, chunk_ringbuffer): chunk_ringbuffer.get() assert chunk_ringbuffer.empty() - @pytest.mark.parametrize('method', [recv.ChunkRingbuffer.get, recv.ChunkRingbuffer.get_nowait]) + @pytest.mark.parametrize("method", [recv.ChunkRingbuffer.get, recv.ChunkRingbuffer.get_nowait]) def test_stop(self, chunk_ringbuffer, method): chunk_ringbuffer.put(make_chunk()) chunk_ringbuffer.stop() @@ -326,7 +334,7 @@ def free_ring(self): recv.Chunk( present=np.zeros(HEAPS_PER_CHUNK, np.uint8), data=np.zeros(CHUNK_PAYLOAD_SIZE, np.uint8), - extra=np.zeros(HEAPS_PER_CHUNK, np.int64) + extra=np.zeros(HEAPS_PER_CHUNK, np.int64), ) ) return ring @@ -334,9 +342,9 @@ def free_ring(self): @pytest.fixture def item_group(self): ig = spead2.send.ItemGroup() - ig.add_item(0x1000, 'position', 'position in stream', (), format=[('u', 32)]) - ig.add_item(0x1001, 'payload', 'payload data', (HEAP_PAYLOAD_SIZE,), dtype=np.uint8) - ig.add_item(0x1002, 'extra', 'extra data to capture', (), format=[('u', 32)]) + ig.add_item(0x1000, "position", "position in stream", (), format=[("u", 32)]) + ig.add_item(0x1001, "payload", "payload data", (HEAP_PAYLOAD_SIZE,), dtype=np.uint8) + ig.add_item(0x1002, "extra", "extra data to capture", (), format=[("u", 32)]) return ig @pytest.fixture @@ -356,7 +364,7 @@ def recv_stream(self, request, data_ring, free_ring, queue): place=request.param, ), data_ring, - free_ring + free_ring, ) stream.add_inproc_reader(queue) yield stream @@ -378,25 +386,25 @@ def make_heap_payload(self, position): heap_payload[1::2] = range(HEAP_PAYLOAD_SIZE // 4) return heap_payload.view(np.uint8) - @pytest.mark.parametrize('send_end', [True, False]) + @pytest.mark.parametrize("send_end", [True, False]) def test_cleanup(self, send_stream, recv_stream, item_group, send_end): """Send some heaps and don't retrieve the chunks, making sure cleanup works.""" - send_stream.send_heap(item_group.get_heap(descriptors='all', data='none')) + send_stream.send_heap(item_group.get_heap(descriptors="all", data="none")) for i in range(1000): - item_group['position'].value = i - item_group['payload'].value = self.make_heap_payload(i) - send_stream.send_heap(item_group.get_heap(descriptors='none', data='all')) + item_group["position"].value = i + item_group["payload"].value = self.make_heap_payload(i) + send_stream.send_heap(item_group.get_heap(descriptors="none", data="all")) if send_end: send_stream.send_heap(item_group.get_end()) def send_heaps(self, send_stream, item_group, positions): """Send heaps with given numbers.""" - send_stream.send_heap(item_group.get_heap(descriptors='all', data='none')) + send_stream.send_heap(item_group.get_heap(descriptors="all", data="none")) for i in positions: - item_group['position'].value = i - item_group['payload'].value = self.make_heap_payload(i) - item_group['extra'].value = i ^ 0xBEEF - send_stream.send_heap(item_group.get_heap(descriptors='none', data='all')) + item_group["position"].value = i + item_group["payload"].value = self.make_heap_payload(i) + item_group["extra"].value = i ^ 0xBEEF + send_stream.send_heap(item_group.get_heap(descriptors="none", data="all")) send_stream.send_heap(item_group.get_end()) def check_chunk(self, chunk, expected_chunk_id, expected_present, extra=False): @@ -409,7 +417,7 @@ def check_chunk(self, chunk, expected_chunk_id, expected_present, extra=False): position = chunk.chunk_id * HEAPS_PER_CHUNK + i np.testing.assert_equal( chunk.data[i * HEAP_PAYLOAD_SIZE : (i + 1) * HEAP_PAYLOAD_SIZE], - self.make_heap_payload(position) + self.make_heap_payload(position), ) if extra: assert chunk.extra[i] == position ^ 0xBEEF @@ -427,15 +435,13 @@ def check_chunk_packets(self, chunk, expected_chunk_id, expected_present): end = (packet_index + 1) * PACKET_SIZE np.testing.assert_equal( chunk.data[i * PACKET_SIZE : (i + 1) * PACKET_SIZE], - self.make_heap_payload(heap_index)[start:end] + self.make_heap_payload(heap_index)[start:end], ) @pytest.mark.parametrize( - 'recv_stream, extra', - [ - (place_plain_llc, False), - (place_extra_llc, True) - ], indirect=['recv_stream'] + "recv_stream, extra", + [(place_plain_llc, False), (place_extra_llc, True)], + indirect=["recv_stream"], ) def test_basic(self, send_stream, recv_stream, item_group, extra): n_heaps = 103 @@ -491,12 +497,12 @@ def test_heap_too_old(self, send_stream, recv_stream, item_group): self.check_chunk(chunk, i, expected_present) seen += 1 recv_stream.add_free_chunk(chunk) - assert seen == 5 # Will see chunk 0 with no heaps, but won't see it again + assert seen == 5 # Will see chunk 0 with no heaps, but won't see it again recv_stream.stop() # Ensure that stats are brought up to date assert recv_stream.stats["too_old_heaps"] == 1 assert recv_stream.stats["rejected_heaps"] == 2 # Descriptors and stop heap - @pytest.mark.parametrize('recv_stream', [place_extra_llc], indirect=True) + @pytest.mark.parametrize("recv_stream", [place_extra_llc], indirect=True) def test_extra(self, send_stream, recv_stream, item_group): """Test writing extra data about each heap.""" n_heaps = 103 @@ -517,12 +523,10 @@ def test_shared_ringbuffer(self, send_stream, recv_stream, item_group): spead2.ThreadPool(), spead2.recv.StreamConfig(stream_id=1), spead2.recv.ChunkStreamConfig( - items=[0x1000, spead2.HEAP_LENGTH_ID], - max_chunks=4, - place=place_plain_llc + items=[0x1000, spead2.HEAP_LENGTH_ID], max_chunks=4, place=place_plain_llc ), recv_stream.data_ringbuffer, - recv_stream.free_ringbuffer + recv_stream.free_ringbuffer, ) queue2 = spead2.InprocQueue() recv_stream2.add_inproc_reader(queue2) @@ -551,7 +555,7 @@ def test_missing_place_callback(self, data_ring, free_ring): spead2.recv.StreamConfig(), spead2.recv.ChunkStreamConfig(items=[0x1000, spead2.HEAP_LENGTH_ID]), data_ring, - free_ring + free_ring, ) def make_packet(self, position, start, end): @@ -570,17 +574,17 @@ def make_packet(self, position, start, end): heap_payload = self.make_heap_payload(position) parts = [ # Magic, version, item ID bytes, heap address bytes, flags, number of items - struct.pack('>BBBBHH', 0x53, 4, 2, 6, 0, 6), + struct.pack(">BBBBHH", 0x53, 4, 2, 6, 0, 6), # Item ID (and immediate flag), item value/offset - struct.pack('>HxxI', 0x8000 | spead2.HEAP_CNT_ID, position), - struct.pack('>HxxI', 0x8000 | spead2.PAYLOAD_OFFSET_ID, start), - struct.pack('>HxxI', 0x8000 | spead2.PAYLOAD_LENGTH_ID, end - start), - struct.pack('>HxxI', 0x8000 | spead2.HEAP_LENGTH_ID, HEAP_PAYLOAD_SIZE), - struct.pack('>HxxI', 0x8000 | 0x1000, position), - struct.pack('>HxxI', 0x1001, 0), - heap_payload[start:end].tobytes() + struct.pack(">HxxI", 0x8000 | spead2.HEAP_CNT_ID, position), + struct.pack(">HxxI", 0x8000 | spead2.PAYLOAD_OFFSET_ID, start), + struct.pack(">HxxI", 0x8000 | spead2.PAYLOAD_LENGTH_ID, end - start), + struct.pack(">HxxI", 0x8000 | spead2.HEAP_LENGTH_ID, HEAP_PAYLOAD_SIZE), + struct.pack(">HxxI", 0x8000 | 0x1000, position), + struct.pack(">HxxI", 0x1001, 0), + heap_payload[start:end].tobytes(), ] - return b''.join(parts) + return b"".join(parts) def test_packet_too_old(self, recv_stream, queue): """Test a packet that adds to an existing heap whose chunk was already aged out.""" @@ -616,7 +620,7 @@ def test_packet_presence(self, data_ring, queue): free_ring.put( recv.Chunk( present=np.zeros(HEAPS_PER_CHUNK * PACKETS_PER_HEAP, np.uint8), - data=np.zeros(CHUNK_PAYLOAD_SIZE, np.uint8) + data=np.zeros(CHUNK_PAYLOAD_SIZE, np.uint8), ) ) @@ -629,7 +633,8 @@ def test_packet_presence(self, data_ring, queue): place_bind_llc = scipy.LowLevelCallable( place_bind.ctypes, user_data=user_data.ctypes.data_as(ctypes.c_void_p), - signature='void (void *, size_t, void *)') + signature="void (void *, size_t, void *)", + ) stream = spead2.recv.ChunkRingStream( spead2.ThreadPool(), stream_config, @@ -639,7 +644,7 @@ def test_packet_presence(self, data_ring, queue): place=place_bind_llc, ).enable_packet_presence(PACKET_SIZE), data_ring, - free_ring + free_ring, ) stream.add_inproc_reader(queue) @@ -650,9 +655,13 @@ def test_packet_presence(self, data_ring, queue): chunks = list(stream.data_ringbuffer) assert len(chunks) == 2 self.check_chunk_packets( - chunks[0], 0, - np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], np.uint8)) + chunks[0], + 0, + np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], np.uint8), + ) self.check_chunk_packets( - chunks[1], 1, - np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], np.uint8)) + chunks[1], + 1, + np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], np.uint8), + ) assert stream.stats["placed_heaps"] == 2 diff --git a/tests/test_recv_chunk_stream_group.py b/tests/test_recv_chunk_stream_group.py index af574d469..765351414 100644 --- a/tests/test_recv_chunk_stream_group.py +++ b/tests/test_recv_chunk_stream_group.py @@ -45,7 +45,8 @@ @numba.cfunc( types.void(types.CPointer(chunk_place_data), types.uintp, types.CPointer(types.int64)), - nopython=True) + nopython=True, +) def place_bias(data_ptr, data_size, user_data_ptr): # Biases the chunk_id by the user parameter data = numba.carray(data_ptr, 1) @@ -60,7 +61,8 @@ def place_bias(data_ptr, data_size, user_data_ptr): place_bias_llc = scipy.LowLevelCallable( - place_bias.ctypes, signature='void (void *, size_t, void *)') + place_bias.ctypes, signature="void (void *, size_t, void *)" +) class TestChunkStreamGroupConfig: @@ -94,7 +96,7 @@ def make_group(self, n_streams): group = spead2.recv.ChunkStreamRingGroup( spead2.recv.ChunkStreamGroupConfig(), spead2.recv.ChunkRingbuffer(4), - spead2.recv.ChunkRingbuffer(4) + spead2.recv.ChunkRingbuffer(4), ) streams = [] for _ in range(n_streams): @@ -102,7 +104,7 @@ def make_group(self, n_streams): group.emplace_back( spead2.ThreadPool(), spead2.recv.StreamConfig(), - spead2.recv.ChunkStreamConfig(place=place_plain_llc) + spead2.recv.ChunkStreamConfig(place=place_plain_llc), ) ) return group, streams @@ -198,7 +200,7 @@ def free_ring(self): ring.put( recv.Chunk( present=np.zeros(HEAPS_PER_CHUNK, np.uint8), - data=np.zeros(CHUNK_PAYLOAD_SIZE, np.uint8) + data=np.zeros(CHUNK_PAYLOAD_SIZE, np.uint8), ) ) return ring @@ -224,7 +226,8 @@ def group(self, eviction_mode, data_ring, free_ring, queues, chunk_id_bias): place_llc = scipy.LowLevelCallable( place_bias.ctypes, user_data=chunk_id_bias.ctypes.data_as(ctypes.c_void_p), - signature='void (void *, size_t, void *)') + signature="void (void *, size_t, void *)", + ) chunk_stream_config = spead2.recv.ChunkStreamConfig( items=[0x1000, spead2.HEAP_LENGTH_ID], max_chunks=4, @@ -232,9 +235,7 @@ def group(self, eviction_mode, data_ring, free_ring, queues, chunk_id_bias): ) for queue in queues: group.emplace_back( - spead2.ThreadPool(), - config=config, - chunk_stream_config=chunk_stream_config + spead2.ThreadPool(), config=config, chunk_stream_config=chunk_stream_config ) for stream, queue in zip(group, queues): stream.add_inproc_reader(queue) @@ -251,18 +252,18 @@ def _send_data(self, send_stream, data, eviction_mode, heaps=None): To send only a subset of heaps (or to send out of order), pass the indices to skip in `heaps`. """ - lossy = (eviction_mode == recv.ChunkStreamGroupConfig.EvictionMode.LOSSY) + lossy = eviction_mode == recv.ChunkStreamGroupConfig.EvictionMode.LOSSY data_by_heap = data.reshape(-1, HEAP_PAYLOAD_SIZE) ig = spead2.send.ItemGroup() - ig.add_item(0x1000, 'position', 'position in stream', (), format=[('u', 32)]) - ig.add_item(0x1001, 'payload', 'payload data', (HEAP_PAYLOAD_SIZE,), dtype=np.uint8) + ig.add_item(0x1000, "position", "position in stream", (), format=[("u", 32)]) + ig.add_item(0x1001, "payload", "payload data", (HEAP_PAYLOAD_SIZE,), dtype=np.uint8) # In lossy mode the behaviour is inherently non-deterministic. # We just feed the data in slowly enough that we expect heaps provided # before a sleep to be processed before those after the sleep. for i in heaps: - ig['position'].value = i - ig['payload'].value = data_by_heap[i] - heap = ig.get_heap(data='all', descriptors='none') + ig["position"].value = i + ig["payload"].value = data_by_heap[i] + heap = ig.get_heap(data="all", descriptors="none") send_stream.send_heap(heap, substream_index=i % STREAMS) if lossy: time.sleep(0.001) @@ -333,7 +334,8 @@ def test_half_missing_stream(self, group, send_stream): """Skip sending data to one of the streams after a certain point.""" chunks = 20 heaps = [ - i for i in range(chunks * HEAPS_PER_CHUNK) + i + for i in range(chunks * HEAPS_PER_CHUNK) if i < 7 * HEAPS_PER_CHUNK or i % STREAMS != 2 ] self._test_simple(group, send_stream, chunks, heaps) diff --git a/tests/test_send.py b/tests/test_send.py index e3b36105b..6f2bcd834 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -36,15 +36,15 @@ def hexlify(data): chunks = [] for i in range(0, len(data), 8): part = data[i : min(i + 8, len(data))] - chunks.append(b':'.join([binascii.hexlify(part[i : i + 1]) for i in range(len(part))])) - return b'\n'.join(chunks).decode('ascii') + chunks.append(b":".join([binascii.hexlify(part[i : i + 1]) for i in range(len(part))])) + return b"\n".join(chunks).decode("ascii") def encode_be(size, value): """Encodes `value` as big-endian in `size` bytes""" assert size <= 8 - packed = struct.pack('>Q', value) - return packed[8 - size:] + packed = struct.pack(">Q", value) + return packed[8 - size :] class Flavour(spead2.Flavour): @@ -55,37 +55,38 @@ def make_header(self, num_items): address_size = self.heap_address_bits // 8 item_size = 8 - address_size return struct.pack( - '>Q', - 0x5304000000000000 | (address_size << 32) | (item_size << 40) | num_items) + ">Q", 0x5304000000000000 | (address_size << 32) | (item_size << 40) | num_items + ) def make_immediate(self, item_id, value): - return struct.pack('>Q', 2**63 | (item_id << self.heap_address_bits) | value) + return struct.pack(">Q", 2**63 | (item_id << self.heap_address_bits) | value) def make_address(self, item_id, address): - return struct.pack('>Q', (item_id << self.heap_address_bits) | address) + return struct.pack(">Q", (item_id << self.heap_address_bits) | address) def make_shape(self, shape): # TODO: extend for bug_compat flavours - assert not (self.bug_compat & - (spead2.BUG_COMPAT_DESCRIPTOR_WIDTHS | spead2.BUG_COMPAT_SHAPE_BIT_1)) + assert not ( + self.bug_compat & (spead2.BUG_COMPAT_DESCRIPTOR_WIDTHS | spead2.BUG_COMPAT_SHAPE_BIT_1) + ) ans = [] for size in shape: if size < 0: - ans.append(struct.pack('B', 1)) + ans.append(struct.pack("B", 1)) ans.append(encode_be(self.heap_address_bits // 8, 0)) else: - ans.append(struct.pack('B', 0)) + ans.append(struct.pack("B", 0)) ans.append(encode_be(self.heap_address_bits // 8, size)) - return b''.join(ans) + return b"".join(ans) def make_format(self, format): # TODO: extend for bug_compat flavours assert not self.bug_compat & spead2.BUG_COMPAT_DESCRIPTOR_WIDTHS ans = [] - for (code, length) in format: - ans.append(struct.pack('B', ord(code))) + for code, length in format: + ans.append(struct.pack("B", ord(code))) ans.append(encode_be(8 - self.heap_address_bits // 8, length)) - return b''.join(ans) + return b"".join(ans) def items_to_bytes(self, items, descriptors=None, max_packet_size=1500, repeat_pointers=False): if descriptors is None: @@ -117,28 +118,32 @@ def setup_method(self): def test_empty(self): """An empty heap must still generate a packet""" expected = [ - b''.join([ - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), - self.flavour.make_address(spead2.NULL_ID, 0), - struct.pack('B', 0)]) + b"".join( + [ + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), + self.flavour.make_address(spead2.NULL_ID, 0), + struct.pack("B", 0), + ] + ) ] packet = self.flavour.items_to_bytes([]) assert hexlify(packet) == hexlify(expected) def test_lifetime(self): """Heap must hold references to item values""" - item = spead2.Item(id=0x2345, name='name', description='description', - shape=(2, 3), dtype=np.uint16) + item = spead2.Item( + id=0x2345, name="name", description="description", shape=(2, 3), dtype=np.uint16 + ) item.value = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16) weak = weakref.ref(item.value) heap = send.Heap(self.flavour) heap.add_item(item) del item - packets = list(send.PacketGenerator(heap, 0x123456, 1472)) # noqa: F841 + packets = list(send.PacketGenerator(heap, 0x123456, 1472)) # noqa: F841 assert weak() is not None del heap # pypy needs multiple gc passes to wind it all up @@ -148,53 +153,58 @@ def test_lifetime(self): def make_descriptor_numpy(self, id, name, description, shape, dtype_str, fortran_order): payload_fields = [ - b'name', - b'description', - b'', + b"name", + b"description", + b"", self.flavour.make_shape(shape), "{{'descr': {!r}, 'fortran_order': {!r}, 'shape': {!r}}}".format( - str(dtype_str), bool(fortran_order), tuple(shape)).encode() + str(dtype_str), bool(fortran_order), tuple(shape) + ).encode(), ] - payload = b''.join(payload_fields) + payload = b"".join(payload_fields) offsets = offset_generator(payload_fields) - descriptor = b''.join([ - self.flavour.make_header(10), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, len(payload)), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, len(payload)), - self.flavour.make_immediate(spead2.DESCRIPTOR_ID_ID, id), - self.flavour.make_address(spead2.DESCRIPTOR_NAME_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_DESCRIPTION_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_FORMAT_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_SHAPE_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_DTYPE_ID, next(offsets)), - payload - ]) + descriptor = b"".join( + [ + self.flavour.make_header(10), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, len(payload)), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, len(payload)), + self.flavour.make_immediate(spead2.DESCRIPTOR_ID_ID, id), + self.flavour.make_address(spead2.DESCRIPTOR_NAME_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_DESCRIPTION_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_FORMAT_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_SHAPE_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_DTYPE_ID, next(offsets)), + payload, + ] + ) return descriptor def make_descriptor_fallback(self, id, name, description, shape, format): payload_fields = [ - b'name', - b'description', + b"name", + b"description", self.flavour.make_format(format), self.flavour.make_shape(shape), ] - payload = b''.join(payload_fields) + payload = b"".join(payload_fields) offsets = offset_generator(payload_fields) - descriptor = b''.join([ - self.flavour.make_header(9), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, len(payload)), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, len(payload)), - self.flavour.make_immediate(spead2.DESCRIPTOR_ID_ID, id), - self.flavour.make_address(spead2.DESCRIPTOR_NAME_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_DESCRIPTION_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_FORMAT_ID, next(offsets)), - self.flavour.make_address(spead2.DESCRIPTOR_SHAPE_ID, next(offsets)), - payload - ]) + descriptor = b"".join( + [ + self.flavour.make_header(9), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, len(payload)), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, len(payload)), + self.flavour.make_immediate(spead2.DESCRIPTOR_ID_ID, id), + self.flavour.make_address(spead2.DESCRIPTOR_NAME_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_DESCRIPTION_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_FORMAT_ID, next(offsets)), + self.flavour.make_address(spead2.DESCRIPTOR_SHAPE_ID, next(offsets)), + payload, + ] + ) return descriptor def test_numpy_simple(self): @@ -203,26 +213,29 @@ def test_numpy_simple(self): shape = (2, 3) data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16) payload_fields = [ - self.make_descriptor_numpy(id, 'name', 'description', shape, '2B', 4, 5) + payload = struct.pack(">2B", 4, 5) expected = [ - b''.join([ - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, len(payload)), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, len(payload)), - self.flavour.make_address(id, 0), - payload - ]) + b"".join( + [ + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, len(payload)), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, len(payload)), + self.flavour.make_address(id, 0), + payload, + ] + ) ] - item = spead2.Item(id=id, name='name', description='description', - shape=shape, format=[('u', 8)]) + item = spead2.Item( + id=id, name="name", description="description", shape=shape, format=[("u", 8)] + ) item.value = data packet = self.flavour.items_to_bytes([item], []) assert hexlify(packet) == hexlify(expected) @@ -359,28 +387,30 @@ def test_small_variable(self): def test_numpy_zero_length(self): """A zero-length numpy type raises :exc:`ValueError`""" with pytest.raises(ValueError): - spead2.Item(id=0x2345, name='name', description='description', - shape=(), dtype=np.str_) + spead2.Item(id=0x2345, name="name", description="description", shape=(), dtype=np.str_) def test_fallback_zero_length(self): """A zero-length type raises :exc:`ValueError`""" with pytest.raises(ValueError): - spead2.Item(id=0x2345, name='name', description='description', - shape=(), format=[('u', 0)]) + spead2.Item( + id=0x2345, name="name", description="description", shape=(), format=[("u", 0)] + ) def test_start(self): """Tests sending a start-of-stream marker.""" expected = [ - b''.join([ - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START), - self.flavour.make_address(spead2.NULL_ID, 0), - struct.pack('B', 0) - ]) + b"".join( + [ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START), + self.flavour.make_address(spead2.NULL_ID, 0), + struct.pack("B", 0), + ] + ) ] heap = send.Heap(self.flavour) heap.add_start() @@ -390,16 +420,18 @@ def test_start(self): def test_end(self): """Tests sending an end-of-stream marker.""" expected = [ - b''.join([ - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_STOP), - self.flavour.make_address(spead2.NULL_ID, 0), - struct.pack('B', 0) - ]) + b"".join( + [ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_STOP), + self.flavour.make_address(spead2.NULL_ID, 0), + struct.pack("B", 0), + ] + ) ] heap = send.Heap(self.flavour) heap.add_end() @@ -410,35 +442,51 @@ def test_replicate_pointers(self): """Tests sending a heap with replicate_pointers set to true""" id = 0x2345 data = np.arange(32, dtype=np.uint8) - item1 = spead2.Item(id=id, name='item1', description='addressed item', - shape=data.shape, dtype=data.dtype, value=data) - item2 = spead2.Item(id=id + 1, name='item2', description='inline item', - shape=(), format=[('u', self.flavour.heap_address_bits)], - value=0xdeadbeef) + item1 = spead2.Item( + id=id, + name="item1", + description="addressed item", + shape=data.shape, + dtype=data.dtype, + value=data, + ) + item2 = spead2.Item( + id=id + 1, + name="item2", + description="inline item", + shape=(), + format=[("u", self.flavour.heap_address_bits)], + value=0xDEADBEEF, + ) expected = [ - b''.join([ - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 32), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 16), - self.flavour.make_address(id, 0), - self.flavour.make_immediate(id + 1, 0xdeadbeef), - data.tobytes()[0:16] - ]), - b''.join([ - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 32), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 16), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 16), - self.flavour.make_address(id, 0), - self.flavour.make_immediate(id + 1, 0xdeadbeef), - data.tobytes()[16:32] - ]) + b"".join( + [ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 32), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 16), + self.flavour.make_address(id, 0), + self.flavour.make_immediate(id + 1, 0xDEADBEEF), + data.tobytes()[0:16], + ] + ), + b"".join( + [ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 32), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 16), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 16), + self.flavour.make_address(id, 0), + self.flavour.make_immediate(id + 1, 0xDEADBEEF), + data.tobytes()[16:32], + ] + ), ] - packets = self.flavour.items_to_bytes([item1, item2], [], max_packet_size=72, - repeat_pointers=True) + packets = self.flavour.items_to_bytes( + [item1, item2], [], max_packet_size=72, repeat_pointers=True + ) assert hexlify(packets) == hexlify(expected) def test_zero_copy(self): @@ -448,30 +496,43 @@ def test_zero_copy(self): """ id = 0x2345 data = np.arange(8, dtype=np.uint8) - imm = np.zeros((), dtype='>u8') - item1 = spead2.Item(id=id, name='item1', description='addressed item', - shape=data.shape, dtype=data.dtype, value=data) - item2 = spead2.Item(id=id + 1, name='item2', description='inline item', - shape=(), format=[('u', self.flavour.heap_address_bits)], - value=imm) + imm = np.zeros((), dtype=">u8") + item1 = spead2.Item( + id=id, + name="item1", + description="addressed item", + shape=data.shape, + dtype=data.dtype, + value=data, + ) + item2 = spead2.Item( + id=id + 1, + name="item2", + description="inline item", + shape=(), + format=[("u", self.flavour.heap_address_bits)], + value=imm, + ) heap = spead2.send.Heap(self.flavour) heap.add_item(item1) heap.add_item(item2) # Now change the values after they've been added to the heap. data *= 2 - imm[()] = 0xdeadbeef + imm[()] = 0xDEADBEEF expected = [ - b''.join([ - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 8), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 8), - self.flavour.make_address(id, 0), - self.flavour.make_immediate(id + 1, 0xdeadbeef), - data.tobytes() - ]) + b"".join( + [ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 8), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 8), + self.flavour.make_address(id, 0), + self.flavour.make_immediate(id + 1, 0xDEADBEEF), + data.tobytes(), + ] + ) ] gen = send.PacketGenerator(heap, 0x123456, 1500) @@ -529,12 +590,12 @@ def setup_method(self): # A slow stream, so that we can test overflowing the queue self.flavour = Flavour(4, 64, 48, 0) self.stream = send.BytesStream( - spead2.ThreadPool(), - send.StreamConfig(rate=1e6, max_heaps=2)) + spead2.ThreadPool(), send.StreamConfig(rate=1e6, max_heaps=2) + ) # A large heap ig = send.ItemGroup(flavour=self.flavour) - ig.add_item(0x1000, 'test', 'A large item', shape=(256 * 1024,), dtype=np.uint8) - ig['test'].value = np.zeros((256 * 1024,), np.uint8) + ig.add_item(0x1000, "test", "A large item", shape=(256 * 1024,), dtype=np.uint8) + ig["test"].value = np.zeros((256 * 1024,), np.uint8) self.heap = ig.get_heap() self.threads = [] @@ -560,8 +621,11 @@ def test_send_error(self): # Create a stream with a packet size that is bigger than the likely # MTU. It should cause an error. stream = send.UdpStream( - spead2.ThreadPool(), [("localhost", 8888)], - send.StreamConfig(max_packet_size=100000), buffer_size=0) + spead2.ThreadPool(), + [("localhost", 8888)], + send.StreamConfig(max_packet_size=100000), + buffer_size=0, + ) with pytest.raises(IOError): stream.send_heap(self.heap) @@ -575,26 +639,27 @@ def test_send_explicit_cnt(self): self.stream.send_heap(ig.get_start()) self.stream.set_cnt_sequence(0x1111111111, 0x1234512345) self.stream.send_heap(ig.get_start()) - self.stream.send_heap(ig.get_start(), 0x9876543210ab) + self.stream.send_heap(ig.get_start(), 0x9876543210AB) self.stream.send_heap(ig.get_start()) self.stream.set_cnt_sequence(2**48 - 1, 1) self.stream.send_heap(ig.get_start()) self.stream.send_heap(ig.get_start()) - expected_cnts = [1, 0x1111111111, 0x9876543210ab, 0x2345623456, - 2**48 - 1, 0] - expected = b'' + expected_cnts = [1, 0x1111111111, 0x9876543210AB, 0x2345623456, 2**48 - 1, 0] + expected = b"" for cnt in expected_cnts: - expected = b''.join([ - expected, - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, cnt), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START), - self.flavour.make_address(spead2.NULL_ID, 0), - struct.pack('B', 0) - ]) + expected = b"".join( + [ + expected, + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, cnt), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START), + self.flavour.make_address(spead2.NULL_ID, 0), + struct.pack("B", 0), + ] + ) assert hexlify(self.stream.getvalue()) == hexlify(expected) def test_invalid_cnt(self): @@ -615,26 +680,29 @@ def test_send_heaps_unwind(self): self.stream.send_heaps( [ send.HeapReference(ig.get_start(), substream_index=0), - send.HeapReference(ig.get_start(), substream_index=100) + send.HeapReference(ig.get_start(), substream_index=100), ], - send.GroupMode.ROUND_ROBIN) + send.GroupMode.ROUND_ROBIN, + ) self.stream.send_heap(ig.get_end(), substream_index=0) expected_cnts = [1, 2] expected_ctrl = [spead2.CTRL_STREAM_START, spead2.CTRL_STREAM_STOP] - expected = b'' + expected = b"" for cnt, ctrl in zip(expected_cnts, expected_ctrl): - expected = b''.join([ - expected, - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, cnt), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.STREAM_CTRL_ID, ctrl), - self.flavour.make_address(spead2.NULL_ID, 0), - struct.pack('B', 0) - ]) + expected = b"".join( + [ + expected, + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, cnt), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.STREAM_CTRL_ID, ctrl), + self.flavour.make_address(spead2.NULL_ID, 0), + struct.pack("B", 0), + ] + ) assert hexlify(self.stream.getvalue()) == hexlify(expected) def test_round_robin(self): @@ -642,61 +710,65 @@ def test_round_robin(self): igs = [send.ItemGroup(flavour=self.flavour) for i in range(n)] for i in range(n): data = np.arange(i, i + i * 64 + 8).astype(np.uint8) - igs[i].add_item(0x1000 + i, 'test', 'Test item', - shape=data.shape, dtype=data.dtype, value=data) + igs[i].add_item( + 0x1000 + i, "test", "Test item", shape=data.shape, dtype=data.dtype, value=data + ) self.stream = send.BytesStream( - spead2.ThreadPool(), - send.StreamConfig(max_heaps=n, max_packet_size=88)) + spead2.ThreadPool(), send.StreamConfig(max_heaps=n, max_packet_size=88) + ) self.stream.send_heaps( - [send.HeapReference(ig.get_heap(descriptors='none', data='all')) for ig in igs], - send.GroupMode.ROUND_ROBIN) - expected = b''.join([ - # Heap 0, packet 0 (only packet) - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 8), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 8), - self.flavour.make_address(0x1000, 0), - igs[0]['test'].value.tobytes(), - # Heap 1, packet 0 - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), - self.flavour.make_address(0x1001, 0), - igs[1]['test'].value.tobytes()[0 : 40], - # Heap 2, packet 0 - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), - self.flavour.make_address(0x1002, 0), - igs[2]['test'].value.tobytes()[0 : 40], - # Heap 1, packet 1 - self.flavour.make_header(4), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 32), - igs[1]['test'].value.tobytes()[40 : 72], - # Heap 2, packet 1 - self.flavour.make_header(4), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), - igs[2]['test'].value.tobytes()[40 : 88], - # Heap 2, packet 2 - self.flavour.make_header(4), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 88), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), - igs[2]['test'].value.tobytes()[88 : 136] - ]) + [send.HeapReference(ig.get_heap(descriptors="none", data="all")) for ig in igs], + send.GroupMode.ROUND_ROBIN, + ) + expected = b"".join( + [ + # Heap 0, packet 0 (only packet) + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 8), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 8), + self.flavour.make_address(0x1000, 0), + igs[0]["test"].value.tobytes(), + # Heap 1, packet 0 + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), + self.flavour.make_address(0x1001, 0), + igs[1]["test"].value.tobytes()[0:40], + # Heap 2, packet 0 + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), + self.flavour.make_address(0x1002, 0), + igs[2]["test"].value.tobytes()[0:40], + # Heap 1, packet 1 + self.flavour.make_header(4), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 32), + igs[1]["test"].value.tobytes()[40:72], + # Heap 2, packet 1 + self.flavour.make_header(4), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), + igs[2]["test"].value.tobytes()[40:88], + # Heap 2, packet 2 + self.flavour.make_header(4), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 88), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), + igs[2]["test"].value.tobytes()[88:136], + ] + ) assert hexlify(self.stream.getvalue()) == hexlify(expected) def test_serial(self): @@ -704,61 +776,65 @@ def test_serial(self): igs = [send.ItemGroup(flavour=self.flavour) for i in range(n)] for i in range(n): data = np.arange(i, i + i * 64 + 8).astype(np.uint8) - igs[i].add_item(0x1000 + i, 'test', 'Test item', - shape=data.shape, dtype=data.dtype, value=data) + igs[i].add_item( + 0x1000 + i, "test", "Test item", shape=data.shape, dtype=data.dtype, value=data + ) self.stream = send.BytesStream( - spead2.ThreadPool(), - send.StreamConfig(max_heaps=n, max_packet_size=88)) + spead2.ThreadPool(), send.StreamConfig(max_heaps=n, max_packet_size=88) + ) self.stream.send_heaps( - [send.HeapReference(ig.get_heap(descriptors='none', data='all')) for ig in igs], - send.GroupMode.SERIAL) - expected = b''.join([ - # Heap 0, packet 0 (only packet) - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 8), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 8), - self.flavour.make_address(0x1000, 0), - igs[0]['test'].value.tobytes(), - # Heap 1, packet 0 - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), - self.flavour.make_address(0x1001, 0), - igs[1]['test'].value.tobytes()[0 : 40], - # Heap 1, packet 1 - self.flavour.make_header(4), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 32), - igs[1]['test'].value.tobytes()[40 : 72], - # Heap 2, packet 0 - self.flavour.make_header(5), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), - self.flavour.make_address(0x1002, 0), - igs[2]['test'].value.tobytes()[0 : 40], - # Heap 2, packet 1 - self.flavour.make_header(4), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), - igs[2]['test'].value.tobytes()[40 : 88], - # Heap 2, packet 2 - self.flavour.make_header(4), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 88), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), - igs[2]['test'].value.tobytes()[88 : 136] - ]) + [send.HeapReference(ig.get_heap(descriptors="none", data="all")) for ig in igs], + send.GroupMode.SERIAL, + ) + expected = b"".join( + [ + # Heap 0, packet 0 (only packet) + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 1), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 8), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 8), + self.flavour.make_address(0x1000, 0), + igs[0]["test"].value.tobytes(), + # Heap 1, packet 0 + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), + self.flavour.make_address(0x1001, 0), + igs[1]["test"].value.tobytes()[0:40], + # Heap 1, packet 1 + self.flavour.make_header(4), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 2), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 72), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 32), + igs[1]["test"].value.tobytes()[40:72], + # Heap 2, packet 0 + self.flavour.make_header(5), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 40), + self.flavour.make_address(0x1002, 0), + igs[2]["test"].value.tobytes()[0:40], + # Heap 2, packet 1 + self.flavour.make_header(4), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 40), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), + igs[2]["test"].value.tobytes()[40:88], + # Heap 2, packet 2 + self.flavour.make_header(4), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 3), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 136), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 88), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 48), + igs[2]["test"].value.tobytes()[88:136], + ] + ) assert hexlify(self.stream.getvalue()) == hexlify(expected) def test_heap_reference_list_type_error(self): @@ -769,7 +845,7 @@ def test_heap_reference_list_type_error(self): class TestTcpStream: def test_failed_connect(self): with pytest.raises(IOError): - send.TcpStream(spead2.ThreadPool(), [('127.0.0.1', 8887)]) + send.TcpStream(spead2.ThreadPool(), [("127.0.0.1", 8887)]) class TestInprocStream: @@ -785,12 +861,12 @@ def test_stopped_queue(self): self.stream.send_heap(ig.get_end()) -@pytest.mark.skipif(not hasattr(spead2, 'IbvContext'), reason='IBV support not compiled in') +@pytest.mark.skipif(not hasattr(spead2, "IbvContext"), reason="IBV support not compiled in") class TestUdpIbvConfig: def test_default_construct(self): config = send.UdpIbvConfig() assert config.endpoints == [] - assert config.interface_address == '' + assert config.interface_address == "" assert config.buffer_size == send.UdpIbvConfig.DEFAULT_BUFFER_SIZE assert config.ttl == 1 assert config.comp_vector == 0 @@ -801,15 +877,16 @@ def test_kwargs_construct(self): data1 = bytearray(10) data2 = bytearray(10) config = send.UdpIbvConfig( - endpoints=[('hello', 1234), ('goodbye', 2345)], - interface_address='1.2.3.4', + endpoints=[("hello", 1234), ("goodbye", 2345)], + interface_address="1.2.3.4", buffer_size=100, ttl=2, comp_vector=-1, max_poll=1000, - memory_regions=[data1, data2]) - assert config.endpoints == [('hello', 1234), ('goodbye', 2345)] - assert config.interface_address == '1.2.3.4' + memory_regions=[data1, data2], + ) + assert config.endpoints == [("hello", 1234), ("goodbye", 2345)] + assert config.interface_address == "1.2.3.4" assert config.buffer_size == 100 assert config.ttl == 2 assert config.comp_vector == -1 @@ -832,10 +909,9 @@ def test_empty_memory_region(self): data = bytearray(0) config = send.StreamConfig() udp_ibv_config = send.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)], - interface_address='10.0.0.1', - memory_regions=[data]) - with pytest.raises(ValueError, match='memory region must have non-zero size'): + endpoints=[("239.255.88.88", 8888)], interface_address="10.0.0.1", memory_regions=[data] + ) + with pytest.raises(ValueError, match="memory region must have non-zero size"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_overlapping_memory_regions(self): @@ -844,60 +920,59 @@ def test_overlapping_memory_regions(self): part2 = data[5:] config = send.StreamConfig() udp_ibv_config = send.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)], - interface_address='10.0.0.1', - memory_regions=[part1, part2]) - with pytest.raises(ValueError, match='memory regions overlap'): + endpoints=[("239.255.88.88", 8888)], + interface_address="10.0.0.1", + memory_regions=[part1, part2], + ) + with pytest.raises(ValueError, match="memory regions overlap"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_no_endpoints(self): config = send.StreamConfig() - udp_ibv_config = send.UdpIbvConfig(interface_address='10.0.0.1') - with pytest.raises(ValueError, match='endpoints is empty'): + udp_ibv_config = send.UdpIbvConfig(interface_address="10.0.0.1") + with pytest.raises(ValueError, match="endpoints is empty"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_ipv6_endpoints(self): config = send.StreamConfig() - udp_ibv_config = send.UdpIbvConfig( - endpoints=[('::1', 8888)], - interface_address='10.0.0.1') - with pytest.raises(ValueError, match='endpoint is not an IPv4 multicast address'): + udp_ibv_config = send.UdpIbvConfig(endpoints=[("::1", 8888)], interface_address="10.0.0.1") + with pytest.raises(ValueError, match="endpoint is not an IPv4 multicast address"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_unicast_endpoints(self): config = send.StreamConfig() udp_ibv_config = send.UdpIbvConfig( - endpoints=[('10.0.0.1', 8888)], - interface_address='10.0.0.1') - with pytest.raises(ValueError, match='endpoint is not an IPv4 multicast address'): + endpoints=[("10.0.0.1", 8888)], interface_address="10.0.0.1" + ) + with pytest.raises(ValueError, match="endpoint is not an IPv4 multicast address"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_no_interface_address(self): config = send.StreamConfig() - udp_ibv_config = send.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)]) - with pytest.raises(ValueError, match='interface address'): + udp_ibv_config = send.UdpIbvConfig(endpoints=[("239.255.88.88", 8888)]) + with pytest.raises(ValueError, match="interface address"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_bad_interface_address(self): config = send.StreamConfig() udp_ibv_config = send.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)], - interface_address='this is not an interface address') - with pytest.raises(RuntimeError, match='Host not found'): + endpoints=[("239.255.88.88", 8888)], + interface_address="this is not an interface address", + ) + with pytest.raises(RuntimeError, match="Host not found"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) def test_ipv6_interface_address(self): config = send.StreamConfig() udp_ibv_config = send.UdpIbvConfig( - endpoints=[('239.255.88.88', 8888)], - interface_address='::1') - with pytest.raises(ValueError, match='interface address'): + endpoints=[("239.255.88.88", 8888)], interface_address="::1" + ) + with pytest.raises(ValueError, match="interface address"): send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config) @pytest.mark.skipif( - platform.python_implementation() == 'PyPy', - reason='Deprecations not being report on PyPy due to pybind/pybind11#3110' + platform.python_implementation() == "PyPy", + reason="Deprecations not being report on PyPy due to pybind/pybind11#3110", ) def test_deprecated_constants(self): with pytest.deprecated_call(): diff --git a/tests/test_send_asyncio.py b/tests/test_send_asyncio.py index c9790cbb8..f6d7d7d98 100644 --- a/tests/test_send_asyncio.py +++ b/tests/test_send_asyncio.py @@ -32,10 +32,10 @@ class TestUdpStream: def setup_method(self): # Make a stream slow enough that we can test async interactions config = spead2.send.StreamConfig(rate=5e6) - self.stream = UdpStream(spead2.ThreadPool(), [('localhost', 8888)], config) + self.stream = UdpStream(spead2.ThreadPool(), [("localhost", 8888)], config) self.ig = spead2.send.ItemGroup() - self.ig.add_item(0x1000, 'test', 'Test item', shape=(256 * 1024,), dtype=np.uint8) - self.ig['test'].value = np.zeros((256 * 1024,), np.uint8) + self.ig.add_item(0x1000, "test", "Test item", shape=(256 * 1024,), dtype=np.uint8) + self.ig["test"].value = np.zeros((256 * 1024,), np.uint8) self.heap = self.ig.get_heap() async def _test_async_flush(self): @@ -67,8 +67,11 @@ async def test_send_error(self): # Create a stream with a packet size that is bigger than the likely # MTU. It should cause an error. stream = UdpStream( - spead2.ThreadPool(), [("localhost", 8888)], - spead2.send.StreamConfig(max_packet_size=100000), buffer_size=0) + spead2.ThreadPool(), + [("localhost", 8888)], + spead2.send.StreamConfig(max_packet_size=100000), + buffer_size=0, + ) with pytest.raises(IOError): await stream.async_send_heap(self.heap) @@ -84,8 +87,9 @@ async def test_async_send_heap_refcount(self): async def test_async_send_heaps_refcount(self): """async_send_heaps must release the reference to the heap.""" weak = weakref.ref(self.heap) - future = self.stream.async_send_heaps([spead2.send.HeapReference(weak())], - spead2.send.GroupMode.ROUND_ROBIN) + future = self.stream.async_send_heaps( + [spead2.send.HeapReference(weak())], spead2.send.GroupMode.ROUND_ROBIN + ) self.heap = None await future for i in range(5): # Try extra hard to make PyPy release things @@ -111,4 +115,4 @@ class TestTcpStream: async def test_connect_failed(self): thread_pool = spead2.ThreadPool() with pytest.raises(IOError): - await spead2.send.asyncio.TcpStream.connect(thread_pool, [('127.0.0.1', 8887)]) + await spead2.send.asyncio.TcpStream.connect(thread_pool, [("127.0.0.1", 8887)]) From 428da5cb3d0b85f1b248725ec1ebb8329af09226 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 10:38:18 +0200 Subject: [PATCH 10/14] Add changelog entry for black, isort, pre-commit --- doc/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index 39331c43e..c027c0d63 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. rubric:: Development version + +- Reformat the Python codebase using black_ and isort_. +- Add `pre-commit`_ configuration. + +.. _black: https://black.readthedocs.io/en/stable/ +.. _isort: https://pycqa.github.io/isort/ +.. _pre-commit: https://pre-commit.com/ + .. rubric:: 3.12.0 - Add support for :doc:`recv-chunk-group` to assemble chunks in parallel. From 99f7c9f0279e8e932e709c61e5134f69430b17f8 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 10:39:20 +0200 Subject: [PATCH 11/14] Add missing r prefix to regex string in conf.py Found by pyupgrade. --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index e3a8e7fc4..d1edb188b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -43,7 +43,7 @@ def get_version(): code = f.read() exec(code, globals_) release = globals_["__version__"] - match = re.match("^(\d+)\.(\d+)", release) + match = re.match(r"^(\d+)\.(\d+)", release) return match.group(0), release From 3063c3aa2d4689e50e9caa5040361ea59db36307 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 10:45:28 +0200 Subject: [PATCH 12/14] Run pyupgrade on codebase --- gen/gen_loader.py | 2 +- src/spead2/__init__.py | 8 ++++---- src/spead2/tools/bench_asyncio.py | 2 +- src/spead2/tools/recv_asyncio.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/gen/gen_loader.py b/gen/gen_loader.py index 5d679204e..e60791e04 100755 --- a/gen/gen_loader.py +++ b/gen/gen_loader.py @@ -121,7 +121,7 @@ class RenameVisitor(c_ast.NodeVisitor): """ def __init__(self, old_name, new_name): - super(RenameVisitor, self).__init__() + super().__init__() self.old_name = old_name self.new_name = new_name diff --git a/src/spead2/__init__.py b/src/spead2/__init__.py index cb5bd4542..970d252c0 100644 --- a/src/spead2/__init__.py +++ b/src/spead2/__init__.py @@ -457,9 +457,9 @@ def _load_recursive(self, shape, gen): elif length == 64: field = _np.uint64(raw).view(_np.float64) else: - raise ValueError("unhandled float length {}".format((code, length))) + raise ValueError(f"unhandled float length {(code, length)}") else: - raise ValueError("unhandled format {}".format((code, length))) + raise ValueError(f"unhandled format {(code, length)}") fields.append(field) if len(fields) == 1: ans = fields[0] @@ -498,9 +498,9 @@ def _store_recursive(self, dims, value, gen): elif length == 64: raw = _np.float64(field).view(_np.uint64) else: - raise ValueError("unhandled float length {}".format((code, length))) + raise ValueError(f"unhandled float length {(code, length)}") else: - raise ValueError("unhandled format {}".format((code, length))) + raise ValueError(f"unhandled format {(code, length)}") gen.send((raw, length)) def set_from_raw(self, raw_item, new_order="="): diff --git a/src/spead2/tools/bench_asyncio.py b/src/spead2/tools/bench_asyncio.py index cf277f36c..969e460be 100644 --- a/src/spead2/tools/bench_asyncio.py +++ b/src/spead2/tools/bench_asyncio.py @@ -270,7 +270,7 @@ async def run_master(args, protocol, sender, receiver): print("Limited by send speed") best_actual = actual_rate else: - print("Send rate: {:.3f} Gbps".format(actual_rate * 8e-9)) + print(f"Send rate: {actual_rate * 8e-9:.3f} Gbps") low = 0.0 high = actual_rate while high - low > high * 0.02: diff --git a/src/spead2/tools/recv_asyncio.py b/src/spead2/tools/recv_asyncio.py index 3c5edbd0a..e9edc77ec 100644 --- a/src/spead2/tools/recv_asyncio.py +++ b/src/spead2/tools/recv_asyncio.py @@ -94,7 +94,7 @@ async def run_stream(stream, name, args): print(f"Shutting down stream {name} after {num_heaps} heaps") stats = stream.stats for key, value in stats.items(): - print("{}: {}".format(key, getattr(stats, key))) + print(f"{key}: {getattr(stats, key)}") break finally: stream.stop() From e60faff4237cd226646af0b831e16afa3cde483b Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 10:45:53 +0200 Subject: [PATCH 13/14] Simplify _make_numpy_header Let Python do the work of turning a dict into a repr thereof. --- src/spead2/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/spead2/__init__.py b/src/spead2/__init__.py index 970d252c0..3f8c0cf33 100644 --- a/src/spead2/__init__.py +++ b/src/spead2/__init__.py @@ -206,8 +206,12 @@ def _parse_numpy_header(cls, header): @classmethod def _make_numpy_header(self, shape, dtype, order): - return "{{'descr': {!r}, 'fortran_order': {!r}, 'shape': {!r}}}".format( - _np.lib.format.dtype_to_descr(dtype), order == "F", tuple(shape) + return repr( + { + "descr": _np.lib.format.dtype_to_descr(dtype), + "fortran_order": order == "F", + "shape": tuple(shape), + } ) @classmethod From 0c1f30e0c0a976a8941b54ab3219aeb0fb997548 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 2 Aug 2023 10:52:03 +0200 Subject: [PATCH 14/14] Manually eliminate some calls to str.format --- src/spead2/__init__.py | 8 ++------ src/spead2/tools/bench_asyncio.py | 7 ++----- src/spead2/tools/recv_asyncio.py | 14 ++++++-------- src/spead2/tools/send_asyncio.py | 6 +----- tests/test_send.py | 8 ++++++-- 5 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/spead2/__init__.py b/src/spead2/__init__.py index 3f8c0cf33..51d6aea76 100644 --- a/src/spead2/__init__.py +++ b/src/spead2/__init__.py @@ -515,9 +515,7 @@ def set_from_raw(self, raw_item, new_order="="): elements = _shape_elements(shape) if elements > max_elements: raise ValueError( - "Item {} has too few elements for shape ({} < {})".format( - self.name, max_elements, elements - ) + f"Item {self.name} has too few elements for shape ({max_elements} < {elements})" ) size_bytes = elements * self._internal_dtype.itemsize if raw_item.is_immediate: @@ -550,9 +548,7 @@ def set_from_raw(self, raw_item, new_order="="): bits = elements * itemsize_bits if elements > max_elements: raise ValueError( - "Item {} has too few elements for shape ({} < {})".format( - self.name, max_elements, elements - ) + f"Item {self.name} has too few elements for shape ({max_elements} < {elements})" ) if raw_item.is_immediate: # Immediates get head padding instead of tail padding diff --git a/src/spead2/tools/bench_asyncio.py b/src/spead2/tools/bench_asyncio.py index 969e460be..66605d572 100644 --- a/src/spead2/tools/bench_asyncio.py +++ b/src/spead2/tools/bench_asyncio.py @@ -282,11 +282,8 @@ async def run_master(args, protocol, sender, receiver): args, protocol, sender, receiver, rate, num_heaps, num_heaps - 1 ) if not args.quiet: - print( - "Rate: {:.3f} Gbps ({:.3f} actual): {}".format( - rate * 8e-9, actual_rate * 8e-9, "GOOD" if good else "BAD" - ) - ) + result = "GOOD" if good else "BAD" + print(f"Rate: {rate * 8e-9:.3f} Gbps ({actual_rate * 8e-9:.3f} actual): {result}") if good: low = rate best_actual = actual_rate diff --git a/src/spead2/tools/recv_asyncio.py b/src/spead2/tools/recv_asyncio.py index e9edc77ec..1d887fa3a 100644 --- a/src/spead2/tools/recv_asyncio.py +++ b/src/spead2/tools/recv_asyncio.py @@ -73,14 +73,12 @@ async def run_stream(stream, name, args): for raw_descriptor in heap.get_descriptors(): descriptor = spead2.Descriptor.from_raw(raw_descriptor, heap.flavour) print( - """\ - Descriptor for {0.name} ({0.id:#x}) - description: {0.description} - format: {0.format} - dtype: {0.dtype} - shape: {0.shape}""".format( - descriptor - ) + f"""\ + Descriptor for {descriptor.name} ({descriptor.id:#x}) + description: {descriptor.description} + format: {descriptor.format} + dtype: {descriptor.dtype} + shape: {descriptor.shape}""" ) changed = item_group.update(heap) for key, item in changed.items(): diff --git a/src/spead2/tools/send_asyncio.py b/src/spead2/tools/send_asyncio.py index 1f6d9b3cf..f26c89a86 100644 --- a/src/spead2/tools/send_asyncio.py +++ b/src/spead2/tools/send_asyncio.py @@ -116,11 +116,7 @@ async def run(item_group, stream, args): elapsed = time.time() - start_time if last_error is not None: logging.warn("%d errors, last one: %s", n_errors, last_error) - print( - "Sent {} bytes in {:.6f}s, {:.6f} Gb/s".format( - n_bytes, elapsed, n_bytes * 8 / elapsed / 1e9 - ) - ) + print(f"Sent {n_bytes} bytes in {elapsed:.6f}s, {n_bytes * 8 / elapsed / 1e9:.6f} Gb/s") async def async_main(): diff --git a/tests/test_send.py b/tests/test_send.py index 6f2bcd834..3f6c642cc 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -157,8 +157,12 @@ def make_descriptor_numpy(self, id, name, description, shape, dtype_str, fortran b"description", b"", self.flavour.make_shape(shape), - "{{'descr': {!r}, 'fortran_order': {!r}, 'shape': {!r}}}".format( - str(dtype_str), bool(fortran_order), tuple(shape) + repr( + { + "descr": str(dtype_str), + "fortran_order": bool(fortran_order), + "shape": tuple(shape), + } ).encode(), ] payload = b"".join(payload_fields)