Skip to content

Commit

Permalink
Merge branch 'master' into pre-commit-ci-update-config
Browse files Browse the repository at this point in the history
  • Loading branch information
CagtayFabry authored Aug 5, 2024
2 parents c757d0a + 336c835 commit 9b02ac1
Show file tree
Hide file tree
Showing 13 changed files with 105 additions and 73 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
init-shell: >-
bash
powershell
cache-environment: true
cache-environment: false

- name: activate build env
run: micromamba activate rtd
Expand Down
2 changes: 1 addition & 1 deletion .ruff.toml
Original file line number Diff line number Diff line change
Expand Up @@ -77,5 +77,5 @@ required-imports = [
"from __future__ import annotations",
]

[flake8-import-conventions]
[lint.flake8-import-conventions]
extend-aliases = { xarray = "xr" }
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
### Fixes

- rename (fix typo) argument to `lcs_child_in_parent` in `CoordinateSystemManager.add_cs` \[{pull}`936`\].
- replace usages of `pkg_resources` with `importlib.metadata` \[{pull}`941`\].
- replace usages of `copy_arrays` with `memmap` for `asdf>=3.1.0` \[{pull}`940`\].

### Dependencies

- pin `weldx-widgets>=0.2.3` for viz \[{pull}`939`\].
- pin `pint>=0.21` \[{pull}`941`\].

## 0.6.8 (07.06.2024)

Expand Down
4 changes: 2 additions & 2 deletions doc/rtd_environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,5 @@ dependencies:
# pip packages
- pip
- pip:
- ../
- ./json_mime_render_plugin/
- weldx @ file:/../..//
- json_mime_render_plugin @ file:/..//json_mime_render_plugin
12 changes: 11 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,23 @@ dependencies = [
"networkx>=2.8.2",
"numpy>=1.20,<2",
"pandas>=1.5",
"pint>=0.18",
"pint>=0.21",
"pint-xarray>=0.3",
"psutil",
"scipy>=1.6.2",
"sympy>=1.6",
"xarray>=2022.9",
]
optional-dependencies.docs = [
"docutils>=0.19",
"numpydoc>=0.5",
"pydata-sphinx-theme<0.15", # parallel-write-unsafe
"sphinx>=4.1.1,==7.2",
"sphinx-autodoc-typehints>=1.21.8,==2",
"sphinx-copybutton==0.5",
"typing-extensions",
"urllib3<2",
]
optional-dependencies.media = [
"av",
"dask-image",
Expand Down
28 changes: 20 additions & 8 deletions weldx/asdf/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import copy
import importlib.metadata
import io
import pathlib
import warnings
Expand Down Expand Up @@ -42,12 +43,23 @@
__all__ = [
"WeldxFile",
"DEFAULT_ARRAY_COMPRESSION",
"DEFAULT_ARRAY_COPYING",
"DEFAULT_MEMORY_MAPPING",
"DEFAULT_ARRAY_INLINE_THRESHOLD",
"_PROTECTED_KEYS",
]


def asdf_open_memory_mapping_kwarg(memmap: bool) -> dict:
if tuple(importlib.metadata.version("asdf").split(".")) >= (
"3",
"1",
"0",
):
return {"memmap": memmap}
else:
return {"copy_arrays": not memmap}


@contextmanager
def reset_file_position(fh: SupportsFileReadWrite):
"""Reset the internal position of the given file after leaving the context.
Expand All @@ -66,8 +78,8 @@ def reset_file_position(fh: SupportsFileReadWrite):
DEFAULT_ARRAY_COMPRESSION = "input"
"""All arrays will be compressed using this algorithm, if not specified by user."""

DEFAULT_ARRAY_COPYING = True
"""Stored Arrays will be copied to memory, or not. If False, use memory mapping."""
DEFAULT_MEMORY_MAPPING = False
"""Stored Arrays will be memory-mapped, or not. If True, use memory mapping."""

DEFAULT_ARRAY_INLINE_THRESHOLD = 10
"""Arrays with less or equal elements will be inlined (stored as string, not binary)."""
Expand Down Expand Up @@ -148,8 +160,8 @@ class WeldxFile(_ProtectedViewDict):
- ``lz4``: Use lz4 compression.
- ``input``: Use the same compression as in the file read.
If there is no prior file, acts as None.
copy_arrays :
When `False`, when reading files, attempt to memory map (memmap) underlying data
memmap :
When `True`, when reading files, attempt to memory map (memmap) underlying data
arrays when possible. This avoids blowing the memory when working with very
large datasets.
array_inline_threshold :
Expand Down Expand Up @@ -219,19 +231,19 @@ def __init__(
) = None,
software_history_entry: Mapping = None,
compression: str = DEFAULT_ARRAY_COMPRESSION,
copy_arrays: bool = DEFAULT_ARRAY_COPYING,
memmap: bool = DEFAULT_MEMORY_MAPPING,
array_inline_threshold: int = DEFAULT_ARRAY_INLINE_THRESHOLD,
):
if write_kwargs is None:
write_kwargs = dict(all_array_compression=compression)

if asdffile_kwargs is None:
asdffile_kwargs = dict(copy_arrays=copy_arrays)
asdffile_kwargs = asdf_open_memory_mapping_kwarg(memmap=memmap)

# this parameter is now (asdf-2.8) a asdf.config parameter, so we store it here.
self._array_inline_threshold = array_inline_threshold

# TODO: ensure no mismatching args for compression and copy_arrays.
# TODO: ensure no mismatching args for compression and memmap.
self._write_kwargs = write_kwargs
self._asdffile_kwargs = asdffile_kwargs

Expand Down
17 changes: 12 additions & 5 deletions weldx/asdf/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import importlib.metadata
from collections.abc import Callable, Hashable, Mapping, MutableMapping, Set
from contextlib import contextmanager
from io import BytesIO, TextIOBase
Expand Down Expand Up @@ -149,7 +150,7 @@ def read_buffer_context(
Buffer containing ASDF file contents
open_kwargs
Additional keywords to pass to `asdf.AsdfFile.open`
Extensions are always set, ``copy_arrays=True`` is set by default.
Extensions are always set, ``memmap=False`` is set by default.
Returns
-------
Expand All @@ -158,7 +159,13 @@ def read_buffer_context(
"""
if open_kwargs is None:
open_kwargs = {"copy_arrays": True, "lazy_load": False}
open_kwargs = {"memmap": False, "lazy_load": False}

if "memmap" in open_kwargs and tuple(
importlib.metadata.version("asdf").split(".")
) < ("3", "1", "0"):
open_kwargs["copy_arrays"] = not open_kwargs["memmap"]
del open_kwargs["memmap"]

buffer.seek(0)

Expand Down Expand Up @@ -190,7 +197,7 @@ def read_buffer(
Buffer containing ASDF file contents
open_kwargs
Additional keywords to pass to `asdf.AsdfFile.open`
Extensions are always set, ``copy_arrays=True`` is set by default.
Extensions are always set, ``memmap=False`` is set by default.
Returns
-------
Expand Down Expand Up @@ -220,7 +227,7 @@ def write_read_buffer_context(
Extensions are always set.
open_kwargs
Additional keywords to pass to `asdf.AsdfFile.open`
Extensions are always set, ``copy_arrays=True`` is set by default.
Extensions are always set, ``memmap=False`` is set by default.
Returns
-------
Expand Down Expand Up @@ -248,7 +255,7 @@ def write_read_buffer(
Extensions are always set.
open_kwargs
Additional keywords to pass to `asdf.AsdfFile.open`
Extensions are always set, ``copy_arrays=True`` is set by default.
Extensions are always set, ``memmap=False`` is set by default.
Returns
-------
Expand Down
11 changes: 9 additions & 2 deletions weldx/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

from __future__ import annotations

import importlib.metadata
import sys
from pathlib import Path

import asdf
import pkg_resources
import yaml
from asdf.config import ResourceMappingProxy
from asdf.versioning import AsdfVersion, split_tag_version
Expand Down Expand Up @@ -175,7 +176,13 @@ def enable_quality_standard(name: str, version: AsdfVersion | str = None):
@staticmethod
def load_installed_standards():
"""Load all standards that are installed to the active virtual environment."""
for entry_point in pkg_resources.iter_entry_points("weldx.standard"):
if sys.version_info < (3, 10):
entry_points = importlib.metadata.entry_points().get("weldx.standard", [])
else:
entry_points = importlib.metadata.entry_points().select(
group="weldx.standard"
)
for entry_point in entry_points:
standards = entry_point.load()()
if not isinstance(standards, list):
standards = [standards]
Expand Down
1 change: 0 additions & 1 deletion weldx/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

WELDX_UNIT_REGISTRY = pint.UnitRegistry(
preprocessors=[
lambda string: string.replace("%", "percent"), # allow %-sign
lambda string: string.replace("Δ°", "delta_deg"), # parse Δ° for temperature
],
force_ndarray_like=True,
Expand Down
3 changes: 1 addition & 2 deletions weldx/tests/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import numpy as np
import pint
from pkg_resources import get_distribution

from weldx.constants import Q_
from weldx.geometry import _vector_is_close as vector_is_close
Expand Down Expand Up @@ -130,7 +129,7 @@ def matrix_is_close(mat_a, mat_b, abs_tol=1e-9) -> bool:
return False

atol_unit = 1.0
if isinstance(mat_b, pint.Quantity) and get_distribution("pint").version >= "0.21":
if isinstance(mat_b, pint.Quantity):
atol_unit = mat_b.u

return np.all(np.isclose(mat_a, mat_b, atol=abs_tol * atol_unit)).__bool__()
Loading

0 comments on commit 9b02ac1

Please sign in to comment.