From afd0d4ec20494747b4f22f050eefd36031090e04 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Thu, 11 Apr 2024 18:40:48 -0700 Subject: [PATCH 1/5] removed last of pkg_resources. --- compliance_checker/base.py | 7 ++--- compliance_checker/cf/util.py | 6 ++++- compliance_checker/cfutil.py | 5 +++- compliance_checker/suite.py | 36 +++++++++++++++++--------- compliance_checker/tests/conftest.py | 5 +++- compliance_checker/tests/resources.py | 5 +++- compliance_checker/tests/test_cli.py | 3 ++- compliance_checker/tests/test_suite.py | 8 ++++-- requirements.txt | 1 + 9 files changed, 54 insertions(+), 22 deletions(-) diff --git a/compliance_checker/base.py b/compliance_checker/base.py index c2674e85..64c2685c 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -191,9 +191,10 @@ def __del__(self): are cleared before the next checker uses it. Some caches were inadvertently mutated by other functions. """ - - cfutil.get_geophysical_variables.cache_clear() - cfutil.get_time_variables.cache_clear() + # odd errors -- module getting deleted before this object? + if cfutil is not None: + cfutil.get_geophysical_variables.cache_clear() + cfutil.get_time_variables.cache_clear() class BaseNCCheck: diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py index 6f100653..61e44f75 100644 --- a/compliance_checker/cf/util.py +++ b/compliance_checker/cf/util.py @@ -5,7 +5,11 @@ import requests from cf_units import Unit -from importlib_resources import files +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files + from lxml import etree from netCDF4 import Dataset diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index 245c6fd3..2f16ac22 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -9,7 +9,10 @@ from functools import lru_cache, partial from cf_units import Unit -from importlib_resources import files +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files _UNITLESS_DB = None _SEA_NAMES = None diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index 7d922110..fd29af1b 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -13,6 +13,7 @@ import warnings from collections import defaultdict from datetime import datetime, timezone +from packaging import version from operator import itemgetter from pathlib import Path from urllib.parse import urlparse @@ -22,8 +23,10 @@ from netCDF4 import Dataset from owslib.sos import SensorObservationService from owslib.swe.sensor.sml import SensorML -from packaging.version import parse -from pkg_resources import working_set +if sys.version_info >= (3, 9): + import importlib.metadata as impmd +else: + import importlib_metadata as impmd from compliance_checker import __version__, tempnc from compliance_checker.base import BaseCheck, GenericFile, Result, fix_return_value @@ -71,10 +74,11 @@ def _get_generator_plugins(cls): Return a list of classes from external plugins that are used to generate checker classes """ - + # NOTE: updated to not use pkg_resources, but + # not tested -- it is ever used? if not hasattr(cls, "suite_generators"): - gens = working_set.iter_entry_points("compliance_checker.generators") - cls.suite_generators = [x.resolve() for x in gens] + gens = impmd.entry_points(group='compliance_checker.generators') + cls.suite_generators = [x.load() for x in gens] return cls.suite_generators @@ -136,7 +140,9 @@ def load_all_available_checkers(cls): Helper method to retrieve all sub checker classes derived from various base classes. """ - cls._load_checkers(working_set.iter_entry_points("compliance_checker.suites")) + checkers = impmd.entry_points(group='compliance_checker.suites') + cls._load_checkers(checkers) + @classmethod def _load_checkers(cls, checkers): @@ -147,7 +153,8 @@ def _load_checkers(cls, checkers): for c in checkers: try: - check_obj = c.resolve() + # check_obj = c.resolve() + check_obj = c.load() if hasattr(check_obj, "_cc_spec") and hasattr( check_obj, "_cc_spec_version", @@ -186,8 +193,8 @@ def _load_checkers(cls, checkers): for spec, versions in itertools.groupby(ver_checkers, itemgetter(0)): version_nums = [v[-1] for v in versions] try: - latest_version = str(max(parse(v) for v in version_nums)) - # if the version can't be parsed, do it according to character collation + latest_version = str(max(version.parse(v) for v in version_nums)) + # if the version can't be parsed, sort according to character collation except ValueError: latest_version = max(version_nums) cls.checkers[spec] = cls.checkers[spec + ":latest"] = cls.checkers[ @@ -764,9 +771,14 @@ def generate_dataset(self, cdl_path): :param str cdl_path: Absolute path to cdl file that is used to generate netCDF file """ - if isinstance(cdl_path, str): - cdl_path = Path(cdl_path) - ds_str = cdl_path.with_suffix(".nc") + # better to update the following code with Path object -- some day + cdl_path = os.fspath(cdl_path) + if ( + ".cdl" in cdl_path + ): # it's possible the filename doesn't have the .cdl extension + ds_str = cdl_path.replace(".cdl", ".nc") + else: + ds_str = cdl_path + ".nc" # generate netCDF-4 file iostat = subprocess.run( diff --git a/compliance_checker/tests/conftest.py b/compliance_checker/tests/conftest.py index 2c662c16..178ab73a 100644 --- a/compliance_checker/tests/conftest.py +++ b/compliance_checker/tests/conftest.py @@ -3,7 +3,10 @@ from itertools import chain import pytest -from importlib_resources import files +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files from netCDF4 import Dataset from compliance_checker.cf import util diff --git a/compliance_checker/tests/resources.py b/compliance_checker/tests/resources.py index 7bbca9d6..e61fa290 100644 --- a/compliance_checker/tests/resources.py +++ b/compliance_checker/tests/resources.py @@ -1,6 +1,9 @@ import subprocess -from importlib_resources import files +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files def get_filename(path): diff --git a/compliance_checker/tests/test_cli.py b/compliance_checker/tests/test_cli.py index fc6b4f94..cb3c21a7 100644 --- a/compliance_checker/tests/test_cli.py +++ b/compliance_checker/tests/test_cli.py @@ -91,7 +91,8 @@ def checker_1(): def checker_2(): return Namespace(_cc_spec="checker_2", _cc_spec_version="2.2") - mock_checkers = [Namespace(resolve=checker_1), Namespace(resolve=checker_2)] + mock_checkers = [Namespace(load=checker_1), + Namespace(load=checker_2)] with pytest.warns(DeprecationWarning): CheckSuite._load_checkers(mock_checkers) diff --git a/compliance_checker/tests/test_suite.py b/compliance_checker/tests/test_suite.py index 8c8987df..7faa2eea 100644 --- a/compliance_checker/tests/test_suite.py +++ b/compliance_checker/tests/test_suite.py @@ -3,7 +3,11 @@ from pathlib import Path import numpy as np -from importlib_resources import files + +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files from compliance_checker.acdd import ACDDBaseCheck from compliance_checker.base import BaseCheck, GenericFile, Result @@ -83,7 +87,7 @@ def test_generate_dataset_netCDF4(self): # create netCDF4 file ds_name = self.cs.generate_dataset(static_files["netCDF4"]) # check if correct name is return - assert ds_name == static_files["netCDF4"].with_suffix(".nc") + assert ds_name == str(static_files["netCDF4"].with_suffix(".nc")) # check if netCDF4 file was created assert os.path.isfile(static_files["netCDF4"].with_suffix(".nc")) diff --git a/requirements.txt b/requirements.txt index 6277c633..46bbf078 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ cf-units>=2 cftime>=1.1.0 importlib-resources # drop this when dropping Python 3.8 +importlib-metadata # drop this when dropping Python 3.8 isodate>=0.6.1 jinja2>=2.7.3 lxml>=3.2.1 From f03b3b60ad581157d7e5babaf672fb99470efa71 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Thu, 11 Apr 2024 18:50:24 -0700 Subject: [PATCH 2/5] using importlib_metadata for 3.10_ only --- compliance_checker/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index fd29af1b..ec3631d0 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -23,7 +23,7 @@ from netCDF4 import Dataset from owslib.sos import SensorObservationService from owslib.swe.sensor.sml import SensorML -if sys.version_info >= (3, 9): +if sys.version_info >= (3, 10): import importlib.metadata as impmd else: import importlib_metadata as impmd From b957367322394e7bc450b2bc4eb7064f72d98769 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Sat, 13 Apr 2024 14:41:15 +0200 Subject: [PATCH 3/5] update pre-commits --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0f1a089..805ce058 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - test_requirements.txt - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.4.0 hooks: - id: black language_version: python3 @@ -31,7 +31,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.5 + rev: v0.3.7 hooks: - id: ruff From ebe9fc8123d973b5c46bb682ba8fa069c1dcf267 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Sat, 13 Apr 2024 14:41:20 +0200 Subject: [PATCH 4/5] only essential pkg_resource fixes --- compliance_checker/base.py | 7 +++-- compliance_checker/cf/util.py | 6 +---- compliance_checker/cfutil.py | 5 +--- compliance_checker/suite.py | 36 ++++++++++---------------- compliance_checker/tests/conftest.py | 5 +--- compliance_checker/tests/resources.py | 5 +--- compliance_checker/tests/test_cli.py | 3 +-- compliance_checker/tests/test_suite.py | 8 ++---- 8 files changed, 24 insertions(+), 51 deletions(-) diff --git a/compliance_checker/base.py b/compliance_checker/base.py index 64c2685c..c2674e85 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -191,10 +191,9 @@ def __del__(self): are cleared before the next checker uses it. Some caches were inadvertently mutated by other functions. """ - # odd errors -- module getting deleted before this object? - if cfutil is not None: - cfutil.get_geophysical_variables.cache_clear() - cfutil.get_time_variables.cache_clear() + + cfutil.get_geophysical_variables.cache_clear() + cfutil.get_time_variables.cache_clear() class BaseNCCheck: diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py index 61e44f75..6f100653 100644 --- a/compliance_checker/cf/util.py +++ b/compliance_checker/cf/util.py @@ -5,11 +5,7 @@ import requests from cf_units import Unit -try: - from importlib.resources import files -except ImportError: - from importlib_resources import files - +from importlib_resources import files from lxml import etree from netCDF4 import Dataset diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index 2f16ac22..245c6fd3 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -9,10 +9,7 @@ from functools import lru_cache, partial from cf_units import Unit -try: - from importlib.resources import files -except ImportError: - from importlib_resources import files +from importlib_resources import files _UNITLESS_DB = None _SEA_NAMES = None diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index ec3631d0..d261a1cc 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -13,20 +13,17 @@ import warnings from collections import defaultdict from datetime import datetime, timezone -from packaging import version from operator import itemgetter from pathlib import Path from urllib.parse import urlparse +import importlib_metadata import requests from lxml import etree as ET from netCDF4 import Dataset from owslib.sos import SensorObservationService from owslib.swe.sensor.sml import SensorML -if sys.version_info >= (3, 10): - import importlib.metadata as impmd -else: - import importlib_metadata as impmd +from packaging.version import parse from compliance_checker import __version__, tempnc from compliance_checker.base import BaseCheck, GenericFile, Result, fix_return_value @@ -74,10 +71,11 @@ def _get_generator_plugins(cls): Return a list of classes from external plugins that are used to generate checker classes """ - # NOTE: updated to not use pkg_resources, but - # not tested -- it is ever used? + if not hasattr(cls, "suite_generators"): - gens = impmd.entry_points(group='compliance_checker.generators') + gens = importlib_metadata.entry_points( + groups="compliance_checker.generators", + ) cls.suite_generators = [x.load() for x in gens] return cls.suite_generators @@ -140,9 +138,9 @@ def load_all_available_checkers(cls): Helper method to retrieve all sub checker classes derived from various base classes. """ - checkers = impmd.entry_points(group='compliance_checker.suites') - cls._load_checkers(checkers) - + cls._load_checkers( + importlib_metadata.entry_points(group="compliance_checker.suites"), + ) @classmethod def _load_checkers(cls, checkers): @@ -153,7 +151,6 @@ def _load_checkers(cls, checkers): for c in checkers: try: - # check_obj = c.resolve() check_obj = c.load() if hasattr(check_obj, "_cc_spec") and hasattr( check_obj, @@ -193,8 +190,8 @@ def _load_checkers(cls, checkers): for spec, versions in itertools.groupby(ver_checkers, itemgetter(0)): version_nums = [v[-1] for v in versions] try: - latest_version = str(max(version.parse(v) for v in version_nums)) - # if the version can't be parsed, sort according to character collation + latest_version = str(max(parse(v) for v in version_nums)) + # if the version can't be parsed, do it according to character collation except ValueError: latest_version = max(version_nums) cls.checkers[spec] = cls.checkers[spec + ":latest"] = cls.checkers[ @@ -771,14 +768,9 @@ def generate_dataset(self, cdl_path): :param str cdl_path: Absolute path to cdl file that is used to generate netCDF file """ - # better to update the following code with Path object -- some day - cdl_path = os.fspath(cdl_path) - if ( - ".cdl" in cdl_path - ): # it's possible the filename doesn't have the .cdl extension - ds_str = cdl_path.replace(".cdl", ".nc") - else: - ds_str = cdl_path + ".nc" + if isinstance(cdl_path, str): + cdl_path = Path(cdl_path) + ds_str = cdl_path.with_suffix(".nc") # generate netCDF-4 file iostat = subprocess.run( diff --git a/compliance_checker/tests/conftest.py b/compliance_checker/tests/conftest.py index 178ab73a..2c662c16 100644 --- a/compliance_checker/tests/conftest.py +++ b/compliance_checker/tests/conftest.py @@ -3,10 +3,7 @@ from itertools import chain import pytest -try: - from importlib.resources import files -except ImportError: - from importlib_resources import files +from importlib_resources import files from netCDF4 import Dataset from compliance_checker.cf import util diff --git a/compliance_checker/tests/resources.py b/compliance_checker/tests/resources.py index e61fa290..7bbca9d6 100644 --- a/compliance_checker/tests/resources.py +++ b/compliance_checker/tests/resources.py @@ -1,9 +1,6 @@ import subprocess -try: - from importlib.resources import files -except ImportError: - from importlib_resources import files +from importlib_resources import files def get_filename(path): diff --git a/compliance_checker/tests/test_cli.py b/compliance_checker/tests/test_cli.py index cb3c21a7..c86cd897 100644 --- a/compliance_checker/tests/test_cli.py +++ b/compliance_checker/tests/test_cli.py @@ -91,8 +91,7 @@ def checker_1(): def checker_2(): return Namespace(_cc_spec="checker_2", _cc_spec_version="2.2") - mock_checkers = [Namespace(load=checker_1), - Namespace(load=checker_2)] + mock_checkers = [Namespace(load=checker_1), Namespace(load=checker_2)] with pytest.warns(DeprecationWarning): CheckSuite._load_checkers(mock_checkers) diff --git a/compliance_checker/tests/test_suite.py b/compliance_checker/tests/test_suite.py index 7faa2eea..8c8987df 100644 --- a/compliance_checker/tests/test_suite.py +++ b/compliance_checker/tests/test_suite.py @@ -3,11 +3,7 @@ from pathlib import Path import numpy as np - -try: - from importlib.resources import files -except ImportError: - from importlib_resources import files +from importlib_resources import files from compliance_checker.acdd import ACDDBaseCheck from compliance_checker.base import BaseCheck, GenericFile, Result @@ -87,7 +83,7 @@ def test_generate_dataset_netCDF4(self): # create netCDF4 file ds_name = self.cs.generate_dataset(static_files["netCDF4"]) # check if correct name is return - assert ds_name == str(static_files["netCDF4"].with_suffix(".nc")) + assert ds_name == static_files["netCDF4"].with_suffix(".nc") # check if netCDF4 file was created assert os.path.isfile(static_files["netCDF4"].with_suffix(".nc")) From 064aac92f9287af62950edb93141bd89068f2632 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Sat, 13 Apr 2024 14:41:32 +0200 Subject: [PATCH 5/5] sort --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 46bbf078..fd3bbe9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ cf-units>=2 cftime>=1.1.0 -importlib-resources # drop this when dropping Python 3.8 importlib-metadata # drop this when dropping Python 3.8 +importlib-resources # drop this when dropping Python 3.8 isodate>=0.6.1 jinja2>=2.7.3 lxml>=3.2.1