diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..8462b37 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,30 @@ +name: Lint + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + lint: + name: "Lint: code quality and formatting checks" + runs-on: ubuntu-latest + steps: + - name: Clone repo + uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + cache: 'pip' + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install isort black ruff + - name: isort + run: python -m isort . --check --diff + # - name: black + # run: python -m black --check --diff . + - name: ruff + run: ruff check . diff --git a/.github/workflows/pytest-collect.yaml b/.github/workflows/pytest-collect.yaml new file mode 100644 index 0000000..4a5f33b --- /dev/null +++ b/.github/workflows/pytest-collect.yaml @@ -0,0 +1,31 @@ +name: Pytest-collect + +on: + push: + branches: [ main ] + paths: + - 'tests/**' + pull_request: + branches: [ main ] + paths: + - 'tests/**' + +jobs: + pytest-collect: + name: "Pytest: check test collection" + runs-on: ubuntu-latest + steps: + - name: Clone repo + uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install .[dev] + - name: "pytest: check test collection" + run: pytest --collect-only diff --git a/pyproject.toml b/pyproject.toml index 3f8e9c2..eb16ffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "h5netcdf>=1.2.0", "openeo[localprocessing]", "cftime", - "pytest-dependency", + "pytest-depends", "pyarrow", "fastparquet" ] @@ -60,5 +60,20 @@ testpaths = [ "tests", ] addopts = [ - "--import-mode=importlib", + "--import-mode=prepend", ] + +[tool.isort] +profile = "black" + + +[tool.ruff] +# line-length = 100 + +[tool.ruff.lint] +select = ["E", "F"] +ignore = [ + "E501", # Ignore "line-too-long" issues, let black handle that. +] + + diff --git a/src/openeo_gfmap/__init__.py b/src/openeo_gfmap/__init__.py index 9055193..0e1dbe7 100644 --- a/src/openeo_gfmap/__init__.py +++ b/src/openeo_gfmap/__init__.py @@ -8,7 +8,7 @@ from .backend import Backend, BackendContext from .metadata import FakeMetadata -from .spatial import SpatialContext, BoundingBoxExtent +from .spatial import BoundingBoxExtent, SpatialContext from .temporal import TemporalContext __all__ = [ diff --git a/src/openeo_gfmap/fetching/__init__.py b/src/openeo_gfmap/fetching/__init__.py index c36a793..f1bbd9f 100644 --- a/src/openeo_gfmap/fetching/__init__.py +++ b/src/openeo_gfmap/fetching/__init__.py @@ -6,8 +6,8 @@ """ from .fetching import CollectionFetcher, FetchType -from .s2 import build_sentinel2_l2a_extractor from .s1 import build_sentinel1_grd_extractor +from .s2 import build_sentinel2_l2a_extractor __all__ = [ "build_sentinel2_l2a_extractor", "CollectionFetcher", "FetchType", diff --git a/src/openeo_gfmap/fetching/s1.py b/src/openeo_gfmap/fetching/s1.py index 672943a..dda8be9 100644 --- a/src/openeo_gfmap/fetching/s1.py +++ b/src/openeo_gfmap/fetching/s1.py @@ -1,23 +1,22 @@ """ Collection fetching of S1 features, supporting different backends. """ -from typing import Callable from functools import partial +from typing import Callable import openeo from geojson import GeoJSON from openeo_gfmap.backend import Backend, BackendContext -from openeo_gfmap.spatial import SpatialContext, BoundingBoxExtent +from openeo_gfmap.spatial import SpatialContext from openeo_gfmap.temporal import TemporalContext -from .fetching import CollectionFetcher, FetchType - from .commons import ( convert_band_names, load_collection, rename_bands, resample_reproject, ) +from .fetching import CollectionFetcher, FetchType BASE_SENTINEL1_GRD_MAPPING = { "VH": "S1-VH", diff --git a/src/openeo_gfmap/preprocessing/__init__.py b/src/openeo_gfmap/preprocessing/__init__.py index 6b52c82..9f83f02 100644 --- a/src/openeo_gfmap/preprocessing/__init__.py +++ b/src/openeo_gfmap/preprocessing/__init__.py @@ -2,9 +2,14 @@ right after the extraction and the execution of the features UDF. """ -from openeo_gfmap.preprocessing.cloudmasking import mask_scl_dilation, get_bap_score, bap_masking, get_bap_mask +from openeo_gfmap.preprocessing.cloudmasking import ( + bap_masking, + get_bap_mask, + get_bap_score, + mask_scl_dilation, +) +from openeo_gfmap.preprocessing.compositing import mean_compositing, median_compositing from openeo_gfmap.preprocessing.interpolation import linear_interpolation -from openeo_gfmap.preprocessing.compositing import median_compositing, mean_compositing __all__ = [ "mask_scl_dilation", diff --git a/src/openeo_gfmap/preprocessing/cloudmasking.py b/src/openeo_gfmap/preprocessing/cloudmasking.py index 89a09b1..487b705 100644 --- a/src/openeo_gfmap/preprocessing/cloudmasking.py +++ b/src/openeo_gfmap/preprocessing/cloudmasking.py @@ -1,7 +1,8 @@ """Different cloud masking strategies for an OpenEO datacubes.""" -from typing import Union from pathlib import Path +from typing import Union + import openeo from openeo.processes import if_, is_nan diff --git a/src/openeo_gfmap/preprocessing/interpolation.py b/src/openeo_gfmap/preprocessing/interpolation.py index 9159ea9..79fe256 100644 --- a/src/openeo_gfmap/preprocessing/interpolation.py +++ b/src/openeo_gfmap/preprocessing/interpolation.py @@ -3,6 +3,7 @@ """ import openeo + def linear_interpolation(cube: openeo.DataCube,) -> openeo.DataCube: """Perform linear interpolation on the given datacube.""" return cube.apply_dimension( diff --git a/src/openeo_gfmap/preprocessing/udf_rank.py b/src/openeo_gfmap/preprocessing/udf_rank.py index 2a7e6cf..a9011b6 100644 --- a/src/openeo_gfmap/preprocessing/udf_rank.py +++ b/src/openeo_gfmap/preprocessing/udf_rank.py @@ -1,9 +1,9 @@ -import xarray as xr -from pathlib import Path -import numpy as np +import numpy as np +import xarray as xr from openeo.udf import XarrayDataCube + def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube: """For a cube having the BAP score, and a given period of list of intervals, create a binary mask that for each pixel, is True if the BAP score is the diff --git a/src/openeo_gfmap/preprocessing/udf_score.py b/src/openeo_gfmap/preprocessing/udf_score.py index aaf38f6..c2873d2 100644 --- a/src/openeo_gfmap/preprocessing/udf_score.py +++ b/src/openeo_gfmap/preprocessing/udf_score.py @@ -1,14 +1,10 @@ +import math + import numpy as np -import time, math import xarray as xr -import datetime - -from scipy.ndimage import distance_transform_cdt -from skimage.morphology import footprints -from skimage.morphology import binary_erosion, binary_dilation from openeo.udf import XarrayDataCube -from xarray.ufuncs import isnan as ufuncs_isnan - +from scipy.ndimage import distance_transform_cdt +from skimage.morphology import binary_erosion, footprints def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube: diff --git a/src/openeo_gfmap/spatial.py b/src/openeo_gfmap/spatial.py index 48bb2b0..1a38c49 100644 --- a/src/openeo_gfmap/spatial.py +++ b/src/openeo_gfmap/spatial.py @@ -3,7 +3,6 @@ from typing import Union from geojson import GeoJSON -from typing import Dict @dataclass diff --git a/src/openeo_gfmap/temporal.py b/src/openeo_gfmap/temporal.py index 503b988..7fdb2c3 100644 --- a/src/openeo_gfmap/temporal.py +++ b/src/openeo_gfmap/temporal.py @@ -1,7 +1,5 @@ """ Definitions of temporal context""" from dataclasses import dataclass -from typing import Tuple - from datetime import datetime diff --git a/src/openeo_gfmap/utils/__init__.py b/src/openeo_gfmap/utils/__init__.py index 4d163c2..b9f6608 100644 --- a/src/openeo_gfmap/utils/__init__.py +++ b/src/openeo_gfmap/utils/__init__.py @@ -1,14 +1,14 @@ """This sub-module contains utilitary function and tools for OpenEO-GFMap""" from openeo_gfmap.utils.build_df import load_json +from openeo_gfmap.utils.intervals import quintad_intervals from openeo_gfmap.utils.tile_processing import ( + array_bounds, + arrays_cosine_similarity, normalize_array, select_optical_bands, - array_bounds, select_sar_bands, - arrays_cosine_similarity ) -from openeo_gfmap.utils.intervals import quintad_intervals __all__ = [ "load_json", "normalize_array", "select_optical_bands", "array_bounds", diff --git a/src/openeo_gfmap/utils/build_df.py b/src/openeo_gfmap/utils/build_df.py index aeb0d35..7627189 100644 --- a/src/openeo_gfmap/utils/build_df.py +++ b/src/openeo_gfmap/utils/build_df.py @@ -2,7 +2,6 @@ based job. Usefull to collect the output of point based extraction. """ -from datetime import datetime from pathlib import Path import pandas as pd diff --git a/src/openeo_gfmap/utils/catalogue.py b/src/openeo_gfmap/utils/catalogue.py index 92d3e95..b552da4 100644 --- a/src/openeo_gfmap/utils/catalogue.py +++ b/src/openeo_gfmap/utils/catalogue.py @@ -1,10 +1,11 @@ """Functionalities to interract with product catalogues.""" import requests - -from shapely.geometry import shape from geojson import GeoJSON +from shapely.geometry import shape + from openeo_gfmap import SpatialContext, TemporalContext + def _check_cdse_catalogue( collection: str, spatial_extent: SpatialContext, diff --git a/src/openeo_gfmap/utils/intervals.py b/src/openeo_gfmap/utils/intervals.py index b4dfa5e..022ca64 100644 --- a/src/openeo_gfmap/utils/intervals.py +++ b/src/openeo_gfmap/utils/intervals.py @@ -2,9 +2,9 @@ methods. """ -from openeo_gfmap import TemporalContext +from datetime import timedelta -from datetime import datetime, timedelta +from openeo_gfmap import TemporalContext def quintad_intervals(temporal_extent: TemporalContext) -> list: diff --git a/src/openeo_gfmap/utils/tile_processing.py b/src/openeo_gfmap/utils/tile_processing.py index ddec31f..c7add8a 100644 --- a/src/openeo_gfmap/utils/tile_processing.py +++ b/src/openeo_gfmap/utils/tile_processing.py @@ -1,7 +1,8 @@ """Utilitaries to process data tiles.""" -import xarray as xr import numpy as np +import xarray as xr + def normalize_array( inarr: xr.DataArray, diff --git a/tests/test_openeo_gfmap/__init__.py b/tests/test_openeo_gfmap/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_openeo_gfmap/test_cloud_masking.py b/tests/test_openeo_gfmap/test_cloud_masking.py index edc037d..e0ba500 100644 --- a/tests/test_openeo_gfmap/test_cloud_masking.py +++ b/tests/test_openeo_gfmap/test_cloud_masking.py @@ -1,18 +1,17 @@ from pathlib import Path -from typing import Union - import pytest -from openeo_gfmap.backend import Backend, BackendContext, BACKEND_CONNECTIONS -from openeo_gfmap.temporal import TemporalContext -from openeo_gfmap.spatial import BoundingBoxExtent +from openeo_gfmap.backend import BACKEND_CONNECTIONS, Backend, BackendContext from openeo_gfmap.fetching import FetchType, build_sentinel2_l2a_extractor - from openeo_gfmap.preprocessing import ( - get_bap_score, bap_masking, median_compositing, get_bap_mask + bap_masking, + get_bap_mask, + get_bap_score, + median_compositing, ) - +from openeo_gfmap.spatial import BoundingBoxExtent +from openeo_gfmap.temporal import TemporalContext from openeo_gfmap.utils import quintad_intervals backends = [Backend.TERRASCOPE, Backend.CDSE] diff --git a/tests/test_openeo_gfmap/test_intervals.py b/tests/test_openeo_gfmap/test_intervals.py index fc80347..2ce833d 100644 --- a/tests/test_openeo_gfmap/test_intervals.py +++ b/tests/test_openeo_gfmap/test_intervals.py @@ -1,5 +1,6 @@ -from openeo_gfmap.utils import quintad_intervals from openeo_gfmap.temporal import TemporalContext +from openeo_gfmap.utils import quintad_intervals + def test_quintad_january(): start_date = "2023-01-01" diff --git a/tests/test_openeo_gfmap/test_s1_fetchers.py b/tests/test_openeo_gfmap/test_s1_fetchers.py index 7034d4f..9d81fb2 100644 --- a/tests/test_openeo_gfmap/test_s1_fetchers.py +++ b/tests/test_openeo_gfmap/test_s1_fetchers.py @@ -1,32 +1,37 @@ """ Test the data extractors for Sentinel1 data. """ from pathlib import Path -import pytest - import geojson -import openeo import geopandas as gpd -import xarray as xr +import openeo +import pytest import rioxarray +import xarray as xr -from openeo_gfmap.backend import BACKEND_CONNECTIONS from openeo_gfmap import ( - SpatialContext, TemporalContext, Backend, BackendContext, BoundingBoxExtent + Backend, + BackendContext, + BoundingBoxExtent, + SpatialContext, + TemporalContext, ) - +from openeo_gfmap.backend import BACKEND_CONNECTIONS from openeo_gfmap.fetching import ( - CollectionFetcher, build_sentinel1_grd_extractor, FetchType + CollectionFetcher, + FetchType, + build_sentinel1_grd_extractor, ) - from openeo_gfmap.utils import ( - array_bounds, normalize_array, select_sar_bands, arrays_cosine_similarity, - load_json + array_bounds, + arrays_cosine_similarity, + load_json, + normalize_array, + select_sar_bands, ) # Retrieve the test parameters from the s2 fetcher tests -from .test_s2_fetchers import ( - test_configurations, test_backends, POINT_EXTRACTION_DF -) +from .test_s2_fetchers import POINT_EXTRACTION_DF, test_backends, test_configurations + class TestS1Extractors: """Build collection extractor for different S1 collections on different diff --git a/tests/test_openeo_gfmap/test_s2_fetchers.py b/tests/test_openeo_gfmap/test_s2_fetchers.py index 9afb062..fed1bc3 100644 --- a/tests/test_openeo_gfmap/test_s2_fetchers.py +++ b/tests/test_openeo_gfmap/test_s2_fetchers.py @@ -1,23 +1,27 @@ """ Tests for data extractors for Sentinel2 data. """ from pathlib import Path -from typing import Union import geojson import geopandas as gpd -import numpy as np import openeo import pytest import rioxarray import xarray as xr -from openeo_gfmap import SpatialContext, TemporalContext, BoundingBoxExtent +from openeo_gfmap import BoundingBoxExtent, SpatialContext, TemporalContext from openeo_gfmap.backend import BACKEND_CONNECTIONS, Backend, BackendContext from openeo_gfmap.fetching import ( CollectionFetcher, FetchType, build_sentinel2_l2a_extractor, ) -from openeo_gfmap.utils import load_json, normalize_array, select_optical_bands, array_bounds, arrays_cosine_similarity +from openeo_gfmap.utils import ( + array_bounds, + arrays_cosine_similarity, + load_json, + normalize_array, + select_optical_bands, +) # Fields close to TAP, Belgium SPATIAL_EXTENT_1 = {