Merge branch 'main' into issue9-jenkins-auth

.github/workflows/lint.yaml

@@ -0,0 +1,30 @@
+name: Lint
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  lint:
+    name: "Lint: code quality and formatting checks"
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone repo
+      uses: actions/checkout@v2
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+        cache: 'pip'
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install isort black ruff
+    - name: isort
+      run: python -m isort . --check --diff
+    # - name: black
+    #   run: python -m black --check --diff .
+    - name: ruff
+      run: ruff check .

.github/workflows/pytest-collect.yaml

@@ -0,0 +1,31 @@
+name: Pytest-collect
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'tests/**'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'tests/**'
+
+jobs:
+  pytest-collect:
+    name: "Pytest: check test collection"
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone repo
+      uses: actions/checkout@v2
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest
+        python -m pip install .[dev]
+    - name: "pytest: check test collection"
+      run: pytest --collect-only

pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     "h5netcdf>=1.2.0",
     "openeo[localprocessing]",
     "cftime",
-    "pytest-dependency",
+    "pytest-depends",
     "pyarrow",
     "fastparquet"
 ]
@@ -60,5 +60,20 @@ testpaths = [
   "tests",
 ]
 addopts = [
-  "--import-mode=importlib",
+  "--import-mode=prepend",
 ]
+
+[tool.isort]
+profile = "black"
+
+
+[tool.ruff]
+# line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F"]
+ignore = [
+  "E501",  # Ignore "line-too-long" issues, let black handle that.
+]
+
+

src/openeo_gfmap/__init__.py

@@ -8,7 +8,7 @@
 
 from .backend import Backend, BackendContext
 from .metadata import FakeMetadata
-from .spatial import SpatialContext, BoundingBoxExtent
+from .spatial import BoundingBoxExtent, SpatialContext
 from .temporal import TemporalContext
 
 __all__ = [

src/openeo_gfmap/fetching/__init__.py

@@ -6,8 +6,8 @@
 """
 
 from .fetching import CollectionFetcher, FetchType
-from .s2 import build_sentinel2_l2a_extractor
 from .s1 import build_sentinel1_grd_extractor
+from .s2 import build_sentinel2_l2a_extractor
 
 __all__ = [
     "build_sentinel2_l2a_extractor", "CollectionFetcher", "FetchType",

src/openeo_gfmap/fetching/s1.py

@@ -1,23 +1,22 @@
 """ Collection fetching of S1 features, supporting different backends.
 """
-from typing import Callable
 from functools import partial
+from typing import Callable
 
 import openeo
 from geojson import GeoJSON
 
 from openeo_gfmap.backend import Backend, BackendContext
-from openeo_gfmap.spatial import SpatialContext, BoundingBoxExtent
+from openeo_gfmap.spatial import SpatialContext
 from openeo_gfmap.temporal import TemporalContext
 
-from .fetching import CollectionFetcher, FetchType
-
 from .commons import (
     convert_band_names,
     load_collection,
     rename_bands,
     resample_reproject,
 )
+from .fetching import CollectionFetcher, FetchType
 
 BASE_SENTINEL1_GRD_MAPPING = {
     "VH": "S1-VH",

src/openeo_gfmap/preprocessing/__init__.py

@@ -2,9 +2,14 @@
 right after the extraction and the execution of the features UDF.
 """
 
-from openeo_gfmap.preprocessing.cloudmasking import mask_scl_dilation, get_bap_score, bap_masking, get_bap_mask
+from openeo_gfmap.preprocessing.cloudmasking import (
+    bap_masking,
+    get_bap_mask,
+    get_bap_score,
+    mask_scl_dilation,
+)
+from openeo_gfmap.preprocessing.compositing import mean_compositing, median_compositing
 from openeo_gfmap.preprocessing.interpolation import linear_interpolation
-from openeo_gfmap.preprocessing.compositing import median_compositing, mean_compositing
 
 __all__ = [
     "mask_scl_dilation",

src/openeo_gfmap/preprocessing/cloudmasking.py

@@ -1,7 +1,8 @@
 """Different cloud masking strategies for an OpenEO datacubes."""
 
-from typing import Union
 from pathlib import Path
+from typing import Union
+
 import openeo
 from openeo.processes import if_, is_nan
 

src/openeo_gfmap/preprocessing/interpolation.py

@@ -3,6 +3,7 @@
 """
 import openeo
 
+
 def linear_interpolation(cube: openeo.DataCube,) -> openeo.DataCube:
     """Perform linear interpolation on the given datacube."""
     return cube.apply_dimension(

src/openeo_gfmap/preprocessing/udf_rank.py

@@ -1,9 +1,9 @@
-import xarray as xr
-from pathlib import Path
-import numpy as np
 
+import numpy as np
+import xarray as xr
 from openeo.udf import XarrayDataCube
 
+
 def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube:
     """For a cube having the BAP score, and a given period of list of intervals,
     create a binary mask that for each pixel, is True if the BAP score is the

src/openeo_gfmap/preprocessing/udf_score.py

@@ -1,14 +1,10 @@
+import math
+
 import numpy as np
-import time, math
 import xarray as xr
-import datetime
-
-from scipy.ndimage import distance_transform_cdt
-from skimage.morphology import footprints
-from skimage.morphology import binary_erosion, binary_dilation
 from openeo.udf import XarrayDataCube
-from xarray.ufuncs import isnan as ufuncs_isnan
-
+from scipy.ndimage import distance_transform_cdt
+from skimage.morphology import binary_erosion, footprints
 
 
 def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube:

src/openeo_gfmap/spatial.py

@@ -3,7 +3,6 @@
 from typing import Union
 
 from geojson import GeoJSON
-from typing import Dict
 
 
 @dataclass

src/openeo_gfmap/temporal.py

@@ -1,7 +1,5 @@
 """ Definitions of temporal context"""
 from dataclasses import dataclass
-from typing import Tuple
-
 from datetime import datetime
 
 

src/openeo_gfmap/utils/__init__.py

@@ -1,14 +1,14 @@
 """This sub-module contains utilitary function and tools for OpenEO-GFMap"""
 
 from openeo_gfmap.utils.build_df import load_json
+from openeo_gfmap.utils.intervals import quintad_intervals
 from openeo_gfmap.utils.tile_processing import (
+    array_bounds,
+    arrays_cosine_similarity,
     normalize_array,
     select_optical_bands,
-    array_bounds,
     select_sar_bands,
-    arrays_cosine_similarity
 )
-from openeo_gfmap.utils.intervals import quintad_intervals
 
 __all__ = [
     "load_json", "normalize_array", "select_optical_bands", "array_bounds",

src/openeo_gfmap/utils/build_df.py

@@ -2,7 +2,6 @@
 based job. Usefull to collect the output of point based extraction.
 """
 
-from datetime import datetime
 from pathlib import Path
 
 import pandas as pd

src/openeo_gfmap/utils/catalogue.py

@@ -1,10 +1,11 @@
 """Functionalities to interract with product catalogues."""
 import requests
-
-from shapely.geometry import shape
 from geojson import GeoJSON
+from shapely.geometry import shape
+
 from openeo_gfmap import SpatialContext, TemporalContext
 
+
 def _check_cdse_catalogue(
     collection: str,
     spatial_extent: SpatialContext,

src/openeo_gfmap/utils/intervals.py

@@ -2,9 +2,9 @@
 methods.
 """
 
-from openeo_gfmap import TemporalContext
+from datetime import timedelta
 
-from datetime import datetime, timedelta
+from openeo_gfmap import TemporalContext
 
 
 def quintad_intervals(temporal_extent: TemporalContext) -> list:

src/openeo_gfmap/utils/tile_processing.py

@@ -1,7 +1,8 @@
 """Utilitaries to process data tiles."""
 
-import xarray as xr
 import numpy as np
+import xarray as xr
+
 
 def normalize_array(
     inarr: xr.DataArray,

tests/test_openeo_gfmap/test_cloud_masking.py

@@ -1,18 +1,17 @@
 from pathlib import Path
 
-from typing import Union
-
 import pytest
 
-from openeo_gfmap.backend import Backend, BackendContext, BACKEND_CONNECTIONS
-from openeo_gfmap.temporal import TemporalContext
-from openeo_gfmap.spatial import BoundingBoxExtent
+from openeo_gfmap.backend import BACKEND_CONNECTIONS, Backend, BackendContext
 from openeo_gfmap.fetching import FetchType, build_sentinel2_l2a_extractor
-
 from openeo_gfmap.preprocessing import (
-    get_bap_score, bap_masking, median_compositing, get_bap_mask
+    bap_masking,
+    get_bap_mask,
+    get_bap_score,
+    median_compositing,
 )
-
+from openeo_gfmap.spatial import BoundingBoxExtent
+from openeo_gfmap.temporal import TemporalContext
 from openeo_gfmap.utils import quintad_intervals
 
 backends = [Backend.TERRASCOPE, Backend.CDSE]

tests/test_openeo_gfmap/test_intervals.py

@@ -1,5 +1,6 @@
-from openeo_gfmap.utils import quintad_intervals
 from openeo_gfmap.temporal import TemporalContext
+from openeo_gfmap.utils import quintad_intervals
+
 
 def test_quintad_january():
     start_date = "2023-01-01"

tests/test_openeo_gfmap/test_s1_fetchers.py

@@ -1,32 +1,37 @@
 """ Test the data extractors for Sentinel1 data. """
 from pathlib import Path
 
-import pytest
-
 import geojson
-import openeo
 import geopandas as gpd
-import xarray as xr
+import openeo
+import pytest
 import rioxarray
+import xarray as xr
 
-from openeo_gfmap.backend import BACKEND_CONNECTIONS
 from openeo_gfmap import (
-    SpatialContext, TemporalContext, Backend, BackendContext, BoundingBoxExtent
+    Backend,
+    BackendContext,
+    BoundingBoxExtent,
+    SpatialContext,
+    TemporalContext,
 )
-
+from openeo_gfmap.backend import BACKEND_CONNECTIONS
 from openeo_gfmap.fetching import (
-    CollectionFetcher, build_sentinel1_grd_extractor, FetchType
+    CollectionFetcher,
+    FetchType,
+    build_sentinel1_grd_extractor,
 )
-
 from openeo_gfmap.utils import (
-    array_bounds, normalize_array, select_sar_bands, arrays_cosine_similarity,
-    load_json
+    array_bounds,
+    arrays_cosine_similarity,
+    load_json,
+    normalize_array,
+    select_sar_bands,
 )
 
 # Retrieve the test parameters from the s2 fetcher tests
-from .test_s2_fetchers import (
-    test_configurations, test_backends, POINT_EXTRACTION_DF
-)
+from .test_s2_fetchers import POINT_EXTRACTION_DF, test_backends, test_configurations
+
 
 class TestS1Extractors:
     """Build collection extractor for different S1 collections on different