diff --git a/src/openeo_gfmap/fetching/__init__.py b/src/openeo_gfmap/fetching/__init__.py index e6009f1..1101b82 100644 --- a/src/openeo_gfmap/fetching/__init__.py +++ b/src/openeo_gfmap/fetching/__init__.py @@ -7,15 +7,27 @@ import logging -from .fetching import CollectionFetcher, FetchType -from .s1 import build_sentinel1_grd_extractor -from .s2 import build_sentinel2_l2a_extractor - _log = logging.getLogger(__name__) +_log.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) + +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +ch.setFormatter(formatter) + +_log.addHandler(ch) __all__ = [ "build_sentinel2_l2a_extractor", "CollectionFetcher", "FetchType", "build_sentinel1_grd_extractor", + "build_generic_extractor", + "build_generic_extractor_stac", ] + +from .fetching import CollectionFetcher, FetchType # noqa: E402 +from .generic import build_generic_extractor, build_generic_extractor_stac # noqa: E402 +from .s1 import build_sentinel1_grd_extractor # noqa: E402 +from .s2 import build_sentinel2_l2a_extractor # noqa: E402 diff --git a/src/openeo_gfmap/fetching/generic.py b/src/openeo_gfmap/fetching/generic.py index 46edb74..bc388d5 100644 --- a/src/openeo_gfmap/fetching/generic.py +++ b/src/openeo_gfmap/fetching/generic.py @@ -28,11 +28,23 @@ "vapour-pressure": "AGERA5-VAPOUR", "wind-speed": "AGERA5-WIND", } +AGERA5_STAC_MAPPING = { + "dewpoint_temperature_mean": "AGERA5-DEWTEMP", + "total_precipitation": "AGERA5-PRECIP", + "solar_radiation_flux": "AGERA5-SOLRAD", + "2m_temperature_max": "AGERA5-TMAX", + "2m_temperature_mean": "AGERA5-TMEAN", + "2m_temperature_min": "AGERA5-TMIN", + "vapour_pressure": "AGERA5-VAPOUR", + "wind_speed": "AGERA5-WIND", +} KNOWN_UNTEMPORAL_COLLECTIONS = ["COPERNICUS_30"] +AGERA5_TERRASCOPE_STAC = "https://stac.openeo.vito.be/collections/agera5_daily" + def _get_generic_fetcher( - collection_name: str, fetch_type: FetchType, backend: Backend + collection_name: str, fetch_type: FetchType, backend: Backend, is_stac: bool ) -> Callable: band_mapping: Optional[dict] = None @@ -40,6 +52,8 @@ def _get_generic_fetcher( band_mapping = BASE_DEM_MAPPING elif collection_name == "AGERA5": band_mapping = BASE_WEATHER_MAPPING + elif is_stac and (AGERA5_TERRASCOPE_STAC in collection_name): + band_mapping = AGERA5_STAC_MAPPING def generic_default_fetcher( connection: openeo.Connection, @@ -68,6 +82,7 @@ def generic_default_fetcher( spatial_extent, temporal_extent, fetch_type, + is_stac=is_stac, **params, ) except OpenEoApiError as e: @@ -86,7 +101,9 @@ def generic_default_fetcher( return generic_default_fetcher -def _get_generic_processor(collection_name: str, fetch_type: FetchType) -> Callable: +def _get_generic_processor( + collection_name: str, fetch_type: FetchType, is_stac: bool +) -> Callable: """Builds the preprocessing function from the collection name as it stored in the target backend. """ @@ -95,6 +112,8 @@ def _get_generic_processor(collection_name: str, fetch_type: FetchType) -> Calla band_mapping = BASE_DEM_MAPPING elif collection_name == "AGERA5": band_mapping = BASE_WEATHER_MAPPING + elif is_stac and (AGERA5_TERRASCOPE_STAC in collection_name): + band_mapping = AGERA5_STAC_MAPPING def generic_default_processor(cube: openeo.DataCube, **params): """Default collection preprocessing method for generic datasets. @@ -126,9 +145,29 @@ def build_generic_extractor( fetch_type: FetchType, collection_name: str, **params, +) -> CollectionFetcher: + """Creates a generic extractor adapted to the given backend. Provides band mappings for known + collections, such as AGERA5 available on Terrascope/FED and Copernicus 30m DEM in all backends. + """ + fetcher = _get_generic_fetcher( + collection_name, fetch_type, backend_context.backend, False + ) + preprocessor = _get_generic_processor(collection_name, fetch_type, False) + + return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params) + + +def build_generic_extractor_stac( + backend_context: BackendContext, + bands: list, + fetch_type: FetchType, + collection_url: str, + **params, ) -> CollectionFetcher: """Creates a generic extractor adapted to the given backend. Currently only tested with VITO backend""" - fetcher = _get_generic_fetcher(collection_name, fetch_type, backend_context.backend) - preprocessor = _get_generic_processor(collection_name, fetch_type) + fetcher = _get_generic_fetcher( + collection_url, fetch_type, backend_context.backend, True + ) + preprocessor = _get_generic_processor(collection_url, fetch_type, True) return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params) diff --git a/src/openeo_gfmap/fetching/meteo.py b/src/openeo_gfmap/fetching/meteo.py deleted file mode 100644 index a10c545..0000000 --- a/src/openeo_gfmap/fetching/meteo.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Meteo data fetchers.""" - -from functools import partial - -import openeo -from geojson import GeoJSON - -from openeo_gfmap import ( - Backend, - BackendContext, - FetchType, - SpatialContext, - TemporalContext, -) -from openeo_gfmap.fetching import CollectionFetcher -from openeo_gfmap.fetching.commons import convert_band_names -from openeo_gfmap.fetching.generic import ( - _get_generic_fetcher, - _get_generic_processor, - _load_collection, -) - -WEATHER_MAPPING_TERRASCOPE = { - "dewpoint-temperature": "AGERA5-DEWTEMP", - "precipitation-flux": "AGERA5-PRECIP", - "solar-radiation-flux": "AGERA5-SOLRAD", - "temperature-max": "AGERA5-TMAX", - "temperature-mean": "AGERA5-TMEAN", - "temperature-min": "AGERA5-TMIN", - "vapour-pressure": "AGERA5-VAPOUR", - "wind-speed": "AGERA5-WIND", -} - -WEATHER_MAPPING_STAC = { - "dewpoint_temperature_mean": "AGERA5-DEWTEMP", - "total_precipitation": "AGERA5-PRECIP", - "solar_radiataion_flux": "AGERA5-SOLRAD", - "2m_temperature_max": "AGERA5-TMAX", - "2m_temperature_mean": "AGERA5-TMEAN", - "2m_temperature_min": "AGERA5-TMIN", - "vapour_pressure": "AGERA5-VAPOUR", - "wind_speed": "AGERA5-WIND", -} - - -def stac_fetcher( - connection: openeo.Connection, - spatial_extent: SpatialContext, - temporal_extent: TemporalContext, - bands: list, - fetch_type: FetchType, - **params, -) -> openeo.DataCube: - bands = convert_band_names(bands, WEATHER_MAPPING_STAC) - - cube = _load_collection( - connection, - bands, - "https://stac.openeo.vito.be/collections/agera5_daily", - spatial_extent, - temporal_extent, - fetch_type, - is_stac=True, - **params, - ) - - if isinstance(spatial_extent, GeoJSON) and fetch_type == FetchType.POLYGON: - cube = cube.filter_spatial(spatial_extent) - - return cube - - -METEO_BACKEND_MAP = { - Backend.TERRASCOPE: { - "fetch": partial( - _get_generic_fetcher, - collection_name="AGERA5", - band_mapping=WEATHER_MAPPING_TERRASCOPE, - ), - "preprocessor": partial( - _get_generic_processor, - collection_name="AGERA5", - band_mapping=WEATHER_MAPPING_TERRASCOPE, - ), - }, - Backend.CDSE: { - "fetch": stac_fetcher, - "preprocessor": partial( - _get_generic_processor, - collection_name="AGERA5", - band_mapping=WEATHER_MAPPING_STAC, - ), - }, - Backend.CDSE_STAGING: { - "fetch": stac_fetcher, - "preprocessor": partial( - _get_generic_processor, - collection_name="AGERA5", - band_mapping=WEATHER_MAPPING_STAC, - ), - }, - Backend.FED: { - "fetch": stac_fetcher, - "preprocessor": partial( - _get_generic_processor, - collection_name="AGERA5", - band_mapping=WEATHER_MAPPING_STAC, - ), - }, -} - - -def build_meteo_extractor( - backend_context: BackendContext, - bands: list, - fetch_type: FetchType, - **params, -) -> CollectionFetcher: - backend_functions = METEO_BACKEND_MAP.get(backend_context.backend) - - fetcher, preprocessor = ( - partial(backend_functions["fetch"], fetch_type=fetch_type), - backend_functions["preprocessor"](fetch_type=fetch_type), - ) - - return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params)