Skip to content

Commit

Permalink
Sentinel2 l2a (#11)
Browse files Browse the repository at this point in the history 
* Sentinel2 L2A processing capabilities added
  • Loading branch information
@felixlapalma
felixlapalma authored Dec 17, 2021
1 parent eba0137 commit 49ef972
Show file tree
Hide file tree
Showing 10 changed files with 1,686 additions and 315 deletions.
5 changes: 3 additions & 2 deletions eo_forge/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ def set_default_logger(logger=None):
default_logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)

fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s",
datefmt="%Y/%m/%d %H:%M:%S")
fmt = logging.Formatter(
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y/%m/%d %H:%M:%S"
)

handler.setFormatter(fmt)

Expand Down
205 changes: 43 additions & 162 deletions eo_forge/io/GenLoader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,7 @@ def __init__(
self.raw_metadata = {}

if not os.path.isdir(folder):
msg = f"folder {folder} does not exist"
raise ValueError(msg)
raise ValueError(f"folder {folder} does not exist")

self.archive_folder = folder
self.bbox = bbox
Expand All @@ -109,37 +108,24 @@ def __init__(
self.bands.append(band)

@abstractmethod
def _get_product_path(
self,
product_id,
):
def _get_product_path(self, product_id):
"""
Returns the local path where the product id is stored, relative to the archive's
root folder. This path mimics exactly the Google Cloud storage structure.
"""
raise NotImplementedError

@abstractmethod
def _read_metadata(
self,
product_path,
):
def _read_metadata(self, product_path):
"""Returns a dictionary with the product metadata."""
raise NotImplementedError

def post_process_band(
self,
raster,
band,
):
def post_process_band(self, raster, band):
"""Postprocess the band data (ndarray input)."""
return raster

@abstractmethod
def _clean_product_id(
self,
product_id,
):
def _clean_product_id(self, product_id):
"""Cleans product id."""
raise NotImplementedError

Expand All @@ -149,11 +135,7 @@ def _get_is_valid_data(self, raster):
raise NotImplementedError

@abstractmethod
def _preprocess_clouds_mask(
self,
product_path,
**kwargs,
):
def _preprocess_clouds_mask(self, product_path, **kwargs):
"""Returns raster cloud mask."""
raise NotImplementedError

Expand Down Expand Up @@ -213,38 +195,14 @@ def _get_bands_data(
base_bands_data_profiles = {}
base_bands_match_ = {}

enable_transform = kwargs.get(
"enable_transform",
True,
)
crop = kwargs.pop(
"crop",
True,
)
hard_bbox = kwargs.get(
"hard_bbox",
False,
)
bbox = kwargs.get(
"bbox",
self.bbox,
)
nodata = kwargs.get(
"nodata",
0,
)
reproject = kwargs.get(
"reproject",
True,
)
all_touched = kwargs.get(
"all_touched",
True,
)
calibrate = kwargs.get(
"calibrate",
True,
)
enable_transform = kwargs.get("enable_transform", True)
crop = kwargs.pop("crop", True)
hard_bbox = kwargs.get("hard_bbox", False)
bbox = kwargs.get("bbox", self.bbox)
nodata = kwargs.get("nodata", 0)
reproject = kwargs.get("reproject", True)
all_touched = kwargs.get("all_touched", True)
calibrate = kwargs.get("calibrate", True)

clipping_flag = bbox is not None
self.logger.info(f"Using clipping flag: {clipping_flag}")
Expand Down Expand Up @@ -461,42 +419,15 @@ def _get_cloud_mask(
data_profile: raster cloud data profile
"""
cloud_band = kwargs.get(
"cloud_band",
"BQA",
)
enable_transform = kwargs.get(
"enable_transform",
True,
)
crop = kwargs.pop(
"crop",
True,
)
hard_bbox = kwargs.get(
"hard_bbox",
False,
)
bbox = kwargs.get(
"bbox",
self.bbox,
)
nodata = kwargs.get(
"nodata",
0,
)
reproject = kwargs.get(
"reproject",
True,
)
all_touched = kwargs.get(
"all_touched",
True,
)
calibrate = kwargs.get(
"calibrate",
True,
)
cloud_band = kwargs.get("cloud_band", "BQA")
enable_transform = kwargs.get("enable_transform", True)
crop = kwargs.pop("crop", True)
hard_bbox = kwargs.get("hard_bbox", False)
bbox = kwargs.get("bbox", self.bbox)
nodata = kwargs.get("nodata", 0)
reproject = kwargs.get("reproject", True)
all_touched = kwargs.get("all_touched", True)
calibrate = kwargs.get("calibrate", True)

clipping_flag = bbox is not None

Expand Down Expand Up @@ -590,39 +521,24 @@ def _get_cloud_mask(
if resample_flag:
self.logger.info(f"resampling full band")
raster_dataset = resample_raster(
raster_dataset,
scale_factor,
close=True,
raster_dataset, scale_factor, close=True
)
self.logger.info(f"no bbox - full match: {full_match} - area: {area}")

if calibrate:
self.logger.info(f"calibrating band {cloud_band}")
# Apply postprocessing (calibration)
raster_dataset = self.post_process_band(
raster_dataset,
cloud_band,
)
raster_dataset = self.post_process_band(raster_dataset, cloud_band)

if reproject:
self.logger.info(f"reprojecting band {cloud_band}")
raster_dataset = reproject_raster_north_south(
raster_dataset,
close=True,
)
raster_dataset = reproject_raster_north_south(raster_dataset, close=True)

# Get Data
(
data,
data_profile,
) = get_raster_data_and_profile(raster_dataset)
data, data_profile = get_raster_data_and_profile(raster_dataset)

# Help python a little bit with the memory management
gc.collect()
return (
data.astype(rasterio.ubyte),
data_profile,
)
return data.astype(rasterio.ubyte), data_profile

def execute(
self,
Expand Down Expand Up @@ -671,23 +587,12 @@ def execute(

self.folder_proc_ = folder_proc_

bbox = kwargs.pop(
"bbox",
self.bbox,
)
bbox = kwargs.pop("bbox", self.bbox)

if os.path.isdir(product_id):
product_path = product_id
elif os.path.isdir(
os.path.join(
self.archive_folder,
product_id,
)
):
product_path = os.path.join(
self.archive_folder,
product_id,
)
elif os.path.isdir(os.path.join(self.archive_folder, product_id)):
product_path = os.path.join(self.archive_folder, product_id)
else:
product_path = self._get_product_path(product_id)

Expand All @@ -708,14 +613,11 @@ def execute(
base_bands_data,
base_bands_data_profiles,
base_band_match,
) = self._get_bands_data(
metadata,
bbox=bbox,
**kwargs,
)
) = self._get_bands_data(metadata, bbox=bbox, **kwargs)

##
# Store the base bands in a single array
# The bands order is the one expected by eolearn.
ordered_bands = [
band for band in self._ordered_bands if band in base_bands_data
]
Expand All @@ -727,10 +629,7 @@ def execute(
_base_bands_match_ = [base_band_match[band] for band in ordered_bands]

# [bands x n x m ] -> rasterio order
data = np.stack(
_base_bands_data,
axis=0,
)
data = np.stack(_base_bands_data, axis=0)
data_dtype = _base_bands_data_profiles[0]["dtype"]

# For export
Expand Down Expand Up @@ -761,22 +660,15 @@ def execute(
)

if not write_file:
raster = write_mem_raster(
data,
**base_profile,
)
raster = write_mem_raster(data, **base_profile)
file_ = None
self.logger.info(f"Leaving raster processed data IN-MEMORY")
else:
file_ = os.path.join(
self.folder_proc_,
product_id_cleaned + write_file + write_end,
)
raster = write_raster(
file_,
data,
**base_profile,
)
raster = write_raster(file_, data, **base_profile)

self.logger.info(f"Writting raster processed data to {file_}")
# clean
Expand All @@ -787,16 +679,10 @@ def execute(
# Quality Band
if process_clouds:
cloud_raster = self._preprocess_clouds_mask(
metadata,
**{
"raster_base": raster,
"no_data": 0,
},
metadata, **{"raster_base": raster, "no_data": 0}
)
(cloud_data, cloud_profile,) = self._get_cloud_mask(
cloud_raster,
bbox=bbox,
**kwargs,
cloud_data, cloud_profile = self._get_cloud_mask(
cloud_raster, bbox=bbox, **kwargs
)

# make raster_cloud
Expand All @@ -809,10 +695,8 @@ def execute(
}
)
if not write_file:
raster_cloud = write_mem_raster(
cloud_data,
**cloud_profile,
)
raster_cloud = write_mem_raster(cloud_data, **cloud_profile)

self.logger.info(
f"Leaving raster cloud processed data IN-MEMORY",
)
Expand All @@ -821,11 +705,8 @@ def execute(
self.folder_proc_,
product_id_cleaned + "_CLOUDS" + write_file + write_end,
)
raster_cloud = write_raster(
file_cloud,
cloud_data,
**cloud_profile,
)
raster_cloud = write_raster(file_cloud, cloud_data, **cloud_profile)

self.logger.info(
f"Writting raster cloud processed data to {file_cloud}",
)
Expand Down
4 changes: 2 additions & 2 deletions eo_forge/io/LandsatLoaders.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
"""

import glob

import os
import rasterio
import warnings
from datetime import datetime

import rasterio

from eo_forge.io.GenLoader import BaseGenericLoader
from eo_forge.utils.landsat import (
LANDSAT5_BANDS_RESOLUTION,
Expand Down
Loading

0 comments on commit 49ef972

Please sign in to comment.