Merge pull request #23 from fmidev/feature-add-granule-layer-template

Add option to specify layer name w/template in add_granule

examples/granules.yaml

@@ -31,6 +31,9 @@ file_pattern: "{start_time:%Y%m%d_%H%M}_{platform_name}_{areaname}_{productname}
 
 # The filename part to use for layer identification
 layer_id: productname
+# OR use a template for the layer name with items from file_pattern
+# if layer_id given, this is ignored
+# layer_name_template: optclass_{radar_name}
 
 # For delete_old_granules_and_files.py
 #    Maximum age for granules, in minutes
@@ -52,6 +55,24 @@ layers:
   airmass: satellite_geo_europe_seviri-15min_airmass
   ash: satellite_geo_europe_seviri-15min_ash
 
+# Layer options for delete granule layers when using layer_name_template instead of layer_id
+# delete_granule_layer_options:
+#   radar:
+#     [
+#       fivim,
+#       fikan,
+#       fikes,
+#       fikor,
+#       fikuo,
+#       filuo,
+#       finur,
+#       fipet,
+#       fiuta,
+#       fianj,
+#       fivih,
+#     ]
+#   quantity: [dbzh, vrad]
+
 log_config:
   version: 1
   formatters:

georest/__init__.py

@@ -372,8 +372,14 @@ def _get_store_name_from_filename(config, fname):
     """Parse store name from filename."""
     file_pattern = config["file_pattern"]
     file_parts = trollsift.parse(file_pattern, os.path.basename(fname))
-    layer_id = file_parts[config["layer_id"]]
-    return config["layers"][layer_id]
+
+    if "layer_id" in config:
+        layer_id = file_parts[config["layer_id"]]
+        return config["layers"][layer_id]
+    if "layer_name_template" in config:
+        layer_name = trollsift.compose(config["layer_name_template"], file_parts)
+        return layer_name
+    raise ValueError("Either 'layer_id' or 'layer_name_template' must be defined in config")
 
 
 def delete_file_from_mosaic(config, fname):
@@ -416,7 +422,8 @@ def delete_old_files_from_mosaics_and_fs(config):
     cat = connect_to_gs_catalog(config)
     workspace = config["workspace"]
     max_age = dt.datetime.utcnow() - dt.timedelta(minutes=config["max_age"])
-    for store in config["layers"].values():
+
+    for store in utils.get_layers_for_delete_granules(config):
         store_obj = cat.get_store(store, workspace)
         logger.debug("Getting coverage for %s", store)
         coverage = get_layer_coverage(cat, store, store_obj)

georest/tests/test_geoserver.py

@@ -8,7 +8,7 @@
 """Unittests for Geoserver REST methods."""
 
 from copy import deepcopy
-from unittest import mock
+from unittest import mock, TestCase
 
 
 @mock.patch("georest.Catalog")
@@ -287,6 +287,27 @@ def test_get_layer_coverage():
     "layers": {"airmass": "airmass_store"},
 }
 
+ADD_FILE_TO_MOSAIC_LAYERTEMPLATE_CONFIG = {
+    "host": "http://host/",
+    "user": "user",
+    "passwd": "passwd",
+    "workspace": "satellite",
+    "geoserver_target_dir": "/mnt/data",
+    "keep_subpath": False,
+    "file_pattern": "{area}_{productname}.tif",
+    "layer_name_template": "{productname}_{area}",
+}
+
+ADD_FILE_TO_MOSAIC_FAIL_CONFIG = {
+    "host": "http://host/",
+    "user": "user",
+    "passwd": "passwd",
+    "workspace": "satellite",
+    "geoserver_target_dir": "/mnt/data",
+    "keep_subpath": False,
+    "file_pattern": "{area}_{productname}.tif",
+}
+
 
 @mock.patch("georest.utils.file_in_granules")
 @mock.patch("georest.connect_to_gs_catalog")
@@ -307,8 +328,30 @@ def test_add_file_to_mosaic(connect_to_gs_catalog, file_in_granules):
     add_file_to_mosaic(config, fname_in)
 
     connect_to_gs_catalog.assert_called_with(config)
-    add_granule.assert_called_with("/mnt/data/europe_airmass.tif",
-                                   "airmass_store", "satellite")
+    add_granule.assert_called_with("/mnt/data/europe_airmass.tif", "airmass_store", "satellite")
+    add_file_to_mosaic(config, fname_in)
+
+
+@mock.patch("georest.utils.file_in_granules")
+@mock.patch("georest.connect_to_gs_catalog")
+def test_add_file_to_mosaic_layertemplate(connect_to_gs_catalog, file_in_granules):
+    """Test adding files to image mosaic."""
+    from georest import add_file_to_mosaic
+
+    config = deepcopy(ADD_FILE_TO_MOSAIC_LAYERTEMPLATE_CONFIG)
+
+    # Returns False if the file isn't in database
+    file_in_granules.return_value = False
+    add_granule = mock.MagicMock()
+    cat = mock.MagicMock(add_granule=add_granule)
+    connect_to_gs_catalog.return_value = cat
+
+    fname_in = "/path/to/europe_airmass.tif"
+
+    add_file_to_mosaic(config, fname_in)
+
+    connect_to_gs_catalog.assert_called_with(config)
+    add_granule.assert_called_with("/mnt/data/europe_airmass.tif", "airmass_europe", "satellite")
     add_file_to_mosaic(config, fname_in)
 
 
@@ -352,6 +395,25 @@ def test_add_file_to_mosaic_failed_request(connect_to_gs_catalog):
     add_file_to_mosaic(config, fname_in)
 
 
+@mock.patch("georest.connect_to_gs_catalog")
+def test_add_file_to_mosaic_missing_config(connect_to_gs_catalog):
+    """Test that a failed file addition is handled."""
+
+    from georest import add_file_to_mosaic
+
+    config = deepcopy(ADD_FILE_TO_MOSAIC_FAIL_CONFIG)
+
+    add_granule = mock.MagicMock()
+    cat = mock.MagicMock(add_granule=add_granule)
+    connect_to_gs_catalog.return_value = cat
+
+    fname_in = "/path/to/europe_airmass.tif"
+
+    # Check that failed request is handled
+    with TestCase().assertRaises(ValueError):
+        add_file_to_mosaic(config, fname_in)
+
+
 @mock.patch("georest.requests")
 @mock.patch("georest.utils.file_in_granules")
 @mock.patch("georest.connect_to_gs_catalog")
@@ -409,16 +471,62 @@ def test_delete_file_from_mosaic(connect_to_gs_catalog):
     cat.delete_granule.assert_not_called()
 
     # This is the structure returned by cat.list_granules()
-    granules = {"features":
-                [{"properties": {"location": "/mnt/data/europe_airmass.tif"},
-                  "id": "file-id"}]
-                }
+    granules = {"features": [{"properties": {"location": "/mnt/data/europe_airmass.tif"}, "id": "file-id"}]}
     cat.list_granules.return_value = granules
 
     delete_file_from_mosaic(config, fname)
     cat.delete_granule.assert_called()
 
 
+@mock.patch("georest.connect_to_gs_catalog")
+def test_delete_file_from_mosaic_layertemplate(connect_to_gs_catalog):
+    """Test deleting files from image mosaic."""
+    from georest import delete_file_from_mosaic
+
+    # This is the structure returned by cat.mosaic_coverages()
+    coverages = {
+        "coverages": {
+            "coverage": [
+                {"name": "airmass1_europe"},
+                {"name": "airmass2_europe"},
+                {"name": "airmass1_global"},
+                {"name": "airmass2_global"},
+            ],
+        }
+    }
+    cat = mock.MagicMock()
+    cat.mosaic_coverages.return_value = coverages
+    connect_to_gs_catalog.return_value = cat
+
+    config = {
+        "workspace": "satellite",
+        "geoserver_target_dir": "/mnt/data",
+        "file_pattern": "{area}_{productname}.tif",
+        "layer_name_template": "{productname}_{area}",
+        "delete_granule_layer_options": {"area": ["europe", "global"], "productname": ["airmass1", "airmass2"]},
+    }
+    fnames = ["europe_airmass1.tif", "global_airmass1.tif", "europe_airmass2.tif", "global_airmass2.tif"]
+
+    for fname in fnames:
+        delete_file_from_mosaic(config, fname)
+
+    connect_to_gs_catalog.assert_called_with(config)
+    assert cat.get_store.call_count == len(fnames)
+    assert "call('airmass1_europe', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass2_europe', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass1_global', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass2_global', 'satellite')" in str(cat.get_store.call_args_list)
+    cat.list_granules.assert_called()
+    cat.delete_granule.assert_not_called()
+
+    # This is the structure returned by cat.list_granules()
+    granules = {"features": [{"properties": {"location": "/mnt/data/europe_airmass1.tif"}, "id": "file-id"}]}
+    cat.list_granules.return_value = granules
+
+    delete_file_from_mosaic(config, fnames[0])
+    cat.delete_granule.assert_called()
+
+
 @mock.patch("georest.requests")
 def test_create_s3_layers(requests):
     """Test creating layers from S3 data."""

georest/tests/test_utils.py

@@ -364,3 +364,28 @@ def test_convert_file_path_keep_subpath_inverse():
     res = convert_file_path(config, "/geoserver/internal/path/subpath/file.tif", inverse=True, keep_subpath=True)
 
     assert res == "/external/path/subpath/file.tif"
+
+
+def test_get_layers_for_delete_granules():
+    """Test the function that provides layer names for delete granules."""
+    from georest.utils import get_layers_for_delete_granules
+
+    # Case 1: layer_id and layers provided
+    config = {"layer_id": "layer_id", "layers": {"layer_1": "layer_name_1", "layer_2": "layer_name_2"}}
+    res = get_layers_for_delete_granules(config)
+
+    assert res == ["layer_name_1", "layer_name_2"]
+
+    # Case 2: layer_name_template and delete_granule_layer_options provided
+    config = {
+        "layer_name_template": "{opt1}_{opt2}",
+        "delete_granule_layer_options": {"opt1": ["option1_1", "option1_2"], "opt2": ["layer_name_2"]},
+    }
+    res = get_layers_for_delete_granules(config)
+
+    assert res == ["option1_1_layer_name_2", "option1_2_layer_name_2"]
+
+    # Case 3: no layer_id or layer_name_template provided
+    config = {}
+    with pytest.raises(ValueError):
+        get_layers_for_delete_granules(config)

georest/utils.py

@@ -15,6 +15,7 @@
 import shutil
 import tempfile
 import zipfile
+import itertools
 
 import trollsift
 import yaml
@@ -129,6 +130,20 @@ def get_exposed_layer_directories(config):
     return dirs
 
 
+def get_layers_for_delete_granules(config):
+    """Get list of layers for deleting granules."""
+    if config.get("layer_id", False):
+        return list(config["layers"].values())
+    if config.get("layer_name_template", False) and config.get("delete_granule_layer_options", False):
+        keys = sorted(config["delete_granule_layer_options"].keys())
+        combinations = list(itertools.product(*[config["delete_granule_layer_options"][k] for k in keys]))
+        options = [dict(zip(keys, l)) for l in combinations]
+        return [config["layer_name_template"].format(**opt) for opt in options]
+    raise ValueError(
+        "Either 'layer_id' or 'layer_name_template' (with 'delete_granule_layer_options') must be defined in config"
+    )
+
+
 def write_wkt(config, image_fname):
     """Write WKT text besides the image file.