From 4133525ebcc4e475a12e63a7d15dd4e682936c7c Mon Sep 17 00:00:00 2001
From: Jenna Ritvanen <jenna.ritvanen@fmi.fi>
Date: Mon, 13 Nov 2023 14:49:00 +0200
Subject: [PATCH 1/7] Add option to specify layer name w/template

This will make it easier to add granules to radar layers,
where whe can have multiple similar layers (e.g. dbzh, vrad, zdr)
for multiple radar sites
---
 examples/granules.yaml |  3 +++
 georest/__init__.py    | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/examples/granules.yaml b/examples/granules.yaml
index e0c5646..50cf24d 100644
--- a/examples/granules.yaml
+++ b/examples/granules.yaml
@@ -31,6 +31,9 @@ file_pattern: "{start_time:%Y%m%d_%H%M}_{platform_name}_{areaname}_{productname}
 
 # The filename part to use for layer identification
 layer_id: productname
+# OR use a template for the layer name with items from file_pattern
+# if layer_id given, this is ignored
+# layer_name_template: optclass_{radar_name}
 
 # For delete_old_granules_and_files.py
 #    Maximum age for granules, in minutes
diff --git a/georest/__init__.py b/georest/__init__.py
index 59305bb..38532f4 100644
--- a/georest/__init__.py
+++ b/georest/__init__.py
@@ -372,8 +372,15 @@ def _get_store_name_from_filename(config, fname):
     """Parse store name from filename."""
     file_pattern = config["file_pattern"]
     file_parts = trollsift.parse(file_pattern, os.path.basename(fname))
-    layer_id = file_parts[config["layer_id"]]
-    return config["layers"][layer_id]
+
+    if "layer_id" in config:
+        layer_id = file_parts[config["layer_id"]]
+        return config["layers"][layer_id]
+    elif "layer_name_template" in config:
+        layer_name = trollsift.compose(config["layer_name_template"], file_parts)
+        return layer_name
+    else:
+        raise ValueError("Either 'layer_id' or 'layer_name_template' must be defined in config")
 
 
 def delete_file_from_mosaic(config, fname):

From 05108ff39a1f249008fbf46679a9056fc95c93ec Mon Sep 17 00:00:00 2001
From: Jenna Ritvanen <jenna.ritvanen@fmi.fi>
Date: Tue, 14 Nov 2023 15:08:58 +0200
Subject: [PATCH 2/7] Update layer parse for delete_old_granules_and_files

---
 examples/granules.yaml | 18 ++++++++++++++++++
 georest/__init__.py    | 20 +++++++++++++++++++-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/examples/granules.yaml b/examples/granules.yaml
index 50cf24d..422572d 100644
--- a/examples/granules.yaml
+++ b/examples/granules.yaml
@@ -55,6 +55,24 @@ layers:
   airmass: satellite_geo_europe_seviri-15min_airmass
   ash: satellite_geo_europe_seviri-15min_ash
 
+# Layer options for delete granule layers when using layer_name_template instead of layer_id
+# delete_granule_layer_options:
+#   radar:
+#     [
+#       fivim,
+#       fikan,
+#       fikes,
+#       fikor,
+#       fikuo,
+#       filuo,
+#       finur,
+#       fipet,
+#       fiuta,
+#       fianj,
+#       fivih,
+#     ]
+#   quantity: [dbzh, vrad]
+
 log_config:
   version: 1
   formatters:
diff --git a/georest/__init__.py b/georest/__init__.py
index 38532f4..e989766 100644
--- a/georest/__init__.py
+++ b/georest/__init__.py
@@ -11,6 +11,7 @@
 import datetime as dt
 import logging
 import os
+import itertools
 
 try:
     import requests
@@ -414,6 +415,22 @@ def delete_granule(cat, workspace, store, fname):
         logger.info("Granule '%s' removed", fname)
 
 
+def get_layers_for_delete_granules(config):
+    """Get list of layers for deleting granules."""
+    if config.get("layer_id", False):
+        layers = list(config["layers"].values())
+    elif config.get("layer_name_template", False) and config.get("delete_granule_layer_options", False):
+        keys = sorted(config["delete_granule_layer_options"].keys())
+        combinations = list(itertools.product(*[config["delete_granule_layer_options"][k] for k in keys]))
+        options = [dict(zip(keys, l)) for l in combinations]
+        layers = [config["layer_name_template"].format(**opt) for opt in options]
+    else:
+        raise ValueError(
+            "Either 'layer_id' or 'layer_name_template' (with 'delete_granule_layer_options') must be defined in config"
+        )
+    return layers
+
+
 def delete_old_files_from_mosaics_and_fs(config):
     """Delete a file from image mosaic.
 
@@ -423,7 +440,8 @@ def delete_old_files_from_mosaics_and_fs(config):
     cat = connect_to_gs_catalog(config)
     workspace = config["workspace"]
     max_age = dt.datetime.utcnow() - dt.timedelta(minutes=config["max_age"])
-    for store in config["layers"].values():
+
+    for store in get_layers_for_delete_granules(config):
         store_obj = cat.get_store(store, workspace)
         logger.debug("Getting coverage for %s", store)
         coverage = get_layer_coverage(cat, store, store_obj)

From 0a8e687171676e0ac29712e57e591539bf98a311 Mon Sep 17 00:00:00 2001
From: Jenna Ritvanen <jenna.ritvanen@fmi.fi>
Date: Mon, 20 Nov 2023 08:55:55 +0200
Subject: [PATCH 3/7] Add tests for adding/deleting granules with layer name
 template

---
 georest/tests/test_geoserver.py | 83 ++++++++++++++++++++++++++++++---
 1 file changed, 77 insertions(+), 6 deletions(-)

diff --git a/georest/tests/test_geoserver.py b/georest/tests/test_geoserver.py
index c9c789c..7703098 100644
--- a/georest/tests/test_geoserver.py
+++ b/georest/tests/test_geoserver.py
@@ -287,6 +287,17 @@ def test_get_layer_coverage():
     "layers": {"airmass": "airmass_store"},
 }
 
+ADD_FILE_TO_MOSAIC_LAYERTEMPLATE_CONFIG = {
+    "host": "http://host/",
+    "user": "user",
+    "passwd": "passwd",
+    "workspace": "satellite",
+    "geoserver_target_dir": "/mnt/data",
+    "keep_subpath": False,
+    "file_pattern": "{area}_{productname}.tif",
+    "layer_name_template": "{productname}_{area}",
+}
+
 
 @mock.patch("georest.utils.file_in_granules")
 @mock.patch("georest.connect_to_gs_catalog")
@@ -307,8 +318,30 @@ def test_add_file_to_mosaic(connect_to_gs_catalog, file_in_granules):
     add_file_to_mosaic(config, fname_in)
 
     connect_to_gs_catalog.assert_called_with(config)
-    add_granule.assert_called_with("/mnt/data/europe_airmass.tif",
-                                   "airmass_store", "satellite")
+    add_granule.assert_called_with("/mnt/data/europe_airmass.tif", "airmass_store", "satellite")
+    add_file_to_mosaic(config, fname_in)
+
+
+@mock.patch("georest.utils.file_in_granules")
+@mock.patch("georest.connect_to_gs_catalog")
+def test_add_file_to_mosaic_layertemplate(connect_to_gs_catalog, file_in_granules):
+    """Test adding files to image mosaic."""
+    from georest import add_file_to_mosaic
+
+    config = deepcopy(ADD_FILE_TO_MOSAIC_LAYERTEMPLATE_CONFIG)
+
+    # Returns False if the file isn't in database
+    file_in_granules.return_value = False
+    add_granule = mock.MagicMock()
+    cat = mock.MagicMock(add_granule=add_granule)
+    connect_to_gs_catalog.return_value = cat
+
+    fname_in = "/path/to/europe_airmass.tif"
+
+    add_file_to_mosaic(config, fname_in)
+
+    connect_to_gs_catalog.assert_called_with(config)
+    add_granule.assert_called_with("/mnt/data/europe_airmass.tif", "airmass_europe", "satellite")
     add_file_to_mosaic(config, fname_in)
 
 
@@ -409,10 +442,48 @@ def test_delete_file_from_mosaic(connect_to_gs_catalog):
     cat.delete_granule.assert_not_called()
 
     # This is the structure returned by cat.list_granules()
-    granules = {"features":
-                [{"properties": {"location": "/mnt/data/europe_airmass.tif"},
-                  "id": "file-id"}]
-                }
+    granules = {"features": [{"properties": {"location": "/mnt/data/europe_airmass.tif"}, "id": "file-id"}]}
+    cat.list_granules.return_value = granules
+
+    delete_file_from_mosaic(config, fname)
+    cat.delete_granule.assert_called()
+
+
+@mock.patch("georest.connect_to_gs_catalog")
+def test_delete_file_from_mosaic_layertemplate(connect_to_gs_catalog):
+    """Test deleting files from image mosaic."""
+    from georest import delete_file_from_mosaic
+
+    # This is the structure returned by cat.mosaic_coverages()
+    coverages = {
+        "coverages": {
+            "coverage": [
+                {"name": "airmass_europe"},
+            ]
+        }
+    }
+    cat = mock.MagicMock()
+    cat.mosaic_coverages.return_value = coverages
+    connect_to_gs_catalog.return_value = cat
+
+    config = {
+        "workspace": "satellite",
+        "geoserver_target_dir": "/mnt/data",
+        "file_pattern": "{area}_{productname}.tif",
+        "layer_name_template": "{productname}_{area}",
+        "delete_granule_layer_options": {"area": ["europe"], "productname": ["airmass"]},
+    }
+    fname = "europe_airmass.tif"
+
+    delete_file_from_mosaic(config, fname)
+
+    connect_to_gs_catalog.assert_called_with(config)
+    cat.get_store.assert_called_with("airmass_europe", "satellite")
+    cat.list_granules.assert_called()
+    cat.delete_granule.assert_not_called()
+
+    # This is the structure returned by cat.list_granules()
+    granules = {"features": [{"properties": {"location": "/mnt/data/europe_airmass.tif"}, "id": "file-id"}]}
     cat.list_granules.return_value = granules
 
     delete_file_from_mosaic(config, fname)

From f3b71152749c342658cda61660c05a063e8905c2 Mon Sep 17 00:00:00 2001
From: Jenna Ritvanen <jenna.ritvanen@fmi.fi>
Date: Mon, 20 Nov 2023 09:22:49 +0200
Subject: [PATCH 4/7] Move get layer names func to utils

---
 georest/__init__.py         | 19 +------------------
 georest/tests/test_utils.py | 25 +++++++++++++++++++++++++
 georest/utils.py            | 17 +++++++++++++++++
 3 files changed, 43 insertions(+), 18 deletions(-)

diff --git a/georest/__init__.py b/georest/__init__.py
index e989766..34fb9f2 100644
--- a/georest/__init__.py
+++ b/georest/__init__.py
@@ -11,7 +11,6 @@
 import datetime as dt
 import logging
 import os
-import itertools
 
 try:
     import requests
@@ -415,22 +414,6 @@ def delete_granule(cat, workspace, store, fname):
         logger.info("Granule '%s' removed", fname)
 
 
-def get_layers_for_delete_granules(config):
-    """Get list of layers for deleting granules."""
-    if config.get("layer_id", False):
-        layers = list(config["layers"].values())
-    elif config.get("layer_name_template", False) and config.get("delete_granule_layer_options", False):
-        keys = sorted(config["delete_granule_layer_options"].keys())
-        combinations = list(itertools.product(*[config["delete_granule_layer_options"][k] for k in keys]))
-        options = [dict(zip(keys, l)) for l in combinations]
-        layers = [config["layer_name_template"].format(**opt) for opt in options]
-    else:
-        raise ValueError(
-            "Either 'layer_id' or 'layer_name_template' (with 'delete_granule_layer_options') must be defined in config"
-        )
-    return layers
-
-
 def delete_old_files_from_mosaics_and_fs(config):
     """Delete a file from image mosaic.
 
@@ -441,7 +424,7 @@ def delete_old_files_from_mosaics_and_fs(config):
     workspace = config["workspace"]
     max_age = dt.datetime.utcnow() - dt.timedelta(minutes=config["max_age"])
 
-    for store in get_layers_for_delete_granules(config):
+    for store in utils.get_layers_for_delete_granules(config):
         store_obj = cat.get_store(store, workspace)
         logger.debug("Getting coverage for %s", store)
         coverage = get_layer_coverage(cat, store, store_obj)
diff --git a/georest/tests/test_utils.py b/georest/tests/test_utils.py
index 4a8e0ce..078fc55 100644
--- a/georest/tests/test_utils.py
+++ b/georest/tests/test_utils.py
@@ -364,3 +364,28 @@ def test_convert_file_path_keep_subpath_inverse():
     res = convert_file_path(config, "/geoserver/internal/path/subpath/file.tif", inverse=True, keep_subpath=True)
 
     assert res == "/external/path/subpath/file.tif"
+
+
+def test_get_layers_for_delete_granules():
+    """Test the function that provides layer names for delete granules."""
+    from georest.utils import get_layers_for_delete_granules
+
+    # Case 1: layer_id and layers provided
+    config = {"layer_id": "layer_id", "layers": {"layer_1": "layer_name_1", "layer_2": "layer_name_2"}}
+    res = get_layers_for_delete_granules(config)
+
+    assert res == ["layer_name_1", "layer_name_2"]
+
+    # Case 2: layer_name_template and delete_granule_layer_options provided
+    config = {
+        "layer_name_template": "{opt1}_{opt2}",
+        "delete_granule_layer_options": {"opt1": ["option1_1", "option1_2"], "opt2": ["layer_name_2"]},
+    }
+    res = get_layers_for_delete_granules(config)
+
+    assert res == ["option1_1_layer_name_2", "option1_2_layer_name_2"]
+
+    # Case 3: no layer_id or layer_name_template provided
+    config = {}
+    with pytest.raises(ValueError):
+        get_layers_for_delete_granules(config)
diff --git a/georest/utils.py b/georest/utils.py
index 519ee83..0324d4f 100644
--- a/georest/utils.py
+++ b/georest/utils.py
@@ -15,6 +15,7 @@
 import shutil
 import tempfile
 import zipfile
+import itertools
 
 import trollsift
 import yaml
@@ -129,6 +130,22 @@ def get_exposed_layer_directories(config):
     return dirs
 
 
+def get_layers_for_delete_granules(config):
+    """Get list of layers for deleting granules."""
+    if config.get("layer_id", False):
+        layers = list(config["layers"].values())
+    elif config.get("layer_name_template", False) and config.get("delete_granule_layer_options", False):
+        keys = sorted(config["delete_granule_layer_options"].keys())
+        combinations = list(itertools.product(*[config["delete_granule_layer_options"][k] for k in keys]))
+        options = [dict(zip(keys, l)) for l in combinations]
+        layers = [config["layer_name_template"].format(**opt) for opt in options]
+    else:
+        raise ValueError(
+            "Either 'layer_id' or 'layer_name_template' (with 'delete_granule_layer_options') must be defined in config"
+        )
+    return layers
+
+
 def write_wkt(config, image_fname):
     """Write WKT text besides the image file.
 

From 9394ee31de28556af0eac876c411d52dd8f9ba29 Mon Sep 17 00:00:00 2001
From: Jenna Ritvanen <jenna.ritvanen@fmi.fi>
Date: Mon, 20 Nov 2023 09:23:34 +0200
Subject: [PATCH 5/7] Add test for missing config

---
 georest/tests/test_geoserver.py | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/georest/tests/test_geoserver.py b/georest/tests/test_geoserver.py
index 7703098..33b2679 100644
--- a/georest/tests/test_geoserver.py
+++ b/georest/tests/test_geoserver.py
@@ -8,7 +8,7 @@
 """Unittests for Geoserver REST methods."""
 
 from copy import deepcopy
-from unittest import mock
+from unittest import mock, TestCase
 
 
 @mock.patch("georest.Catalog")
@@ -298,6 +298,16 @@ def test_get_layer_coverage():
     "layer_name_template": "{productname}_{area}",
 }
 
+ADD_FILE_TO_MOSAIC_FAIL_CONFIG = {
+    "host": "http://host/",
+    "user": "user",
+    "passwd": "passwd",
+    "workspace": "satellite",
+    "geoserver_target_dir": "/mnt/data",
+    "keep_subpath": False,
+    "file_pattern": "{area}_{productname}.tif",
+}
+
 
 @mock.patch("georest.utils.file_in_granules")
 @mock.patch("georest.connect_to_gs_catalog")
@@ -385,6 +395,25 @@ def test_add_file_to_mosaic_failed_request(connect_to_gs_catalog):
     add_file_to_mosaic(config, fname_in)
 
 
+@mock.patch("georest.connect_to_gs_catalog")
+def test_add_file_to_mosaic_missing_config(connect_to_gs_catalog):
+    """Test that a failed file addition is handled."""
+
+    from georest import add_file_to_mosaic
+
+    config = deepcopy(ADD_FILE_TO_MOSAIC_FAIL_CONFIG)
+
+    add_granule = mock.MagicMock()
+    cat = mock.MagicMock(add_granule=add_granule)
+    connect_to_gs_catalog.return_value = cat
+
+    fname_in = "/path/to/europe_airmass.tif"
+
+    # Check that failed request is handled
+    with TestCase().assertRaises(ValueError):
+        add_file_to_mosaic(config, fname_in)
+
+
 @mock.patch("georest.requests")
 @mock.patch("georest.utils.file_in_granules")
 @mock.patch("georest.connect_to_gs_catalog")

From 18fc16112cf16079807c4bc6ea2580a1139cdf8f Mon Sep 17 00:00:00 2001
From: Jenna Ritvanen <jenna.ritvanen@fmi.fi>
Date: Mon, 20 Nov 2023 11:00:03 +0200
Subject: [PATCH 6/7] Simplify funcs & add test cases to delete files test

---
 georest/__init__.py             |  5 ++---
 georest/tests/test_geoserver.py | 24 ++++++++++++++++--------
 georest/utils.py                | 14 ++++++--------
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/georest/__init__.py b/georest/__init__.py
index 34fb9f2..e76fbc8 100644
--- a/georest/__init__.py
+++ b/georest/__init__.py
@@ -376,11 +376,10 @@ def _get_store_name_from_filename(config, fname):
     if "layer_id" in config:
         layer_id = file_parts[config["layer_id"]]
         return config["layers"][layer_id]
-    elif "layer_name_template" in config:
+    if "layer_name_template" in config:
         layer_name = trollsift.compose(config["layer_name_template"], file_parts)
         return layer_name
-    else:
-        raise ValueError("Either 'layer_id' or 'layer_name_template' must be defined in config")
+    raise ValueError("Either 'layer_id' or 'layer_name_template' must be defined in config")
 
 
 def delete_file_from_mosaic(config, fname):
diff --git a/georest/tests/test_geoserver.py b/georest/tests/test_geoserver.py
index 33b2679..8686219 100644
--- a/georest/tests/test_geoserver.py
+++ b/georest/tests/test_geoserver.py
@@ -487,8 +487,11 @@ def test_delete_file_from_mosaic_layertemplate(connect_to_gs_catalog):
     coverages = {
         "coverages": {
             "coverage": [
-                {"name": "airmass_europe"},
-            ]
+                {"name": "airmass1_europe"},
+                {"name": "airmass2_europe"},
+                {"name": "airmass1_global"},
+                {"name": "airmass2_global"},
+            ],
         }
     }
     cat = mock.MagicMock()
@@ -500,22 +503,27 @@ def test_delete_file_from_mosaic_layertemplate(connect_to_gs_catalog):
         "geoserver_target_dir": "/mnt/data",
         "file_pattern": "{area}_{productname}.tif",
         "layer_name_template": "{productname}_{area}",
-        "delete_granule_layer_options": {"area": ["europe"], "productname": ["airmass"]},
+        "delete_granule_layer_options": {"area": ["europe", "global"], "productname": ["airmass1", "airmass2"]},
     }
-    fname = "europe_airmass.tif"
+    fnames = ["europe_airmass1.tif", "global_airmass1.tif", "europe_airmass2.tif", "global_airmass2.tif"]
 
-    delete_file_from_mosaic(config, fname)
+    for fname in fnames:
+        delete_file_from_mosaic(config, fname)
 
     connect_to_gs_catalog.assert_called_with(config)
-    cat.get_store.assert_called_with("airmass_europe", "satellite")
+    assert cat.get_store.call_count == len(fnames)
+    assert "call('airmass1_europe', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass2_europe', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass1_global', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass1_global', 'satellite')" in str(cat.get_store.call_args_list)
     cat.list_granules.assert_called()
     cat.delete_granule.assert_not_called()
 
     # This is the structure returned by cat.list_granules()
-    granules = {"features": [{"properties": {"location": "/mnt/data/europe_airmass.tif"}, "id": "file-id"}]}
+    granules = {"features": [{"properties": {"location": "/mnt/data/europe_airmass1.tif"}, "id": "file-id"}]}
     cat.list_granules.return_value = granules
 
-    delete_file_from_mosaic(config, fname)
+    delete_file_from_mosaic(config, fnames[0])
     cat.delete_granule.assert_called()
 
 
diff --git a/georest/utils.py b/georest/utils.py
index 0324d4f..7e289bb 100644
--- a/georest/utils.py
+++ b/georest/utils.py
@@ -133,17 +133,15 @@ def get_exposed_layer_directories(config):
 def get_layers_for_delete_granules(config):
     """Get list of layers for deleting granules."""
     if config.get("layer_id", False):
-        layers = list(config["layers"].values())
-    elif config.get("layer_name_template", False) and config.get("delete_granule_layer_options", False):
+        return list(config["layers"].values())
+    if config.get("layer_name_template", False) and config.get("delete_granule_layer_options", False):
         keys = sorted(config["delete_granule_layer_options"].keys())
         combinations = list(itertools.product(*[config["delete_granule_layer_options"][k] for k in keys]))
         options = [dict(zip(keys, l)) for l in combinations]
-        layers = [config["layer_name_template"].format(**opt) for opt in options]
-    else:
-        raise ValueError(
-            "Either 'layer_id' or 'layer_name_template' (with 'delete_granule_layer_options') must be defined in config"
-        )
-    return layers
+        return [config["layer_name_template"].format(**opt) for opt in options]
+    raise ValueError(
+        "Either 'layer_id' or 'layer_name_template' (with 'delete_granule_layer_options') must be defined in config"
+    )
 
 
 def write_wkt(config, image_fname):

From 639e44abce0a302ddfd3745d2f3192a6c02bda7e Mon Sep 17 00:00:00 2001
From: Panu Lahtinen <113439939+lahtinep@users.noreply.github.com>
Date: Mon, 20 Nov 2023 10:02:36 +0000
Subject: [PATCH 7/7] Fix typo in test result

---
 georest/tests/test_geoserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/georest/tests/test_geoserver.py b/georest/tests/test_geoserver.py
index 8686219..7540e0e 100644
--- a/georest/tests/test_geoserver.py
+++ b/georest/tests/test_geoserver.py
@@ -515,7 +515,7 @@ def test_delete_file_from_mosaic_layertemplate(connect_to_gs_catalog):
     assert "call('airmass1_europe', 'satellite')" in str(cat.get_store.call_args_list)
     assert "call('airmass2_europe', 'satellite')" in str(cat.get_store.call_args_list)
     assert "call('airmass1_global', 'satellite')" in str(cat.get_store.call_args_list)
-    assert "call('airmass1_global', 'satellite')" in str(cat.get_store.call_args_list)
+    assert "call('airmass2_global', 'satellite')" in str(cat.get_store.call_args_list)
     cat.list_granules.assert_called()
     cat.delete_granule.assert_not_called()