CMR-8506 Adding Bulk granule update for ECHO10 OnlineAccessURLs (#1843)

* CMR-8506 Adding Bulk granule update for ECHO10 OnlineAccessURLs * Update ingest-app/resources/granule_bulk_update_schema.json Co-authored-by: K. Lucia Z. <[email protected]> * Update ingest-app/resources/granule_bulk_update_schema.json Co-authored-by: K. Lucia Z. <[email protected]> * Updated to fix comment. --------- Co-authored-by: K. Lucia Z. <[email protected]>
nasa · Feb 1, 2023 · 21d4bab · 21d4bab
1 parent 86a118b
commit 21d4bab
Show file tree

Hide file tree

Showing 6 changed files with 238 additions and 360 deletions.
diff --git a/ingest-app/resources/granule_bulk_update_schema.json b/ingest-app/resources/granule_bulk_update_schema.json
@@ -15,6 +15,7 @@
         "Checksum",
         "Format",
         "MimeType",
+        "OnlineAccessURL",
         "OnlineResourceURL",
         "OPeNDAPLink",
         "S3Link",
@@ -49,6 +50,24 @@
         }
       }
     },
+    "UpdateURLType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Describes an existing URL and replacement URL.",
+      "properties": {
+        "from": {
+          "description": "The existing URL to be updated.",
+          "type": "string",
+          "minLength": 1
+        },
+        "to": {
+          "description": "The new URL to be substituted",
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": [ "from", "to" ]
+    },
     "FileType": {
       "type": "object",
       "additionalProperties": false,
@@ -82,23 +101,11 @@
         "MimeType": {
           "type": "string"
         },
+        "OnlineAccessURL": {
+          "$ref": "#/definitions/UpdateURLType"
+        },
         "OnlineResourceURL": {
-          "type": "object",
-          "additionalProperties": false,
-          "description": "Describes an OnlineResource existing URL and replacment URL.",
-          "properties": {
-            "from": {
-              "description": "The existing URL to updated.",
-              "type": "string",
-              "minLength": 1
-            },
-            "to": {
-              "description": "The new URL to be substituted",
-              "type": "string",
-              "minLength": 1
-            }
-          },
-          "required": [ "from", "to" ]
+          "$ref": "#/definitions/UpdateURLType"
         },
         "Checksum": {
           "type": "object",

diff --git a/ingest-app/src/cmr/ingest/services/granule_bulk_update/mime_type/echo10.clj b/ingest-app/src/cmr/ingest/services/granule_bulk_update/mime_type/echo10.clj
@@ -5,67 +5,10 @@
    [clojure.data.xml :as xml]
    [clojure.set :as set]
    [clojure.string :as string]
-   [clojure.zip :as zip]
    [cmr.common.services.errors :as errors]
    [cmr.common.xml :as cx]
    [cmr.ingest.services.granule-bulk-update.utils.echo10 :as echo10-utils]))
 
-(defn- xml-elem->online-access-url
-  "Parses and returns XML element for OnlineAccessURL"
-  [elem]
-  (let [url (cx/string-at-path elem [:URL])
-        description (cx/string-at-path elem [:URLDescription])
-        mime-type (cx/string-at-path elem [:MimeType])]
-    {:url url
-     :url-description description
-     :mime-type mime-type}))
-
-(defn- update-accesses
-  "Constructs the new OnlineAccessURLs node in zipper representation"
-  [online-access-urls url-map]
-  (let [edn-access-urls (map xml-elem->online-access-url online-access-urls)
-        access-urls (map #(merge %
-                                 (when-let [mime-type (get url-map (:url %))]
-                                   {:mime-type mime-type}))
-                         edn-access-urls)]
-    (xml/element
-     :OnlineAccessURLs {}
-     (for [r access-urls]
-       (let [{:keys [url url-description mime-type]} r]
-         (xml/element :OnlineAccessURL {}
-                      (xml/element :URL {} url)
-                      (when url-description (xml/element :URLDescription {} url-description))
-                      (when mime-type (xml/element :MimeType {} mime-type))))))))
-
-(defn- replace-in-tree
-  "Take a parsed granule xml, replace the given node with the provided replacement
-  Returns the zipper representation of the updated xml."
-  [tree element-tag replacement]
-  (let [zipper (zip/xml-zip tree)
-        start-loc (zip/down zipper)]
-    (loop [loc start-loc done false]
-      (if done
-        (zip/root loc)
-        (if-let [right-loc (zip/right loc)]
-          (cond
-            ;; at an OnlineResources element, replace the node with updated value
-            (= element-tag (-> right-loc zip/node :tag))
-            (recur (zip/replace right-loc replacement) true)
-
-            ;; no action needs to be taken, move to the next node
-            :else
-            (recur right-loc false))
-          (recur loc true))))))
-
-(defn- update-online-access-urls
-  "Return an OnlineAccessURLs node where MimeType is updated where the URL has a matching entry in the url-map."
-  [tree url-map]
-  (let [online-access-urls (cx/elements-at-path
-                            tree
-                            [:OnlineAccessURLs :OnlineAccessURL])]
-    (when (seq online-access-urls)
-      (update-accesses online-access-urls url-map))))
-
 (defn update-mime-type
   "Update the the MimeType for elements within OnlineResources and OnlineAccess in echo10
    granule metadata and return granule."
@@ -93,10 +36,10 @@
                    "]")]))
 
         online-resources (echo10-utils/update-online-resources parsed :url :mime-type url-map)
-        access-urls (update-online-access-urls parsed url-map)
+        access-urls (echo10-utils/update-online-access-urls parsed :url :mime-type url-map)
 
         updated-metadata (-> parsed
-                             (replace-in-tree :OnlineResources online-resources)
-                             (replace-in-tree :OnlineAccessURLs access-urls)
+                             (echo10-utils/replace-in-tree :OnlineResources online-resources)
+                             (echo10-utils/replace-in-tree :OnlineAccessURLs access-urls)
                              xml/indent-str)]
     (assoc concept :metadata updated-metadata)))
diff --git a/ingest-app/src/cmr/ingest/services/granule_bulk_update/online_resource_url/echo10.clj b/ingest-app/src/cmr/ingest/services/granule_bulk_update/online_resource_url/echo10.clj
@@ -5,32 +5,10 @@
    [clojure.data.xml :as xml]
    [clojure.set :as set]
    [clojure.string :as string]
-   [clojure.zip :as zip]
-   [cmr.common.log :refer [info debug]]
    [cmr.common.services.errors :as errors]
    [cmr.common.xml :as cx]
    [cmr.ingest.services.granule-bulk-update.utils.echo10 :as echo10-utils]))
 
-(defn- replace-in-tree
-  "Take a parsed granule xml, replace the given node with the provided replacement
-  Returns the zipper representation of the updated xml."
-  [tree element-tag replacement]
-  (let [zipper (zip/xml-zip tree)
-        start-loc (zip/down zipper)]
-    (loop [loc start-loc done false]
-      (if done
-        (zip/root loc)
-        (if-let [right-loc (zip/right loc)]
-          (cond
-            ;; at an OnlineResources element, replace the node with updated value
-            (= element-tag (-> right-loc zip/node :tag))
-            (recur (zip/replace right-loc replacement) true)
-
-            ;; no action needs to be taken, move to the next node
-            :else
-            (recur right-loc false))
-          (recur loc true))))))
-
 (defn update-online-resource-url
   "Update the the URL for elements within OnlineResources in echo10
   granule metadata and return metadata."
@@ -57,5 +35,34 @@
 
         online-resources (echo10-utils/update-online-resources parsed :url :url url-map)]
     (-> parsed
-        (replace-in-tree :OnlineResources online-resources)
+        (echo10-utils/replace-in-tree :OnlineResources online-resources)
+        xml/indent-str)))
+
+(defn update-online-access-url
+  "Update the the URL for elements within OnlineAccesses in echo10
+  granule metadata and return metadata."
+  [concept links]
+  (let [parsed (xml/parse-str (:metadata concept))
+        url-map (apply merge (map #(hash-map (:from %) (:to %)) links))
+
+        existing-urls (map #(cx/string-at-path % [:URL])
+                           (cx/elements-at-path parsed [:OnlineAccessURLs :OnlineAccessURL]))
+        _ (when-not (set/superset? (set existing-urls) (set (keys url-map)))
+            (errors/throw-service-errors
+             :invalid-data
+             [(str "Update failed - please only specify URLs contained in the"
+                   " existing granule OnlineAccessURLs ["
+                   (string/join ", " (set/difference (set (keys url-map)) (set existing-urls)))
+                   "] were not found")]))
+
+        _ (when-not (= (count (set (keys url-map))) (count links))
+            (errors/throw-service-errors
+             :invalid-data
+             [(str "Update failed - duplicate URLs provided for granule update ["
+                   (string/join ", " (for [[v freq] (frequencies (map :URL links)) :when (> freq 1)] v))
+                   "]")]))
+
+        online-access-urls (echo10-utils/update-online-access-urls parsed :url :url url-map)]
+    (-> parsed
+        (echo10-utils/replace-in-tree :OnlineAccessURLs online-access-urls)
         xml/indent-str)))
diff --git a/ingest-app/src/cmr/ingest/services/granule_bulk_update/utils/echo10.clj b/ingest-app/src/cmr/ingest/services/granule_bulk_update/utils/echo10.clj
@@ -2,6 +2,7 @@
   "Contains functions for updating ECHO10 granule xml metadata."
   (:require
    [clojure.data.xml :as xml]
+   [clojure.zip :as zip]
    [cmr.common.xml :as cx]))
 
 (defn xml-elem->online-resource
@@ -43,3 +44,60 @@
                           [:OnlineResources :OnlineResource])]
     (when (seq online-resources)
       (update-resources online-resources locator-field update-field value-map))))
+
+(defn- xml-elem->online-access-url
+  "Parses and returns XML element for OnlineAccessURL"
+  [elem]
+  (let [url (cx/string-at-path elem [:URL])
+        description (cx/string-at-path elem [:URLDescription])
+        mime-type (cx/string-at-path elem [:MimeType])]
+    {:url url
+     :url-description description
+     :mime-type mime-type}))
+
+(defn- update-accesses
+  "Constructs the new OnlineAccessURLs node in zipper representation"
+  [online-access-urls locator-field value-field value-map]
+  (let [edn-access-urls (map xml-elem->online-access-url online-access-urls)
+        access-urls (map #(merge %
+                               (when-let [replacement (get value-map (get % locator-field))]
+                                 (hash-map value-field replacement)))
+                       edn-access-urls)]
+    (xml/element
+     :OnlineAccessURLs {}
+     (for [r access-urls]
+       (let [{:keys [url url-description mime-type]} r]
+         (xml/element :OnlineAccessURL {}
+                      (xml/element :URL {} url)
+                      (when url-description (xml/element :URLDescription {} url-description))
+                      (when mime-type (xml/element :MimeType {} mime-type))))))))
+
+(defn update-online-access-urls
+  "Return an OnlineAccessURLs node where UPDATE-FIELD is updated where the
+  LOCATOR-FIELD has a matching key in the VALUE-MAP."
+  [xml-tree locator-field value-field value-map]
+  (let [online-access-urls (cx/elements-at-path
+                            xml-tree
+                            [:OnlineAccessURLs :OnlineAccessURL])]
+    (when (seq online-access-urls)
+      (update-accesses online-access-urls locator-field value-field value-map))))
+
+(defn replace-in-tree
+  "Take a parsed granule xml, replace the given node with the provided replacement
+  Returns the zipper representation of the updated xml."
+  [tree element-tag replacement]
+  (let [zipper (zip/xml-zip tree)
+        start-loc (zip/down zipper)]
+    (loop [loc start-loc done false]
+      (if done
+        (zip/root loc)
+        (if-let [right-loc (zip/right loc)]
+          (cond
+            ;; at an OnlineResources element, replace the node with updated value
+            (= element-tag (-> right-loc zip/node :tag))
+            (recur (zip/replace right-loc replacement) true)
+
+            ;; no action needs to be taken, move to the next node
+            :else
+            (recur right-loc false))
+          (recur loc true))))))