add subType for OPERA image files

podaac · May 11, 2024 · 513c7bc · 513c7bc
1 parent c179ad4
commit 513c7bc
Show file tree

Hide file tree

Showing 48 changed files with 4,704 additions and 178 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- **PODAAC-6181**
+  - add relatedUrl subType BROWSE IMAGE SOURCE
+### Deprecated
+### Removed
+### Fixed
+### Security
+
+
 ## [8.6.0]
 ### Added
 - **PODAAC-5876**

diff --git a/pom.xml b/pom.xml
@@ -44,13 +44,13 @@
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
-      <version>1.12.701</version>
+      <version>1.12.719</version>
     </dependency>
     <!-- For AWS Secret Manager -->
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-secretsmanager</artifactId>
-      <version>1.12.701</version>
+      <version>1.12.719</version>
     </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/Constants.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/Constants.java
@@ -6,5 +6,5 @@ public static class Metadata{
         public static final String START_ORBIT = "startorbit";
         public static final String END_ORBIT = "endorbit";
     }
-    public static final String UMMG_VERSION = "1.6.5";
+    public static final String UMMG_VERSION = "1.6.6";
 }
diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java
@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.math.BigInteger;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
 import java.security.KeyManagementException;
 import java.security.KeyStoreException;
@@ -15,9 +16,12 @@
 import java.util.Hashtable;
 import java.util.List;
 
+import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
+import gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory;
 import gov.nasa.cumulus.metadata.aggregator.processor.DMRPPProcessor;
 import gov.nasa.cumulus.metadata.aggregator.processor.FootprintProcessor;
 import gov.nasa.cumulus.metadata.aggregator.processor.ImageProcessor;
+import gov.nasa.cumulus.metadata.aggregator.processor.RelatedUrlsProcessor;
 import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
 import gov.nasa.cumulus.metadata.state.WorkflowTypeEnum;
 import gov.nasa.cumulus.metadata.util.S3Utils;
@@ -57,11 +61,7 @@ public String PerformFunction(String input, Context context) throws Exception {
 		 * this will help the logic in postIngestProcess function.
 		 */
 		this.setWorkFlowType((String) config.get("stateMachine"));
-		// This is a switch to determine, shall footprint, orbit or boundingbox shall be processed from iso.xml
-		// while ingesting swot collections
-		JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
-		HashSet isoXMLSpatialTypeHashSet = createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray);
-
+		TaskConfigBO taskConfigBO = TaskConfigFactory.createTaskConfigBO(config);
 
 		String isoRegex = (String) config.get("isoRegex");
 		String archiveXmlRegex = (String) config.get("archiveXmlRegex");
@@ -154,8 +154,7 @@ public String PerformFunction(String input, Context context) throws Exception {
 
 		MetadataFilesToEcho mtfe;
         boolean isIsoFile = (iso != null);
-
-        mtfe = new MetadataFilesToEcho(isIsoFile, isoXMLSpatialTypeHashSet);
+		mtfe = new MetadataFilesToEcho(isIsoFile, taskConfigBO.getIsoXMLSpatialTypeHashSet());
 		//set the name/granuleId
 		mtfe.getGranule().setName(granuleId);
         mtfe.setDatasetValues(collectionName, collectionVersion, rangeIs360, boundingBox, additionalAttributes);
@@ -190,20 +189,27 @@ public String PerformFunction(String input, Context context) throws Exception {
 			}
 		}
 
-		//write UMM-G to file
+		/**
+		 * generate the ummg json object.  Use RelatedUrlsProcessor to append more items to RelateUrls.
+		 * See RelatedUrls java doc to know detailed logic to process payload.granules[0].files[]
+		 */
+		String cmrFilePath = "/tmp/" + granuleId + ".cmr.json";
 		try {
-			mtfe.writeJson("/tmp/" + granuleId + ".cmr.json");
+			JSONObject granuleJson = mtfe.createJson();
+			RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor();
+			granuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson, taskConfigBO, files);
+			FileUtils.writeStringToFile(new File(cmrFilePath), granuleJson.toJSONString(), StandardCharsets.UTF_8);
 		} catch (IOException e) {
 			AdapterLogger.LogError(this.className + " mtfe.writeJson error:" + e.getMessage());
 			e.printStackTrace();
 			throw e;
 		}
 
-		//copy new file to S3
 		/*
+		 * Upload cmr.json to s3 bucket
 		 * Add UMM-G file to payload/file objects
 		 */
-		File cmrFile = new File("/tmp/" + granuleId + ".cmr.json");
+		File cmrFile = new File(cmrFilePath);
 		s3Utils.upload(region, internalBucket,
 				Paths.get(stagingDirectory, "/" + granuleId + ".cmr.json").toString(),
 				cmrFile);
@@ -237,34 +243,6 @@ public String PerformFunction(String input, Context context) throws Exception {
 		}
 		return returnable.toJSONString();
 	}
-
-	public HashSet<MENDsIsoXMLSpatialTypeEnum> createIsoXMLSpatialTypeSet(JSONArray isoXMLSpatialTypeConfigJSONArray) throws IllegalArgumentException{
-		HashSet<MENDsIsoXMLSpatialTypeEnum> isoSpatialTypes = new HashSet<>();
-		// if not containing isoXMLTypes, then return an empty HashSet
-		if(isoXMLSpatialTypeConfigJSONArray == null || isoXMLSpatialTypeConfigJSONArray.size()==0) {
-			return isoSpatialTypes;
-		}
-		isoXMLSpatialTypeConfigJSONArray.forEach(item -> {
-			String t = (String) item;
-			MENDsIsoXMLSpatialTypeEnum en = MENDsIsoXMLSpatialTypeEnum.getEnum(getIsoXMLSpatialTypeStr(t));
-			isoSpatialTypes.add(en);
-		});
-		AdapterLogger.LogDebug(this.className + " isoSpatialTypes HashSet: " + isoSpatialTypes);
-		return isoSpatialTypes;
-	}
-
-	public String getIsoXMLSpatialTypeStr(String token) {
-		final String trimmedToken = StringUtils.trim(token);
-		String s;
-		try {
-			s = MENDsIsoXMLSpatialTypeEnum.getEnumValuList().stream()
-					.filter(e -> StringUtils.equals(trimmedToken, e)).findFirst().get();
-		} catch (java.util.NoSuchElementException e) {
-			s = "";
-		}
-		return s;
-	}
-
 	/**
 	 * get S3 fileStaging direction from S3 full key
 	 *

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataFilesToEcho.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataFilesToEcho.java
@@ -1074,13 +1074,13 @@ public JSONObject createJson()
 		granule.setIngestTime(new Date());
 		UMMGranuleFile granuleFile = new UMMGranuleFile(granule, dataset, rangeIs360, this.isoXMLSpatialTypeEnumHashSet);
 		JSONObject granuleJson = granuleFile.defineGranule();
+		JSONUtils.cleanJSON(granuleJson);
 		return granuleJson;
 	}
 
 	public void writeJson(String outputLocation)
 			throws IOException, ParseException, URISyntaxException{
 		JSONObject granuleJson = createJson();
-        JSONUtils.cleanJSON(granuleJson);
         FileUtils.writeStringToFile(new File(outputLocation), granuleJson.toJSONString(), StandardCharsets.UTF_8);
 	}
 

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/bo/TaskConfigBO.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/bo/TaskConfigBO.java
@@ -0,0 +1,30 @@
+package gov.nasa.cumulus.metadata.aggregator.bo;
+
+import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+
+public class TaskConfigBO {
+    HashSet<MENDsIsoXMLSpatialTypeEnum> isoXMLSpatialTypeHashSet;
+    ArrayList<HashMap<String, String>> subTypeHashArray = new ArrayList<>();
+
+    public TaskConfigBO() {}
+
+    public HashSet<MENDsIsoXMLSpatialTypeEnum> getIsoXMLSpatialTypeHashSet() {
+        return isoXMLSpatialTypeHashSet;
+    }
+
+    public void setIsoXMLSpatialTypeHashSet(HashSet<MENDsIsoXMLSpatialTypeEnum> isoXMLSpatialTypeHashSet) {
+        this.isoXMLSpatialTypeHashSet = isoXMLSpatialTypeHashSet;
+    }
+
+    public ArrayList<HashMap<String, String>> getSubTypeHashArray() {
+        return subTypeHashArray;
+    }
+
+    public void setSubTypeHashArray(ArrayList<HashMap<String, String>> subTypeHashArray) {
+        this.subTypeHashArray = subTypeHashArray;
+    }
+}
diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java
@@ -0,0 +1,71 @@
+package gov.nasa.cumulus.metadata.aggregator.factory;
+
+import cumulus_message_adapter.message_parser.AdapterLogger;
+import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
+import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
+import org.apache.commons.lang3.StringUtils;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+
+public class TaskConfigFactory {
+    static String className = "gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory";
+
+    public static TaskConfigBO createTaskConfigBO(JSONObject config) {
+        TaskConfigBO taskConfigBO = new TaskConfigBO();
+        //Construct isoXMLSpatialTypeHashset
+        JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
+        taskConfigBO.setIsoXMLSpatialTypeHashSet(createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray));
+        //Construct subTypeHashArray
+        JSONArray subTypeTypeJsonArray = (JSONArray) config.get("relatedUrlSubTypeMap");
+        taskConfigBO.setSubTypeHashArray(createSubTypeHashArray(subTypeTypeJsonArray));
+        return taskConfigBO;
+
+    }
+
+    public static HashSet<MENDsIsoXMLSpatialTypeEnum> createIsoXMLSpatialTypeSet(JSONArray isoXMLSpatialTypeConfigJSONArray) throws IllegalArgumentException{
+        HashSet<MENDsIsoXMLSpatialTypeEnum> isoSpatialTypes = new HashSet<>();
+        // if not containing isoXMLTypes, then return an empty HashSet
+        if(isoXMLSpatialTypeConfigJSONArray == null || isoXMLSpatialTypeConfigJSONArray.size()==0) {
+            return isoSpatialTypes;
+        }
+        isoXMLSpatialTypeConfigJSONArray.forEach(item -> {
+            String t = (String) item;
+            MENDsIsoXMLSpatialTypeEnum en = MENDsIsoXMLSpatialTypeEnum.getEnum(getIsoXMLSpatialTypeStr(t));
+            isoSpatialTypes.add(en);
+        });
+        AdapterLogger.LogDebug(className + " isoSpatialTypes HashSet: " + isoSpatialTypes);
+        return isoSpatialTypes;
+    }
+
+    public static String getIsoXMLSpatialTypeStr(String token) {
+        final String trimmedToken = StringUtils.trim(token);
+        String s;
+        try {
+            s = MENDsIsoXMLSpatialTypeEnum.getEnumValuList().stream()
+                    .filter(e -> StringUtils.equals(trimmedToken, e)).findFirst().get();
+        } catch (java.util.NoSuchElementException e) {
+            s = "";
+        }
+        return s;
+    }
+
+    public static ArrayList<HashMap<String, String>> createSubTypeHashArray(JSONArray jsonArray) {
+        ArrayList<HashMap<String, String>> subTypeHashArray = new ArrayList<>();
+        if(jsonArray !=null && jsonArray.size()>0) {
+            jsonArray.forEach(item -> {
+                String regex = ((JSONObject) item).get("regex").toString();
+                String subType = ((JSONObject) item).get("subType").toString();
+                HashMap<String, String> mapper = new HashMap<>();
+                mapper.put("regex", regex);
+                mapper.put("subType", subType);
+                subTypeHashArray.add(mapper);
+            });
+        }
+        return subTypeHashArray;
+    }
+
+}
diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java
@@ -0,0 +1,54 @@
+package gov.nasa.cumulus.metadata.aggregator.processor;
+
+import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * this class is a post-processing of created umm-g
+ * by appending relatedUrl items into RelatedUrls array based on
+ *
+ * if any item within payload.granules[0].files[] matched collection configurations meta.subTypeMap array's
+ * regex
+ */
+public class RelatedUrlsProcessor {
+    public void RelatedUrlsProcessor() {
+    }
+
+    public JSONObject appendSubTypes(JSONObject granuleJson, TaskConfigBO taskConfigBO, JSONArray files) {
+        ArrayList<HashMap<String, String>> subTypeHashArray=taskConfigBO.getSubTypeHashArray();
+        JSONArray relateUrlsArray =  (JSONArray) granuleJson.get("RelatedUrls");
+        // 1: Loop through the existing RelatedUrls array to find any match for the subType regex.
+        //    for the finding match, set Subtype to
+        for(int i=0; i<relateUrlsArray.size(); i++) {
+            JSONObject relatedUrlJson = (JSONObject) relateUrlsArray.get(i);
+            String URLStr=relatedUrlJson.get("URL").toString();
+            for(HashMap<String, String> subTypeHash : subTypeHashArray) {
+                if(URLStr.matches(subTypeHash.get("regex"))) {
+                    relatedUrlJson.put("Subtype", subTypeHash.get("subType"));
+                }
+            }
+        }
+        // 2: Loop through the existing Input files:[] array to find any match for the subType regex.
+        //    for the finding match, append a new RelatedUrl item with Subtype set to subTypeHash.get("subType")
+        for(Object file: files) {
+            JSONObject fileJson = (JSONObject) file;
+            String source = fileJson.get("source").toString();
+            for(HashMap<String, String> subTypeHash : subTypeHashArray) {
+                if(source.matches(subTypeHash.get("regex"))) {
+                    JSONObject relatedUrl = new JSONObject();
+                    relatedUrl.put("URL", source);
+                    relatedUrl.put("Type", "GET DATA");
+                    relatedUrl.put("Subtype", subTypeHash.get("subType"));
+                    relateUrlsArray.add(relatedUrl);
+                }
+            }
+        }
+        return granuleJson;
+    }
+
+}
diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/AdditionalAttributeType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/AdditionalAttributeType.java
@@ -3,6 +3,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.processing.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -11,6 +12,7 @@
  * A reference to an additional attribute in the parent collection. The attribute reference may contain a granule specific value that will override the value in the parent collection for this granule. An attribute with the  same name must exist in the parent collection.
  * 
  */
+@Generated("jsonschema2pojo")
 public class AdditionalAttributeType {
 
     /**

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundaryType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundaryType.java
@@ -3,6 +3,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.processing.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -11,6 +12,7 @@
  * A boundary is set of points connected by straight lines representing a polygon on the earth. It takes a minimum of three points to make a boundary. Points must be specified in counter-clockwise order and closed (the first and last vertices are the same).
  * 
  */
+@Generated("jsonschema2pojo")
 public class BoundaryType {
 
     /**

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundingRectangleType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundingRectangleType.java
@@ -1,6 +1,7 @@
 
 package gov.nasa.cumulus.metadata.umm.generated;
 
+import javax.annotation.processing.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -9,6 +10,7 @@
  * This entity holds the horizontal spatial coverage of a bounding box.
  * 
  */
+@Generated("jsonschema2pojo")
 public class BoundingRectangleType {
 
     /**

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/ExclusiveZoneType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/ExclusiveZoneType.java
@@ -3,6 +3,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.processing.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -11,6 +12,7 @@
  * Contains the excluded boundaries from the GPolygon.
  * 
  */
+@Generated("jsonschema2pojo")
 public class ExclusiveZoneType {
 
     /**