Skip to content

Commit

Permalink
add subType for OPERA image files (#76)
Browse files Browse the repository at this point in the history
* add subType for OPERA image files

* drop umm166.java for it contains un-checked in generated files.

* enhance code guard to only append relatedUrls when subTypeHashArray is not null or size zero.  more unit test.

---------

Co-authored-by: Yen, David (398B-Affiliate) <[email protected]>
Co-authored-by: Stepheny Perez <[email protected]>
  • Loading branch information
3 people authored May 29, 2024
1 parent 12f986e commit 99698f0
Show file tree
Hide file tree
Showing 49 changed files with 4,368 additions and 183 deletions.
4 changes: 3 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]
### Added
- **PODAAC-6181**
- add relatedUrl subType BROWSE IMAGE SOURCE
### Deprecated
### Removed
### Fixed
### Security
-

## [8.6.0]
### Added
- **PODAAC-5876**
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.701</version>
<version>1.12.719</version>
</dependency>
<!-- For AWS Secret Manager -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-secretsmanager</artifactId>
<version>1.12.701</version>
<version>1.12.719</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ public static class Metadata{
public static final String START_ORBIT = "startorbit";
public static final String END_ORBIT = "endorbit";
}
public static final String UMMG_VERSION = "1.6.5";
public static final String UMMG_VERSION = "1.6.6";
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,23 @@
import java.io.IOException;
import java.math.BigInteger;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;

import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
import gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory;
import gov.nasa.cumulus.metadata.aggregator.processor.DMRPPProcessor;
import gov.nasa.cumulus.metadata.aggregator.processor.FootprintProcessor;
import gov.nasa.cumulus.metadata.aggregator.processor.ImageProcessor;
import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
import gov.nasa.cumulus.metadata.aggregator.processor.RelatedUrlsProcessor;
import gov.nasa.cumulus.metadata.state.WorkflowTypeEnum;
import gov.nasa.cumulus.metadata.util.S3Utils;
import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -57,11 +59,7 @@ public String PerformFunction(String input, Context context) throws Exception {
* this will help the logic in postIngestProcess function.
*/
this.setWorkFlowType((String) config.get("stateMachine"));
// This is a switch to determine, shall footprint, orbit or boundingbox shall be processed from iso.xml
// while ingesting swot collections
JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
HashSet isoXMLSpatialTypeHashSet = createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray);

TaskConfigBO taskConfigBO = TaskConfigFactory.createTaskConfigBO(config);

String isoRegex = (String) config.get("isoRegex");
String archiveXmlRegex = (String) config.get("archiveXmlRegex");
Expand Down Expand Up @@ -154,8 +152,7 @@ public String PerformFunction(String input, Context context) throws Exception {

MetadataFilesToEcho mtfe;
boolean isIsoFile = (iso != null);

mtfe = new MetadataFilesToEcho(isIsoFile, isoXMLSpatialTypeHashSet);
mtfe = new MetadataFilesToEcho(isIsoFile, taskConfigBO.getIsoXMLSpatialTypeHashSet());
//set the name/granuleId
mtfe.getGranule().setName(granuleId);
mtfe.setDatasetValues(collectionName, collectionVersion, rangeIs360, boundingBox, additionalAttributes);
Expand Down Expand Up @@ -190,20 +187,29 @@ public String PerformFunction(String input, Context context) throws Exception {
}
}

//write UMM-G to file
/**
* generate the ummg json object. Use RelatedUrlsProcessor to append more items to RelateUrls.
* See RelatedUrls java doc to know detailed logic to process payload.granules[0].files[]
*/
String cmrFilePath = "/tmp/" + granuleId + ".cmr.json";
try {
mtfe.writeJson("/tmp/" + granuleId + ".cmr.json");
JSONObject granuleJson = mtfe.createJson();
if(taskConfigBO.getSubTypeHashArray().size() > 0) {
RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor();
granuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson, taskConfigBO, files);
}
FileUtils.writeStringToFile(new File(cmrFilePath), granuleJson.toJSONString(), StandardCharsets.UTF_8);
} catch (IOException e) {
AdapterLogger.LogError(this.className + " mtfe.writeJson error:" + e.getMessage());
e.printStackTrace();
throw e;
}

//copy new file to S3
/*
* Upload cmr.json to s3 bucket
* Add UMM-G file to payload/file objects
*/
File cmrFile = new File("/tmp/" + granuleId + ".cmr.json");
File cmrFile = new File(cmrFilePath);
s3Utils.upload(region, internalBucket,
Paths.get(stagingDirectory, "/" + granuleId + ".cmr.json").toString(),
cmrFile);
Expand Down Expand Up @@ -237,34 +243,6 @@ public String PerformFunction(String input, Context context) throws Exception {
}
return returnable.toJSONString();
}

public HashSet<MENDsIsoXMLSpatialTypeEnum> createIsoXMLSpatialTypeSet(JSONArray isoXMLSpatialTypeConfigJSONArray) throws IllegalArgumentException{
HashSet<MENDsIsoXMLSpatialTypeEnum> isoSpatialTypes = new HashSet<>();
// if not containing isoXMLTypes, then return an empty HashSet
if(isoXMLSpatialTypeConfigJSONArray == null || isoXMLSpatialTypeConfigJSONArray.size()==0) {
return isoSpatialTypes;
}
isoXMLSpatialTypeConfigJSONArray.forEach(item -> {
String t = (String) item;
MENDsIsoXMLSpatialTypeEnum en = MENDsIsoXMLSpatialTypeEnum.getEnum(getIsoXMLSpatialTypeStr(t));
isoSpatialTypes.add(en);
});
AdapterLogger.LogDebug(this.className + " isoSpatialTypes HashSet: " + isoSpatialTypes);
return isoSpatialTypes;
}

public String getIsoXMLSpatialTypeStr(String token) {
final String trimmedToken = StringUtils.trim(token);
String s;
try {
s = MENDsIsoXMLSpatialTypeEnum.getEnumValuList().stream()
.filter(e -> StringUtils.equals(trimmedToken, e)).findFirst().get();
} catch (java.util.NoSuchElementException e) {
s = "";
}
return s;
}

/**
* get S3 fileStaging direction from S3 full key
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1074,13 +1074,13 @@ public JSONObject createJson()
granule.setIngestTime(new Date());
UMMGranuleFile granuleFile = new UMMGranuleFile(granule, dataset, rangeIs360, this.isoXMLSpatialTypeEnumHashSet);
JSONObject granuleJson = granuleFile.defineGranule();
JSONUtils.cleanJSON(granuleJson);
return granuleJson;
}

public void writeJson(String outputLocation)
throws IOException, ParseException, URISyntaxException{
JSONObject granuleJson = createJson();
JSONUtils.cleanJSON(granuleJson);
FileUtils.writeStringToFile(new File(outputLocation), granuleJson.toJSONString(), StandardCharsets.UTF_8);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package gov.nasa.cumulus.metadata.aggregator.bo;

import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

public class TaskConfigBO {
HashSet<MENDsIsoXMLSpatialTypeEnum> isoXMLSpatialTypeHashSet;
ArrayList<HashMap<String, String>> subTypeHashArray = new ArrayList<>();

public TaskConfigBO() {}

public HashSet<MENDsIsoXMLSpatialTypeEnum> getIsoXMLSpatialTypeHashSet() {
return isoXMLSpatialTypeHashSet;
}

public void setIsoXMLSpatialTypeHashSet(HashSet<MENDsIsoXMLSpatialTypeEnum> isoXMLSpatialTypeHashSet) {
this.isoXMLSpatialTypeHashSet = isoXMLSpatialTypeHashSet;
}

public ArrayList<HashMap<String, String>> getSubTypeHashArray() {
return subTypeHashArray;
}

public void setSubTypeHashArray(ArrayList<HashMap<String, String>> subTypeHashArray) {
this.subTypeHashArray = subTypeHashArray;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package gov.nasa.cumulus.metadata.aggregator.factory;

import cumulus_message_adapter.message_parser.AdapterLogger;
import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

public class TaskConfigFactory {
static String className = "gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory";

public static TaskConfigBO createTaskConfigBO(JSONObject config) {
TaskConfigBO taskConfigBO = new TaskConfigBO();
//Construct isoXMLSpatialTypeHashset
JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
if(isoXMLSpatialTypeJsonArray !=null && isoXMLSpatialTypeJsonArray.size()>0) {
taskConfigBO.setIsoXMLSpatialTypeHashSet(createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray));
}
//Construct subTypeHashArray
JSONArray subTypeTypeJsonArray = (JSONArray) config.get("relatedUrlSubTypeMap");
if(subTypeTypeJsonArray!=null && subTypeTypeJsonArray.size()>0) {
taskConfigBO.setSubTypeHashArray(createSubTypeHashArray(subTypeTypeJsonArray));
}
return taskConfigBO;

}

public static HashSet<MENDsIsoXMLSpatialTypeEnum> createIsoXMLSpatialTypeSet(JSONArray isoXMLSpatialTypeConfigJSONArray) throws IllegalArgumentException{
HashSet<MENDsIsoXMLSpatialTypeEnum> isoSpatialTypes = new HashSet<>();
// if not containing isoXMLTypes, then return an empty HashSet
if(isoXMLSpatialTypeConfigJSONArray == null || isoXMLSpatialTypeConfigJSONArray.size()==0) {
return isoSpatialTypes;
}
isoXMLSpatialTypeConfigJSONArray.forEach(item -> {
String t = (String) item;
MENDsIsoXMLSpatialTypeEnum en = MENDsIsoXMLSpatialTypeEnum.getEnum(getIsoXMLSpatialTypeStr(t));
isoSpatialTypes.add(en);
});
AdapterLogger.LogDebug(className + " isoSpatialTypes HashSet: " + isoSpatialTypes);
return isoSpatialTypes;
}

public static String getIsoXMLSpatialTypeStr(String token) {
final String trimmedToken = StringUtils.trim(token);
String s;
try {
s = MENDsIsoXMLSpatialTypeEnum.getEnumValuList().stream()
.filter(e -> StringUtils.equals(trimmedToken, e)).findFirst().get();
} catch (java.util.NoSuchElementException e) {
s = "";
}
return s;
}

public static ArrayList<HashMap<String, String>> createSubTypeHashArray(JSONArray jsonArray) {
ArrayList<HashMap<String, String>> subTypeHashArray = new ArrayList<>();
if(jsonArray !=null && jsonArray.size()>0) {
jsonArray.forEach(item -> {
String regex = ((JSONObject) item).get("regex").toString();
String subType = ((JSONObject) item).get("subType").toString();
HashMap<String, String> mapper = new HashMap<>();
mapper.put("regex", regex);
mapper.put("subType", subType);
subTypeHashArray.add(mapper);
});
}
return subTypeHashArray;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package gov.nasa.cumulus.metadata.aggregator.processor;

import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;

import java.util.ArrayList;
import java.util.HashMap;

/**
* this class is a post-processing of created umm-g
* by appending relatedUrl items into RelatedUrls array based on
*
* if any item within payload.granules[0].files[] matched collection configurations meta.subTypeMap array's
* regex
*/
public class RelatedUrlsProcessor {
public void RelatedUrlsProcessor() {
}

public JSONObject appendSubTypes(JSONObject granuleJson, TaskConfigBO taskConfigBO, JSONArray files) {
ArrayList<HashMap<String, String>> subTypeHashArray=taskConfigBO.getSubTypeHashArray();
if(subTypeHashArray==null || subTypeHashArray.size()==0) {
return granuleJson;
}
JSONArray relateUrlsArray = (JSONArray) granuleJson.get("RelatedUrls");
// 1: Loop through the existing RelatedUrls array to find any match for the subType regex.
// for the finding match, set Subtype to
for(int i=0; i<relateUrlsArray.size(); i++) {
JSONObject relatedUrlJson = (JSONObject) relateUrlsArray.get(i);
String URLStr=relatedUrlJson.get("URL").toString();
for(HashMap<String, String> subTypeHash : subTypeHashArray) {
if(URLStr.matches(subTypeHash.get("regex"))) {
relatedUrlJson.put("Subtype", subTypeHash.get("subType"));
}
}
}
// 2: Loop through the existing Input files:[] array to find any match for the subType regex.
// for the finding match, append a new RelatedUrl item with Subtype set to subTypeHash.get("subType")
for(Object file: files) {
JSONObject fileJson = (JSONObject) file;
String source = fileJson.get("source").toString();
for(HashMap<String, String> subTypeHash : subTypeHashArray) {
if(source.matches(subTypeHash.get("regex"))) {
JSONObject relatedUrl = new JSONObject();
relatedUrl.put("URL", source);
relatedUrl.put("Type", "GET DATA");
relatedUrl.put("Subtype", subTypeHash.get("subType"));
relateUrlsArray.add(relatedUrl);
}
}
}
return granuleJson;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.processing.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -11,6 +12,7 @@
* A reference to an additional attribute in the parent collection. The attribute reference may contain a granule specific value that will override the value in the parent collection for this granule. An attribute with the same name must exist in the parent collection.
*
*/
@Generated("jsonschema2pojo")
public class AdditionalAttributeType {

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.processing.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -11,6 +12,7 @@
* A boundary is set of points connected by straight lines representing a polygon on the earth. It takes a minimum of three points to make a boundary. Points must be specified in counter-clockwise order and closed (the first and last vertices are the same).
*
*/
@Generated("jsonschema2pojo")
public class BoundaryType {

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

package gov.nasa.cumulus.metadata.umm.generated;

import javax.annotation.processing.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -9,6 +10,7 @@
* This entity holds the horizontal spatial coverage of a bounding box.
*
*/
@Generated("jsonschema2pojo")
public class BoundingRectangleType {

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.processing.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -11,6 +12,7 @@
* Contains the excluded boundaries from the GPolygon.
*
*/
@Generated("jsonschema2pojo")
public class ExclusiveZoneType {

/**
Expand Down
Loading

0 comments on commit 99698f0

Please sign in to comment.