Cmr 9264 - adding InstanceInformation to UMM-Var for version 1.9.

ingest-app/src/cmr/ingest/config.clj

@@ -38,7 +38,7 @@
 (defconfig variable-umm-version
   "Defines the latest variable umm version accepted by ingest - it's the latest official version.
    This environment variable needs to be manually set when newer UMM version becomes official"
-  {:default "1.8.2"})
+  {:default "1.9.0"})
 
 (defconfig service-umm-version
   "Defines the latest service umm version accepted by ingest - it's the latest official version.

umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-search-results-json-schema.json

@@ -0,0 +1,286 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "definitions": {
+    "ItemType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Represents a single item found in search results. It contains some metadata about the item found along with the UMM representing the item. UMM won't be present if the item represents a tombstone.",
+      "properties": {
+        "meta": {
+          "$ref": "#/definitions/MetaType"
+        },
+        "umm": {
+          "$ref": "umm-var-json-schema.json"
+        },
+        "associations": {
+          "$ref": "#/definitions/AssociationsInItemType"
+        }
+      },
+      "required": ["meta"]
+    },
+    "MetaType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "CMR level metadata about the item found. This represents data not found in the actual metadata.",
+      "properties": {
+        "provider-id": {
+          "description": "The identity of the provider in the CMR",
+          "type": "string",
+          "minLength": 1,
+          "pattern": "[A-Z0-9_]+"
+        },
+        "concept-type": {
+          "description": "The type of item found.",
+          "type": "string",
+          "enum": ["variable"]
+        },
+        "native-id": {
+          "description": "The id used by the provider to identify this item during ingest.",
+          "type": "string",
+          "minLength": 1
+        },
+        "concept-id": {
+          "description": "The concept id of the item found.",
+          "type": "string",
+          "minLength": 4,
+          "pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
+        },
+        "revision-id": {
+          "description": "A number >= 1 that indicates which revision of the item.",
+          "type": "number"
+        },
+        "revision-date": {
+          "description": "The date this revision was created. This would be the creation or update date of the item in the CMR.",
+          "format": "date-time",
+          "type": "string"
+        },
+        "user-id": {
+          "description": "The id of the user who created this item revision.",
+          "type": "string",
+          "minLength": 1
+        },
+        "deleted": {
+          "description": "Indicates if the item represents a tombstone",
+          "type": "boolean"
+        },
+        "format": {
+          "description": "The mime type of the original metadata",
+          "type": "string",
+          "minLength": 1
+        },
+        "associations": {
+          "description": "Contains concept ids of concepts associated with the variable",
+          "$ref": "#/definitions/AssociationsType"
+        },
+        "association-details": {
+          "description": "Contains the details of each association.",
+          "$ref": "#/definitions/AssociationDetailsType"
+        }
+      },
+      "required": ["provider-id", "concept-type", "native-id", "concept-id", "revision-id", "revision-date"]
+    },
+    "AssociationsInItemType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Any concept associated with the variable.",
+      "properties": {
+        "variables": {
+          "description": "The concept ids of variables associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "collections": {
+          "description": "The concept ids of collections associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "tools": {
+          "description": "The concept ids of tools associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "data-quality-summaries": {
+          "description": "The concept ids of data quality summary concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "order-options": {
+          "description": "The concept ids of order option concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "service-options": {
+          "description": "The concept ids of service option concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "service-entries": {
+          "description": "The concept ids of service entry concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        },
+        "grids": {
+          "description": "The concept ids of grid concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptRevisionType"
+          },
+          "minItems": 1
+        }
+      }
+    },
+    "AssociationsType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Any concept associated with the variable.",
+      "properties": {
+        "variables": {
+          "description": "The concept ids of variables associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "collections": {
+          "description": "The concept ids of collections associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "tools": {
+          "description": "The concept ids of tools associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "data-quality-summaries": {
+          "description": "The concept ids of data quality summary concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "order-options": {
+          "description": "The concept ids of order option concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "service-options": {
+          "description": "The concept ids of service option concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "service-entries": {
+          "description": "The concept ids of service entry concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "grids": {
+          "description": "The concept ids of grid concepts associated with the variable",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        }
+      }
+    },
+    "AssociationDetailsType": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Association details of concepts associated with the variable.",
+      "properties": {
+        "concept-id": {
+          "description": "The concept id of the associated concept.",
+          "type": "string"
+        },
+        "revision-id": {
+          "description": "The revision id of the associated concept.",
+          "type": "string"
+        }
+      }
+    },
+    "ConceptIdType": {
+      "description": "The concept id of a concept.",
+      "type": "string",
+      "minLength": 4,
+      "pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
+    },
+    "ConceptRevisionType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Concept id and revision id.",
+      "properties": {
+        "concept-id": {
+          "description": "The concept id of the item found.",
+          "type": "string",
+          "minLength": 4,
+          "pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
+        },
+        "revision-id": {
+          "description": "A number >= 1 that indicates which revision of the item.",
+          "type": "number"
+        }
+      },
+      "required": ["concept-id"]
+    }
+  },
+  "title": "UMM Search Results",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "hits": {
+      "description": "The total number of items that matched the search.",
+      "type": "number"
+    },
+    "took": {
+      "description": "How long the search took in milliseconds from the time the CMR received the request until it had generated the response. This does not include network traffic time to send the request or return the response.",
+      "type": "number"
+    },
+    "items": {
+      "description": "The list of items matching the result in this page.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ItemType"
+      },
+      "minItems": 0
+    }
+  },
+  "required": ["hits", "took", "items"]
+}

umm-spec-lib/src/cmr/umm_spec/migration/version/variable.clj

@@ -258,3 +258,18 @@
                                                    (assoc % :Size -1)
                                                    %))
       (m-spec/update-version :variable "1.8.1")))
+
+;; migrations for 1.9.0 **********************************************************
+
+(defmethod interface/migrate-umm-version [:variable "1.8.2" "1.9.0"]
+  [_context umm-v & _]
+  ;; update the MetadataSpecification
+  (-> umm-v
+      (m-spec/update-version :variable "1.9.0")))
+
+(defmethod interface/migrate-umm-version [:variable "1.9.0" "1.8.2"]
+  [_context umm-v & _]
+  ;; Update the MetadataSpecification and remove InstanceInformation
+  (-> umm-v
+      (dissoc :InstanceInformation)
+      (m-spec/update-version :variable "1.8.2")))

umm-spec-lib/src/cmr/umm_spec/models/umm_variable_models.clj

@@ -5,6 +5,10 @@
 
 (defrecord UMM-Var
   [
+   ;; Describes a store (zarr) where a variable has been separated from its original data files and
+   ;; saved as its own entity.
+   InstanceInformation
+
    ;; Valid ranges of variable data values.
    ValidRanges
 
@@ -49,7 +53,9 @@
    Definition
 
    ;; Controlled Science Keywords describing the measurements/variables. The controlled vocabulary
-   ;; for Science Keywords is maintained in the Keyword Management System (KMS).
+   ;; for Science Keywords is maintained in the Keyword Management System (KMS). The valid values
+   ;; can be found at the KMS website:
+   ;; https://gcmdservices.gsfc.nasa.gov/kms/concepts/concept_scheme/sciencekeywords?format=csv.
    ScienceKeywords
 
    ;; The name of a variable.
@@ -145,6 +151,34 @@
   ])
 (record-pretty-printer/enable-record-pretty-printing SamplingIdentifierType)
 
+;; This object describes a store for a variable instance. A variable instance is when the variable
+;; is extracted from the original data files and stored somewhere.
+(defrecord InstanceInformationType
+  [
+   ;; The internet location of the variable instance store.
+   URL
+
+   ;; Describes the format of the URL's data content so that users and applications know how to read
+   ;; and use the content. At this point in time, an industry standard mime-type does not exist, so
+   ;; a mime-type element will not be included. The controlled vocabulary for formats is maintained
+   ;; in the Keyword Management System (KMS):
+   ;; https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/DataFormat
+   Format
+
+   ;; Brief description of the store or any other useful information about the store.
+   Description
+
+   ;; This element allows end users to get direct access to data products that are stored in the
+   ;; Amazon Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a
+   ;; documentation URL as well as bucket prefix names and an AWS region.
+   DirectDistributionInformation
+
+   ;; Description of the chunking strategy for the store. Chunking information such as the chunk
+   ;; sizes should be documented.
+   ChunkingInformation
+  ])
+(record-pretty-printer/enable-record-pretty-printing InstanceInformationType)
+
 ;; Enables specification of Earth science keywords related to the collection. The Earth Science
 ;; keywords are chosen from a controlled keyword hierarchy maintained in the Keyword Management
 ;; System (KMS). The valid values can be found at the KMS website:
@@ -167,6 +201,27 @@
   ])
 (record-pretty-printer/enable-record-pretty-printing ScienceKeywordType)
 
+;; This element allows end users to get direct access to data products that are stored in the Amazon
+;; Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a
+;; documentation URL as well as bucket prefix names and an AWS region.
+(defrecord DirectDistributionInformationType
+  [
+   ;; Defines the possible values for the Amazon Web Service US Regions where the data product
+   ;; resides.
+   Region
+
+   ;; Defines the possible values for the Amazon Web Service US S3 bucket and/or object prefix
+   ;; names.
+   S3BucketAndObjectPrefixNames
+
+   ;; Defines the URL where the credentials are stored.
+   S3CredentialsAPIEndpoint
+
+   ;; Defines the URL where the credential documentation are stored.
+   S3CredentialsAPIDocumentationURL
+  ])
+(record-pretty-printer/enable-record-pretty-printing DirectDistributionInformationType)
+
 ;; Valid range data value of a variable: minimum and maximum values. For example, '-100, 5000'.
 (defrecord ValidRangeType
   [