Skip to content

Commit

Permalink
Cmr 9264 - adding InstanceInformation to UMM-Var for version 1.9. (#1928
Browse files Browse the repository at this point in the history
)

* CMR-9264 variable 1.8.2 schema and search results as baseline for what changed for 1.9

* CMR-9264 Adding in UMM-Var 1.9 which adds InstanceInformation to UMM-Var.

* CMR-9264 Fixing unit test

* CMR-9264 Fixing PR requests.

* CMR-9264 Fixing PR comment for new line at end of file.

* CMR-9264 Fixing unit test.
  • Loading branch information
eereiter authored Jul 12, 2023
1 parent 3081fe5 commit 9b08eb7
Show file tree
Hide file tree
Showing 8 changed files with 1,000 additions and 3 deletions.
2 changes: 1 addition & 1 deletion ingest-app/src/cmr/ingest/config.clj
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
(defconfig variable-umm-version
"Defines the latest variable umm version accepted by ingest - it's the latest official version.
This environment variable needs to be manually set when newer UMM version becomes official"
{:default "1.8.2"})
{:default "1.9.0"})

(defconfig service-umm-version
"Defines the latest service umm version accepted by ingest - it's the latest official version.
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"definitions": {
"ItemType": {
"type": "object",
"additionalProperties": false,
"description": "Represents a single item found in search results. It contains some metadata about the item found along with the UMM representing the item. UMM won't be present if the item represents a tombstone.",
"properties": {
"meta": {
"$ref": "#/definitions/MetaType"
},
"umm": {
"$ref": "umm-var-json-schema.json"
},
"associations": {
"$ref": "#/definitions/AssociationsInItemType"
}
},
"required": ["meta"]
},
"MetaType": {
"type": "object",
"additionalProperties": false,
"description": "CMR level metadata about the item found. This represents data not found in the actual metadata.",
"properties": {
"provider-id": {
"description": "The identity of the provider in the CMR",
"type": "string",
"minLength": 1,
"pattern": "[A-Z0-9_]+"
},
"concept-type": {
"description": "The type of item found.",
"type": "string",
"enum": ["variable"]
},
"native-id": {
"description": "The id used by the provider to identify this item during ingest.",
"type": "string",
"minLength": 1
},
"concept-id": {
"description": "The concept id of the item found.",
"type": "string",
"minLength": 4,
"pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
},
"revision-id": {
"description": "A number >= 1 that indicates which revision of the item.",
"type": "number"
},
"revision-date": {
"description": "The date this revision was created. This would be the creation or update date of the item in the CMR.",
"format": "date-time",
"type": "string"
},
"user-id": {
"description": "The id of the user who created this item revision.",
"type": "string",
"minLength": 1
},
"deleted": {
"description": "Indicates if the item represents a tombstone",
"type": "boolean"
},
"format": {
"description": "The mime type of the original metadata",
"type": "string",
"minLength": 1
},
"associations": {
"description": "Contains concept ids of concepts associated with the variable",
"$ref": "#/definitions/AssociationsType"
},
"association-details": {
"description": "Contains the details of each association.",
"$ref": "#/definitions/AssociationDetailsType"
}
},
"required": ["provider-id", "concept-type", "native-id", "concept-id", "revision-id", "revision-date"]
},
"AssociationsInItemType": {
"type": "object",
"additionalProperties": false,
"description": "Any concept associated with the variable.",
"properties": {
"variables": {
"description": "The concept ids of variables associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"collections": {
"description": "The concept ids of collections associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"tools": {
"description": "The concept ids of tools associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"data-quality-summaries": {
"description": "The concept ids of data quality summary concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"order-options": {
"description": "The concept ids of order option concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"service-options": {
"description": "The concept ids of service option concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"service-entries": {
"description": "The concept ids of service entry concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
},
"grids": {
"description": "The concept ids of grid concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptRevisionType"
},
"minItems": 1
}
}
},
"AssociationsType": {
"type": "object",
"additionalProperties": false,
"description": "Any concept associated with the variable.",
"properties": {
"variables": {
"description": "The concept ids of variables associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"collections": {
"description": "The concept ids of collections associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"tools": {
"description": "The concept ids of tools associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"data-quality-summaries": {
"description": "The concept ids of data quality summary concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"order-options": {
"description": "The concept ids of order option concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"service-options": {
"description": "The concept ids of service option concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"service-entries": {
"description": "The concept ids of service entry concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
},
"grids": {
"description": "The concept ids of grid concepts associated with the variable",
"type": "array",
"items": {
"$ref": "#/definitions/ConceptIdType"
},
"minItems": 1
}
}
},
"AssociationDetailsType": {
"type": "object",
"additionalProperties": true,
"description": "Association details of concepts associated with the variable.",
"properties": {
"concept-id": {
"description": "The concept id of the associated concept.",
"type": "string"
},
"revision-id": {
"description": "The revision id of the associated concept.",
"type": "string"
}
}
},
"ConceptIdType": {
"description": "The concept id of a concept.",
"type": "string",
"minLength": 4,
"pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
},
"ConceptRevisionType": {
"type": "object",
"additionalProperties": false,
"description": "Concept id and revision id.",
"properties": {
"concept-id": {
"description": "The concept id of the item found.",
"type": "string",
"minLength": 4,
"pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
},
"revision-id": {
"description": "A number >= 1 that indicates which revision of the item.",
"type": "number"
}
},
"required": ["concept-id"]
}
},
"title": "UMM Search Results",
"type": "object",
"additionalProperties": false,
"properties": {
"hits": {
"description": "The total number of items that matched the search.",
"type": "number"
},
"took": {
"description": "How long the search took in milliseconds from the time the CMR received the request until it had generated the response. This does not include network traffic time to send the request or return the response.",
"type": "number"
},
"items": {
"description": "The list of items matching the result in this page.",
"type": "array",
"items": {
"$ref": "#/definitions/ItemType"
},
"minItems": 0
}
},
"required": ["hits", "took", "items"]
}
15 changes: 15 additions & 0 deletions umm-spec-lib/src/cmr/umm_spec/migration/version/variable.clj
Original file line number Diff line number Diff line change
Expand Up @@ -258,3 +258,18 @@
(assoc % :Size -1)
%))
(m-spec/update-version :variable "1.8.1")))

;; migrations for 1.9.0 **********************************************************

(defmethod interface/migrate-umm-version [:variable "1.8.2" "1.9.0"]
[_context umm-v & _]
;; update the MetadataSpecification
(-> umm-v
(m-spec/update-version :variable "1.9.0")))

(defmethod interface/migrate-umm-version [:variable "1.9.0" "1.8.2"]
[_context umm-v & _]
;; Update the MetadataSpecification and remove InstanceInformation
(-> umm-v
(dissoc :InstanceInformation)
(m-spec/update-version :variable "1.8.2")))
57 changes: 56 additions & 1 deletion umm-spec-lib/src/cmr/umm_spec/models/umm_variable_models.clj
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@

(defrecord UMM-Var
[
;; Describes a store (zarr) where a variable has been separated from its original data files and
;; saved as its own entity.
InstanceInformation

;; Valid ranges of variable data values.
ValidRanges

Expand Down Expand Up @@ -49,7 +53,9 @@
Definition

;; Controlled Science Keywords describing the measurements/variables. The controlled vocabulary
;; for Science Keywords is maintained in the Keyword Management System (KMS).
;; for Science Keywords is maintained in the Keyword Management System (KMS). The valid values
;; can be found at the KMS website:
;; https://gcmdservices.gsfc.nasa.gov/kms/concepts/concept_scheme/sciencekeywords?format=csv.
ScienceKeywords

;; The name of a variable.
Expand Down Expand Up @@ -145,6 +151,34 @@
])
(record-pretty-printer/enable-record-pretty-printing SamplingIdentifierType)

;; This object describes a store for a variable instance. A variable instance is when the variable
;; is extracted from the original data files and stored somewhere.
(defrecord InstanceInformationType
[
;; The internet location of the variable instance store.
URL

;; Describes the format of the URL's data content so that users and applications know how to read
;; and use the content. At this point in time, an industry standard mime-type does not exist, so
;; a mime-type element will not be included. The controlled vocabulary for formats is maintained
;; in the Keyword Management System (KMS):
;; https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/DataFormat
Format

;; Brief description of the store or any other useful information about the store.
Description

;; This element allows end users to get direct access to data products that are stored in the
;; Amazon Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a
;; documentation URL as well as bucket prefix names and an AWS region.
DirectDistributionInformation

;; Description of the chunking strategy for the store. Chunking information such as the chunk
;; sizes should be documented.
ChunkingInformation
])
(record-pretty-printer/enable-record-pretty-printing InstanceInformationType)

;; Enables specification of Earth science keywords related to the collection. The Earth Science
;; keywords are chosen from a controlled keyword hierarchy maintained in the Keyword Management
;; System (KMS). The valid values can be found at the KMS website:
Expand All @@ -167,6 +201,27 @@
])
(record-pretty-printer/enable-record-pretty-printing ScienceKeywordType)

;; This element allows end users to get direct access to data products that are stored in the Amazon
;; Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a
;; documentation URL as well as bucket prefix names and an AWS region.
(defrecord DirectDistributionInformationType
[
;; Defines the possible values for the Amazon Web Service US Regions where the data product
;; resides.
Region

;; Defines the possible values for the Amazon Web Service US S3 bucket and/or object prefix
;; names.
S3BucketAndObjectPrefixNames

;; Defines the URL where the credentials are stored.
S3CredentialsAPIEndpoint

;; Defines the URL where the credential documentation are stored.
S3CredentialsAPIDocumentationURL
])
(record-pretty-printer/enable-record-pretty-printing DirectDistributionInformationType)

;; Valid range data value of a variable: minimum and maximum values. For example, '-100, 5000'.
(defrecord ValidRangeType
[
Expand Down
Loading

0 comments on commit 9b08eb7

Please sign in to comment.