From 9b08eb771c09b368257bc2ccf1472d1ad389ccbb Mon Sep 17 00:00:00 2001 From: eereiter Date: Wed, 12 Jul 2023 17:22:28 -0400 Subject: [PATCH] Cmr 9264 - adding InstanceInformation to UMM-Var for version 1.9. (#1928) * CMR-9264 variable 1.8.2 schema and search results as baseline for what changed for 1.9 * CMR-9264 Adding in UMM-Var 1.9 which adds InstanceInformation to UMM-Var. * CMR-9264 Fixing unit test * CMR-9264 Fixing PR requests. * CMR-9264 Fixing PR comment for new line at end of file. * CMR-9264 Fixing unit test. --- ingest-app/src/cmr/ingest/config.clj | 2 +- .../umm/v1.9.0/umm-var-json-schema.json | 609 ++++++++++++++++++ .../umm-var-search-results-json-schema.json | 286 ++++++++ .../umm_spec/migration/version/variable.clj | 15 + .../umm_spec/models/umm_variable_models.clj | 57 +- umm-spec-lib/src/cmr/umm_spec/versioning.clj | 2 +- .../test/migration/version/variable.clj | 31 + .../umm_spec_variable_validation_tests.clj | 1 + 8 files changed, 1000 insertions(+), 3 deletions(-) create mode 100644 umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-json-schema.json create mode 100644 umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-search-results-json-schema.json diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 905d132ccb..9e28129613 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -38,7 +38,7 @@ (defconfig variable-umm-version "Defines the latest variable umm version accepted by ingest - it's the latest official version. This environment variable needs to be manually set when newer UMM version becomes official" - {:default "1.8.2"}) + {:default "1.9.0"}) (defconfig service-umm-version "Defines the latest service umm version accepted by ingest - it's the latest official version. diff --git a/umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-json-schema.json b/umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-json-schema.json new file mode 100644 index 0000000000..40d6649699 --- /dev/null +++ b/umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-json-schema.json @@ -0,0 +1,609 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "UMM-Var", + "type": "object", + "additionalProperties": false, + "properties": { + "Name": { + "description": "The name of a variable.", + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "StandardName": { + "description": "This is the more formal or scientific name, .e.g., the CF Standard Name.", + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "AdditionalIdentifiers": { + "description": "Any additional identifiers of a variable.", + "type": "array", + "items": { + "$ref": "#/definitions/AdditionalIdentifierType" + }, + "minItems": 1 + }, + "LongName": { + "description": "The expanded or long name related to the variable Name.", + "$ref": "#/definitions/LongNameType" + }, + "Definition": { + "description": "The definition of the variable.", + "type": "string", + "minLength": 1, + "maxLength": 10000 + }, + "Units": { + "description": "The units associated with a variable.", + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "DataType": { + "description": "Specify data type of a variable. These types can be either: uint8, uint16, etc.", + "$ref": "#/definitions/DataTypeEnum" + }, + "Dimensions": { + "description": "A variable consists of one or more dimensions. An example of a dimension name is 'XDim'. An example of a dimension size is '1200'. Variables are rarely one dimensional.", + "type": "array", + "items": { + "$ref": "#/definitions/DimensionType" + }, + "minItems": 1 + }, + "ValidRanges": { + "description": "Valid ranges of variable data values.", + "type": "array", + "items": { + "$ref": "#/definitions/ValidRangeType" + }, + "minItems": 0 + }, + "Scale": { + "description": "The scale is the numerical factor by which all values in the stored data field are multiplied in order to obtain the original values. May be used together with Offset. An example of a scale factor is '0.002'", + "type": "number" + }, + "Offset": { + "description": "The offset is the value which is either added to or subtracted from all values in the stored data field in order to obtain the original values. May be used together with Scale. An example of an offset is '0.49'.", + "type": "number" + }, + "FillValues": { + "description": "The fill value of the variable in the data file. It is generally a value which falls outside the valid range. For example, if the valid range is '0, 360', the fill value may be '-1'. The fill value type is data provider-defined. For example, 'Out of Valid Range'.", + "type": "array", + "items": { + "$ref": "#/definitions/FillValueType" + }, + "minItems": 0 + }, + "VariableType": { + "description": "Specify basic type of a variable.", + "$ref": "#/definitions/VariableTypeEnum" + }, + "VariableSubType": { + "description": "Specifies the sub type of a variable.", + "$ref": "#/definitions/VariableSubTypeEnum" + }, + "IndexRanges": { + "description": "This element describes the x and y dimension ranges for this variable. Typically these values are 2 latitude and longitude ranges, but they don't necessarily have to be.", + "$ref": "#/definitions/IndexRangesType" + }, + "MeasurementIdentifiers": { + "description": "The measurement information of a variable.", + "type": "array", + "items": { + "$ref": "#/definitions/MeasurementIdentifierType" + }, + "minItems": 0 + }, + "SamplingIdentifiers": { + "description": "The sampling information of a variable.", + "type": "array", + "items": { + "$ref": "#/definitions/SamplingIdentifierType" + }, + "minItems": 0 + }, + "ScienceKeywords": { + "description": "Controlled Science Keywords describing the measurements/variables. The controlled vocabulary for Science Keywords is maintained in the Keyword Management System (KMS). The valid values can be found at the KMS website: https://gcmdservices.gsfc.nasa.gov/kms/concepts/concept_scheme/sciencekeywords?format=csv.", + "type": "array", + "items": { + "$ref": "#/definitions/ScienceKeywordType" + }, + "minItems": 0 + }, + "Sets": { + "description": "The set information of a variable. The variable is grouped within a set. The set is defined by the name, type, size and index. For example, Name: 'Data_Fields', Type: 'General', Size: '15', Index: '7' for the case of the variable named 'LST_Day_1km'.", + "type": "array", + "items": { + "$ref": "#/definitions/SetType" + }, + "minItems": 1 + }, + "RelatedURLs": { + "description": "A described URL associated with the a web resource, or interface. e.g., the home page for the variable provider.", + "type": "array", + "items": { + "$ref": "#/definitions/RelatedURLType" + }, + "minItems": 1 + }, + "InstanceInformation": { + "description": "Describes a store (zarr) where a variable has been separated from its original data files and saved as its own entity.", + "$ref": "#/definitions/InstanceInformationType" + }, + "MetadataSpecification": { + "description": "Requires the client, or user, to add in schema information into every variable record. It includes the schema's name, version, and URL location. The information is controlled through enumerations at the end of this schema.", + "$ref": "#/definitions/MetadataSpecificationType" + } + }, + "required": [ + "Name", + "LongName", + "Definition", + "MetadataSpecification" + ], + "definitions": { + "AdditionalIdentifierType": { + "type": "object", + "additionalProperties": false, + "description": "The elements of this section apply to an additional identifier.", + "properties": { + "Identifier": { + "description": "The actual identifier.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "Description": { + "description": "This element describes to a person or machine what the identifier is called. e.g., if the identifier is 1057.2345/asfb then the Description should be DOI or Digital Object Identifier.", + "type": "string", + "minLength": 0, + "maxLength": 4000 + } + }, + "required": ["Identifier"] + }, + "IndexRangesType": { + "type": "object", + "additionalProperties": false, + "description": "The index ranges consist of a LatRange and a LonRange.", + "properties": { + "LatRange": { + "description": "The LatRange consists of an index range for latitude.", + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2, + "maxItems": 2 + }, + "LonRange": { + "description": "The LonRange consists of an index range for longitude.", + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2, + "maxItems": 2 + } + }, + "required": ["LatRange", "LonRange"] + }, + "MeasurementIdentifierType": { + "type": "object", + "additionalProperties": false, + "description": "The elements of this section allow authors to provide community sourced words or phrases to further describe the variable data.", + "properties": { + "MeasurementContextMedium": { + "description": "This element describes the context/medium within which the measurement was made.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "MeasurementContextMediumURI": { + "description": "This element contains the URI for the context/medium.", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 4000 + }, + "MeasurementObject": { + "description": "This element describes the object which was measured.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "MeasurementObjectURI": { + "description": "This element contains the URI for the object which was measured.", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 4000 + }, + "MeasurementQuantities": { + "description": "This element contains the quantity or quantities which was/were measured.", + "type": "array", + "items": { + "$ref": "#/definitions/MeasurementQuantityType" + }, + "minItems": 0 + } + }, + "required": ["MeasurementContextMedium", "MeasurementObject"] + }, + "MeasurementQuantityType": { + "type": "object", + "additionalProperties": false, + "description": "The elements of this section apply to a measurement name. The measurement name is structured according to the form defined by Scott Peckham. This is: _.", + "properties": { + "Value": { + "description": "This element describes the value for the quantity which was measured.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "MeasurementQuantityURI": { + "description": "This element contains the URI for the quantity which was measured.", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 4000 + } + } + }, + "SamplingIdentifierType": { + "type": "object", + "additionalProperties": false, + "description": "The elements of this section apply to a measurement.", + "properties": { + "SamplingMethod": { + "description": "The name of the sampling method used for the measurement. For example, 'radiometric detection within the visible and infra-red ranges of the electromagnetic spectrum.", + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "MeasurementConditions": { + "description": "The measurement conditions of the variable. For example, 'Sampled Particle Size Range: 90 - 600 nm'.", + "type": "string", + "minLength": 0, + "maxLength": 4000 + }, + "ReportingConditions": { + "description": "The reporting conditions of the variable. The conditions over which the measurements of the variable are valid. For example, 'STP: 1013 mb and 273 K'.", + "type": "string", + "minLength": 0, + "maxLength": 4000 + } + }, + "required": ["SamplingMethod", "MeasurementConditions"] + }, + "FillValueType": { + "description": "The fill value, fill value type and fill value description of the variable in the data file. The fill value is generally a value which falls outside the valid range. For example, if the valid range is '0, 360', the fill value may be '-1'. The elements of this section apply to the fill value of a variable.", + "type": "object", + "additionalProperties": false, + "properties": { + "Value": { + "description": "The fill value of the variable in the data file.", + "type": "number" + }, + "Type": { + "description": "Type of the fill value of the variable in the data file.", + "type": "string", + "enum": ["SCIENCE_FILLVALUE", "QUALITY_FILLVALUE", "ANCILLARY_FILLVALUE", "OTHER"] + }, + "Description": { + "description": "Description of the fill value of the variable in the data file.", + "type": "string", + "minLength": 1, + "maxLength": 160 + } + }, + "required": ["Value", "Type"] + }, + "SetType": { + "type": "object", + "additionalProperties": false, + "description": "The elements of this section apply to variable sets.", + "properties": { + "Name": { + "description": "This element enables specification of set name. For example, 'Data_Fields'.", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "Type": { + "description": "This element enables specification of set type. For example, if the variables have been grouped together based on a particular theme, such as wavelength, then the type should be set to that theme, otherwise it should be set to 'General'.", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "Size": { + "description": "This element specifies the number of variables in the set. For example, if the number of variables in the set is fifteen, the size should be set to '15'.", + "type": "number" + }, + "Index": { + "description": "This element specifies the index value within the set for this variable, For example, if this varible is the third variable in the set, the index value should be set to '3'.", + "type": "number" + } + }, + "required": ["Name", "Type", "Size", "Index"] + }, + "ValidRangeType": { + "type": "object", + "additionalProperties": false, + "description": "Valid range data value of a variable: minimum and maximum values. For example, '-100, 5000'.", + "properties": { + "Min": { + "description": "Minimum data value of a variable. For example, '-100'.", + "type": "number" + }, + "Max": { + "description": "Maximum data value of a variable. For example, '5000'.", + "type": "number" + }, + "CodeSystemIdentifierMeaning": { + "description": "This element can be used to specify a code system identifier meaning. For example, 'Open Shrubland' corresponds to '7'.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 60 + }, + "minItems": 0 + }, + "CodeSystemIdentifierValue": { + "description": "The code system identifier value is the textual or numerical value assigned to each meaning.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 30 + }, + "minItems": 0 + } + } + }, + "DataTypeEnum": { + "description": "This element is used to identify the data type of the variable.", + "type": "string", + "enum": ["byte", "float", "float32", "float64", "double", "ubyte", "ushort", "uint", "uchar", "string", "char8", "uchar8", "short", "long", "int", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", "OTHER"] + }, + "DimensionType": { + "type": "object", + "additionalProperties": false, + "description": "A variable consists of one or more dimensions. An example of a dimension name is 'XDim'. An example of a dimension size is '1200'. Variables are rarely one dimensional.", + "properties": { + "Name": { + "description": "The name of the dimension of the variable represented in the data field. For example, 'XDim.", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "Size": { + "oneOf": [{ + "description": "The size of the dimension of the variable represented in the data field. For example, '1200'.", + "title": "Enter the dimension size.", + "type": "number" + }, { + "description": "A value to designate that the dimension size of a variable in each granule is not uniform, but that it varies across granules.", + "title": "Dimension size varies.", + "type": "string", + "enum": ["Varies"] + }] + }, + "Type": { + "description": "The type of the dimension of the variable represented in the data field. For example, if the dimension has a special meaning, i.e., a latitude, longitude, pressure, height (or depth) or time, then the type should be set to either 'LATITUDE_DIMENSION', 'LONGITUDE_DIMENSION', 'PRESSURE_DIMENSION', 'HEIGHT_DIMENSION', 'DEPTH_DIMENSION' or 'TIME_DIMENSION', otherwise it should be set to 'OTHER'.", + "type": "string", + "enum": ["LATITUDE_DIMENSION", "LONGITUDE_DIMENSION", "ALONG_TRACK_DIMENSION", "CROSS_TRACK_DIMENSION", "PRESSURE_DIMENSION", "HEIGHT_DIMENSION", "DEPTH_DIMENSION", "TIME_DIMENSION", "OTHER"] + } + }, + "required": ["Name", "Size", "Type"] + }, + "VariableTypeEnum": { + "description": "This element is used to classify the variable. If a variable does not contain this field, it will be assumed to be a science variable.", + "type": "string", + "enum": ["SCIENCE_VARIABLE", "QUALITY_VARIABLE", "ANCILLARY_VARIABLE", "COORDINATE", "OTHER"] + }, + "VariableSubTypeEnum": { + "description": "This element is used to further classify the variable. If a variable does not contain this field, it will be assumed to be a science vector.", + "type": "string", + "enum": ["SCIENCE_SCALAR", "SCIENCE_VECTOR", "SCIENCE_ARRAY", "SCIENCE_EVENTFLAG", "LATITUDE", "LONGITUDE", "TIME", "OTHER"] + }, + "ScienceKeywordType": { + "type": "object", + "additionalProperties": false, + "description": "Enables specification of Earth science keywords related to the collection. The Earth Science keywords are chosen from a controlled keyword hierarchy maintained in the Keyword Management System (KMS). The valid values can be found at the KMS website: https://gcmdservices.gsfc.nasa.gov/kms/concepts/concept_scheme/sciencekeywords?format=csv.", + "properties": { + "Category": { + "$ref": "#/definitions/KeywordStringType" + }, + "Topic": { + "$ref": "#/definitions/KeywordStringType" + }, + "Term": { + "$ref": "#/definitions/KeywordStringType" + }, + "VariableLevel1": { + "$ref": "#/definitions/KeywordStringType" + }, + "VariableLevel2": { + "$ref": "#/definitions/KeywordStringType" + }, + "VariableLevel3": { + "$ref": "#/definitions/KeywordStringType" + }, + "DetailedVariable": { + "$ref": "#/definitions/KeywordStringType" + } + }, + "required": ["Category", "Topic", "Term"] + }, + "LongNameType": { + "description": "The expanded or long name related to the variable name main element.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "KeywordStringType": { + "description": "The keyword string type for science keywords. The pattern below helps to minimize the types of characters allowed in the keywords to help users minimize keyword issues.", + "type": "string", + "minLength": 1, + "maxLength": 80, + "pattern": "[\\w\\-&'()\\[\\]/.\"#$%\\^@!*+=,][\\w\\-&'()\\[\\]/.\"#$%\\^@!*+=, ]{1,79}" + }, + "RelatedURLType": { + "type": "object", + "additionalProperties": false, + "description": "Represents Internet sites that contain information related to the data, as well as related Internet sites such as project home pages, variable colormaps, metadata extensions, etc.", + "properties": { + "Description": { + "description": "Description of the web resource at this URL.", + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "URLContentType": { + "description": "A keyword describing the distinct content type of the online resource to this resource. (e.g., 'DATACENTER URL', 'DATA CONTACT URL', 'Visualization URL'). The valid values are contained in the KMS System: https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/all/8759ab63-ac04-4136-bc25-0c00eece1096", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "Type": { + "description": "A keyword describing the type of the online resource to this resource. This helps the GUI to know what to do with this resource. (e.g., 'COLORMAP', 'GET VISUALIZATION'). The valid values are contained in the KMS System and are dependent on the URLContentType: https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/all/8759ab63-ac04-4136-bc25-0c00eece1096", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "Subtype": { + "description": "A keyword describing the subtype of the online resource to this resource. This further helps the GUI to know what to do with this resource. (e.g., 'MEDIA', 'BROWSE', 'OPENDAP', 'OPENSEARCH', 'GITC', etc. ). The valid values are contained in the KMS System and are dependent on the Type: https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/all/8759ab63-ac04-4136-bc25-0c00eece1096", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "URL": { + "description": "The URL for the relevant web page (e.g., the URL of the responsible organization's home page, the URL of the colormap server, etc.).", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 1024 + }, + "Format": { + "description": "Describes the organization of the data content so that users and applications know how to read and use the content. The controlled vocabulary for formats is maintained in the Keyword Management System (KMS): https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/DataFormat?gtm_scheme=DataFormat", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "MimeType": { + "description": "The multi-purpose internet mail extensions indicates the nature and format of the data that is accessed through the URL. The controlled vocabulary for MimeTypes is maintained in the Keyword Management System (KMS): https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/MimeType?gtm_scheme=MimeType", + "type": "string", + "minLength": 1, + "maxLength": 80 + } + }, + "required": ["URLContentType", "Type", "URL"] + }, + "InstanceInformationType": { + "type": "object", + "additionalProperties": false, + "description": "This object describes a store for a variable instance. A variable instance is when the variable is extracted from the original data files and stored somewhere.", + "properties": { + "URL": { + "description": "The internet location of the variable instance store.", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 1024 + }, + "Format": { + "description": "Describes the format of the URL's data content so that users and applications know how to read and use the content. At this point in time, an industry standard mime-type does not exist, so a mime-type element will not be included. The controlled vocabulary for formats is maintained in the Keyword Management System (KMS): https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/DataFormat", + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "Description": { + "description": "Brief description of the store or any other useful information about the store.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "DirectDistributionInformation": { + "description": "This element allows end users to get direct access to data products that are stored in the Amazon Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a documentation URL as well as bucket prefix names and an AWS region.", + "$ref": "#/definitions/DirectDistributionInformationType" + }, + "ChunkingInformation": { + "description": "Description of the chunking strategy for the store. Chunking information such as the chunk sizes should be documented.", + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "required": ["URL", "Format"] + }, + "DirectDistributionInformationType": { + "type": "object", + "additionalProperties": false, + "description": "This element allows end users to get direct access to data products that are stored in the Amazon Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a documentation URL as well as bucket prefix names and an AWS region.", + "properties": { + "Region": { + "description": "Defines the possible values for the Amazon Web Service US Regions where the data product resides.", + "$ref": "#/definitions/DirectDistributionInformationRegionEnum" + }, + "S3BucketAndObjectPrefixNames": { + "description": "Defines the possible values for the Amazon Web Service US S3 bucket and/or object prefix names.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "pattern": "[!-~]{1,1024}" + }, + "minItems": 1 + }, + "S3CredentialsAPIEndpoint": { + "description": "Defines the URL where the credentials are stored.", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 1024 + }, + "S3CredentialsAPIDocumentationURL": { + "description": "Defines the URL where the credential documentation are stored.", + "type": "string", + "format": "uri", + "minLength": 1, + "maxLength": 1024 + } + }, + "required": ["Region", "S3CredentialsAPIEndpoint", "S3CredentialsAPIDocumentationURL"] + }, + "DirectDistributionInformationRegionEnum": { + "description": "Defines the possible values for the Amazon Web Service US Regions where the data product resides.", + "type": "string", + "enum": ["us-east-1", "us-east-2", "us-west-1", "us-west-2"] + }, + "MetadataSpecificationType": { + "type": "object", + "additionalProperties": false, + "description": "This object requires any metadata record that is validated by this schema to provide information about the schema.", + "properties": { + "URL": { + "description": "This element represents the URL where the schema lives. The schema can be downloaded.", + "type": "string", + "enum": ["https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0"] + }, + "Name": { + "description": "This element represents the name of the schema.", + "type": "string", + "enum": ["UMM-Var"] + }, + "Version": { + "description": "This element represents the version of the schema.", + "type": "string", + "enum": ["1.9.0"] + } + }, + "required": ["URL", "Name", "Version"] + } + } +} diff --git a/umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-search-results-json-schema.json b/umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-search-results-json-schema.json new file mode 100644 index 0000000000..6f08c38513 --- /dev/null +++ b/umm-spec-lib/resources/json-schemas/variable/umm/v1.9.0/umm-var-search-results-json-schema.json @@ -0,0 +1,286 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": { + "ItemType": { + "type": "object", + "additionalProperties": false, + "description": "Represents a single item found in search results. It contains some metadata about the item found along with the UMM representing the item. UMM won't be present if the item represents a tombstone.", + "properties": { + "meta": { + "$ref": "#/definitions/MetaType" + }, + "umm": { + "$ref": "umm-var-json-schema.json" + }, + "associations": { + "$ref": "#/definitions/AssociationsInItemType" + } + }, + "required": ["meta"] + }, + "MetaType": { + "type": "object", + "additionalProperties": false, + "description": "CMR level metadata about the item found. This represents data not found in the actual metadata.", + "properties": { + "provider-id": { + "description": "The identity of the provider in the CMR", + "type": "string", + "minLength": 1, + "pattern": "[A-Z0-9_]+" + }, + "concept-type": { + "description": "The type of item found.", + "type": "string", + "enum": ["variable"] + }, + "native-id": { + "description": "The id used by the provider to identify this item during ingest.", + "type": "string", + "minLength": 1 + }, + "concept-id": { + "description": "The concept id of the item found.", + "type": "string", + "minLength": 4, + "pattern": "[A-Z]+\\d+-[A-Z0-9_]+" + }, + "revision-id": { + "description": "A number >= 1 that indicates which revision of the item.", + "type": "number" + }, + "revision-date": { + "description": "The date this revision was created. This would be the creation or update date of the item in the CMR.", + "format": "date-time", + "type": "string" + }, + "user-id": { + "description": "The id of the user who created this item revision.", + "type": "string", + "minLength": 1 + }, + "deleted": { + "description": "Indicates if the item represents a tombstone", + "type": "boolean" + }, + "format": { + "description": "The mime type of the original metadata", + "type": "string", + "minLength": 1 + }, + "associations": { + "description": "Contains concept ids of concepts associated with the variable", + "$ref": "#/definitions/AssociationsType" + }, + "association-details": { + "description": "Contains the details of each association.", + "$ref": "#/definitions/AssociationDetailsType" + } + }, + "required": ["provider-id", "concept-type", "native-id", "concept-id", "revision-id", "revision-date"] + }, + "AssociationsInItemType": { + "type": "object", + "additionalProperties": false, + "description": "Any concept associated with the variable.", + "properties": { + "variables": { + "description": "The concept ids of variables associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "collections": { + "description": "The concept ids of collections associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "tools": { + "description": "The concept ids of tools associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "data-quality-summaries": { + "description": "The concept ids of data quality summary concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "order-options": { + "description": "The concept ids of order option concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "service-options": { + "description": "The concept ids of service option concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "service-entries": { + "description": "The concept ids of service entry concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + }, + "grids": { + "description": "The concept ids of grid concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptRevisionType" + }, + "minItems": 1 + } + } + }, + "AssociationsType": { + "type": "object", + "additionalProperties": false, + "description": "Any concept associated with the variable.", + "properties": { + "variables": { + "description": "The concept ids of variables associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "collections": { + "description": "The concept ids of collections associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "tools": { + "description": "The concept ids of tools associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "data-quality-summaries": { + "description": "The concept ids of data quality summary concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "order-options": { + "description": "The concept ids of order option concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "service-options": { + "description": "The concept ids of service option concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "service-entries": { + "description": "The concept ids of service entry concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + }, + "grids": { + "description": "The concept ids of grid concepts associated with the variable", + "type": "array", + "items": { + "$ref": "#/definitions/ConceptIdType" + }, + "minItems": 1 + } + } + }, + "AssociationDetailsType": { + "type": "object", + "additionalProperties": true, + "description": "Association details of concepts associated with the variable.", + "properties": { + "concept-id": { + "description": "The concept id of the associated concept.", + "type": "string" + }, + "revision-id": { + "description": "The revision id of the associated concept.", + "type": "string" + } + } + }, + "ConceptIdType": { + "description": "The concept id of a concept.", + "type": "string", + "minLength": 4, + "pattern": "[A-Z]+\\d+-[A-Z0-9_]+" + }, + "ConceptRevisionType": { + "type": "object", + "additionalProperties": false, + "description": "Concept id and revision id.", + "properties": { + "concept-id": { + "description": "The concept id of the item found.", + "type": "string", + "minLength": 4, + "pattern": "[A-Z]+\\d+-[A-Z0-9_]+" + }, + "revision-id": { + "description": "A number >= 1 that indicates which revision of the item.", + "type": "number" + } + }, + "required": ["concept-id"] + } + }, + "title": "UMM Search Results", + "type": "object", + "additionalProperties": false, + "properties": { + "hits": { + "description": "The total number of items that matched the search.", + "type": "number" + }, + "took": { + "description": "How long the search took in milliseconds from the time the CMR received the request until it had generated the response. This does not include network traffic time to send the request or return the response.", + "type": "number" + }, + "items": { + "description": "The list of items matching the result in this page.", + "type": "array", + "items": { + "$ref": "#/definitions/ItemType" + }, + "minItems": 0 + } + }, + "required": ["hits", "took", "items"] +} diff --git a/umm-spec-lib/src/cmr/umm_spec/migration/version/variable.clj b/umm-spec-lib/src/cmr/umm_spec/migration/version/variable.clj index 312e8e90f6..a69d90373f 100644 --- a/umm-spec-lib/src/cmr/umm_spec/migration/version/variable.clj +++ b/umm-spec-lib/src/cmr/umm_spec/migration/version/variable.clj @@ -258,3 +258,18 @@ (assoc % :Size -1) %)) (m-spec/update-version :variable "1.8.1"))) + +;; migrations for 1.9.0 ********************************************************** + +(defmethod interface/migrate-umm-version [:variable "1.8.2" "1.9.0"] + [_context umm-v & _] + ;; update the MetadataSpecification + (-> umm-v + (m-spec/update-version :variable "1.9.0"))) + +(defmethod interface/migrate-umm-version [:variable "1.9.0" "1.8.2"] + [_context umm-v & _] + ;; Update the MetadataSpecification and remove InstanceInformation + (-> umm-v + (dissoc :InstanceInformation) + (m-spec/update-version :variable "1.8.2"))) diff --git a/umm-spec-lib/src/cmr/umm_spec/models/umm_variable_models.clj b/umm-spec-lib/src/cmr/umm_spec/models/umm_variable_models.clj index 208d3da144..43f4a85264 100644 --- a/umm-spec-lib/src/cmr/umm_spec/models/umm_variable_models.clj +++ b/umm-spec-lib/src/cmr/umm_spec/models/umm_variable_models.clj @@ -5,6 +5,10 @@ (defrecord UMM-Var [ + ;; Describes a store (zarr) where a variable has been separated from its original data files and + ;; saved as its own entity. + InstanceInformation + ;; Valid ranges of variable data values. ValidRanges @@ -49,7 +53,9 @@ Definition ;; Controlled Science Keywords describing the measurements/variables. The controlled vocabulary - ;; for Science Keywords is maintained in the Keyword Management System (KMS). + ;; for Science Keywords is maintained in the Keyword Management System (KMS). The valid values + ;; can be found at the KMS website: + ;; https://gcmdservices.gsfc.nasa.gov/kms/concepts/concept_scheme/sciencekeywords?format=csv. ScienceKeywords ;; The name of a variable. @@ -145,6 +151,34 @@ ]) (record-pretty-printer/enable-record-pretty-printing SamplingIdentifierType) +;; This object describes a store for a variable instance. A variable instance is when the variable +;; is extracted from the original data files and stored somewhere. +(defrecord InstanceInformationType + [ + ;; The internet location of the variable instance store. + URL + + ;; Describes the format of the URL's data content so that users and applications know how to read + ;; and use the content. At this point in time, an industry standard mime-type does not exist, so + ;; a mime-type element will not be included. The controlled vocabulary for formats is maintained + ;; in the Keyword Management System (KMS): + ;; https://gcmd.earthdata.nasa.gov/KeywordViewer/scheme/DataFormat + Format + + ;; Brief description of the store or any other useful information about the store. + Description + + ;; This element allows end users to get direct access to data products that are stored in the + ;; Amazon Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a + ;; documentation URL as well as bucket prefix names and an AWS region. + DirectDistributionInformation + + ;; Description of the chunking strategy for the store. Chunking information such as the chunk + ;; sizes should be documented. + ChunkingInformation + ]) +(record-pretty-printer/enable-record-pretty-printing InstanceInformationType) + ;; Enables specification of Earth science keywords related to the collection. The Earth Science ;; keywords are chosen from a controlled keyword hierarchy maintained in the Keyword Management ;; System (KMS). The valid values can be found at the KMS website: @@ -167,6 +201,27 @@ ]) (record-pretty-printer/enable-record-pretty-printing ScienceKeywordType) +;; This element allows end users to get direct access to data products that are stored in the Amazon +;; Web Service (AWS) S3 buckets. The sub elements include S3 credentials end point and a +;; documentation URL as well as bucket prefix names and an AWS region. +(defrecord DirectDistributionInformationType + [ + ;; Defines the possible values for the Amazon Web Service US Regions where the data product + ;; resides. + Region + + ;; Defines the possible values for the Amazon Web Service US S3 bucket and/or object prefix + ;; names. + S3BucketAndObjectPrefixNames + + ;; Defines the URL where the credentials are stored. + S3CredentialsAPIEndpoint + + ;; Defines the URL where the credential documentation are stored. + S3CredentialsAPIDocumentationURL + ]) +(record-pretty-printer/enable-record-pretty-printing DirectDistributionInformationType) + ;; Valid range data value of a variable: minimum and maximum values. For example, '-100, 5000'. (defrecord ValidRangeType [ diff --git a/umm-spec-lib/src/cmr/umm_spec/versioning.clj b/umm-spec-lib/src/cmr/umm_spec/versioning.clj index fff93202d3..65837a390f 100644 --- a/umm-spec-lib/src/cmr/umm_spec/versioning.clj +++ b/umm-spec-lib/src/cmr/umm_spec/versioning.clj @@ -14,7 +14,7 @@ "1.16" "1.16.1" "1.16.2" "1.16.3" "1.16.4" "1.16.5" "1.16.6" "1.16.7" "1.17.0" "1.17.1" "1.17.2" "1.17.3"] :granule ["1.4" "1.5" "1.6" "1.6.1" "1.6.2" "1.6.3" "1.6.4"] - :variable ["1.0" "1.1" "1.2" "1.3" "1.4" "1.5" "1.6" "1.7" "1.8" "1.8.1" "1.8.2"] + :variable ["1.0" "1.1" "1.2" "1.3" "1.4" "1.5" "1.6" "1.7" "1.8" "1.8.1" "1.8.2" "1.9.0"] :service ["1.0" "1.1" "1.2" "1.3" "1.3.1" "1.3.2" "1.3.3" "1.3.4" "1.4" "1.4.1" "1.5.0" "1.5.1"] :tool ["1.0" "1.1" "1.1.1" "1.2.0"] :subscription ["1.0" "1.1"]}) diff --git a/umm-spec-lib/test/cmr/umm_spec/test/migration/version/variable.clj b/umm-spec-lib/test/cmr/umm_spec/test/migration/version/variable.clj index 67a4c76df4..b0e0facf4a 100644 --- a/umm-spec-lib/test/cmr/umm_spec/test/migration/version/variable.clj +++ b/umm-spec-lib/test/cmr/umm_spec/test/migration/version/variable.clj @@ -622,3 +622,34 @@ (dissoc :RelatedURLs)) actual (migrate-variable "1.8.2" "1.8.1" vc-1-8-2)] (is (= vc-1-8-1 actual) "Document Match"))) + +(def variable-concept-1-9 + (-> variable-concept-1-8-2 + (assoc :MetadataSpecification {:URL "https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0" + :Name "UMM-Var" + :Version "1.9.0"}) + (assoc :InstanceInformation + {:URL "s3://prod-giovanni-cache.s3.us-west-2.amazonaws.com/zarr/GPM_3IMERGHH_06_precipitationCal" + :Format "Zarr" + :Description "GPM_3IMERGHH_06_precipitationCal variable zarr store. Other brief end user information can go here." + :DirectDistributionInformation + {:Region "us-west-2" + :S3BucketAndObjectPrefixNames ["prod-giovanni-cache" "zarr/GPM_3IMERGHH_06_precipitationCal"] + :S3CredentialsAPIEndpoint "https://api.giovanni.earthdata.nasa.gov/s3credentials" + :S3CredentialsAPIDocumentationURL "https://disc.gsfc.nasa.gov/information/documents?title=In-region%20Direct%20S3%20Zarr%20Cache%20Access"} + :ChunkingInformation "Chunk size for this example is 1MB. It is optimized for time series."}))) + +(deftest migrate-1-8-2->1-9 + ;; Only MetadataSpecification's version is updated. + (let [vc-1-9 (-> variable-concept-1-8-2 + (assoc :MetadataSpecification {:URL "https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0" + :Name "UMM-Var" + :Version "1.9.0"})) + actual (migrate-variable "1.8.2" "1.9.0" variable-concept-1-8-2)] + (is (= vc-1-9 actual) "Document Match"))) + +(deftest migrate-1-9->1-8-2 + ;; InstanceInformation is removed. + ;; MetadataSpecification version is converted. + (let [actual (migrate-variable "1.9.0" "1.8.2" variable-concept-1-9)] + (is (= variable-concept-1-8-2 actual) "Document Match"))) diff --git a/umm-spec-lib/test/cmr/umm_spec/test/validation/umm_spec_variable_validation_tests.clj b/umm-spec-lib/test/cmr/umm_spec/test/validation/umm_spec_variable_validation_tests.clj index 13c62c4828..ff114f7a99 100644 --- a/umm-spec-lib/test/cmr/umm_spec/test/validation/umm_spec_variable_validation_tests.clj +++ b/umm-spec-lib/test/cmr/umm_spec/test/validation/umm_spec_variable_validation_tests.clj @@ -15,6 +15,7 @@ :Dimensions :FillValues :IndexRanges + :InstanceInformation :LongName :MeasurementIdentifiers :MetadataSpecification