Merge pull request #18 from truenas/refine-keys-in-cached-data

Refine keys for app versions and catalog.json

apps_ci/scripts/catalog_update.py

@@ -49,7 +49,7 @@ def validate_train_data(train_data):
     except (json.JSONDecodeError, JsonValidationError) as e:
         verrors.add(
             'catalog_json',
-            f'Failed to validate contents of train data: {e!r}'
+            f'Failed to validate contents of train data ({".".join(list(e.path))}): {e!r}'
         )
     verrors.check()
 

apps_validation/json_schema_utils.py

@@ -3,13 +3,17 @@
     'properties': {
         'name': {'type': 'string'},
         'train': {'type': 'string'},
+        'description': {'type': 'string'},
+        'home': {'type': 'string'},
+        'app_version': {'type': 'string'},
         'annotations': {
             'type': 'object',
             'properties': {
                 'min_scale_version': {'type': 'string'},
                 'max_scale_version': {'type': 'string'},
             },
         },
+        'title': {'type': 'string'},
         'sources': {
             'type': 'array',
             'items': {'type': 'string'},
@@ -21,10 +25,15 @@
                 'properties': {
                     'name': {'type': 'string'},
                     'email': {'type': 'string'},
+                    'url': {'type': 'string'},
                 },
                 'required': ['name', 'email'],
             },
         },
+        'keywords': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
         'version': {
             'type': 'string',
             'pattern': '[0-9]+.[0-9]+.[0-9]+',
@@ -34,9 +43,46 @@
             'pattern': '[0-9]+.[0-9]+.[0-9]+',
         },
         'lib_version_hash': {'type': 'string'},
+        'run_as_context': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'description': {'type': 'string'},
+                    'gid': {'type': 'integer'},
+                    'groupName': {'type': 'string'},
+                    'userName': {'type': 'string'},
+                    'uid': {'type': 'integer'},
+                },
+                'required': ['description'],
+            },
+        },
+        'capabilities': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'description': {'type': 'string'},
+                    'name': {'type': 'string'},
+                },
+                'required': ['description', 'name'],
+            },
+        },
+        'host_mounts': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'description': {'type': 'string'},
+                    'hostPath': {'type': 'string'},
+                },
+                'required': ['description', 'hostPath'],
+            },
+        },
     },
     'required': [
-        'name', 'train', 'version',
+        'name', 'train', 'version', 'app_version', 'title', 'description', 'home',
+        'sources', 'maintainers', 'run_as_context', 'capabilities', 'host_mounts',
     ],
     'if': {
         'properties': {
@@ -192,47 +238,6 @@
         }
     }
 }
-METADATA_JSON_SCHEMA = {
-    'type': 'object',
-    'properties': {
-        'runAsContext': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'description': {'type': 'string'},
-                    'gid': {'type': 'integer'},
-                    'groupName': {'type': 'string'},
-                    'userName': {'type': 'string'},
-                    'uid': {'type': 'integer'},
-                },
-                'required': ['description'],
-            },
-        },
-        'capabilities': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'description': {'type': 'string'},
-                    'name': {'type': 'string'},
-                },
-                'required': ['description', 'name'],
-            },
-        },
-        'hostMounts': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'description': {'type': 'string'},
-                    'hostPath': {'type': 'string'},
-                },
-                'required': ['description', 'hostPath'],
-            },
-        },
-    },
-}
 RECOMMENDED_APPS_JSON_SCHEMA = {
     'type': 'object',
     'patternProperties': {
@@ -279,80 +284,9 @@
                     'type': 'string',
                     'pattern': '[0-9]+.[0-9]+.[0-9]+'
                 },
-                'chart_metadata': {
-                    'type': 'object',
-                    'properties': {
-                        'name': {
-                            'type': 'string'
-                        },
-                        'description': {
-                            'type': 'string'
-                        },
-                        'annotations': {
-                            'type': 'object'
-                        },
-                        'type': {
-                            'type': 'string'
-                        },
-                        'version': {
-                            'type': 'string',
-                            'pattern': '[0-9]+.[0-9]+.[0-9]+'
-                        },
-                        'apiVersion': {
-                            'type': 'string',
-                        },
-                        'appVersion': {
-                            'type': 'string'
-                        },
-                        'kubeVersion': {
-                            'type': 'string'
-                        },
-                        'app_readme': {'type': 'string'},
-                        'detailed_readme': {'type': 'string'},
-                        'changelog': {'type': ['string', 'null']},
-                        'maintainers': {
-                            'type': 'array',
-                            'items': {
-                                'type': 'object',
-                                'properties': {
-                                    'name': {'type': 'string'},
-                                    'url': {'type': ['string', 'null']},
-                                    'email': {'type': 'string'},
-                                },
-                                'required': ['name', 'email'],
-                            }
-                        },
-                        'dependencies': {
-                            'type': 'array',
-                            'items': {
-                                'type': 'object',
-                                'properties': {
-                                    'name': {'type': 'string'},
-                                    'repository': {'type': 'string'},
-                                    'version': {'type': 'string'}
-                                }
-                            }
-                        },
-                        'home': {'type': 'string'},
-                        'icon': {'type': 'string'},
-                        'sources': {
-                            'type': 'array',
-                            'items': {
-                                'type': 'string'
-                            }
-                        },
-                        'keywords': {
-                            'type': 'array',
-                            'items': {
-                                'type': 'string'
-                            }
-                        },
-                    }
-                },
-                'app_metadata': {
-                    **METADATA_JSON_SCHEMA,
-                    'type': ['object', 'null'],
-                },
+                'app_metadata': APP_METADATA_JSON_SCHEMA,
+                'readme': {'type': ['string', 'null']},
+                'changelog': {'type': ['string', 'null']},
                 'schema': {
                     'type': 'object',
                     'properties': {
@@ -398,7 +332,7 @@
             },
             'required': [
                 'healthy', 'supported', 'healthy_error', 'location', 'last_update', 'required_features',
-                'human_version', 'version', 'chart_metadata', 'app_metadata', 'schema',
+                'human_version', 'version', 'app_metadata', 'schema', 'readme', 'changelog',
             ],
         },
     },

apps_validation/validate_app.py

@@ -37,6 +37,7 @@ def validate_catalog_item(catalog_item_path: str, schema: str, train_name: str,
         with open(os.path.join(catalog_item_path, 'item.yaml'), 'r') as f:
             item_config = yaml.safe_load(f.read())
 
+        # TODO: Remove validate key value type function and have json schemas for all of this
         validate_key_value_types(
             item_config, (
                 ('categories', list), ('tags', list, False), ('screenshots', list, False),

apps_validation/validate_app_version.py

@@ -14,7 +14,7 @@
 
 from .app_version import validate_app_version_file
 from .ix_values import validate_ix_values_schema
-from .json_schema_utils import METADATA_JSON_SCHEMA, VERSION_VALIDATION_SCHEMA
+from .json_schema_utils import VERSION_VALIDATION_SCHEMA
 from .validate_questions import validate_questions_yaml
 from .validate_templates import validate_templates
 
@@ -89,13 +89,6 @@ def validate_catalog_item_version(
             except ValidationErrors as v:
                 verrors.extend(v)
 
-    metadata_path = os.path.join(version_path, 'metadata.yaml')
-    if os.path.exists(metadata_path):
-        try:
-            validate_metadata_yaml(metadata_path, f'{schema}.metadata_configuration')
-        except ValidationErrors as v:
-            verrors.extend(v)
-
     # validate_app_migrations(verrors, version_path, f'{schema}.app_migrations')
     # FIXME: Add validation for app migrations
 
@@ -124,19 +117,3 @@ def validate_ix_values_yaml(ix_values_yaml_path: str, schema: str):
         verrors.add(schema, 'Must be a dictionary')
 
     verrors.check()
-
-
-def validate_metadata_yaml(metadata_yaml_path: str, schema: str):
-    verrors = ValidationErrors()
-    with open(metadata_yaml_path, 'r') as f:
-        try:
-            metadata = yaml.safe_load(f.read())
-        except yaml.YAMLError:
-            verrors.add(schema, 'Must be a valid yaml file')
-        else:
-            try:
-                json_schema_validate(metadata, METADATA_JSON_SCHEMA)
-            except JsonValidationError as e:
-                verrors.add(schema, f'Invalid format specified for application metadata: {e}')
-
-    verrors.check()

catalog_reader/app.py

@@ -9,7 +9,7 @@
 from apps_validation.validate_app import validate_catalog_item
 from apps_validation.validate_app_version import validate_catalog_item_version  # FIXME: rename this
 
-from .app_utils import get_default_questions_context, get_app_details_base
+from .app_utils import get_default_questions_context, get_app_details_base, get_human_version
 from .git import get_last_updated_date
 from .questions import normalize_questions
 from .supported_version import version_supported
@@ -53,32 +53,30 @@ def get_app_details(
         'default_values_callable': options.get('default_values_callable'),
     }))
     unhealthy_versions = []
+    desired_keys_mapping = {
+        'maintainers': 'maintainers',
+        'description': 'description',
+        'title': 'title',
+        'home': 'home',
+        'sources': 'sources',
+    }
     for k, v in sorted(item_data['versions'].items(), key=lambda v: parse_version(v[0]), reverse=True):
         if not v['healthy']:
             unhealthy_versions.append(k)
         else:
-            chart_metadata = v['chart_metadata']
-            if not item_data['app_readme']:
-                item_data['app_readme'] = v['readme']
-            if not item_data['maintainers'] and chart_metadata.get('maintainers'):
-                item_data['maintainers'] = chart_metadata['maintainers']
+            app_metadata = v['app_metadata']
+            for desired_key in list(desired_keys_mapping):
+                if not item_data[desired_key]:
+                    item_data[desired_key] = app_metadata[desired_keys_mapping[desired_key]]
+                desired_keys_mapping.pop(desired_key)
+
             if not item_data['latest_version']:
                 item_data['latest_version'] = k
-                item_data['latest_app_version'] = chart_metadata.get('appVersion')
-                item_data['latest_human_version'] = ''
-                if item_data['latest_app_version']:
-                    item_data['latest_human_version'] = f'{item_data["latest_app_version"]}_'
-                item_data['latest_human_version'] += k
-            if not item_data['description'] and chart_metadata.get('description'):
-                item_data['description'] = v['chart_metadata']['description']
-            if item_data['title'] == item_data['name'].capitalize() and chart_metadata.get(
-                'annotations', {}
-            ).get('title'):
-                item_data['title'] = chart_metadata['annotations']['title']
-            if item_data['home'] is None and chart_metadata.get('home'):
-                item_data['home'] = chart_metadata['home']
-            if not item_data['sources'] and chart_metadata.get('sources'):
-                item_data['sources'] = chart_metadata['sources']
+                item_data['latest_app_version'] = app_metadata['app_version']
+                item_data['latest_human_version'] = get_human_version(app_metadata['app_version'], k)
+
+            if not item_data['app_readme']:
+                item_data['app_readme'] = v['readme']
 
     if unhealthy_versions:
         item_data['healthy_error'] = f'Errors were found with {", ".join(unhealthy_versions)} version(s)'
@@ -149,10 +147,9 @@ def get_app_version_details(
 ) -> dict:
     version_data = {'location': version_path, 'required_features': set()}
     for key, filename, parser in (
-        ('chart_metadata', 'app.yaml', yaml.safe_load),
-        ('app_metadata', 'metadata.yaml', yaml.safe_load),
+        ('app_metadata', 'app.yaml', yaml.safe_load),
         ('schema', 'questions.yaml', yaml.safe_load),
-        ('readme', 'README.md', markdown.markdown),  # TODO: Has been changed, make sure json schema accounts for it
+        ('readme', 'README.md', markdown.markdown),
         ('changelog', 'CHANGELOG.md', markdown.markdown),
     ):
         if os.path.exists(os.path.join(version_path, filename)):
@@ -171,8 +168,9 @@ def get_app_version_details(
     })
     if options and options.get('default_values_callable'):
         version_data['values'] = options['default_values_callable'](version_data)
-    chart_metadata = version_data['chart_metadata']
-    if chart_metadata['name'] != 'ix-chart' and chart_metadata.get('appVersion'):
-        version_data['human_version'] = f'{chart_metadata["appVersion"]}_{chart_metadata["version"]}'
+
+    app_metadata = version_data['app_metadata']
+    # TODO: See if this needs to change for our adaptation of ix-chart
+    version_data['human_version'] = get_human_version(app_metadata['app_version'], app_metadata['version'])
 
     return version_data

catalog_reader/app_utils.py

@@ -60,3 +60,7 @@ def get_values(values_path: str) -> dict:
             return yaml.safe_load(f.read())
 
     return {}
+
+
+def get_human_version(app_version: str, version: str) -> str:
+    return f'{app_version}_{version}' if app_version != version else version