Merge pull request #40 from truenas/NAS-130223

NAS-130223 / 24.10 / Don't normalize apps versions data while caching

apps_ci/scripts/catalog_update.py

@@ -26,7 +26,8 @@ def get_trains(location: str) -> typing.Tuple[dict, dict]:
     catalog_data = {}
     versions_data = {}
     for train_name, train_data in retrieve_trains_data(
-        get_apps_in_trains(trains_to_traverse, location), location, preferred_trains, trains_to_traverse
+        get_apps_in_trains(trains_to_traverse, location), location, preferred_trains, trains_to_traverse,
+        normalize_questions=False,
     )[0].items():
         catalog_data[train_name] = {}
         versions_data[train_name] = {}

catalog_reader/app.py

@@ -19,7 +19,7 @@
 
 
 def get_app_details(
-    item_location: str, questions_context: typing.Optional[dict] = None, options: typing.Optional[dict] = None
+    item_location: str, questions_context: typing.Optional[dict] = None, options: typing.Optional[dict] = None,
 ) -> dict:
     catalog_path = item_location.rstrip('/').rsplit('/', 2)[0]
     item = item_location.rsplit('/', 1)[-1]
@@ -51,6 +51,7 @@ def get_app_details(
     item_data.update(get_app_details_impl(item_location, schema, questions_context, {
         'retrieve_latest_version': not retrieve_versions,
         'default_values_callable': options.get('default_values_callable'),
+        'normalize_questions': options.get('normalize_questions', True),
     }))
     unhealthy_versions = []
     desired_keys_mapping = {
@@ -134,7 +135,11 @@ def get_app_details_impl(
 
         version_details.update({
             'healthy': True,
-            **get_app_version_details(version_details['location'], questions_context)
+            **get_app_version_details(
+                version_details['location'], questions_context, {
+                    'normalize_questions': options.get('normalize_questions', True),
+                },
+            )
         })
         if retrieve_latest_version:
             break
@@ -145,6 +150,7 @@ def get_app_details_impl(
 def get_app_version_details(
     version_path: str, questions_context: typing.Optional[dict], options: typing.Optional[dict] = None
 ) -> dict:
+    options = options or {}
     version_data = {'location': version_path, 'required_features': set()}
     for key, filename, parser in (
         ('app_metadata', 'app.yaml', yaml.safe_load),
@@ -160,13 +166,14 @@ def get_app_version_details(
 
     # We will normalise questions now so that if they have any references, we render them accordingly
     # like a field referring to available interfaces on the system
-    normalize_questions(version_data, questions_context or get_default_questions_context())
+    if options.get('normalize_questions', True):
+        normalize_questions(version_data, questions_context or get_default_questions_context())
 
     version_data.update({
         'supported': version_supported(version_data),
         'required_features': list(version_data['required_features']),
     })
-    if options and options.get('default_values_callable'):
+    if options.get('default_values_callable'):
         version_data['values'] = options['default_values_callable'](version_data)
 
     app_metadata = version_data['app_metadata']

catalog_reader/catalog.py

@@ -8,11 +8,15 @@
 from .train_utils import get_train_path, is_train_valid
 
 
-def app_details(apps: dict, location: str, questions_context: typing.Optional[dict], app_key: str) -> dict:
+def app_details(
+    apps: dict, location: str, questions_context: typing.Optional[dict], app_key: str, normalize_questions: bool = True,
+) -> dict:
     train = apps[app_key]
     app = app_key.removesuffix(f'_{train}')
     app_location = os.path.join(get_train_path(location), train, app)
-    return get_app_details(app_location, questions_context, {'retrieve_versions': True})
+    return get_app_details(
+        app_location, questions_context, {'retrieve_versions': True, 'normalize_questions': normalize_questions},
+    )
 
 
 def retrieve_train_names(location: str, all_trains=True, trains_filter=None) -> list:
@@ -37,8 +41,8 @@ def get_apps_in_trains(trains_to_traverse: list, catalog_location: str) -> dict:
 
 
 def retrieve_trains_data(
-    apps: dict, catalog_location: str, preferred_trains: list,
-    trains_to_traverse: list, job: typing.Any = None, questions_context: typing.Optional[dict] = None
+    apps: dict, catalog_location: str, preferred_trains: list, trains_to_traverse: list, job: typing.Any = None,
+    questions_context: typing.Optional[dict] = None, normalize_questions: bool = True,
 ) -> typing.Tuple[dict, set]:
     questions_context = questions_context or get_default_questions_context()
     trains = {
@@ -50,7 +54,9 @@ def retrieve_trains_data(
     total_apps = len(apps)
     with concurrent.futures.ProcessPoolExecutor(max_workers=(5 if total_apps > 10 else 2)) as exc:
         for index, result in enumerate(zip(apps, exc.map(
-            functools.partial(app_details, apps, catalog_location, questions_context),
+            functools.partial(
+                app_details, apps, catalog_location, questions_context, normalize_questions=normalize_questions
+            ),
             apps, chunksize=(10 if total_apps > 10 else 5)
         ))):
             app_key = result[0]