From 09e3681061e26284c94d9ee79315f74cbd206855 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 01:51:09 +0500
Subject: [PATCH 01/17] Add a util to retrieve hash of dir

---
 apps_validation/validation/hash_utils.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 apps_validation/validation/hash_utils.py

diff --git a/apps_validation/validation/hash_utils.py b/apps_validation/validation/hash_utils.py
new file mode 100644
index 0000000..2928bce
--- /dev/null
+++ b/apps_validation/validation/hash_utils.py
@@ -0,0 +1,12 @@
+import subprocess
+
+
+def get_hash_of_directory(directory: str) -> str:
+    """
+    This returns sha256sum of the directory
+    """
+    cp = subprocess.run(
+        f'find {directory} -type f -exec sha256sum {{}} + | sort | sha256sum',
+        capture_output=True, check=True, shell=True,
+    )
+    return cp.stdout.decode().split()[0]

From 9dae97a08408c274ac60fa4789b985e88e4cd7fe Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 02:49:32 +0500
Subject: [PATCH 02/17] Move hash utils to catalog reader

---
 {apps_validation/validation => catalog_reader}/hash_utils.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename {apps_validation/validation => catalog_reader}/hash_utils.py (100%)

diff --git a/apps_validation/validation/hash_utils.py b/catalog_reader/hash_utils.py
similarity index 100%
rename from apps_validation/validation/hash_utils.py
rename to catalog_reader/hash_utils.py

From 64926f744125dbb8887266ca616e771170db4901 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 02:57:01 +0500
Subject: [PATCH 03/17] Add function to get library hashes

---
 catalog_reader/library.py | 11 +++++++++++
 catalog_reader/names.py   |  6 ++++++
 2 files changed, 17 insertions(+)
 create mode 100644 catalog_reader/library.py

diff --git a/catalog_reader/library.py b/catalog_reader/library.py
new file mode 100644
index 0000000..d6d7c82
--- /dev/null
+++ b/catalog_reader/library.py
@@ -0,0 +1,11 @@
+import contextlib
+import yaml
+
+
+def get_library_hashes(library_path: str) -> dict:
+    """
+    This reads from library hashes file and returns the hashes
+    """
+    with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
+        with open(library_path, 'r') as f:
+            return yaml.safe_load(f.read())
diff --git a/catalog_reader/names.py b/catalog_reader/names.py
index b409f08..82729d0 100644
--- a/catalog_reader/names.py
+++ b/catalog_reader/names.py
@@ -1,6 +1,8 @@
+import os
 import typing
 
 
+LIBRARY_HASHES_FILENAME = 'hashes.yaml'
 RECOMMENDED_APPS_FILENAME = 'recommended_apps.yaml'
 TO_KEEP_VERSIONS = 'to_keep_versions.yaml'
 UPGRADE_STRATEGY_FILENAME = 'upgrade_strategy'
@@ -12,3 +14,7 @@ def get_app_library_dir_name_from_version(version: str) -> str:
 
 def get_base_library_dir_name_from_version(version: typing.Optional[str]) -> str:
     return f'base_v{version.replace(".", "_")}' if version else ''
+
+
+def get_library_hashes_path(library_path: str) -> str:
+    return os.path.join(library_path, LIBRARY_HASHES_FILENAME)

From ea45faf29d39282451a2ba59933c8196d41e06fb Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:12:58 +0500
Subject: [PATCH 04/17] Add a script to generate hashes of library versions

---
 catalog_reader/names.py               |  4 ++
 catalog_reader/scripts/__init__.py    |  0
 catalog_reader/scripts/apps_hashes.py | 53 +++++++++++++++++++++++++++
 setup.py                              |  1 +
 4 files changed, 58 insertions(+)
 create mode 100644 catalog_reader/scripts/__init__.py
 create mode 100644 catalog_reader/scripts/apps_hashes.py

diff --git a/catalog_reader/names.py b/catalog_reader/names.py
index 82729d0..1c9c6af 100644
--- a/catalog_reader/names.py
+++ b/catalog_reader/names.py
@@ -18,3 +18,7 @@ def get_base_library_dir_name_from_version(version: typing.Optional[str]) -> str
 
 def get_library_hashes_path(library_path: str) -> str:
     return os.path.join(library_path, LIBRARY_HASHES_FILENAME)
+
+
+def get_library_path(catalog_path: str) -> str:
+    return os.path.join(catalog_path, 'library')
diff --git a/catalog_reader/scripts/__init__.py b/catalog_reader/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/catalog_reader/scripts/apps_hashes.py b/catalog_reader/scripts/apps_hashes.py
new file mode 100644
index 0000000..8783e25
--- /dev/null
+++ b/catalog_reader/scripts/apps_hashes.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+import argparse
+import os
+import pathlib
+import re
+import yaml
+
+from apps_validation.exceptions import CatalogDoesNotExist, ValidationErrors
+from catalog_reader.hash_utils import get_hash_of_directory
+from catalog_reader.names import get_library_path, get_library_hashes_path
+
+
+RE_VERSION = re.compile(r'^\d+.\d+\.\d+$')
+
+
+def update_catalog_hashes(catalog_path: str) -> None:
+    if not os.path.exists(catalog_path):
+        raise CatalogDoesNotExist(catalog_path)
+
+    verrors = ValidationErrors()
+    library_dir = pathlib.Path(get_library_path(catalog_path))
+    if not library_dir.exists():
+        verrors.add('library', 'Library directory is missing')
+
+    verrors.check()
+
+    hashes = {}
+    for lib_entry in library_dir.iterdir():
+        if not lib_entry.is_dir() or not RE_VERSION.match(lib_entry.name):
+            continue
+
+        hashes[lib_entry.name] = get_hash_of_directory(os.path.join(library_dir.name, lib_entry.name))
+
+    hashes_file_path = get_library_hashes_path(get_library_path(catalog_path))
+    with open(hashes_file_path, 'w') as f:
+        yaml.safe_dump(hashes, f)
+
+    print(f'[\033[92mOK\x1B[0m]\tGenerated hashes for library versions at {hashes_file_path!r}')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--path', help='Specify path of TrueNAS catalog')
+
+    args = parser.parse_args()
+    if not args.path:
+        parser.print_help()
+    else:
+        update_catalog_hashes(args.path)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/setup.py b/setup.py
index 17cbf16..3ff5710 100644
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,7 @@
     platforms='any',
     entry_points={
         'console_scripts': [
+            'apps_catalog_hash_generate = catalog_reader.scripts.apps_hashes:main',
             'apps_catalog_update = apps_validation.scripts.catalog_update:main',
             'apps_catalog_validate = apps_validation.scripts.catalog_validate:main',
             'apps_dev_charts_validate = apps_validation.scripts.dev_apps_validate:main',  # TODO: Remove apps_prefix

From 4e743914b4d6562e6bcaed3e7faaa685f9138cde Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:21:36 +0500
Subject: [PATCH 05/17] Refactor getting hashes logic

---
 catalog_reader/library.py             | 21 +++++++++++++++++++++
 catalog_reader/scripts/apps_hashes.py | 14 ++------------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/catalog_reader/library.py b/catalog_reader/library.py
index d6d7c82..09a083d 100644
--- a/catalog_reader/library.py
+++ b/catalog_reader/library.py
@@ -1,6 +1,15 @@
 import contextlib
+import os
+import pathlib
+import re
 import yaml
 
+from .hash_utils import get_hash_of_directory
+from .names import get_library_path
+
+
+RE_VERSION = re.compile(r'^\d+.\d+\.\d+$')
+
 
 def get_library_hashes(library_path: str) -> dict:
     """
@@ -9,3 +18,15 @@ def get_library_hashes(library_path: str) -> dict:
     with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
         with open(library_path, 'r') as f:
             return yaml.safe_load(f.read())
+
+
+def get_hashes_of_base_lib_versions(catalog_path: str) -> dict:
+    library_dir = pathlib.Path(get_library_path(catalog_path))
+    hashes = {}
+    for lib_entry in library_dir.iterdir():
+        if not lib_entry.is_dir() or not RE_VERSION.match(lib_entry.name):
+            continue
+
+        hashes[lib_entry.name] = get_hash_of_directory(os.path.join(library_dir.name, lib_entry.name))
+
+    return hashes
diff --git a/catalog_reader/scripts/apps_hashes.py b/catalog_reader/scripts/apps_hashes.py
index 8783e25..2f37afe 100644
--- a/catalog_reader/scripts/apps_hashes.py
+++ b/catalog_reader/scripts/apps_hashes.py
@@ -2,17 +2,13 @@
 import argparse
 import os
 import pathlib
-import re
 import yaml
 
 from apps_validation.exceptions import CatalogDoesNotExist, ValidationErrors
-from catalog_reader.hash_utils import get_hash_of_directory
+from catalog_reader.library import get_hashes_of_base_lib_versions
 from catalog_reader.names import get_library_path, get_library_hashes_path
 
 
-RE_VERSION = re.compile(r'^\d+.\d+\.\d+$')
-
-
 def update_catalog_hashes(catalog_path: str) -> None:
     if not os.path.exists(catalog_path):
         raise CatalogDoesNotExist(catalog_path)
@@ -24,13 +20,7 @@ def update_catalog_hashes(catalog_path: str) -> None:
 
     verrors.check()
 
-    hashes = {}
-    for lib_entry in library_dir.iterdir():
-        if not lib_entry.is_dir() or not RE_VERSION.match(lib_entry.name):
-            continue
-
-        hashes[lib_entry.name] = get_hash_of_directory(os.path.join(library_dir.name, lib_entry.name))
-
+    hashes = get_hashes_of_base_lib_versions(catalog_path)
     hashes_file_path = get_library_hashes_path(get_library_path(catalog_path))
     with open(hashes_file_path, 'w') as f:
         yaml.safe_dump(hashes, f)

From e05411641be191f417e5ab3b3ba5807a79ecace7 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:32:49 +0500
Subject: [PATCH 06/17] Add validation utils to make sure we attempt to
 validate base libs

---
 .../validation/json_schema_utils.py           |  8 ++++
 .../validation/validate_library.py            | 46 +++++++++++++++++++
 catalog_reader/library.py                     |  4 +-
 3 files changed, 56 insertions(+), 2 deletions(-)
 create mode 100644 apps_validation/validation/validate_library.py

diff --git a/apps_validation/validation/json_schema_utils.py b/apps_validation/validation/json_schema_utils.py
index 0fd96dc..017876e 100644
--- a/apps_validation/validation/json_schema_utils.py
+++ b/apps_validation/validation/json_schema_utils.py
@@ -73,6 +73,14 @@
         ],
     },
 }
+BASE_LIBRARIES_JSON_SCHEMA = {
+    'type': 'object',
+    'patternProperties': {
+        '[0-9]+.[0-9]+.[0-9]+': {
+            'type': 'string',
+        },
+    },
+}
 CATALOG_JSON_SCHEMA = {
     'type': 'object',
     'patternProperties': {
diff --git a/apps_validation/validation/validate_library.py b/apps_validation/validation/validate_library.py
new file mode 100644
index 0000000..d06e23e
--- /dev/null
+++ b/apps_validation/validation/validate_library.py
@@ -0,0 +1,46 @@
+import os.path
+import pathlib
+
+from jsonschema import validate as json_schema_validate, ValidationError as JsonValidationError
+
+from apps_validation.exceptions import ValidationErrors
+from catalog_reader.library import get_library_hashes, get_hashes_of_base_lib_versions, RE_VERSION
+from catalog_reader.names import get_library_path, get_library_hashes_path
+
+from .json_schema_utils import BASE_LIBRARIES_JSON_SCHEMA
+
+
+def validate_base_libraries(catalog_path: str, verrors: ValidationErrors) -> None:
+    library_path = get_library_path(catalog_path)
+    library_path_obj = pathlib.Path(library_path)
+    if not library_path_obj.exists():
+        return
+
+    if any(
+        entry for entry in library_path_obj.iterdir() if entry.is_dir() and RE_VERSION.match(entry.name)
+    ) and not pathlib.Path(get_library_hashes_path(library_path)).exists():
+        verrors.add('library', 'Library hashes file is missing')
+
+    verrors.check()
+
+    get_local_hashes_contents = get_library_hashes(library_path)
+    try:
+        json_schema_validate(get_local_hashes_contents, BASE_LIBRARIES_JSON_SCHEMA)
+    except JsonValidationError as e:
+        verrors.add('library', f'Invalid format specified for library hashes: {e}')
+
+    verrors.check()
+
+    try:
+        hashes = get_hashes_of_base_lib_versions(catalog_path)
+    except Exception as e:
+        verrors.add('library', f'Error while generating hashes for library versions: {e}')
+    else:
+        if hashes != get_local_hashes_contents:
+            verrors.add(
+                'library',
+                'Generated hashes for library versions do not match with the existing '
+                'hashes file and need to be updated'
+            )
+
+    verrors.check()
diff --git a/catalog_reader/library.py b/catalog_reader/library.py
index 09a083d..e3fb15f 100644
--- a/catalog_reader/library.py
+++ b/catalog_reader/library.py
@@ -5,7 +5,7 @@
 import yaml
 
 from .hash_utils import get_hash_of_directory
-from .names import get_library_path
+from .names import get_library_path, get_library_hashes_path
 
 
 RE_VERSION = re.compile(r'^\d+.\d+\.\d+$')
@@ -16,7 +16,7 @@ def get_library_hashes(library_path: str) -> dict:
     This reads from library hashes file and returns the hashes
     """
     with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
-        with open(library_path, 'r') as f:
+        with open(get_library_hashes_path(library_path), 'r') as f:
             return yaml.safe_load(f.read())
 
 

From 628a7d0c90c6ddf9b697c05b843f1e9c7ee090f2 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:38:46 +0500
Subject: [PATCH 07/17] Make sure we consume validate library checks in catalog
 validation

---
 apps_validation/validation/validate_catalog.py       | 3 +++
 apps_validation/validation/validate_dev_directory.py | 3 +++
 apps_validation/validation/validate_library.py       | 2 +-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/apps_validation/validation/validate_catalog.py b/apps_validation/validation/validate_catalog.py
index 56a13b9..698acc6 100644
--- a/apps_validation/validation/validate_catalog.py
+++ b/apps_validation/validation/validate_catalog.py
@@ -11,6 +11,7 @@
 from .json_schema_utils import CATALOG_JSON_SCHEMA
 from .validate_app_rename_migrations import validate_migrations
 from .validate_app import validate_catalog_item
+from .validate_library import validate_base_libraries
 from .validate_recommended_apps import validate_recommended_apps_file
 from .validate_train import get_train_items, validate_train_structure
 
@@ -41,6 +42,8 @@ def validate_catalog(catalog_path: str):
 
     verrors.check()
 
+    validate_base_libraries(catalog_path, verrors)
+
     for method, params in (
         (validate_recommended_apps_file, (catalog_path,)),
         (validate_migrations, (os.path.join(catalog_path, 'migrations'),)),
diff --git a/apps_validation/validation/validate_dev_directory.py b/apps_validation/validation/validate_dev_directory.py
index fbf410a..e27a7a0 100644
--- a/apps_validation/validation/validate_dev_directory.py
+++ b/apps_validation/validation/validate_dev_directory.py
@@ -13,6 +13,7 @@
 
 from .app_version import validate_app_version_file
 from .validate_app_version import validate_catalog_item_version
+from .validate_library import validate_base_libraries
 
 
 def validate_dev_directory_structure(catalog_path: str, to_check_apps: dict) -> None:
@@ -28,6 +29,8 @@ def validate_dev_directory_structure(catalog_path: str, to_check_apps: dict) ->
         validate_train(
             catalog_path, os.path.join(dev_directory, train_name), f'dev.{train_name}', to_check_apps[train_name]
         )
+
+    validate_base_libraries(catalog_path, verrors)
     verrors.check()
 
 
diff --git a/apps_validation/validation/validate_library.py b/apps_validation/validation/validate_library.py
index d06e23e..1b79eae 100644
--- a/apps_validation/validation/validate_library.py
+++ b/apps_validation/validation/validate_library.py
@@ -1,4 +1,3 @@
-import os.path
 import pathlib
 
 from jsonschema import validate as json_schema_validate, ValidationError as JsonValidationError
@@ -44,3 +43,4 @@ def validate_base_libraries(catalog_path: str, verrors: ValidationErrors) -> Non
             )
 
     verrors.check()
+    # TODO: Should we enforce that no folder is here which does not match the regex version ?

From 6c08a0f8c6a1735df2b446e8445e92c8bdfaa163 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:41:41 +0500
Subject: [PATCH 08/17] Make sure no other folder exists in library folder
 which is not valid

---
 apps_validation/validation/validate_library.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/apps_validation/validation/validate_library.py b/apps_validation/validation/validate_library.py
index 1b79eae..a23cc8d 100644
--- a/apps_validation/validation/validate_library.py
+++ b/apps_validation/validation/validate_library.py
@@ -15,9 +15,16 @@ def validate_base_libraries(catalog_path: str, verrors: ValidationErrors) -> Non
     if not library_path_obj.exists():
         return
 
-    if any(
-        entry for entry in library_path_obj.iterdir() if entry.is_dir() and RE_VERSION.match(entry.name)
-    ) and not pathlib.Path(get_library_hashes_path(library_path)).exists():
+    found_libs = False
+    for entry in filter(lambda e: e.is_dir(), library_path_obj.iterdir()):
+        if RE_VERSION.match(entry.name):
+            found_libs = True
+        else:
+            verrors.add(
+                f'library.{entry.name}', 'Library version folder should conform to semantic versioning i.e 1.0.0'
+            )
+
+    if found_libs and not pathlib.Path(get_library_hashes_path(library_path)).exists():
         verrors.add('library', 'Library hashes file is missing')
 
     verrors.check()
@@ -43,4 +50,3 @@ def validate_base_libraries(catalog_path: str, verrors: ValidationErrors) -> Non
             )
 
     verrors.check()
-    # TODO: Should we enforce that no folder is here which does not match the regex version ?

From 21575d0c0f2073f8d00e42b2e86c5ed0e8bd3111 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:50:06 +0500
Subject: [PATCH 09/17] Do not account for folder names when computing checksum

---
 catalog_reader/hash_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalog_reader/hash_utils.py b/catalog_reader/hash_utils.py
index 2928bce..c38b5f1 100644
--- a/catalog_reader/hash_utils.py
+++ b/catalog_reader/hash_utils.py
@@ -6,7 +6,7 @@ def get_hash_of_directory(directory: str) -> str:
     This returns sha256sum of the directory
     """
     cp = subprocess.run(
-        f'find {directory} -type f -exec sha256sum {{}} + | sort | sha256sum',
+        f'find {directory} -type f -exec sha256sum {{}} + | sort | awk \'{{print $1}}\' | sha256sum',
         capture_output=True, check=True, shell=True,
     )
     return cp.stdout.decode().split()[0]

From 432577aa3a4c5f373f04346ba07c5db0e615ee6e Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 03:55:28 +0500
Subject: [PATCH 10/17] Properly generate hashes of libs

---
 catalog_reader/library.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/catalog_reader/library.py b/catalog_reader/library.py
index e3fb15f..916dc00 100644
--- a/catalog_reader/library.py
+++ b/catalog_reader/library.py
@@ -21,12 +21,13 @@ def get_library_hashes(library_path: str) -> dict:
 
 
 def get_hashes_of_base_lib_versions(catalog_path: str) -> dict:
-    library_dir = pathlib.Path(get_library_path(catalog_path))
+    library_path = get_library_path(catalog_path)
+    library_dir = pathlib.Path(library_path)
     hashes = {}
     for lib_entry in library_dir.iterdir():
         if not lib_entry.is_dir() or not RE_VERSION.match(lib_entry.name):
             continue
 
-        hashes[lib_entry.name] = get_hash_of_directory(os.path.join(library_dir.name, lib_entry.name))
+        hashes[lib_entry.name] = get_hash_of_directory(os.path.join(library_path, lib_entry.name))
 
     return hashes

From c0351e46690f361f242fe6ae780b1f5070b03542 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 20:30:18 +0500
Subject: [PATCH 11/17] If lib_version is specified, ensure lib version hash is
 also present

---
 apps_validation/validation/json_schema_utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apps_validation/validation/json_schema_utils.py b/apps_validation/validation/json_schema_utils.py
index 017876e..516b54f 100644
--- a/apps_validation/validation/json_schema_utils.py
+++ b/apps_validation/validation/json_schema_utils.py
@@ -33,10 +33,14 @@
             'type': 'string',
             'pattern': '[0-9]+.[0-9]+.[0-9]+',
         },
+        'lib_version_hash': {'type': 'string'},
     },
     'required': [
         'name', 'train', 'version',
     ],
+    'dependencies': {
+        'lib_version': ['lib_version_hash'],  # Ensure lib_version_hash exists if lib_version is present
+    },
 }
 APP_MIGRATION_SCHEMA = {
     'type': 'array',

From e3ea8ff605fc7aeaddbdeb0e5f43c1d21c7108e0 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 20:43:16 +0500
Subject: [PATCH 12/17] Fix json schema for conditional enforcement

---
 apps_validation/validation/json_schema_utils.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/apps_validation/validation/json_schema_utils.py b/apps_validation/validation/json_schema_utils.py
index 516b54f..b396712 100644
--- a/apps_validation/validation/json_schema_utils.py
+++ b/apps_validation/validation/json_schema_utils.py
@@ -38,8 +38,14 @@
     'required': [
         'name', 'train', 'version',
     ],
-    'dependencies': {
-        'lib_version': ['lib_version_hash'],  # Ensure lib_version_hash exists if lib_version is present
+    'if': {
+        'properties': {
+            'lib_version': {'type': 'string'},
+        },
+        'required': ['lib_version'],
+    },
+    'then': {
+        'required': ['lib_version_hash'],
     },
 }
 APP_MIGRATION_SCHEMA = {

From 88f7deacf100bc0d339daea8e5371856018d4e5e Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 10 May 2024 21:07:13 +0500
Subject: [PATCH 13/17] Make sure we validate specified hash in app.yaml
 matches with dir's actual hash

---
 apps_validation/validation/validate_app_version.py | 3 +++
 catalog_reader/app_utils.py                        | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/apps_validation/validation/validate_app_version.py b/apps_validation/validation/validate_app_version.py
index 37c7257..6ea4f19 100644
--- a/apps_validation/validation/validate_app_version.py
+++ b/apps_validation/validation/validate_app_version.py
@@ -8,6 +8,7 @@
 
 from apps_validation.exceptions import ValidationErrors
 from catalog_reader.app_utils import get_app_basic_details
+from catalog_reader.hash_utils import get_hash_of_directory
 from catalog_reader.names import get_base_library_dir_name_from_version
 from catalog_reader.questions_util import CUSTOM_PORTALS_KEY
 
@@ -67,6 +68,8 @@ def validate_catalog_item_version(
             )
         elif not base_lib_dir.is_dir():
             verrors.add(f'{schema}.lib_version', f'{base_lib_dir!r} is not a directory')
+        elif get_hash_of_directory(str(base_lib_dir)) != app_basic_details['lib_version_hash']:
+            verrors.add(f'{schema}.lib_version', 'Library version hash does not match with the actual library version')
 
     questions_path = os.path.join(version_path, 'questions.yaml')
     if os.path.exists(questions_path):
diff --git a/catalog_reader/app_utils.py b/catalog_reader/app_utils.py
index d29d856..d349b95 100644
--- a/catalog_reader/app_utils.py
+++ b/catalog_reader/app_utils.py
@@ -47,7 +47,7 @@ def get_app_basic_details(app_path: str) -> dict:
     with contextlib.suppress(FileNotFoundError, yaml.YAMLError, KeyError):
         with open(os.path.join(app_path, 'app.yaml'), 'r') as f:
             app_config = yaml.safe_load(f.read())
-        return {'lib_version': app_config.get('lib_version')} | {
+        return {k: app_config.get(k) for k in ('lib_version', 'lib_version_hash')} | {
             k: app_config[k] for k in ('name', 'train', 'version')
         }
 

From 3e88606a5a15d08cbe065c11868db199360b8636 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Sat, 11 May 2024 00:18:34 +0500
Subject: [PATCH 14/17] Have hash update script to copy over base lib from top
 level to app lib

---
 catalog_reader/scripts/apps_hashes.py | 48 ++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/catalog_reader/scripts/apps_hashes.py b/catalog_reader/scripts/apps_hashes.py
index 2f37afe..b4d894d 100644
--- a/catalog_reader/scripts/apps_hashes.py
+++ b/catalog_reader/scripts/apps_hashes.py
@@ -2,11 +2,13 @@
 import argparse
 import os
 import pathlib
+import shutil
 import yaml
 
 from apps_validation.exceptions import CatalogDoesNotExist, ValidationErrors
+from catalog_reader.dev_directory import get_ci_development_directory
 from catalog_reader.library import get_hashes_of_base_lib_versions
-from catalog_reader.names import get_library_path, get_library_hashes_path
+from catalog_reader.names import get_library_path, get_library_hashes_path, get_base_library_dir_name_from_version
 
 
 def update_catalog_hashes(catalog_path: str) -> None:
@@ -27,6 +29,50 @@ def update_catalog_hashes(catalog_path: str) -> None:
 
     print(f'[\033[92mOK\x1B[0m]\tGenerated hashes for library versions at {hashes_file_path!r}')
 
+    dev_directory = pathlib.Path(get_ci_development_directory(catalog_path))
+    if not dev_directory.is_dir():
+        return
+    elif not hashes:
+        print(f'[\033[92mOK\x1B[0m]\tNo hashes found for library versions, skipping updating apps hashes')
+        return
+
+    for train_dir in dev_directory.iterdir():
+        if not train_dir.is_dir():
+            continue
+
+        for app_dir in train_dir.iterdir():
+            if not app_dir.is_dir():
+                continue
+
+            app_metadata_file = app_dir / 'app.yaml'
+            if not app_metadata_file.is_file():
+                continue
+
+            with open(str(app_metadata_file), 'r') as f:
+                app_config = yaml.safe_load(f.read())
+
+            if (lib_version := app_config.get('lib_version')) and lib_version not in hashes:
+                print(
+                    f'[\033[93mWARN\x1B[0m]\tLibrary version {lib_version!r} not found in hashes, '
+                    f'skipping updating {app_dir.name!r} in {train_dir.name} train'
+                )
+                continue
+
+            base_lib_name = get_base_library_dir_name_from_version(lib_version)
+            app_lib_dir = app_dir / 'templates/library'
+            app_lib_dir.mkdir(exist_ok=True, parents=True)
+            app_base_lib_dir = app_lib_dir / base_lib_name
+            shutil.rmtree(app_base_lib_dir.as_posix(), ignore_errors=True)
+
+            catalog_base_lib_dir_path = os.path.join(library_dir.as_posix(), lib_version)
+            shutil.copytree(catalog_base_lib_dir_path, app_base_lib_dir.as_posix())
+
+            app_config['lib_version_hash'] = hashes[lib_version]
+            with open(str(app_metadata_file), 'w') as f:
+                yaml.safe_dump(app_config, f)
+
+            print(f'[\033[92mOK\x1B[0m]\tUpdated library hash for {app_dir.name!r} in {train_dir.name}')
+
 
 def main():
     parser = argparse.ArgumentParser()

From 590f76129e71c0fb100268feb1b1600e408d2346 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Sat, 11 May 2024 00:23:53 +0500
Subject: [PATCH 15/17] Have ruamel.yaml in requirements.txt

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index ffef472..a863b1d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,5 @@ jinja2
 jsonschema==4.10.3
 markdown
 pyyaml
+ruamel.yaml
 semantic_version

From 8311611acbfd4f97cdc65f8502fd3d50c3619df5 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Sat, 11 May 2024 00:31:34 +0500
Subject: [PATCH 16/17] Preserve yaml structure when updating hashes

---
 catalog_reader/scripts/apps_hashes.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/catalog_reader/scripts/apps_hashes.py b/catalog_reader/scripts/apps_hashes.py
index b4d894d..e6f6b87 100644
--- a/catalog_reader/scripts/apps_hashes.py
+++ b/catalog_reader/scripts/apps_hashes.py
@@ -2,6 +2,7 @@
 import argparse
 import os
 import pathlib
+import ruamel.yaml
 import shutil
 import yaml
 
@@ -11,6 +12,10 @@
 from catalog_reader.names import get_library_path, get_library_hashes_path, get_base_library_dir_name_from_version
 
 
+YAML = ruamel.yaml.YAML()
+YAML.indent(mapping=2, sequence=4, offset=2)
+
+
 def update_catalog_hashes(catalog_path: str) -> None:
     if not os.path.exists(catalog_path):
         raise CatalogDoesNotExist(catalog_path)
@@ -49,7 +54,7 @@ def update_catalog_hashes(catalog_path: str) -> None:
                 continue
 
             with open(str(app_metadata_file), 'r') as f:
-                app_config = yaml.safe_load(f.read())
+                app_config = YAML.load(f)
 
             if (lib_version := app_config.get('lib_version')) and lib_version not in hashes:
                 print(
@@ -69,7 +74,7 @@ def update_catalog_hashes(catalog_path: str) -> None:
 
             app_config['lib_version_hash'] = hashes[lib_version]
             with open(str(app_metadata_file), 'w') as f:
-                yaml.safe_dump(app_config, f)
+                YAML.dump(app_config, f)
 
             print(f'[\033[92mOK\x1B[0m]\tUpdated library hash for {app_dir.name!r} in {train_dir.name}')
 

From 8c7fe0358fa703f9d13ed066ae806ee6e753e169 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Sat, 11 May 2024 00:38:08 +0500
Subject: [PATCH 17/17] Fix flake8

---
 catalog_reader/scripts/apps_hashes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalog_reader/scripts/apps_hashes.py b/catalog_reader/scripts/apps_hashes.py
index e6f6b87..391c336 100644
--- a/catalog_reader/scripts/apps_hashes.py
+++ b/catalog_reader/scripts/apps_hashes.py
@@ -38,7 +38,7 @@ def update_catalog_hashes(catalog_path: str) -> None:
     if not dev_directory.is_dir():
         return
     elif not hashes:
-        print(f'[\033[92mOK\x1B[0m]\tNo hashes found for library versions, skipping updating apps hashes')
+        print('[\033[92mOK\x1B[0m]\tNo hashes found for library versions, skipping updating apps hashes')
         return
 
     for train_dir in dev_directory.iterdir():