Adapt compliance monitor to recent change regarding versions to test (#…

…734) * Adapt compliance monitor to recent change regarding versions to test * rephrase logic for relevant to avoid redundant computation --------- Signed-off-by: Matthias Büchse <[email protected]>
SovereignCloudStack · Sep 4, 2024 · 89f70aa · 89f70aa
1 parent 08f5376
commit 89f70aa
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 10 deletions.
diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml
@@ -46,7 +46,7 @@ modules:
         tags: [mandatory]
         description: >
           Must fulfill all requirements of
-          <https://docs.scs.community/standards/scs-0100-v3-flavor-naming -- plus the list of mandatory>
+          <https://docs.scs.community/standards/scs-0100-v3-flavor-naming> -- plus the list of mandatory
           and recommended flavors found in <https://docs.scs.community/standards/scs-0100-v2-flavor-naming#standard-scs-flavors>
   - id: scs-0100-v3.1
     name: Flavor naming v3.1

diff --git a/compliance-monitor/monitor.py b/compliance-monitor/monitor.py
@@ -77,7 +77,7 @@ def __init__(self):
 #       http://127.0.0.1:8080/reports
 # to achieve this!
 SEP = "-----END SSH SIGNATURE-----\n&"
-ASTERISK_LOOKUP = {'effective': '', 'draft': '*', 'warn': '†'}
+ASTERISK_LOOKUP = {'effective': '', 'draft': '*', 'warn': '†', 'deprecated': '††'}
 
 
 class ViewType(Enum):
@@ -222,6 +222,7 @@ def __init__(self, version):
         self.name = version['version']
         self.suite = compile_suite(self.name, version['include'])
         self.validity = version['validity']
+        self.listed = bool(version['_explicit_validity'])
         self.targets = {
             tname: self.suite.select(tname, target_spec)
             for tname, target_spec in version['targets'].items()
@@ -263,10 +264,14 @@ def evaluate(self, scope_results):
         by_validity = defaultdict(list)
         for vname in scope_results:
             by_validity[self.versions[vname].validity].append(vname)
-        relevant = list(by_validity['effective'])
-        # only show "warn" versions if no effective ones are passed
-        if not any(version_results[vname]['result'] == 1 for vname in relevant):
-            relevant.extend(by_validity['warn'])
+        # go through worsening validity values until a passing version is found
+        relevant = []
+        for validity in ('effective', 'warn', 'deprecated'):
+            vnames = by_validity[validity]
+            relevant.extend(vnames)
+            if any(version_results[vname]['result'] == 1 for vname in vnames):
+                break
+        # always include draft (but only at the end)
         relevant.extend(by_validity['draft'])
         passed = [vname for vname in relevant if version_results[vname]['result'] == 1]
         return {
@@ -294,8 +299,10 @@ def update_lookup(self, target_dict):
         """
         scope_uuid = self.spec['uuid']
         for vname, precomputed_version in self.versions.items():
+            listed = precomputed_version.listed
             for testcase in precomputed_version.suite.testcases:
-                target_dict[(scope_uuid, vname, testcase['id'])] = testcase
+                # put False if listed is False, else put testcase
+                target_dict[(scope_uuid, vname, testcase['id'])] = listed and testcase
 
 
 def import_cert_yaml(yaml_path, target_dict):
@@ -486,8 +493,11 @@ def convert_result_rows_to_dict2(
     missing = set()
     for subject, scope_uuid, version, testcase_id, result, checked_at, report_uuid in rows:
         testcase = scopes_lookup.get((scope_uuid, version, testcase_id))
-        if testcase is None:
-            missing.add((scope_uuid, version, testcase_id))
+        if not testcase:
+            # it can be False (testcase is known but version too old) or None (testcase not known)
+            # only report the latter case
+            if testcase is None:
+                missing.add((scope_uuid, version, testcase_id))
             continue
         # drop value if too old
         expires_at = add_period(checked_at, testcase.get('lifetime'))
@@ -498,7 +508,7 @@ def convert_result_rows_to_dict2(
             tc_result.update(report=report_uuid)
         preliminary[subject][scope_uuid][version][testcase_id] = tc_result
     if missing:
-        logger.warning('missing objects: ' + ', '.join(missing))
+        logger.warning('missing objects: ' + ', '.join(repr(x) for x in missing))
     # make sure the requested subjects and scopes are present (facilitates writing jinja2 templates)
     for subject in subjects:
         for scope in scopes: