feat: allow multiple or all result sets to be specified

This is needed for the Vunnel changes.

Signed-off-by: Will Murphy <[email protected]>

src/yardstick/cli/config.py

@@ -119,6 +119,15 @@ class Application:
     default_max_year: int | None = None
     derive_year_from_cve_only: bool = False
 
+    def max_year_for_any_result_set(self, result_sets: list[str]) -> int | None:
+        years = []
+        for result_set in result_sets:
+            m = self.max_year_for_result_set(result_set)
+            if m is not None:
+                years.append(m)
+
+        return max(years)
+
     def max_year_for_result_set(self, result_set: str) -> int | None:
         """return the max year needed by any validation on the result set, or default_max_year"""
         rs = self.result_sets.get(result_set, None)

src/yardstick/cli/validate.py

@@ -102,36 +102,46 @@ class bcolors:
 @click.option(
     "--result-set",
     "-r",
-    default=default_result_set,
+    "result_sets",
+    multiple=True,
+    default=[default_result_set],
     help="the result set to use for the quality gate",
 )
+@click.option(
+    "--all",
+    "all_result_sets",
+    is_flag=True,
+    default=False,
+    help="validate all known result sets",
+)
 def validate(
     cfg: config.Application,
     images: list[str],
     always_run_label_comparison: bool,
     breakdown_by_ecosystem: bool,
     verbosity: int,
-    result_set: str,
+    result_sets: list[str],
+    all_result_sets: bool,
 ):
     setup_logging(verbosity)
+    if all:
+        result_sets = [r for r in cfg.result_sets.keys()]
 
     # let's not load any more labels than we need to, base this off of the images we're validating
     if not images:
         unique_images = set()
-        result_set_obj = store.result_set.load(name=result_set)
-        for state in result_set_obj.state:
-            unique_images.add(state.config.image)
+        for r in result_sets:
+            result_set_obj = store.result_set.load(name=r)
+            for state in result_set_obj.state:
+                unique_images.add(state.config.image)
         images = sorted(list(unique_images))
 
     print("Loading label entries...", end=" ")
     label_entries = store.labels.load_for_image(
-        images, year_max_limit=cfg.max_year_for_result_set(result_set)
+        images, year_max_limit=cfg.max_year_for_any_result_set(result_sets)
     )
     print(f"done! {len(label_entries)} entries loaded")
 
-    result_sets = [
-        result_set
-    ]  # today only one result set is supported, but more can be added
     gates = []
     for result_set in result_sets:
         rs_config = cfg.result_sets[result_set]

src/yardstick/validate.py

@@ -20,6 +20,8 @@ class GateConfig:
     max_new_false_negatives: int = 0
     max_unlabeled_percent: int = 0
     max_year: int | None = None
+    reference_tool_label: str = "reference"
+    candidate_tool_label: str = "candidate"
 
 
 @dataclass
@@ -161,7 +163,7 @@ def __post_init__(
 
         self.reasons = reasons
 
-    def passed(self):
+    def passed(self) -> bool:
         return len(self.reasons) == 0
 
 
@@ -241,7 +243,6 @@ def results_used(results: Sequence[artifact.ScanResult]) -> list[GateInputDescri
 
 
 def validate_result_set(
-    # cfg: config.Application, # TODO: bad!
     gate_config: GateConfig,
     result_set: str,
     images: list[str],
@@ -289,14 +290,11 @@ def validate_result_set(
 
 
 def validate_image(
-    # result_set_obj: artifact.ResultSet,
     gate_config: GateConfig,
     descriptions: list[str],
     always_run_label_comparison: bool,
     verbosity: int,
     label_entries: Optional[list[artifact.LabelEntry]] = None,
-    reference_tool_label: str = "reference",
-    candidate_tool_label: str = "candidate",
 ):
     # do a relative comparison
     # - show comparison summary (no gating action)
@@ -359,9 +357,9 @@ def validate_image(
     # TODO: should be specified in config
     reference_tool, candidate_tool = None, None
     for r in results:
-        if r.config.tool_label == reference_tool_label:
+        if r.config.tool_label == gate_config.reference_tool_label:
             reference_tool = r.config.tool
-        if r.config.tool_label == candidate_tool_label:
+        if r.config.tool_label == gate_config.candidate_tool_label:
             candidate_tool = r.config.tool
 
     if reference_tool is None or candidate_tool is None: