diff --git a/config/grype-db-manager/include.d/validate.yaml b/config/grype-db-manager/include.d/validate.yaml index 7a043ce4..493f83c7 100644 --- a/config/grype-db-manager/include.d/validate.yaml +++ b/config/grype-db-manager/include.d/validate.yaml @@ -13,16 +13,17 @@ db: gate: # float between 0 and 1, the maximum allowable difference below the OSS F1 score before the gate fails (default 0, # meaning the test F1 score must be equal to or greater than the OSS F1 score to pass the gate) - f1-score-threshold: 0.15 + max_f1_regression: 0.15 # float between 0 and 100, the maximum % of unlabeled matches for a scan result before the gate fails (default 10%, # meaning the test scan must have less than 10% unlabeled matches to pass the gate) - # TODO: this should be at 25.0 after we sunset v1 and v2 schemas - unlabeled-matches-threshold: 50.0 + # TODO: this should be at 25 after we sunset v1 and v2 schemas + max_unlabeled_percent: 50 # integer, the maximum allowable introduced FNs by the test scan (but found by the OSS scan) before the gate fails # (default 0, meaning the test scan must have the same or fewer FNs than the OSS scan to pass the gate) - introduced-fns-threshold: 10 + max_new_false_negatives: 10 + max_year: 2021 # these are the set of images we will capture grype scans for using an existing published DB and a newly build DB. # The assumption is that they should perform similarly or the new DB should perform better. We do allow for the new diff --git a/manager/src/grype_db_manager/cli/config.py b/manager/src/grype_db_manager/cli/config.py index a965231e..12034ce6 100644 --- a/manager/src/grype_db_manager/cli/config.py +++ b/manager/src/grype_db_manager/cli/config.py @@ -10,6 +10,7 @@ import yaml from dataclass_wizard import asdict, fromdict from yamlinclude import YamlIncludeConstructor +from yardstick.validate import GateConfig from grype_db_manager import db, s3utils @@ -50,7 +51,7 @@ class ValidateDB: images: list[str] = field(default_factory=list) grype: Grype = field(default_factory=Grype) default_max_year: int = 2021 - gate: db.validation.GateConfig = field(default_factory=db.validation.GateConfig) + gate: GateConfig = field(default_factory=GateConfig) def __post_init__(self): # flatten elements in images (in case yaml anchors are used) diff --git a/manager/src/grype_db_manager/cli/db.py b/manager/src/grype_db_manager/cli/db.py index 3191ca6f..c23053f2 100644 --- a/manager/src/grype_db_manager/cli/db.py +++ b/manager/src/grype_db_manager/cli/db.py @@ -171,15 +171,22 @@ def validate_db( }, ) - gates = db.validate( + db.capture_results( cfg=yardstick_cfg, - result_set=result_set, db_uuid=db_uuid, - verbosity=verbosity, + result_set=result_set, recapture=recapture, root_dir=cfg.data.root, ) + gates = yardstick.validate.validate_result_set( + gate_config=cfg.validate.db.gate, + result_set=result_set, + images=[], + always_run_label_comparison=False, + verbosity=verbosity, + ) + failure = not all(gate.passed() for gate in gates) if failure: click.echo(f"{Format.BOLD}{Format.FAIL}Validation failed{Format.RESET}") diff --git a/manager/src/grype_db_manager/db/__init__.py b/manager/src/grype_db_manager/db/__init__.py index 626e4e7c..eb159056 100644 --- a/manager/src/grype_db_manager/db/__init__.py +++ b/manager/src/grype_db_manager/db/__init__.py @@ -1,7 +1,7 @@ from . import listing, metadata, schema from .listing import Listing from .metadata import Metadata -from .validation import validate +from .validation import capture_results, validate __all__ = [ "Listing", @@ -10,4 +10,5 @@ "metadata", "validate", "schema", + "capture_results", ] diff --git a/poetry.lock b/poetry.lock index 3d291487..5e85ea2d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "black" @@ -1063,6 +1063,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -1402,7 +1403,7 @@ Colr = "^0.9.1" dataclass-wizard = "^0.22.3" dataclasses-json = "^0.6.7" GitPython = "^3.1.43" -importlib-metadata = ">=7.0.1,<8.0.0" +importlib-metadata = ">=7.0.1,<9.0.0" mergedeep = "^1.3.4" omitempty = "^0.1.1" prompt-toolkit = "^3.0.47" @@ -1415,8 +1416,8 @@ tabulate = "^0.9.0" [package.source] type = "git" url = "https://github.com/anchore/yardstick" -reference = "8ef84656c6618292110c47fbb8805d78fc7b235d" -resolved_reference = "8ef84656c6618292110c47fbb8805d78fc7b235d" +reference = "feat-validate-subcommand" +resolved_reference = "371df21e04089be0131dc3d0e3b5953eecca6871" [[package]] name = "zipp" @@ -1497,4 +1498,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<=3.13" -content-hash = "943f0065c9a9329e092b695fd19a660059c1753f70790eb53fe0cf48e4dcab75" +content-hash = "d167835b517dc4b01049738173ce488068d670a26bd9520b65e601c86bbad6ed" diff --git a/pyproject.toml b/pyproject.toml index 2c163f63..57647698 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,8 +36,8 @@ zstandard = ">=0.21.0, <1" colorlog = "^6.7.0" mergedeep = "^1.3.4" pyyaml = ">=5.0.1, <7" -yardstick = {git = "https://github.com/anchore/yardstick", rev = "8ef84656c6618292110c47fbb8805d78fc7b235d"} -#yardstick = {path = "../yardstick", develop = true} +yardstick = {git = "https://github.com/anchore/yardstick", rev = "feat-validate-subcommand"} +# yardstick = {path = "../yardstick", develop = true} colr = "^0.9.1" pyyaml-include = "^1.3.1" python-magic = "^0.4.27"