From 8744c2052101269391d90ae45040fb2824f86510 Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Mon, 22 Jul 2024 18:28:47 +0300 Subject: [PATCH] [CI] Correction files for conformance tests (#2824) ### Changes Consistent apply of correction files until OV version is greater that version of correction file. ### Reason A new file needs to be added for the any new versions of OV. --- tests/openvino/native/common.py | 5 ++- .../data/wc_reference_data_2024.4.yaml | 6 ---- .../test_quantize_conformance.py | 32 +++++++++++++++---- 3 files changed, 28 insertions(+), 15 deletions(-) delete mode 100644 tests/post_training/data/wc_reference_data_2024.4.yaml diff --git a/tests/openvino/native/common.py b/tests/openvino/native/common.py index 47118916d87..c61d56bdbc2 100644 --- a/tests/openvino/native/common.py +++ b/tests/openvino/native/common.py @@ -88,9 +88,8 @@ def get_openvino_major_minor_version() -> Tuple[int]: def get_openvino_version() -> str: - major_verison, minor_version = get_openvino_major_minor_version() - - return f"{major_verison}.{minor_version}" + major_version, minor_version = get_openvino_major_minor_version() + return f"{major_version}.{minor_version}" def get_actual_reference_for_current_openvino(rel_path: Path) -> Path: diff --git a/tests/post_training/data/wc_reference_data_2024.4.yaml b/tests/post_training/data/wc_reference_data_2024.4.yaml deleted file mode 100644 index 607196fff61..00000000000 --- a/tests/post_training/data/wc_reference_data_2024.4.yaml +++ /dev/null @@ -1,6 +0,0 @@ -tinyllama_data_aware_awq_scale_estimation_backend_OV: - metric_value: 0.84038 -tinyllama_data_aware_awq_scale_estimation_stateful_backend_OV: - metric_value: 0.84038 -tinyllama_data_aware_gptq_backend_OV: - metric_value: 0.81936 diff --git a/tests/post_training/test_quantize_conformance.py b/tests/post_training/test_quantize_conformance.py index 719e01f8dc4..e0f7a107627 100644 --- a/tests/post_training/test_quantize_conformance.py +++ b/tests/post_training/test_quantize_conformance.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re import time import traceback from collections import OrderedDict @@ -19,6 +20,7 @@ import pandas as pd import pytest import yaml +from packaging import version import nncf from tests.openvino.native.common import get_openvino_version @@ -28,6 +30,8 @@ from tests.post_training.pipelines.base import BaseTestPipeline from tests.post_training.pipelines.base import RunInfo +DATA_ROOT = Path(__file__).parent / "data" + @pytest.fixture(scope="session", name="data_dir") def fixture_data(pytestconfig): @@ -76,20 +80,36 @@ def fixture_extra_columns(pytestconfig): return pytestconfig.getoption("extra_columns") +def _parse_version(s: Path): + version_str = re.search(r".*_(\d+\.\d+).(?:yaml|yml)", s.name).group(1) + return version.parse(version_str) + + def ref_data_correction(data: Dict, file_name: str): - correction_data_path = Path(__file__).parent / "data" / f"{file_name}_{get_openvino_version()}.yaml" - if correction_data_path.exists(): - with correction_data_path.open() as f: + """ + Apply corrections from reference YAML files according current of OV version to the provided data dictionary. + + This function reads correction data from YAML files that match the given + file name pattern (ptq|wc)_reference_data_(ov_version).yaml + """ + ov_version = version.parse(get_openvino_version()) + + for file_path in sorted(DATA_ROOT.glob(f"{file_name}_*.yaml"), key=_parse_version): + file_ov_version = _parse_version(file_path) + if file_ov_version > ov_version: + break + with file_path.open() as f: correction_data = yaml.safe_load(f) - for m_name, c_data in correction_data.items(): data[m_name].update(c_data) + print(f"Applied correction file {file_path}") + return data @pytest.fixture(scope="session", name="ptq_reference_data") def fixture_ptq_reference_data(): - path_reference = Path(__file__).parent / "data" / "ptq_reference_data.yaml" + path_reference = DATA_ROOT / "ptq_reference_data.yaml" with path_reference.open() as f: data = yaml.safe_load(f) return ref_data_correction(data, "ptq_reference_data") @@ -97,7 +117,7 @@ def fixture_ptq_reference_data(): @pytest.fixture(scope="session", name="wc_reference_data") def fixture_wc_reference_data(): - path_reference = Path(__file__).parent / "data" / "wc_reference_data.yaml" + path_reference = DATA_ROOT / "wc_reference_data.yaml" with path_reference.open() as f: data = yaml.safe_load(f) fp32_test_cases = defaultdict(dict)