Skip to content

Commit

Permalink
[CI] Correction files for conformance tests (#2824)
Browse files Browse the repository at this point in the history
### Changes

Consistent apply of correction files until OV version is greater that
version of correction file.

### Reason 

A new file needs to be added for the any new versions of OV.
  • Loading branch information
AlexanderDokuchaev authored Jul 22, 2024
1 parent e70befd commit 8744c20
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 15 deletions.
5 changes: 2 additions & 3 deletions tests/openvino/native/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,8 @@ def get_openvino_major_minor_version() -> Tuple[int]:


def get_openvino_version() -> str:
major_verison, minor_version = get_openvino_major_minor_version()

return f"{major_verison}.{minor_version}"
major_version, minor_version = get_openvino_major_minor_version()
return f"{major_version}.{minor_version}"


def get_actual_reference_for_current_openvino(rel_path: Path) -> Path:
Expand Down
6 changes: 0 additions & 6 deletions tests/post_training/data/wc_reference_data_2024.4.yaml

This file was deleted.

32 changes: 26 additions & 6 deletions tests/post_training/test_quantize_conformance.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import time
import traceback
from collections import OrderedDict
Expand All @@ -19,6 +20,7 @@
import pandas as pd
import pytest
import yaml
from packaging import version

import nncf
from tests.openvino.native.common import get_openvino_version
Expand All @@ -28,6 +30,8 @@
from tests.post_training.pipelines.base import BaseTestPipeline
from tests.post_training.pipelines.base import RunInfo

DATA_ROOT = Path(__file__).parent / "data"


@pytest.fixture(scope="session", name="data_dir")
def fixture_data(pytestconfig):
Expand Down Expand Up @@ -76,28 +80,44 @@ def fixture_extra_columns(pytestconfig):
return pytestconfig.getoption("extra_columns")


def _parse_version(s: Path):
version_str = re.search(r".*_(\d+\.\d+).(?:yaml|yml)", s.name).group(1)
return version.parse(version_str)


def ref_data_correction(data: Dict, file_name: str):
correction_data_path = Path(__file__).parent / "data" / f"{file_name}_{get_openvino_version()}.yaml"
if correction_data_path.exists():
with correction_data_path.open() as f:
"""
Apply corrections from reference YAML files according current of OV version to the provided data dictionary.
This function reads correction data from YAML files that match the given
file name pattern (ptq|wc)_reference_data_(ov_version).yaml
"""
ov_version = version.parse(get_openvino_version())

for file_path in sorted(DATA_ROOT.glob(f"{file_name}_*.yaml"), key=_parse_version):
file_ov_version = _parse_version(file_path)
if file_ov_version > ov_version:
break
with file_path.open() as f:
correction_data = yaml.safe_load(f)

for m_name, c_data in correction_data.items():
data[m_name].update(c_data)
print(f"Applied correction file {file_path}")

return data


@pytest.fixture(scope="session", name="ptq_reference_data")
def fixture_ptq_reference_data():
path_reference = Path(__file__).parent / "data" / "ptq_reference_data.yaml"
path_reference = DATA_ROOT / "ptq_reference_data.yaml"
with path_reference.open() as f:
data = yaml.safe_load(f)
return ref_data_correction(data, "ptq_reference_data")


@pytest.fixture(scope="session", name="wc_reference_data")
def fixture_wc_reference_data():
path_reference = Path(__file__).parent / "data" / "wc_reference_data.yaml"
path_reference = DATA_ROOT / "wc_reference_data.yaml"
with path_reference.open() as f:
data = yaml.safe_load(f)
fp32_test_cases = defaultdict(dict)
Expand Down

0 comments on commit 8744c20

Please sign in to comment.