From f4bd077250d6226a5533201efc820e82ed99d5fc Mon Sep 17 00:00:00 2001 From: Nikita Malinin Date: Thu, 20 Jun 2024 07:24:27 +0200 Subject: [PATCH] Tests revision (#2749) ### Changes - Revised `xfailed` tests. ### Reason for changes - Absence of the revision for tests at all. ### Related tickets - N/A ### Tests - N/A --- .../openvino/native/quantization/test_graphs.py | 7 ------- .../quantization/test_weights_compression.py | 6 ------ tests/openvino/native/test_model_transformer.py | 10 ---------- tests/openvino/native/test_smooth_quant.py | 3 ++- .../openvino/native/test_statistic_collector.py | 9 --------- .../test_templates/test_smooth_quant.py | 17 +++++++---------- tests/torch/ptq/test_statistic_collector.py | 5 ----- .../sparsity/movement/test_model_saving.py | 5 +---- 8 files changed, 10 insertions(+), 52 deletions(-) diff --git a/tests/openvino/native/quantization/test_graphs.py b/tests/openvino/native/quantization/test_graphs.py index 1a36a7c0331..bc72efd7912 100644 --- a/tests/openvino/native/quantization/test_graphs.py +++ b/tests/openvino/native/quantization/test_graphs.py @@ -32,7 +32,6 @@ from tests.openvino.native.common import dump_model from tests.openvino.native.common import get_actual_reference_for_current_openvino from tests.openvino.native.common import get_dataset_for_test -from tests.openvino.native.common import get_openvino_major_minor_version from tests.openvino.native.models import SYNTHETIC_MODELS from tests.openvino.native.models import DepthwiseConv3DModel from tests.openvino.native.models import DepthwiseConv4DModel @@ -231,9 +230,6 @@ def test_if_model_fq_placement_ignored_scope(): @pytest.mark.parametrize("q_params", [{}, {"model_type": ModelType.TRANSFORMER}], ids=["default", "transformer"]) def test_scaled_dot_product_attention_placement(q_params, tmp_path): - ov_major_version, ov_minor_version = get_openvino_major_minor_version() - if ov_major_version < 2023 or (ov_major_version == 2023 and ov_minor_version < 3): - pytest.xfail("ScaledDotProductAttention is not supported until 2023.3") model = ScaledDotProductAttentionModel().ov_model quantized_model = quantize_model(model, q_params) @@ -257,9 +253,6 @@ def test_scaled_dot_product_attention_placement(q_params, tmp_path): [SYNTHETIC_MODELS.get("LinearModel"), SYNTHETIC_MODELS.get("ConvModel"), SYNTHETIC_MODELS.get("SharedConvModel")], ) def test_synthetic_models_fc_placement(model_creator_func): - ov_major_version, ov_minor_version = get_openvino_major_minor_version() - if ov_major_version < 2023 or (ov_major_version == 2023 and ov_minor_version < 3): - pytest.xfail("FakeConvert is not supported until 2023.3") model = model_creator_func() quantized_model = quantize_model( model.ov_model, diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 98606b38fb7..384cbb56ffa 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -33,7 +33,6 @@ from nncf.quantization.algorithms.weight_compression.weight_lowering import reshape_weight_for_grouped_quantization from nncf.scopes import IgnoredScope from tests.openvino.native.common import get_actual_reference_for_current_openvino -from tests.openvino.native.common import get_openvino_major_minor_version from tests.openvino.native.models import AWQActMatmulModel from tests.openvino.native.models import AWQMatmulModel from tests.openvino.native.models import GatherAndMatmulShareData @@ -766,11 +765,6 @@ def test_data_type_for_num_weights(mocker): ], ) def test_compression_for_different_dtypes(activation_dtype, weight_dtype): - if weight_dtype == ov.Type.bf16: - ov_major_version, ov_minor_version = get_openvino_major_minor_version() - if ov_major_version < 2024 or (ov_major_version == 2024 and ov_minor_version < 2): - pytest.xfail("const_node.get_data() is not supported until 2024.2") - model = IdentityMatmul(weights_dtype=weight_dtype, activation_dtype=activation_dtype).ov_model compressed_model = compress_weights( model, mode=CompressWeightsMode.INT4_SYM, ratio=1, group_size=1, all_layers=True diff --git a/tests/openvino/native/test_model_transformer.py b/tests/openvino/native/test_model_transformer.py index eaa5dc778bb..46d4a991316 100644 --- a/tests/openvino/native/test_model_transformer.py +++ b/tests/openvino/native/test_model_transformer.py @@ -45,7 +45,6 @@ from nncf.quantization.fake_quantize import FakeQuantizeParameters from tests.openvino.native.common import compare_nncf_graphs from tests.openvino.native.common import get_actual_reference_for_current_openvino -from tests.openvino.native.common import get_openvino_major_minor_version from tests.openvino.native.models import ConvModel from tests.openvino.native.models import ConvNotBiasModel from tests.openvino.native.models import FPModel @@ -473,9 +472,6 @@ def test_fq_insertion_pre_layer(target_layers, ref_fq_names): @pytest.mark.parametrize("target_layers, ref_fс_names", zip(TARGET_INSERT_LAYERS, TARGET_PRE_LAYER_FCS)) def test_fc_insertion_pre_layer(target_layers, ref_fс_names): - ov_major_version, ov_minor_version = get_openvino_major_minor_version() - if ov_major_version < 2023 or (ov_major_version == 2023 and ov_minor_version < 3): - pytest.xfail("FakeConvert is not supported until 2023.3") model = LinearModel().ov_model transformed_model = create_transformed_model( @@ -512,9 +508,6 @@ def test_fq_insertion_post_layer(target_layers, ref_fq_names): @pytest.mark.parametrize("target_layers, ref_fс_names", zip(TARGET_INSERT_LAYERS, TARGET_POST_LAYER_FCS)) def test_fc_insertion_post_layer(target_layers, ref_fс_names): - ov_major_version, ov_minor_version = get_openvino_major_minor_version() - if ov_major_version < 2023 or (ov_major_version == 2023 and ov_minor_version < 3): - pytest.xfail("FakeConvert is not supported until 2023.3") model = LinearModel().ov_model transformed_model = create_transformed_model( @@ -552,9 +545,6 @@ def test_fq_insertion_weights(target_layers, ref_fq_names): @pytest.mark.parametrize("target_layers, ref_fс_names", zip(TARGET_INSERT_LAYERS, TARGET_WEIGHTS_FCS)) def test_fc_insertion_weights(target_layers, ref_fс_names): - ov_major_version, ov_minor_version = get_openvino_major_minor_version() - if ov_major_version < 2023 or (ov_major_version == 2023 and ov_minor_version < 3): - pytest.xfail("FakeConvert is not supported until 2023.3") model = LinearModel().ov_model transformed_model = create_transformed_model( diff --git a/tests/openvino/native/test_smooth_quant.py b/tests/openvino/native/test_smooth_quant.py index caa297ef83d..ef62647d4d7 100644 --- a/tests/openvino/native/test_smooth_quant.py +++ b/tests/openvino/native/test_smooth_quant.py @@ -16,6 +16,7 @@ import pytest import torch +import nncf from nncf.common.graph.transformations.commands import TransformationCommand from nncf.openvino.graph.layer_attributes import OVLayerAttributes from nncf.openvino.graph.layout import OVLayoutElem @@ -130,7 +131,7 @@ def check_scales(model: ov.Model, reference_values: Dict[str, np.ndarray], model (OVMatMulMetatype, OVLayerAttributes({}, inputs_attributes={"transpose": True}), 0, -2), (OVMatMulMetatype, OVLayerAttributes({}, inputs_attributes={"transpose": False}), 1, -2), (OVMatMulMetatype, OVLayerAttributes({}, inputs_attributes={"transpose": True}), 1, -1), - (OVMatMulMetatype, OVLayerAttributes({}, inputs_attributes={"transpose": False}), 2, RuntimeError), + (OVMatMulMetatype, OVLayerAttributes({}, inputs_attributes={"transpose": False}), 2, nncf.InternalError), (OVConvolutionMetatype, OVLayerAttributes({}, inputs_attributes={}), 0, 1), ), ) diff --git a/tests/openvino/native/test_statistic_collector.py b/tests/openvino/native/test_statistic_collector.py index c1a2aa02544..de7685314d0 100644 --- a/tests/openvino/native/test_statistic_collector.py +++ b/tests/openvino/native/test_statistic_collector.py @@ -11,7 +11,6 @@ import numpy as np -import pytest from nncf.experimental.tensor import Tensor from tests.common.experimental.test_statistic_collector import TemplateTestStatisticCollector @@ -20,11 +19,3 @@ class TestOVStatisticCollector(TemplateTestStatisticCollector): def get_nncf_tensor(self, value: np.ndarray) -> Tensor: return Tensor(value) - - @pytest.mark.skip() - def test_median_mad_stat_building(self): - pass - - @pytest.mark.skip - def test_percentile_max_stat_building(self): - pass diff --git a/tests/post_training/test_templates/test_smooth_quant.py b/tests/post_training/test_templates/test_smooth_quant.py index c3827cc81dd..79d43bce711 100644 --- a/tests/post_training/test_templates/test_smooth_quant.py +++ b/tests/post_training/test_templates/test_smooth_quant.py @@ -28,7 +28,6 @@ from nncf.quantization.algorithms.post_training.algorithm import PostTrainingQuantization from nncf.quantization.algorithms.smooth_quant.algorithm import SmoothQuant from nncf.quantization.algorithms.smooth_quant.backend import SmoothQuantAlgoBackend -from nncf.quantization.algorithms.smooth_quant.openvino_backend import OVSmoothQuantAlgoBackend from tests.post_training.test_templates.helpers import ConvTestModel from tests.post_training.test_templates.helpers import LinearMultiShapeModel from tests.post_training.test_templates.helpers import NonZeroLinearModel @@ -213,9 +212,6 @@ def test__get_nodes_to_smooth_data(self, model_cls, references, tmpdir): model = self.backend_specific_model(model_cls(), tmpdir) nncf_graph = NNCFGraphFactory.create(model) - if isinstance(self.get_backend(), OVSmoothQuantAlgoBackend) and model_cls is ShareWeghtsConvAndShareLinearModel: - pytest.xfail("Matmuls don't share one weight in OV ir. Ticket 130912.") - algo = SmoothQuant() algo._set_backend_entity(model) alpha_map = algo._get_alpha_map() @@ -268,13 +264,14 @@ def test_get_activation_channel_axis(self, node_metatype, layer_attributes, port } node = NNCFNode(attributes) - try: # noqa + if reference_value is nncf.InternalError: + with pytest.raises( + nncf.InternalError, match=f"{node.metatype.name} can not take more than 2 input tensors." + ): + backend.get_activation_channel_axis(node, port_id) + else: activation_channel_axis = backend.get_activation_channel_axis(node, port_id) - except nncf.InternalError as e: # noqa - # TODO: this is wrong, should only expect an exception for cases where the exception is known to be raised - # with pytest.raises(...) - pytest.xfail("FIXME") - assert activation_channel_axis == reference_value + assert activation_channel_axis == reference_value def test_get_weight_channel_axis(self, node_metatype, layer_attributes, reference_value): backend = self.get_backend() diff --git a/tests/torch/ptq/test_statistic_collector.py b/tests/torch/ptq/test_statistic_collector.py index 55ec070a53b..963ace9ceb0 100644 --- a/tests/torch/ptq/test_statistic_collector.py +++ b/tests/torch/ptq/test_statistic_collector.py @@ -11,7 +11,6 @@ import numpy as np -import pytest import torch from nncf.experimental.tensor import Tensor @@ -21,7 +20,3 @@ class TestPTStatisticCollector(TemplateTestStatisticCollector): def get_nncf_tensor(self, value: np.ndarray) -> Tensor: return Tensor(torch.tensor(value)) - - @pytest.mark.skip - def test_raw_max_stat_building(self): - pass diff --git a/tests/torch/sparsity/movement/test_model_saving.py b/tests/torch/sparsity/movement/test_model_saving.py index 3eade01181b..27a9655591a 100644 --- a/tests/torch/sparsity/movement/test_model_saving.py +++ b/tests/torch/sparsity/movement/test_model_saving.py @@ -15,7 +15,6 @@ import numpy as np import onnx import onnxruntime -import openvino as ov import pytest import torch from addict import Dict @@ -184,9 +183,7 @@ def test_same_outputs_in_torch_and_exported_onnx(self, tmp_path: Path, recipe: B ), Dict( nncf_weight_ratio=0.43, - ov_weight_ratio=( - 0.29 if version.parse(ov.runtime.__version__.split("-")[0]) < version.parse("2024.2.0") else 0.33 - ), + ov_weight_ratio=0.33, recipe=SwinRunRecipe().model_config_( num_heads=[4], num_labels=1,