Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added ignored RoPE block #3049

Merged
merged 6 commits into from
Nov 6, 2024
Merged

Added ignored RoPE block #3049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1ed567e
Added ignored RoPE block
KodiaqQ Oct 31, 2024
25d629d
Fix tests for ONNX/PT
KodiaqQ Oct 31, 2024
28ba705
Added Torch test
KodiaqQ Nov 4, 2024
5bf3be2
Add ONNX test
KodiaqQ Nov 4, 2024
54c750d
Fix patterns test
KodiaqQ Nov 4, 2024
6a6434d
Rollback ONNX/PT changes
KodiaqQ Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions nncf/common/graph/patterns/patterns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,3 +406,4 @@ class IgnoredPatternNames(Enum):
SE_BLOCK = PatternDesc("se_block")
FC_BN_HSWISH_ACTIVATION = PatternDesc("fc_bn_hswish_activation")
EQUAL_LOGICALNOT = PatternDesc("equal_logicalnot")
ROPE = PatternDesc("rope", model_types=[ModelType.TRANSFORMER])
12 changes: 12 additions & 0 deletions nncf/openvino/graph/metatypes/openvino_metatypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,18 @@ class OVScaledDotProductAttentionMetatype(OVOpMetatype):
target_input_ports = [0, 1]


@OV_OPERATOR_METATYPES.register()
class OVCosMetatype(OVOpMetatype):
name = "CosOp"
op_names = ["Cos"]


@OV_OPERATOR_METATYPES.register()
class OVSinMetatype(OVOpMetatype):
name = "SinOp"
op_names = ["Sin"]


def get_operator_metatypes() -> List[Type[OperatorMetatype]]:
"""
Returns a list of the operator metatypes.
Expand Down
22 changes: 22 additions & 0 deletions nncf/openvino/quantization/ignored_patterns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,3 +169,25 @@ def create_se_block() -> GraphPattern:
pattern.add_edge(activation_node_2, multiply_node)
pattern.add_edge(any_node, multiply_node)
return pattern


@OPENVINO_IGNORED_PATTERNS.register(IgnoredPatternNames.ROPE)
def create_rope() -> GraphPattern:
pattern = GraphPattern()
matmul_node = pattern.add_node(
**{GraphPattern.LABEL_ATTR: "MATMUL", GraphPattern.METATYPE_ATTR: om.OVMatMulMetatype}
)
transpose_node = pattern.add_node(
**{GraphPattern.LABEL_ATTR: "TRANSPOSE", GraphPattern.METATYPE_ATTR: om.OVTransposeMetatype}
)
concat_node = pattern.add_node(
**{GraphPattern.LABEL_ATTR: "CONCAT", GraphPattern.METATYPE_ATTR: om.OVConcatMetatype}
)
cos_node = pattern.add_node(**{GraphPattern.LABEL_ATTR: "COS", GraphPattern.METATYPE_ATTR: om.OVCosMetatype})
sin_node = pattern.add_node(**{GraphPattern.LABEL_ATTR: "SIN", GraphPattern.METATYPE_ATTR: om.OVSinMetatype})

pattern.add_edge(matmul_node, transpose_node)
pattern.add_edge(transpose_node, concat_node)
pattern.add_edge(concat_node, cos_node)
pattern.add_edge(concat_node, sin_node)
return pattern
1 change: 1 addition & 0 deletions tests/onnx/test_pattern_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
IGNORING_IGNORED_PATTERN_REASONS = {
IgnoredPatternNames.FC_BN_HSWISH_ACTIVATION: "Not relevant for ONNX.",
IgnoredPatternNames.EQUAL_LOGICALNOT: "Not relevant for ONNX.",
IgnoredPatternNames.ROPE: "Not relevant for ONNX.",
}


Expand Down
31 changes: 31 additions & 0 deletions tests/openvino/native/data/2024.4/reference_graphs/quantized/RoPEModel.dot
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
strict digraph {
"0 position_ids" [id=0, type=Parameter];
"1 Unsqueeze_3697" [id=1, type=Unsqueeze];
"2 convert" [id=2, type=Convert];
"3 MatMul" [id=3, type=MatMul];
"4 transpose" [id=4, type=Transpose];
"5 concat" [id=5, type=Concat];
"6 cos" [id=6, type=Cos];
"7 sin" [id=7, type=Sin];
"8 cos_result" [id=8, type=Result];
"9 sin_result" [id=9, type=Result];
"10 transpose/Constant_3703" [id=10, type=Constant];
"11 unsqueeze/Constant_3696" [id=11, type=Constant];
"12 broadcast" [id=12, type=Broadcast];
"13 broadcast/Constant_3700" [id=13, type=Constant];
"14 broadcast/Constant_3699" [id=14, type=Constant];
"0 position_ids" -> "1 Unsqueeze_3697" [label="[1, 10]", style=solid];
"1 Unsqueeze_3697" -> "2 convert" [label="[1, 1, 10]", style=solid];
"2 convert" -> "3 MatMul" [label="[1, 1, 10]", style=solid];
"3 MatMul" -> "4 transpose" [label="[1, 5, 10]", style=solid];
"4 transpose" -> "5 concat" [label="[1, 10, 5]", style=solid];
"5 concat" -> "6 cos" [label="[1, 10, 5]", style=solid];
"5 concat" -> "7 sin" [label="[1, 10, 5]", style=solid];
"6 cos" -> "8 cos_result" [label="[1, 10, 5]", style=solid];
"7 sin" -> "9 sin_result" [label="[1, 10, 5]", style=solid];
"10 transpose/Constant_3703" -> "4 transpose" [label="[3]", style=dashed];
"11 unsqueeze/Constant_3696" -> "1 Unsqueeze_3697" [label="[]", style=dashed];
"12 broadcast" -> "3 MatMul" [label="[1, 5, 1]", style=solid];
"13 broadcast/Constant_3700" -> "12 broadcast" [label="[3]", style=dashed];
"14 broadcast/Constant_3699" -> "12 broadcast" [label="[1, 5, 1]", style=solid];
}
23 changes: 23 additions & 0 deletions tests/openvino/native/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1162,3 +1162,26 @@ def _create_ov_model(self):
result = opset.result(conv, name="Result")
model = ov.Model([result], [input])
return model


class RoPEModel(OVReferenceModel):
def _create_ov_model(self):
position_ids = opset.parameter([1, 10], name="position_ids")

unsqueeze = opset.unsqueeze(position_ids, 0, name="unsqueeze")
convert = opset.convert(unsqueeze, ov.Type.f32, name="convert")

broadcast_data = self._rng.random((1, 5, 1)).astype(np.float32)
broadcast_shape = [1, 5, 1]
broadcast = opset.broadcast(broadcast_data, broadcast_shape, name="broadcast")

matmul = opset.matmul(broadcast, convert, transpose_a=False, transpose_b=False, name="MatMul")
transpose = opset.transpose(matmul, [0, 2, 1], name="transpose")
concat = opset.concat([transpose], axis=0, name="concat")
sin = opset.sin(concat, name="sin")
cos = opset.cos(concat, name="cos")
sin_result = opset.result(sin, name="sin_result")
cos_result = opset.result(cos, name="cos_result")

model = ov.Model([sin_result, cos_result], [position_ids])
return model
3 changes: 2 additions & 1 deletion tests/openvino/native/quantization/test_graphs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
from tests.openvino.native.models import IfModel
from tests.openvino.native.models import IfModel_2
from tests.openvino.native.models import MatmulSoftmaxMatmulBlock
from tests.openvino.native.models import RoPEModel
from tests.openvino.native.models import ScaledDotProductAttentionModel
from tests.openvino.native.models import get_torch_model_info
from tests.openvino.native.quantization.test_fq_params_calculation import quantize_model
Expand Down Expand Up @@ -96,7 +97,7 @@ def test_real_models_fq_placement(model_name_params, tmp_path):
compare_nncf_graphs(quantized_model, path_ref_graph)


@pytest.mark.parametrize("model_creator_func", [MatmulSoftmaxMatmulBlock])
@pytest.mark.parametrize("model_creator_func", [MatmulSoftmaxMatmulBlock, RoPEModel])
def test_transformer_models_fq_placement(model_creator_func, tmp_path):
model = model_creator_func()
quantized_model = quantize_model(
Expand Down
1 change: 1 addition & 0 deletions tests/torch/test_pattern_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
IGNORING_IGNORED_PATTERN_REASONS = {
IgnoredPatternNames.FC_BN_HSWISH_ACTIVATION: "Not relevant for Torch.",
IgnoredPatternNames.EQUAL_LOGICALNOT: "Not relevant for Torch.",
IgnoredPatternNames.ROPE: "Not relevant for Torch.",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you explain why this pattern is not relevant for other backends?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are no examples of this pattern in other backends. I can not add this for ONNX/PT since I have no models with this structure from these frameworks.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What models did you use for testing the RoPE pattern for OV backend? If source models are PyTorch models you can reuse them to specify RoPE pattern for ONNX/PT backends.

Copy link
Collaborator Author

@KodiaqQ KodiaqQ Nov 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know who would need this, but OK.
I've set PR to draft until other backends are covered.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added patterns with the corresponding tests as you asked. Need to properly review it, since I've had to create it without strong ONNX/PT research due to lack of time.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On which models have you tested the pattern?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For OV backend: phi-3, llama2-3.

}


Expand Down
Loading