[GHA] conformance test for weight compression

.github/scripts/pytest_md_summary.py

@@ -44,8 +44,12 @@ def parse_xml_report(xml_file) -> None:
         elif testcase.find("error") is not None:
             status = "$${\color{red}Error}$$"
         elif testcase.find("skipped") is not None:
-            status = "$${\color{orange}Skipped}$$"
-            message = testcase.find("skipped").get("message", "")
+            if "xfail" in testcase.find("skipped").get("type", ""):
+                status = "$${\color{orange}xfail}$$"
+                message = testcase.find("skipped").get("message", "")
+            else:
+                status = "$${\color{yellow}Skipped}$$"
+                message = testcase.find("skipped").get("message", "")
         else:
             status = "$${\color{green}Ok}$$"
 

.github/workflows/api_changes_check.yml

@@ -58,7 +58,7 @@ jobs:
           echo '{"pr_number": "${{ github.event.pull_request.number }}", "action": "none"}' > api_status.json
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b #v4.5.0
         with:
           name: api_status
           path: api_status.json

.github/workflows/conformance_weight_compression.yml

@@ -0,0 +1,74 @@
+name: Weight compression
+permissions: read-all
+
+on:
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      pull_request_number:
+        description: 'The pull request number'
+        default: ''
+      pytest_args:
+        description: 'Pytest arguments'
+        default: ''
+
+jobs:
+  examples-cpu:
+    name: Weight compression [${{ matrix.group }}/3]
+    runs-on: ubuntu-22.04-16-cores
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1, 2, 3]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+            lfs: true
+            fetch-depth: 0  # Fetch full history to allow checking out any branch or PR
+      - name: Fetch and Checkout the Pull Request Branch
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pull_request_number != '' }}
+        run: |
+          git fetch origin pull/${{ github.event.inputs.pull_request_number }}/head:pr-${{ github.event.inputs.pull_request_number }}
+          git checkout pr-${{ github.event.inputs.pull_request_number }}
+      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        with:
+          python-version: 3.10.14
+          cache: pip
+      - name: cpuinfo
+        run: cat /proc/cpuinfo
+      - name: Install NNCF and test requirements
+        run: pip install -e . -r tests/post_training/requirements.txt
+      - name: Print installed modules
+        run: pip list
+      - name: Run examples test scope
+        run: |
+          python -m pytest -s -ra tests/post_training/test_quantize_conformance.py::test_weight_compression \
+            --junit-xml=pytest-results.xml \
+            --durations-path=tests/post_training/data/wc_test_durations.json \
+            --splitting-algorithm=least_duration \
+            --splits 3 \
+            --group ${{ matrix.group }} \
+            ${{ github.event.inputs.pytest_args || '' }}
+          ret=$?
+          [ $ret -eq 5 ] && [ -n "${{ github.event.inputs.pytest_args || '' }}" ]  && exit 0 || exit $ret
+        env:
+          TQDM_DISABLE: 1
+          HOME_HF: "/home/runner/hf_home"
+      - name: Print results.csv
+        if: ${{ !cancelled() }}
+        run: column -s, -t < tmp/results.csv || echo "no file"
+      - name: Upload artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b #v4.5.0
+        with:
+          name: wc_results_${{ matrix.group }}
+          path: tmp/results.csv
+      - name: Test Summary
+        if: ${{ !cancelled() }}
+        run: |
+          pip install defusedxml==0.7.1
+          python .github/scripts/pytest_md_summary.py pytest-results.xml >> $GITHUB_STEP_SUMMARY

.github/workflows/nightly.yml

@@ -30,11 +30,11 @@ jobs:
           python-version: 3.10.14
           cache: pip
       - name: Install NNCF and test requirements
-        run: |
-          pip install -e .
-          pip install -r tests/tensorflow/requirements.txt
+        run: pip install -e . -r tests/tensorflow/requirements.txt
       - name: Print installed modules
         run: pip list
       - name: Run TensorFlow precommit
         run:  pytest tests/tensorflow -m 'nightly'
 
+  weight-compression:
+    uses: ./.github/workflows/conformance_weight_compression.yml

tests/post_training/data/wc_test_durations.json

@@ -0,0 +1,15 @@
+{
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_awq_backup_mode_none_backend_OV]": 269,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_aware_awq_scale_estimation_backend_OV]": 421,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_aware_awq_scale_estimation_stateful_backend_OV]": 374,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_aware_awq_stateful_backend_OV]": 243,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_aware_backend_OV]": 190,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_aware_gptq_scale_estimation_stateful_backend_OV]": 1463,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_aware_lora_stateful_backend_OV]": 483,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_free_backend_FP32]": 0,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_data_free_backend_OV]": 196,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_int4_data_free_backend_TORCH]": 133,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_int8_data_free_backend_TORCH]": 154,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_NF4_scale_estimation_stateful_per_channel_backend_OV]": 256,
+    "tests/post_training/test_quantize_conformance.py::test_weight_compression[tinyllama_scale_estimation_per_channel_backend_OV]": 258
+}

tests/post_training/requirements.txt

@@ -6,6 +6,7 @@ onnxruntime
 openvino
 pytest
 pytest-forked
+pytest-split
 
 librosa==0.10.0
 memory-profiler==0.61.0

tests/post_training/test_quantize_conformance.py

@@ -9,6 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
 import time
 import traceback
@@ -33,6 +34,19 @@
 DATA_ROOT = Path(__file__).parent / "data"
 
 
+@pytest.fixture(scope="function", name="use_avx2")
+def fixture_use_avx2():
+    old_value = os.environ.get("ONEDNN_MAX_CPU_ISA")
+    os.environ["ONEDNN_MAX_CPU_ISA"] = "AVX2"
+    if old_value is not None and old_value != "AVX2":
+        print(f"Warning: ONEDNN_MAX_CPU_ISA is overriding to AVX2, was {old_value}")
+    yield
+    if old_value is None:
+        del os.environ["ONEDNN_MAX_CPU_ISA"]
+    else:
+        os.environ["ONEDNN_MAX_CPU_ISA"] = old_value
+
+
 @pytest.fixture(scope="session", name="data_dir")
 def fixture_data(pytestconfig):
     if pytestconfig.getoption("data") is None:
@@ -132,7 +146,7 @@ def fixture_wc_reference_data():
     fp32_test_cases = defaultdict(dict)
     for test_case_name in data:
         if "atol" not in data[test_case_name]:
-            data[test_case_name]["atol"] = 1e-5
+            data[test_case_name]["atol"] = 1e-4
         reported_name = test_case_name.split("_backend_")[0]
         fp32_case_name = f"{reported_name}_backend_FP32"
         fp32_test_cases[fp32_case_name]["metric_value"] = 1
@@ -328,7 +342,6 @@ def test_ptq_quantization(
 def test_weight_compression(
     wc_reference_data: dict,
     test_case_name: str,
-    data_dir: Path,
     output_dir: Path,
     wc_result_data: Dict[str, RunInfo],
     no_eval: bool,
@@ -340,6 +353,7 @@ def test_weight_compression(
     capsys: pytest.CaptureFixture,
     extra_columns: bool,
     memory_monitor: bool,
+    use_avx2: None,
 ):
     pipeline = None
     err_msg = None
@@ -355,7 +369,7 @@ def test_weight_compression(
         pipeline_kwargs.update(
             {
                 "output_dir": output_dir,
-                "data_dir": data_dir,
+                "data_dir": None,
                 "no_eval": no_eval,
                 "run_benchmark_app": run_benchmark_app,
                 "batch_size": batch_size,