diff --git a/.github/workflows/conformance_weight_compression.yml b/.github/workflows/conformance_weight_compression.yml new file mode 100644 index 00000000000..31b586a065a --- /dev/null +++ b/.github/workflows/conformance_weight_compression.yml @@ -0,0 +1,67 @@ +name: Weight compression +permissions: read-all + +on: + pull_request: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + pull_request_number: + description: 'The pull request number' + default: '' + pytest_args: + description: 'Pytest arguments' + default: '' + +jobs: + examples-cpu: + name: Test exmaples CPU [${{ matrix.group }}/4] + runs-on: ubuntu-22.04-16-cores + strategy: + fail-fast: false + matrix: + group: [1, 2] + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + lfs: true + fetch-depth: 0 # Fetch full history to allow checking out any branch or PR + - name: Fetch and Checkout the Pull Request Branch + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pull_request_number != '' }} + run: | + git fetch origin pull/${{ github.event.inputs.pull_request_number }}/head:pr-${{ github.event.inputs.pull_request_number }} + git checkout pr-${{ github.event.inputs.pull_request_number }} + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + with: + python-version: 3.10.14 + cache: pip + - name: cpuinfo + run: cat /proc/cpuinfo + - name: Install NNCF and test requirements + run: | + pip install -e . + pip install -r tests/post_training/requirements.txt + - name: Print installed modules + run: pip list + - name: Run examples test scope + run: | + python -m pytest -ras tests/post_training/test_quantize_conformance.py::test_weight_compression \ + --junit-xml=pytest-results-${{ matrix.group }}.xml \ + --durations-path=tests/post_training/.test_durations \ + --splitting-algorithm=least_duration \ + --splits 2 \ + --group ${{ matrix.group }} \ + ${{ github.event.inputs.pytest_args || '' }} + env: + TQDM_DISABLE: 1 + - name: Upload artifact + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: pytest-results-${{ matrix.group }} + path: pytest-results-${{ matrix.group }}.xml + overwrite: True diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index cbed985e3de..92e98eb666c 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -1,213 +1,213 @@ -name: precommit -permissions: read-all +# name: precommit +# permissions: read-all -on: - pull_request: - types: - - opened - - reopened - - synchronize - paths-ignore: - - '**/*.md' - - 'docs/**/*' - - 'tests/post_training/*' # post_training tests runs on Jenkins - - 'tests/torch/sota_checkpoints_eval.json' # reference for PT e2e - - 'tests/tensorflow/sota_checkpoints_eval.json' # reference for TF e2e - - 'tests/cross_fw/examples/*' # examples tests runs in separate workflow -jobs: - common: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Install NNCF and test requirements - run: make install-common-test - - name: Print installed modules - run: pip list - - name: Run common precommit test scope - run: make test-common - env: - NUM_WORKERS: 2 +# on: +# pull_request: +# types: +# - opened +# - reopened +# - synchronize +# paths-ignore: +# - '**/*.md' +# - 'docs/**/*' +# - 'tests/post_training/*' # post_training tests runs on Jenkins +# - 'tests/torch/sota_checkpoints_eval.json' # reference for PT e2e +# - 'tests/tensorflow/sota_checkpoints_eval.json' # reference for TF e2e +# - 'tests/cross_fw/examples/*' # examples tests runs in separate workflow +# jobs: +# common: +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Install NNCF and test requirements +# run: make install-common-test +# - name: Print installed modules +# run: pip list +# - name: Run common precommit test scope +# run: make test-common +# env: +# NUM_WORKERS: 2 - onnx: - runs-on: ubuntu-20.04-8-cores - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - - name: Install NNCF and test requirements - run: make install-onnx-test - - name: Print installed modules - run: pip list - - name: Run ONNX precommit test scope - run: make test-onnx - env: - NUM_WORKERS: 4 +# onnx: +# runs-on: ubuntu-20.04-8-cores +# steps: +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# - name: Install NNCF and test requirements +# run: make install-onnx-test +# - name: Print installed modules +# run: pip list +# - name: Run ONNX precommit test scope +# run: make test-onnx +# env: +# NUM_WORKERS: 4 - openvino: - runs-on: ubuntu-20.04-8-cores - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Install NNCF and test requirements - run: make install-openvino-test - - name: Print installed modules - run: pip list - - name: Run OV precommit test scope - run: make test-openvino - env: - NUM_WORKERS: 4 +# openvino: +# runs-on: ubuntu-20.04-8-cores +# steps: +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Install NNCF and test requirements +# run: make install-openvino-test +# - name: Print installed modules +# run: pip list +# - name: Run OV precommit test scope +# run: make test-openvino +# env: +# NUM_WORKERS: 4 - pytorch-cpu: - timeout-minutes: 40 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-8-cores - env: - DEBIAN_FRONTEND: noninteractive - steps: - - name: Install dependencies - run : | - sudo apt-get update - sudo apt-get --assume-yes install gcc g++ build-essential ninja-build libgl1-mesa-dev libglib2.0-0 - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Runner info - continue-on-error: true - run: | - cat /etc/*release - cat /proc/cpuinfo - - name: Install NNCF and test requirements - run: make install-torch-test - - name: Print installed modules - run: pip list - - name: Run PyTorch precommit test scope - run: | - make test-torch-cpu - env: - NUM_WORKERS: 4 +# pytorch-cpu: +# timeout-minutes: 40 +# defaults: +# run: +# shell: bash +# runs-on: ubuntu-20.04-8-cores +# env: +# DEBIAN_FRONTEND: noninteractive +# steps: +# - name: Install dependencies +# run : | +# sudo apt-get update +# sudo apt-get --assume-yes install gcc g++ build-essential ninja-build libgl1-mesa-dev libglib2.0-0 +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Runner info +# continue-on-error: true +# run: | +# cat /etc/*release +# cat /proc/cpuinfo +# - name: Install NNCF and test requirements +# run: make install-torch-test +# - name: Print installed modules +# run: pip list +# - name: Run PyTorch precommit test scope +# run: | +# make test-torch-cpu +# env: +# NUM_WORKERS: 4 - pytorch-cuda: - timeout-minutes: 40 - defaults: - run: - shell: bash - runs-on: aks-linux-4-cores-28gb-gpu-tesla-t4 - env: - DEBIAN_FRONTEND: noninteractive - steps: - - name: Install dependencies - run : | - sudo apt-get update - sudo apt-get --assume-yes install build-essential ninja-build libgl1-mesa-dev libglib2.0-0 wget make - - name: Download CUDA - run: | - wget -q https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_530.30.02_linux.run - sudo sh cuda_12.1.1_530.30.02_linux.run --toolkit --silent - - name: Runner info - continue-on-error: true - run: | - export PATH=/usr/local/cuda-12.1/bin${PATH:+:${PATH}} - export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - nvidia-smi - cat /proc/cpuinfo - nvcc --version - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Install NNCF and test requirements - run: make install-torch-test - - name: Print installed modules - run: pip list - - name: Check CUDA - run: | - python -c "import torch; print(torch.cuda.is_available())" - - name: Run PyTorch precommit test scope - run: | - export PATH=/usr/local/cuda-12.1/bin${PATH:+:${PATH}} - export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - make test-torch-cuda +# pytorch-cuda: +# timeout-minutes: 40 +# defaults: +# run: +# shell: bash +# runs-on: aks-linux-4-cores-28gb-gpu-tesla-t4 +# env: +# DEBIAN_FRONTEND: noninteractive +# steps: +# - name: Install dependencies +# run : | +# sudo apt-get update +# sudo apt-get --assume-yes install build-essential ninja-build libgl1-mesa-dev libglib2.0-0 wget make +# - name: Download CUDA +# run: | +# wget -q https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_530.30.02_linux.run +# sudo sh cuda_12.1.1_530.30.02_linux.run --toolkit --silent +# - name: Runner info +# continue-on-error: true +# run: | +# export PATH=/usr/local/cuda-12.1/bin${PATH:+:${PATH}} +# export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +# nvidia-smi +# cat /proc/cpuinfo +# nvcc --version +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Install NNCF and test requirements +# run: make install-torch-test +# - name: Print installed modules +# run: pip list +# - name: Check CUDA +# run: | +# python -c "import torch; print(torch.cuda.is_available())" +# - name: Run PyTorch precommit test scope +# run: | +# export PATH=/usr/local/cuda-12.1/bin${PATH:+:${PATH}} +# export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +# make test-torch-cuda - tensorflow: - timeout-minutes: 40 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-8-cores - env: - DEBIAN_FRONTEND: noninteractive - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Install NNCF and test requirements - run: make install-tensorflow-test - - name: Print installed modules - run: pip list - - name: Run TensorFlow precommit test scope - run: make test-tensorflow - env: - NUM_WORKERS: 6 +# tensorflow: +# timeout-minutes: 40 +# defaults: +# run: +# shell: bash +# runs-on: ubuntu-20.04-8-cores +# env: +# DEBIAN_FRONTEND: noninteractive +# steps: +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Install NNCF and test requirements +# run: make install-tensorflow-test +# - name: Print installed modules +# run: pip list +# - name: Run TensorFlow precommit test scope +# run: make test-tensorflow +# env: +# NUM_WORKERS: 6 - tools: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Install NNCF and test requirements - run: pip install -r tests/tools/requirements.txt - - name: Print installed modules - run: pip list - - name: Run tools precommit test scope - run: PYTHONPATH=./ pytest -ra tests/tools +# tools: +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Install NNCF and test requirements +# run: pip install -r tests/tools/requirements.txt +# - name: Print installed modules +# run: pip list +# - name: Run tools precommit test scope +# run: PYTHONPATH=./ pytest -ra tests/tools - pytorch2-cpu: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - with: - lfs: true - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: 3.10.14 - cache: pip - - name: Install NNCF and test requirements - run: | - pip install -e . - pip install -r tests/torch2/requirements.txt - - name: Print installed modules - run: pip list - - name: Run torch2 precommit test scope - run: pytest -ra tests/torch2 -m "not cuda" +# pytorch2-cpu: +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 +# with: +# lfs: true +# - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 +# with: +# python-version: 3.10.14 +# cache: pip +# - name: Install NNCF and test requirements +# run: | +# pip install -e . +# pip install -r tests/torch2/requirements.txt +# - name: Print installed modules +# run: pip list +# - name: Run torch2 precommit test scope +# run: pytest -ra tests/torch2 -m "not cuda" diff --git a/tests/post_training/.test_durations b/tests/post_training/.test_durations new file mode 100644 index 00000000000..2c63c085104 --- /dev/null +++ b/tests/post_training/.test_durations @@ -0,0 +1,2 @@ +{ +} diff --git a/tests/post_training/requirements.txt b/tests/post_training/requirements.txt index 93103066aab..1aa7c162065 100644 --- a/tests/post_training/requirements.txt +++ b/tests/post_training/requirements.txt @@ -6,6 +6,7 @@ onnxruntime openvino pytest pytest-forked +pytest-split librosa==0.10.0 memory-profiler==0.61.0 diff --git a/tests/post_training/test_quantize_conformance.py b/tests/post_training/test_quantize_conformance.py index 5c4fa176ad6..0b0ad0e9d40 100644 --- a/tests/post_training/test_quantize_conformance.py +++ b/tests/post_training/test_quantize_conformance.py @@ -324,7 +324,6 @@ def test_ptq_quantization( def test_weight_compression( wc_reference_data: dict, test_case_name: str, - data_dir: Path, output_dir: Path, wc_result_data: Dict[str, RunInfo], no_eval: bool, @@ -351,7 +350,7 @@ def test_weight_compression( pipeline_kwargs.update( { "output_dir": output_dir, - "data_dir": data_dir, + "data_dir": None, "no_eval": no_eval, "run_benchmark_app": run_benchmark_app, "batch_size": batch_size,