From 8bf5a12f9f523ed1e5a3968cc9be7b1ff787c6f9 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Mon, 21 Aug 2023 13:26:50 +0200 Subject: [PATCH 1/3] Pass codecov token to reusable workflow (#2071) --- .github/workflows/post_pr_merge.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/post_pr_merge.yml b/.github/workflows/post_pr_merge.yml index accfc040119..69b7d9c5293 100644 --- a/.github/workflows/post_pr_merge.yml +++ b/.github/workflows/post_pr_merge.yml @@ -24,6 +24,8 @@ jobs: merge_commit_sha: ${{ github.event.pull_request.merge_commit_sha }} last_sha_in_pr: ${{ github.event.pull_request.head.sha }} coverage_artifact_name_in_pr: coverage_common + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} upload-coverage-onnx: if: github.event.pull_request.merged == true uses: ./.github/workflows/upload_coverage_for_develop.yml @@ -31,3 +33,5 @@ jobs: merge_commit_sha: ${{ github.event.pull_request.merge_commit_sha }} last_sha_in_pr: ${{ github.event.pull_request.head.sha }} coverage_artifact_name_in_pr: coverage_onnx + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From 046ded3d0a811f2031af26fdc347eff276c15c89 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Mon, 21 Aug 2023 13:33:24 +0200 Subject: [PATCH 2/3] Adjust codecov.yml (#2067) ### Changes Makes Codecov wait for 2 reports being uploaded before printing a comment, since we have both onnx and common precommits submitting coverage reports now via GH action runs. ### Reason for changes Currently the common precommit finishes much faster than the ONNX, and Codecov posts coverage based off that run before waiting for the ONNX precommit to finish, which is necessarily lower and confusing. ### Related tickets N/A ### Tests N/A --- codecov.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/codecov.yml b/codecov.yml index e96667f9686..a5b2c837c92 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,10 +3,10 @@ ignore: - "tests" - "tools" -#codecov: -# notify: -# after_n_builds: 4 -# wait_for_ci: yes +codecov: + notify: + after_n_builds: 2 + wait_for_ci: no coverage: status: @@ -25,7 +25,8 @@ coverage: informational: true only_pulls: true paths: - - "nncf/onnx" # extend this once we collect coverage reports for more than just onnx part of precommit + - "nncf/onnx" + - "nncf/common" # extend this once we collect coverage reports for more than just onnx and common part of precommit comment: layout: "diff, flags, files" From 5fdcc465f081cc08ea79fb7e2631fdb196b43d8b Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 22 Aug 2023 16:23:11 +0200 Subject: [PATCH 3/3] Only launch pytest with coverage in Makefile if env var is set (#2072) ### Changes Setting `NNCF_COVERAGE` env var will make commands such as `make test-torch` run pytest with `coverage` reports generated, otherwise the pytest runs without `coverage. ### Reason for changes Devs can now launch Makefile commands without running coverage by default, and if they really need to, they can prefix the make command: `NNCF_COVERAGE=1 make test-torch`. ### Related tickets N/A ### Tests precommit --- .github/workflows/precommit.yml | 4 ++++ Makefile | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 518f8d32665..1652fcb2eb3 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -21,6 +21,8 @@ jobs: run: make install-common-test - name: Run common precommit test scope run: make test-common + env: + NNCF_COVERAGE: 1 - name: Upload coverage report as artifact uses: actions/upload-artifact@v3 with: @@ -44,6 +46,8 @@ jobs: run: make install-onnx-test - name: Run ONNX precommit test scope run: make test-onnx + env: + NNCF_COVERAGE: 1 - name: Upload coverage report as artifact uses: actions/upload-artifact@v3 with: diff --git a/Makefile b/Makefile index f2cc30f60c8..173d7d57738 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ JUNITXML_PATH ?= nncf-tests.xml -COVERAGE ?= --cov=./ --cov-report=xml + +ifdef NNCF_COVERAGE + COVERAGE_ARGS ?= --cov=./ --cov-report=xml +else + COVERAGE_ARGS := +endif ifdef DATA DATA_ARG := --data $(DATA) @@ -26,7 +31,7 @@ install-onnx-dev: install-onnx-test install-pre-commit install-pylint pip install -r examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt test-onnx: - pytest ${COVERAGE} tests/onnx $(DATA_ARG) --junitxml ${JUNITXML_PATH} + pytest ${COVERAGE_ARGS} tests/onnx $(DATA_ARG) --junitxml ${JUNITXML_PATH} pylint-onnx: pylint --rcfile .pylintrc \ @@ -60,8 +65,7 @@ install-openvino-dev: install-openvino-test install-pre-commit install-pylint pip install -r examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/requirements.txt test-openvino: - # omitting ${COVERAGE} for internal runs since they seem to introduce a major slowdown - pytest tests/openvino $(DATA_ARG) --junitxml ${JUNITXML_PATH} + pytest ${COVERAGE_ARGS} tests/openvino $(DATA_ARG) --junitxml ${JUNITXML_PATH} pylint-openvino: pylint --rcfile .pylintrc \ @@ -91,8 +95,7 @@ install-tensorflow-dev: install-tensorflow-test install-pre-commit install-pylin pip install -r examples/post_training_quantization/tensorflow/mobilenet_v2/requirements.txt test-tensorflow: - # omitting ${COVERAGE} for internal runs since they seem to introduce a major slowdown - pytest tests/common tests/tensorflow \ + pytest ${COVERAGE_ARGS} tests/common tests/tensorflow \ --junitxml ${JUNITXML_PATH} \ $(DATA_ARG) @@ -121,8 +124,7 @@ install-torch-dev: install-torch-test install-pre-commit install-pylint pip install -r examples/post_training_quantization/torch/ssd300_vgg16/requirements.txt test-torch: - # omitting ${COVERAGE} for internal runs since they seem to introduce a major slowdown - pytest tests/common tests/torch --junitxml ${JUNITXML_PATH} $(DATA_ARG) + pytest ${COVERAGE_ARGS} tests/common tests/torch --junitxml ${JUNITXML_PATH} $(DATA_ARG) COMMON_PYFILES := $(shell python3 tools/collect_pylint_input_files_for_backend.py common) pylint-torch: @@ -160,7 +162,7 @@ pylint-common: $(COMMON_PYFILES) test-common: - pytest ${COVERAGE} tests/common $(DATA_ARG) --junitxml ${JUNITXML_PATH} + pytest ${COVERAGE_ARGS} tests/common $(DATA_ARG) --junitxml ${JUNITXML_PATH} test-examples: pytest tests/cross_fw/examples -s --junitxml ${JUNITXML_PATH}