Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prechecks for asv #2107

Merged
merged 29 commits into from
Jan 15, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
more fixes on comments
  • Loading branch information
Georgi Rusev authored and Georgi Rusev committed Jan 15, 2025
commit 40c49e345aef71b75e8aad6c25146145ec4ceb3f
55 changes: 55 additions & 0 deletions .github/workflows/analysis_workflow.yml
Original file line number Diff line number Diff line change
@@ -103,6 +103,61 @@ jobs:
python build_tooling/transform_asv_results.py --mode extract
python -m asv publish -v
python -m asv gh-pages -v --rewrite


run-asv-check-script:
timeout-minutes: 120
runs-on: ubuntu-latest
container: ghcr.io/man-group/arcticdb-dev:latest
env:
SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} # Setting this env var enables the caching
VCPKG_NUGET_USER: ${{secrets.VCPKG_NUGET_USER || github.repository_owner}}
VCPKG_NUGET_TOKEN: ${{secrets.VCPKG_NUGET_TOKEN || secrets.GITHUB_TOKEN}}
CMAKE_C_COMPILER_LAUNCHER: sccache
CMAKE_CXX_COMPILER_LAUNCHER: sccache
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
defaults:
run: {shell: bash}
steps:
- uses: actions/checkout@v3.3.0
with:
lfs: 'true'
fetch-depth: 0
submodules: recursive
token: ${{ secrets.ARCTICDB_TEST_PAT }}
ref: ${{ github.event.pull_request.head.sha }}

- name: Configure sccache
uses: mozilla-actions/sccache-action@v0.0.3
with:
version: "v0.4.0"

- name: Extra envs
shell: bash -l {0}
run: |
. build_tooling/vcpkg_caching.sh # Linux follower needs another call in CIBW
echo -e "VCPKG_BINARY_SOURCES=$VCPKG_BINARY_SOURCES
VCPKG_ROOT=$PLATFORM_VCPKG_ROOT" | tee -a $GITHUB_ENV
cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV
env:
CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}}

- name: Install ASV
shell: bash -el {0}
run: |
git config --global --add safe.directory .
python -m pip install --upgrade pip
pip install asv
asv machine --yes

- name: Build project for ASV
run: |
python -m pip install -ve .

- name: Run ASV Tests Check script
run: |
python python/utils/asv_checks.py
continue-on-error: false

# code_coverage:
# runs-on: "ubuntu-22.04"
79 changes: 0 additions & 79 deletions .github/workflows/asv_checks.yml

This file was deleted.

57 changes: 33 additions & 24 deletions python/utils/asv_checks.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
import hashlib
import logging
import os
import re
import subprocess
import sys
from typing import List


logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("ASV Linter")

def error(mes):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use logging not print statements in all PRs please

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will start using it primarily

print("-" * 80)
print(f"ERROR :{mes}", file=sys.stderr )
print(f"ERROR (same error printed on stdout also)): {mes}")
print("-" * 80)
logger.error("-" * 80)
logger.error(f"ERROR :{mes}")
logger.error("-" * 80)


def run_command(command: List[str], cwd: str, ok_errors_list: List[str] = None) -> int:
@@ -25,18 +28,18 @@ def run_command(command: List[str], cwd: str, ok_errors_list: List[str] = None)
err_output = result.stderr
error_code = result.returncode

if not err_output is None:
if err_output is not None:
error(err_output)
if error_code == 0:
print("ABOVE ERRORS DOES NOT AFFECT FINAL ERROR CODE = 0")
logger.info("ABOVE ERRORS DO NOT AFFECT FINAL ERROR CODE = 0")

if not output is None:
print("Standard Output:")
print(output)
if output is not None:
logger.info("Standard Output:")
logger.info(output)

if error_code != 0:
print(f"Error Code Returned: {error_code}")
if not ok_errors_list is None:
logger.error(f"Error Code Returned: {error_code}")
if ok_errors_list is not None:
for ok_error in ok_errors_list:
err_output.replace(ok_error, "")
err_output = err_output.strip()
@@ -67,7 +70,7 @@ def file_unchanged(filepath, last_check_time):

def get_project_root():
file_location = os.path.abspath(__file__)
return file_location.split("/python/benchmarks")[0]
return file_location.split("/python/utils")[0]


def perform_asv_checks() -> int:
@@ -85,29 +88,35 @@ def perform_asv_checks() -> int:
benchmark_config = f"{path}/python/.asv/results/benchmarks.json"
orig_hash = compute_file_hash(benchmark_config)

print("_" * 80)
print("""IMPORTANT: The tool checks CURRENT versions of asv tests along with asv.conf.json")
That means that if there are files that are not submitted yet,
they would need to be in order for completion of current PR""")
print("_" * 80)

print("\n\nCheck 1: Executing check for python code of asv tests")
logger.info("_" * 80)
logger.info("""IMPORTANT: The tool checks CURRENT ACTUAL versions of asv benchmark tests along with the one in benchmarks.json file.
That means that if there are files that are not submitted yet (tests and benchmark.json),
they would need to be in order for completion of current PR.
benchmarks.json is updated with a version number calculated as a hash
of the python test method. Thus any change of this method triggers different
version. Hence you would need to update json file also.
It happens automatically if you run following commandline:
> asv run --bench just-discover --python=same """)
logger.info("_" * 80)

logger.info("\n\nCheck 1: Executing check for python code of asv tests")
if run_command(["asv", "check", "--python=same"], path) != 0:
error("Please address all reported errors and submit code in the PR.")
err = 1
else:
print("Relax, no worries. Code is fine!")
logger.info("Relax, no worries. Code is fine!")


print("\n\nCheck 2: Check that benchmarks.json has up to date latest versions of tests.")
logger.info("\n\nCheck 2: Check that benchmarks.json has up to date latest versions of tests.")
if run_command(command = ["asv", "run", "--bench", "just-discover", "--python=same"],
cwd = path,
ok_errors_list = ["Couldn't load asv.plugins._mamba_helpers"]) != 0:
error("There was error getting latest benchmarks. See log")
err = 1
else:
if compute_file_hash(benchmark_config) == orig_hash:
print("Great, there are no new versions of asv test either!")
new_hash = compute_file_hash(benchmark_config)
if new_hash == orig_hash:
logger.info("Great, there are no new versions of asv test either!")
else:
error(f"""\n\n There are changes in asv test versions.
Open file {benchmark_config} compare with previous version and
@@ -121,4 +130,4 @@ def perform_asv_checks() -> int:
error("Errors detected - check output above")
sys.exit(res)
else:
print("SUCCESS! All checks pass")
logger.info("SUCCESS! All checks pass")
Loading