op.sh: lint (#33217)

* redo

* better

* fix this

* clean everything

* all files

* test

* debug

* get info

* revert

* only good files

* allow skip

* also this

* help section

.github/workflows/selfdrive_tests.yaml

@@ -61,13 +61,13 @@ jobs:
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
                         MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
-    - name: static analysis
+    - name: Static analysis
       timeout-minutes: 1
       run: |
         cd $GITHUB_WORKSPACE
         cp pyproject.toml $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "scripts/lint.sh"
+        ${{ env.RUN }} "scripts/lint.sh --skip check_added_large_files"
 
   build:
     strategy:
@@ -140,16 +140,15 @@ jobs:
     runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                    ((github.event_name != 'pull_request') ||
                     (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    env:
+      PYTHONWARNINGS: default
     steps:
     - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: static analysis
+    - name: Setup
+      run: tools/op.sh setup
+    - name: Static analysis
       timeout-minutes: 1
-      run: ${{ env.RUN }} "scripts/lint.sh"
+      run: tools/op.sh lint
 
   unit_tests:
     name: unit tests

pyproject.toml

@@ -88,6 +88,7 @@ testing = [
   "pytest-repeat",
   "ruff",
   "codespell",
+  "pre-commit-hooks",
 ]
 
 dev = [
@@ -169,17 +170,19 @@ quiet-level = 3
 # if you've got a short variable name that's getting flagged, add it here
 ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead"
 builtin = "clear,rare,informal,code,names,en-GB_to_en-US"
-skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, ./selfdrive/ui/translations/*.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*"
+skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, *.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*"
 
 [tool.mypy]
 python_version = "3.11"
 plugins = [
   "numpy.typing.mypy_plugin",
 ]
 exclude = [
-  "body/",
   "cereal/",
+  "msgq/",
+  "msgq_repo/",
   "opendbc/",
+  "opendbc_repo/",
   "panda/",
   "rednose/",
   "rednose_repo/",
@@ -204,6 +207,10 @@ warn_return_any=true
 # allow implicit optionals for default args
 implicit_optional = true
 
+local_partial_types=true
+explicit_package_bases=true
+disable_error_code = "annotation-unchecked"
+
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
 indent-width = 2

scripts/lint.sh

@@ -1,11 +1,99 @@
 #!/bin/bash
 set -e
 
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+UNDERLINE='\033[4m'
+BOLD='\033[1m'
+NC='\033[0m'
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 cd $DIR/../
 
-# TODO: bring back rest of pre-commit checks:
-# https://github.com/commaai/openpilot/blob/4b11c9e914707df9def598616995be2a5d355a6a/.pre-commit-config.yaml#L2
+FAILED=0
+
+IGNORED_FILES="uv\.lock|docs\/CARS.md"
+IGNORED_DIRS="^third_party.*|^msgq.*|^msgq_repo.*|^opendbc.*|^opendbc_repo.*|^cereal.*|^panda.*|^rednose.*|^rednose_repo.*|^tinygrad.*|^tinygrad_repo.*|^teleoprtc.*|^teleoprtc_repo.*"
+
+function run() {
+  shopt -s extglob
+  case $1 in
+    $SKIP ) return 0 ;;
+  esac
+
+  echo -en "$1"
+
+  for ((i=0; i<$((50 - ${#1})); i++)); do
+    echo -n "."
+  done
+
+  shift 1;
+  CMD="$@"
+
+  set +e
+  log="$((eval "$CMD" ) 2>&1)"
+
+  if [[ $? -eq 0 ]]; then
+    echo -e "[${GREEN}✔${NC}]"
+  else
+    echo -e "[${RED}✗${NC}]"
+    echo "$log"
+    FAILED=1
+  fi
+  set -e
+}
+
+function run_tests() {
+  ALL_FILES=$1
+  PYTHON_FILES=$2
+
+  run "ruff" ruff check $PYTHON_FILES --quiet
+  run "lint-imports" lint-imports
+  run "check_added_large_files" python3 -m pre_commit_hooks.check_added_large_files --enforce-all $ALL_FILES --maxkb=120
+  run "check_shebang_scripts_are_executable" python3 -m pre_commit_hooks.check_shebang_scripts_are_executable $ALL_FILES
+
+  if [[ -z "$FAST" ]]; then
+    run "mypy" mypy $PYTHON_FILES
+    run "codespell" codespell $ALL_FILES
+  fi
+
+  return $FAILED
+}
+
+function help() {
+  echo "A fast linter"
+  echo ""
+  echo -e "${BOLD}${UNDERLINE}Usage:${NC} op lint [OPTIONS]"
+  echo ""
+  echo -e "${BOLD}${UNDERLINE}Options:${NC}"
+  echo -e "  ${BOLD}-f, --fast${NC}"
+  echo "          Skip slow tests"
+  echo -e "  ${BOLD}-s, --skip${NC}"
+  echo "          Specify a test to skip"
+  echo ""
+  echo -e "${BOLD}${UNDERLINE}Examples:${NC}"
+  echo "  op lint --skip mypy --skip ruff"
+  echo "          Skip the ruff and mypy check"
+}
+
+while true; do
+  case $1 in
+    -f | --fast ) shift 1; FAST="1" ;;
+    -s | --skip ) shift 1; SKIP+="$1|"; shift 1 ;;
+    -h | --help | -help | --h ) help; exit 0 ;;
+    * )           break ;;
+  esac
+done
+
+SKIP="@($(echo $SKIP | sed 's/|$//'))"
+
+GIT_FILES="$(git ls-files | sed -E "s/$IGNORED_FILES|$IGNORED_DIRS//g")"
+ALL_FILES=""
+for f in $GIT_FILES; do
+  if [[ -f $f ]]; then
+    ALL_FILES+="$f"$'\n'
+  fi
+done
+PYTHON_FILES=$(echo "$ALL_FILES" | grep --color=never '.py$' || true)
 
-ruff check .
-lint-imports
+run_tests "$ALL_FILES" "$PYTHON_FILES"