Add pre-commit and use ruff

.github/workflows/ci.yml

@@ -47,67 +47,15 @@
                 },
                 {
                     "name": "Set up packages",
-                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  flake8 pylint pylint-django pylint-plugin-utils isort black autopep8 \\\n  coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
+                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  pre-commit coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
                 },
                 {
                     "name": "Copy secret.py",
                     "run": "cp intranet/settings/ci_secret.py intranet/settings/secret.py"
                 },
                 {
-                    "name": "Run flake8",
-                    "run": "flake8 --max-line-length 150 --exclude=*/migrations/* intranet/ scripts/ docs/ *.py"
-                },
-                {
-                    "name": "Run pylintx",
-                    "run": "pylint --jobs=0 --disable=fixme,broad-exception-caught,broad-exception-raised,unsupported-binary-operation,global-statement,attribute-defined-outside-init,cyclic-import,consider-using-f-string --django-settings-module=intranet.settings intranet/"
-                }
-            ]
-        },
-        "formatting": {
-            "runs-on": "ubuntu-latest",
-            "strategy": {
-                "matrix": {
-                    "python-version": [
-                        3.8
-                    ]
-                },
-                "fail-fast": false
-            },
-            "steps": [
-                {
-                    "name": "Set up repo",
-                    "uses": "actions/checkout@v2"
-                },
-                {
-                    "name": "Set up Python ${{ matrix.python-version }}",
-                    "uses": "actions/setup-python@v2",
-                    "with": {
-                        "python-version": "${{ matrix.python-version }}"
-                    }
-                },
-                {
-                    "name": "Set up pip cache",
-                    "uses": "actions/cache@v2",
-                    "with": {
-                        "path": "~/.cache/pip",
-                        "key": "pip-${{ matrix.python-version }}"
-                    }
-                },
-                {
-                    "name": "Set up packages",
-                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  flake8 pylint pylint-django pylint-plugin-utils isort black autopep8 \\\n  coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
-                },
-                {
-                    "name": "Copy secret.py",
-                    "run": "cp intranet/settings/ci_secret.py intranet/settings/secret.py"
-                },
-                {
-                    "name": "Format code",
-                    "run": "./scripts/build_ensure_no_changes.sh ./scripts/format.sh"
-                },
-                {
-                    "name": "Format static files and templates",
-                    "run": "./scripts/build_ensure_no_changes.sh ./scripts/static_templates_format.sh"
+                    "name": "Run pre-commit",
+                    "run": "pre-commit run --all-files"
                 }
             ]
         },
@@ -146,7 +94,7 @@
                 },
                 {
                     "name": "Set up packages",
-                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  flake8 pylint pylint-django pylint-plugin-utils isort black autopep8 \\\n  coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
+                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  pre-commit coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
                 },
                 {
                     "name": "Copy secret.py",
@@ -251,7 +199,7 @@
                 },
                 {
                     "name": "Set up packages",
-                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  flake8 pylint pylint-django pylint-plugin-utils isort black autopep8 \\\n  coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
+                    "run": "set -e\n\npip install -U pip setuptools wheel\npip install -U \\\n  pre-commit coveralls pyyaml pytest-django\npip install -U -r requirements.txt\n\necho \"PATH=$PATH\" >> $GITHUB_ENV\n"
                 },
                 {
                     "name": "Copy secret.py",
@@ -328,7 +276,6 @@
         "finish_success": {
             "needs": [
                 "linting",
-                "formatting",
                 "build",
                 "tests"
             ],

.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+default_stages: [commit, push]
+fail_fast: false
+exclude: ^(intranet/middleware|intranet/apps/.*/migrations|intranet/static/.*vendor/.*|intranet/.*/stubs|Ion.egg-info|\.github)
+
+# This is for pre-commit ci
+# After adding pre-commit ci, the "linting" stage of the tests
+# can be safely deleted
+ci:
+  autofix_commit_msg: |
+    chore: auto fixes from pre-commit hooks
+
+    for more information, see https://pre-commit.ci
+  autoupdate_commit_msg: 'build: pre-commit autoupdate'
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-ast
+        name: Validate Python
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+      - id: check-toml
+      - id: check-yaml
+      - id: detect-private-key
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
+    hooks:
+      - id: ruff
+        args: [ "--fix", "--exit-non-zero-on-fix" ]
+        name: ruff lint
+        files: ^intranet/apps/.*
+      - id: ruff-format
+        files: ^intranet/apps/.*
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.5
+    hooks:
+      - id: codespell
+        files: ^.*\.(py|md|rst)$
+        # TODO: Remove after python version >= 3.11
+        additional_dependencies:
+          - tomli

Ion.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@
 .gitattributes
 .gitignore
 .isort.cfg
+.pre-commit-config.yaml
 .pylintrc
 .style.yapf
 CONTRIBUTING.md

ci/spec.yml

@@ -57,8 +57,7 @@ env:
 
       pip install -U pip setuptools wheel
       pip install -U \
-        flake8 pylint pylint-django pylint-plugin-utils isort black autopep8 \
-        coveralls pyyaml pytest-django
+        pre-commit coveralls pyyaml pytest-django
       pip install -U -r requirements.txt
 
       echo "PATH=$PATH" >> $GITHUB_ENV
@@ -83,30 +82,8 @@ jobs:
       - *setup_packages
       - *copy_secret
 
-      - name: Run flake8
-        run: 'flake8 --max-line-length 150 --exclude=*/migrations/* intranet/ scripts/ docs/ *.py'
-      - name: Run pylintx
-        run: pylint --jobs=0 --disable=fixme,broad-exception-caught,broad-exception-raised,unsupported-binary-operation,global-statement,attribute-defined-outside-init,cyclic-import,consider-using-f-string --django-settings-module=intranet.settings intranet/
-
-  formatting:
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        python-version: *python_versions
-      fail-fast: false
-
-    steps:
-      - *repo_setup
-      - *python_setup
-      - *setup_pip_cache
-      - *setup_packages
-      - *copy_secret
-
-      - name: Format code
-        run: ./scripts/build_ensure_no_changes.sh ./scripts/format.sh
-      - name: Format static files and templates
-        run: ./scripts/build_ensure_no_changes.sh ./scripts/static_templates_format.sh
+      - name: Run pre-commit
+        run: 'pre-commit run --all-files'
 
   build:
     runs-on: ubuntu-latest
@@ -256,7 +233,6 @@ jobs:
   finish_success:
     needs:
       - linting
-      - formatting
       - build
       - tests
 

intranet/apps/announcements/notifications.py

@@ -3,14 +3,13 @@
 import re
 
 import requests
-from requests_oauthlib import OAuth1
-from sentry_sdk import capture_exception
-
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth import get_user_model
 from django.core import exceptions
 from django.urls import reverse
+from requests_oauthlib import OAuth1
+from sentry_sdk import capture_exception
 
 from ...utils.date import get_senior_graduation_year
 from ..notifications.tasks import email_send_task

intranet/apps/announcements/views.py

@@ -15,8 +15,13 @@
 from ..groups.models import Group
 from .forms import AnnouncementAdminForm, AnnouncementEditForm, AnnouncementForm, AnnouncementRequestForm
 from .models import Announcement, AnnouncementRequest
-from .notifications import (admin_request_announcement_email, announcement_approved_email, announcement_posted_email, announcement_posted_twitter,
-                            request_announcement_email)
+from .notifications import (
+    admin_request_announcement_email,
+    announcement_approved_email,
+    announcement_posted_email,
+    announcement_posted_twitter,
+    request_announcement_email,
+)
 
 logger = logging.getLogger(__name__)
 

intranet/apps/api/authentication.py

@@ -1,7 +1,6 @@
-from rest_framework import authentication, exceptions
-
 from django.contrib import auth
 from django.views.decorators.debug import sensitive_variables
+from rest_framework import authentication, exceptions
 
 
 class ApiBasicAuthentication(authentication.BasicAuthentication):

intranet/apps/api/tests.py

@@ -2,12 +2,11 @@
 import json
 import urllib.parse
 
-from oauth2_provider.models import AccessToken, get_application_model
-from oauth2_provider.settings import oauth2_settings
-
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 from django.utils import timezone
+from oauth2_provider.models import AccessToken, get_application_model
+from oauth2_provider.settings import oauth2_settings
 
 from ...test.ion_test import IonTestCase
 from ...utils.date import get_senior_graduation_year

intranet/apps/api/utils.py

@@ -1,12 +1,11 @@
 import logging
 
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
 from rest_framework import status
 from rest_framework.response import Response
 from rest_framework.views import exception_handler
 
-from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
-
 from ..eighth import exceptions as eighth_exceptions
 
 logger = logging.getLogger(__name__)

intranet/apps/api/views.py

@@ -1,11 +1,10 @@
 from collections import OrderedDict
 
+from django.urls import reverse
 from rest_framework.decorators import api_view, permission_classes
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 
-from django.urls import reverse
-
 
 def perma_reverse(request, view, *args, **kwargs):
     return request.build_absolute_uri(reverse(view, *args, **kwargs))

intranet/apps/auth/backends.py

@@ -2,11 +2,10 @@
 import logging
 
 import pam
-from prometheus_client import Counter, Summary
-
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.hashers import check_password
+from prometheus_client import Counter, Summary
 
 logger = logging.getLogger(__name__)
 

intranet/apps/auth/helpers.py

@@ -2,7 +2,6 @@
 import re
 
 import pexpect
-
 from django.conf import settings
 
 logger = logging.getLogger(__name__)

intranet/apps/auth/management/commands/grant_admin.py

@@ -12,6 +12,6 @@ def add_arguments(self, parser):
         parser.add_argument("admin_group")
 
     def handle(self, *args, **options):
-        g = Group.objects.get_or_create(name="admin_%s" % options["admin_group"])[0]
+        g = Group.objects.get_or_create(name=f"admin_{options['admin_group']}")[0]
         get_user_model().objects.get_or_create(username=options["username"])[0].groups.add(g)
-        self.stdout.write("Added %s to %s" % (options["username"], options["admin_group"]))
+        self.stdout.write(f"Added {options['username']} to {options['admin_group']}")

intranet/apps/auth/views.py

@@ -6,7 +6,6 @@
 from typing import Container, Tuple
 
 from dateutil.relativedelta import relativedelta
-
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth import authenticate, get_user_model, login, logout
@@ -29,6 +28,7 @@
 from ..events.models import Event
 from ..schedule.views import schedule_context
 from ..sessionmgmt.helpers import trust_session
+
 # Load these so the Prometheus metrics get added
 from . import backends, signals  # pylint: disable=unused-import # noqa F401
 from .forms import AuthenticateForm

intranet/apps/bus/consumers.py

@@ -2,7 +2,6 @@
 
 from asgiref.sync import async_to_sync
 from channels.generic.websocket import JsonWebsocketConsumer
-
 from django.conf import settings
 from django.utils import timezone
 

intranet/apps/cslapps/views.py

@@ -1,8 +1,7 @@
-from oauth2_provider.models import AccessToken
-
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.shortcuts import redirect
+from oauth2_provider.models import AccessToken
 
 from ..auth.decorators import deny_restricted
 from .models import App

intranet/apps/dataimport/management/commands/import_photos.py

@@ -4,10 +4,9 @@
 import sys
 from pathlib import Path
 
-from PIL import Image
-
 from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
+from PIL import Image
 
 from intranet.apps.users.models import Photo
 

intranet/apps/dataimport/management/commands/import_tj_star.py

@@ -27,7 +27,7 @@ def add_arguments(self, parser):
         parser.add_argument("filename")
 
     def handle(self, *args, **options):
-        with open(options["filename"], "r", encoding="utf-8") as csv_file:
+        with open(options["filename"], encoding="utf-8") as csv_file:
             data = csv.DictReader(csv_file)
 
             for idx, row in enumerate(data):

intranet/apps/dataimport/tests.py

@@ -3,7 +3,6 @@
 from unittest.mock import mock_open, patch
 
 import pytz
-
 from django.contrib.auth import get_user_model
 from django.core.management import CommandError, call_command
 from django.utils import timezone

intranet/apps/eighth/admin.py

@@ -1,6 +1,5 @@
-from simple_history.admin import SimpleHistoryAdmin
-
 from django.contrib import admin
+from simple_history.admin import SimpleHistoryAdmin
 
 from .models import EighthActivity, EighthBlock, EighthRoom, EighthScheduledActivity, EighthSignup, EighthSponsor
 

intranet/apps/eighth/exceptions.py

@@ -1,8 +1,7 @@
 from collections import namedtuple
 
-from rest_framework.status import HTTP_403_FORBIDDEN
-
 from django.http import HttpResponse
+from rest_framework.status import HTTP_403_FORBIDDEN
 
 m = namedtuple("Message", ["regular", "admin"])