Update to v1.1.2

Makefile

@@ -1,5 +1,12 @@
 NAME=portstreelint
-SOURCES=src/${NAME}/__init__.py src/${NAME}/main.py src/${NAME}/library.py
+SOURCES=src/${NAME}/__init__.py src/${NAME}/main.py src/${NAME}/library.py \
+        src/${NAME}/load_data.py src/${NAME}/check_categories.py src/${NAME}/check_comment.py \
+        src/${NAME}/check_description_file.py src/${NAME}/check_installation_prefix.py \
+        src/${NAME}/check_maintainer.py src/${NAME}/check_marks.py src/${NAME}/check_plist.py \
+        src/${NAME}/check_port_path.py src/${NAME}/check_unchanging_ports.py \
+        src/${NAME}/check_vulnerabilities.py src/${NAME}/check_www_site.py \
+        src/${NAME}/show_categories.py src/${NAME}/show_maintainers.py \
+        src/${NAME}/show_notifications.py src/${NAME}/show_summary.py
 
 # Default action is to show this help message:
 .help:

README.md

@@ -36,8 +36,8 @@ portstreelint - FreeBSD ports tree lint
 \[--\]
 
 ## DESCRIPTION
-The **portstreelint** utility checks the FreeBSD port tree Index
-and some part of the port's Makefiles for errors and warnings.
+The **portstreelint** utility checks the FreeBSD ports tree Index
+and some parts of the port's Makefiles for errors and warnings.
 
 By default it will scan the whole port tree, but you can select
 a subset of the ports with the options *--cat|-c* for categories,

TODO.md

@@ -1,14 +1,11 @@
 # portstreelint TODOLIST
 
 ## Planned changes
-* Next version: splitting library.py code into multiple files for modularity
-* Version +2: checking distfiles availability
+* Next version: checking distfiles availability
 
 ## Probable evolutions
 
 ## Possible evolutions
-* Providing a mechanism to keep track of external events
-  * For example, the INDEX:www-site being unavailable since date X
 * Having a configuration file
   * Limits setting
   * Checks disabling
@@ -22,6 +19,8 @@
 
 ## Unprobable evolutions
 * Adding an option to notice port maintainers by email => will be better done by a wrapping script using the per-maintainer output
+* Providing a mechanism to keep track of external events => will be better done by a wrapping script using the per-maintainer output
+  * For example, the INDEX:www-site being unavailable since date X
 * Side functionalities => not the direct purpose of this tool
   * Checking reported vulnerabilities in dependencies
   * Checking unavailable ports in dependencies

man/portstreelint.8

@@ -27,8 +27,8 @@
 .Sh DESCRIPTION
 The
 .Nm
-utility checks the FreeBSD port tree Index
-and some part of the port's Makefiles for errors and warnings.
+utility checks the FreeBSD ports tree Index
+and some parts of the port's Makefiles for errors and warnings.
 .Pp
 By default it will scan the whole port tree, but you can select
 a subset of the ports with the options

setup.cfg

@@ -3,7 +3,7 @@ name = pnu-portstreelint
 description = FreeBSD ports tree lint
 long_description = file: README.md
 long_description_content_type = text/markdown
-version = 1.1.1
+version = 1.1.2
 license = BSD 3-Clause License
 license_files = License
 author = Hubert Tournier

src/portstreelint/check_categories.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import logging
+
+from .library import counters, notify_maintainer
+
+OFFICIAL_CATEGORIES = [
+    "accessibility", "afterstep", "arabic", "archivers", "astro", "audio", "benchmarks", "biology",
+    "cad", "chinese", "comms", "converters", "databases", "deskutils", "devel", "dns", "docs",
+    "editors", "education", "elisp", "emulators", "enlightenment", "finance", "french", "ftp",
+    "games", "geography", "german", "gnome", "gnustep", "graphics", "hamradio", "haskell", "hebrew",
+    "hungarian", "irc", "japanese", "java", "kde", "kde-applications", "kde-frameworks",
+    "kde-plasma", "kld", "korean", "lang", "linux", "lisp", "mail", "mate", "math", "mbone", "misc",
+    "multimedia", "net", "net-im", "net-mgmt", "net-p2p", "net-vpn", "news", "parallel", "pear",
+    "perl5", "plan9", "polish", "ports-mgmt", "portuguese", "print", "python", "ruby", "rubygems",
+    "russian", "scheme", "science", "security", "shells", "spanish", "sysutils", "tcl", "textproc",
+    "tk", "ukrainian", "vietnamese", "wayland", "windowmaker", "www", "x11", "x11-clocks",
+    "x11-drivers", "x11-fm", "x11-fonts", "x11-servers", "x11-themes", "x11-toolkits", "x11-wm",
+    "xfce", "zope",
+]
+
+####################################################################################################
+def check_categories(ports):
+    """ Cross-checks the categories field with the Makefile and compliance with rules
+    Rules at https://docs.freebsd.org/en/books/porters-handbook/makefiles/#makefile-categories-definition
+    """
+    for name, port in ports.items():
+        for category in port["categories"].split():
+            if category not in OFFICIAL_CATEGORIES:
+                logging.warning("Unofficial category '%s' in Index for port %s", category, name)
+                counters["Unofficial categories"] += 1
+                notify_maintainer(port["maintainer"], "Unofficial category", name)
+
+        if "CATEGORIES" in port:
+            if '$' in port["CATEGORIES"]:
+                continue # don't try to resolve embedded variables. Ignore check
+
+            if port["categories"] != port["CATEGORIES"]:
+                logging.error("Diverging categories between Index and Makefile for port %s", name)
+                logging.error("... Index:categories    '%s'", port["categories"])
+                logging.error("... Makefile:CATEGORIES '%s'", port["CATEGORIES"])
+                counters["Diverging categories"] += 1
+                notify_maintainer(port["maintainer"], "Diverging categories", name)
+
+            for category in port["CATEGORIES"].split():
+                if category not in OFFICIAL_CATEGORIES:
+                    logging.warning("Unofficial category '%s' in Makefile for port %s", category, name)
+                    counters["Unofficial categories"] += 1
+                    notify_maintainer(port["maintainer"], "Unofficial category", name)
+
+    logging.info("Found %d ports with unofficial categories", counters["Unofficial categories"])
+    logging.info("Found %d ports with diverging categories", counters["Diverging categories"])

src/portstreelint/check_comment.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import logging
+
+from .library import counters, notify_maintainer
+
+####################################################################################################
+def check_comment(ports):
+    """ Cross-checks the comment field with the Makefile and compliance with rules
+    Rules at https://docs.freebsd.org/en/books/porters-handbook/makefiles/#makefile-comment
+    """
+    for name, port in ports.items():
+        if len(port["comment"]) > 70:
+            logging.warning("Over 70 characters comment for port %s", name)
+            counters["Too long comments"] += 1
+            notify_maintainer(port["maintainer"], "Too long comments", name)
+
+        if 'a' <= port["comment"][0] <= 'z':
+            logging.error("Uncapitalized comment for port %s", name)
+            counters["Uncapitalized comments"] += 1
+            notify_maintainer(port["maintainer"], "Uncapitalized comments", name)
+
+        if port["comment"].endswith('.'):
+            logging.error("Dot-ended comment for port %s", name)
+            counters["Dot-ended comments"] += 1
+            notify_maintainer(port["maintainer"], "Dot-ended comments", name)
+
+        if "COMMENT" in port:
+            if '$' in port["COMMENT"]:
+                continue # don't try to resolve embedded variables. Ignore check
+
+            # Do not take into escaping backslashes which are used inconsistently
+            # in both fields
+            if port["comment"].replace("\\", "") != port["COMMENT"].replace("\\", ""):
+                logging.error("Diverging comments between Index and Makefile for port %s", name)
+                logging.error("... Index:comment    '%s'", port["comment"])
+                logging.error("... Makefile:COMMENT '%s'", port["COMMENT"])
+                counters["Diverging comments"] += 1
+                notify_maintainer(port["maintainer"], "Diverging comments", name)
+
+    logging.info("Found %d ports with too long comments", counters["Too long comments"])
+    logging.info("Found %d ports with uncapitalized comments", counters["Uncapitalized comments"])
+    logging.info("Found %d ports with dot-ended comments", counters["Dot-ended comments"])
+    logging.info("Found %d ports with diverging comments", counters["Diverging comments"])

src/portstreelint/check_description_file.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import logging
+import os
+
+from .library import counters, notify_maintainer
+
+####################################################################################################
+def check_description_file(ports):
+    """ Checks the description-file field consistency and existence """
+    for name, port in ports.items():
+        nonexistent = False
+        if not port["description-file"].startswith(port["port-path"]):
+            if not os.path.isfile(port["description-file"]):
+                nonexistent = True
+        elif not os.path.isdir(port["port-path"]):
+            pass # already reported
+        elif not os.path.isfile(port["description-file"]):
+            nonexistent = True
+
+        if nonexistent:
+            logging.error("Nonexistent description-file '%s' for port %s", port["description-file"], name)
+            counters["Nonexistent description-file"] += 1
+            notify_maintainer(port["maintainer"], "Nonexistent description-file", name)
+        else:
+            try:
+                with open(port["description-file"], encoding="utf-8", errors="ignore") as file:
+                    lines = file.read().splitlines()
+            except:
+                lines = []
+
+            if lines:
+                if lines[-1].strip().startswith("https://") or lines[-1].strip().startswith("http://"):
+                    logging.error("URL '%s' ending description-file for port %s", lines[-1].strip(), name)
+                    counters["URL ending description-file"] += 1
+                    notify_maintainer(port["maintainer"], "URL ending description-file", name)
+                    del lines[-1]
+
+                text = " ".join(lines)
+                text = text.strip()
+                if port["comment"] == text:
+                    logging.error("description-file content is identical to comment for port %s", name)
+                    counters["description-file same as comment"] += 1
+                    notify_maintainer(port["maintainer"], "description-file same as comment", name)
+                elif len(text) <= len(port["comment"]):
+                    logging.error("description-file content is no longer than comment for port %s", name)
+                    counters["Too short description-file"] += 1
+                    notify_maintainer(port["maintainer"], "Too short description-file", name)
+
+    logging.info("Found %d ports with nonexistent description-file", counters["Nonexistent description-file"])
+    logging.info("Found %d ports with URL ending description-file", counters["URL ending description-file"])
+    logging.info("Found %d ports with description-file identical to comment", counters["description-file same as comment"])
+    logging.info("Found %d ports with too short description-file", counters["Too short description-file"])

src/portstreelint/check_installation_prefix.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import logging
+
+from .library import counters, notify_maintainer
+
+####################################################################################################
+def check_installation_prefix(ports):
+    """ Checks the installation-prefix field usualness """
+    for name, port in ports.items():
+        if port["installation-prefix"] == "/usr/local":
+            pass
+        elif port["installation-prefix"] == "/compat/linux" and name.startswith("linux"):
+            pass
+        elif port["installation-prefix"] == "/usr/local/FreeBSD_ARM64" and "-aarch64-" in name:
+            pass
+        elif port["installation-prefix"].startswith("/usr/local/android") and "droid" in name:
+            pass
+        elif port["installation-prefix"] == "/var/qmail" and "qmail" in name or name.startswith("queue-fix"):
+            pass
+        elif port["installation-prefix"] == "/usr" and name.startswith("global-tz-") or name.startswith("zoneinfo-"):
+            pass
+        else:
+            logging.warning("Unusual installation-prefix '%s' for port %s", port["installation-prefix"], name)
+            counters["Unusual installation-prefix"] += 1
+            notify_maintainer(port["maintainer"], "Unusual installation-prefix", name)
+
+    logging.info("Found %d ports with unusual installation-prefix", counters["Unusual installation-prefix"])

src/portstreelint/check_maintainer.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import logging
+
+from .library import counters, notify_maintainer
+
+####################################################################################################
+def check_maintainer(ports):
+    """ Cross-checks the maintainer field with the Makefile and compliance with rules
+    Rules at https://docs.freebsd.org/en/books/porters-handbook/makefiles/#makefile-maintainer
+    """
+    for name, port in ports.items():
+        if "MAINTAINER" in port:
+            if '$' in port["MAINTAINER"]:
+                continue # don't try to resolve embedded variables. Ignore check
+
+            if port["maintainer"] != port["MAINTAINER"].lower():
+                logging.error("Diverging maintainers between Index and Makefile for port %s", name)
+                logging.error("... Index:maintainer    '%s'", port["maintainer"])
+                logging.error("... Makefile:MAINTAINER '%s'", port["MAINTAINER"])
+                counters["Diverging maintainers"] += 1
+                notify_maintainer(port["maintainer"], "Diverging maintainers", name)
+                notify_maintainer(port["MAINTAINER"], "Diverging maintainers", name)
+
+    logging.info("Found %d ports with diverging maintainers", counters["Diverging maintainers"])

src/portstreelint/check_marks.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import datetime
+import logging
+
+from .library import counters, notify_maintainer
+
+####################################################################################################
+def check_marks(ports, limits):
+    """ Checks the existence of special marks variables (ie. BROKEN, etc.) in Makefiles """
+    for name, port in ports.items():
+        today = datetime.datetime.now(datetime.timezone.utc)
+
+        if "BROKEN" in port:
+            if port["Last modification"] < today - datetime.timedelta(days=limits["BROKEN since"]):
+                logging.warning("BROKEN mark '%s' for port %s", port["BROKEN"], name)
+                counters["Marked as BROKEN for too long"] += 1
+                notify_maintainer(port["maintainer"], "Marked as BROKEN for too long", name)
+            else:
+                logging.info("BROKEN mark '%s' for port %s", port["BROKEN"], name)
+                counters["Marked as BROKEN"] += 1
+                notify_maintainer(port["maintainer"], "Marked as BROKEN", name)
+
+        if "DEPRECATED" in port:
+            if port["Last modification"] < today - datetime.timedelta(days=limits["DEPRECATED since"]):
+                logging.warning("DEPRECATED mark '%s' for port %s", port["DEPRECATED"], name)
+                counters["Marked as DEPRECATED for too long"] += 1
+                notify_maintainer(port["maintainer"], "Marked as DEPRECATED for too long", name)
+            else:
+                logging.info("DEPRECATED mark '%s' for port %s", port["DEPRECATED"], name)
+                counters["Marked as DEPRECATED"] += 1
+                notify_maintainer(port["maintainer"], "Marked as DEPRECATED", name)
+
+        if "FORBIDDEN" in port:
+            if port["Last modification"] < today - datetime.timedelta(days=limits["FORBIDDEN since"]):
+                logging.warning("FORBIDDEN mark '%s' for port %s", port["FORBIDDEN"], name)
+                counters["Marked as FORBIDDEN for too long"] += 1
+                notify_maintainer(port["maintainer"], "Marked as FORBIDDEN for too long", name)
+            else:
+                logging.info("FORBIDDEN mark '%s' for port %s", port["FORBIDDEN"], name)
+                counters["Marked as FORBIDDEN"] += 1
+                notify_maintainer(port["maintainer"], "Marked as FORBIDDEN", name)
+
+        if "IGNORE" in port:
+            logging.info("IGNORE mark '%s' for port %s", port["IGNORE"], name)
+            counters["Marked as IGNORE"] += 1
+            notify_maintainer(port["maintainer"], "Containing an IGNORE mark", name)
+
+        if "RESTRICTED" in port:
+            logging.info("RESTRICTED mark '%s' for port %s", port["RESTRICTED"], name)
+            counters["Marked as RESTRICTED"] += 1
+            notify_maintainer(port["maintainer"], "Marked as RESTRICTED", name)
+
+        if "EXPIRATION_DATE" in port:
+            logging.warning("EXPIRATION_DATE mark '%s' for port %s", port["EXPIRATION_DATE"], name)
+            counters["Marked with EXPIRATION_DATE"] += 1
+            notify_maintainer(port["maintainer"], "Marked with EXPIRATION_DATE", name)
+
+    logging.info("Found %d ports marked as BROKEN", counters["Marked as BROKEN"])
+    logging.info("Found %d ports marked as BROKEN for too long", counters["Marked as BROKEN for too long"])
+    logging.info("Found %d ports marked as DEPRECATED", counters["Marked as DEPRECATED"])
+    logging.info("Found %d ports marked as DEPRECATED for too long", counters["Marked as DEPRECATED for too long"])
+    logging.info("Found %d ports marked as FORBIDDEN", counters["Marked as FORBIDDEN"])
+    logging.info("Found %d ports marked as FORBIDDEN for too long", counters["Marked as FORBIDDEN for too long"])
+    logging.info("Found %d ports marked as IGNORE", counters["Marked as IGNORE"])
+    logging.info("Found %d ports marked as RESTRICTED", counters["Marked as RESTRICTED"])
+    logging.info("Found %d ports marked with EXPIRATION_DATE", counters["Marked with EXPIRATION_DATE"])