Python: drop <3.11 support (ivre#1670)

* Python: drop <3.11 support * CI/Tests: update value * Linting: switch to py311 for Black * CI/mypy: remove no longer needed "type: ignore" comments
MikeoPerfect · Oct 15, 2024 · aa48cdf · aa48cdf
1 parent e78e0b6
commit aa48cdf
Show file tree

Hide file tree

Showing 89 changed files with 547 additions and 501 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -80,7 +80,7 @@ dependency makes it harder for users to install IVRE.
 
 We try to comply with the some guidelines for new code:
 
--   The code **must** be compatible with Python 3.7 to 3.12.
+-   The code **must** be compatible with Python 3.11 to 3.12.
 
 -   The code should be PEP-8 compliant; you can check your code with
     [pep8](https://pypi.python.org/pypi/pep8).

diff --git a/.github/workflows/elastic.yml b/.github/workflows/elastic.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.12']
+        python-version: ['3.11', '3.12']
         mongodb-version: ['5.0']
         elastic-version: ['7.17.22', '8.0.1', '8.14.1']
 

diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -46,7 +46,7 @@ jobs:
 
     - run: mv ivre ivre_bak
 
-    - run: black -t py37 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
+    - run: black -t py311 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
 
     - run: bandit --severity-level high -r ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
 

diff --git a/.github/workflows/maxmind.yml b/.github/workflows/maxmind.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11', '3.12']
 
     steps:
 

diff --git a/.github/workflows/mongodb.yml b/.github/workflows/mongodb.yml
@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         # https://docs.mongodb.com/drivers/pymongo/#compatibility
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11', '3.12']
         mongodb-version: ['3.6', '5.0']
 
     steps:

diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.12']
+        python-version: ['3.11', '3.12']
         postgresql-version: ['12', '13']
 
     services:

diff --git a/.github/workflows/sqlite.yml b/.github/workflows/sqlite.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.12']
+        python-version: ['3.11', '3.12']
 
     steps:
 

diff --git a/.github/workflows/tinydb.yml b/.github/workflows/tinydb.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11', '3.12']
 
     steps:
 
@@ -40,10 +40,6 @@ jobs:
 
     - run: pip install -r requirements-tinydb.txt
 
-    # this should be a dependency of tinydb
-    - run: pip install typing_extensions
-      if: ${{ matrix.python-version <= 3.7 }}
-
     - name: Install IVRE
       uses: ./.github/actions/install
 

diff --git a/doc/dev/tests.rst b/doc/dev/tests.rst
@@ -87,6 +87,6 @@ GitHub actions
 
 Tests are run with several MongoDB and PostgreSQL versions, as well as
 TinyDB, SQLite and Elasticsearch for each pull requests. The tests run
-with Python 3.7 to 3.12.
+with Python 3.11 to 3.12.
 
 The configurations are in the `.github/workflows/*.yml` YAML files.
diff --git a/doc/install/installation.rst b/doc/install/installation.rst
@@ -40,7 +40,7 @@ your needs:
 Python
 ~~~~~~
 
-To install IVRE, you'll need `Python <http://www.python.org/>`__ 3.7
+To install IVRE, you'll need `Python <http://www.python.org/>`__ 3.11
 minimum, with the following modules:
 
 - `bottle <https://bottlepy.org/>`_.

diff --git a/ivre/active/data.py b/ivre/active/data.py
@@ -893,8 +893,7 @@ def create_http_ls(data: bytes, volname: str = "???") -> Optional[NmapScript]:
     results.
 
     """
-    match = _EXPR_INDEX_OF.search(data)
-    if match is None:
+    if (match := _EXPR_INDEX_OF.search(data)) is None:
         return None
     files = []
     for pattern in _EXPR_FILES:
@@ -1026,8 +1025,7 @@ def handle_http_content(
     data: bytes,
     path: str = "/",
 ) -> None:
-    title_m = _EXPR_TITLE.search(data)
-    if title_m is not None and not any(
+    if (title_m := _EXPR_TITLE.search(data)) is not None and not any(
         s["id"] == "http-title" for s in port.get("scripts", [])
     ):
         title = nmap_encode_data(title_m.groups()[0])

diff --git a/ivre/data/govcloud.py b/ivre/data/govcloud.py
@@ -41,8 +41,7 @@ def get_azure_url() -> str:
         "https://www.microsoft.com/en-us/download/confirmation.aspx?id=57063"
     ) as udesc:
         for line in udesc:
-            match = AZURE_URL.search(line)
-            if match is None:
+            if (match := AZURE_URL.search(line)) is None:
                 continue
             return match.group(1).decode()
     raise ValueError("URL for Azure US Government Cloud not found")

diff --git a/ivre/db/__init__.py b/ivre/db/__init__.py
@@ -2099,16 +2099,11 @@ def getdns(self, addr_or_name, subdomains=False, dnstype=None):
         )  # {name: {types: {"A", "PTR", ...}, sources: {...}, firstseen: ..., lastseen: ...}}
         if dnstype is not None:
             dnstype = dnstype.upper()
-        # Only with Python >= 3.8
-        # if isinstance(addr_or_name, str) and (
-        #     (is_host := utils.IPADDR.search(addr_or_name))
-        #     or utils.NETADDR.search(addr_or_name)
-        # ):
         if isinstance(addr_or_name, str) and (
-            utils.IPADDR.search(addr_or_name) or utils.NETADDR.search(addr_or_name)
+            (is_host := utils.IPADDR.search(addr_or_name))
+            or utils.NETADDR.search(addr_or_name)
         ):
-            # if is_host:
-            if utils.IPADDR.search(addr_or_name):
+            if is_host:
                 flt = self.flt_and(self.searchhost(addr_or_name), self.searchhostname())
             else:
                 flt = self.flt_and(self.searchnet(addr_or_name), self.searchhostname())

diff --git a/ivre/geoiputils.py b/ivre/geoiputils.py
@@ -43,9 +43,10 @@
 def bgp_raw_to_csv(fname: str, outname: str) -> None:
     cur = None
     assert config.GEOIP_PATH is not None
-    with open(os.path.join(config.GEOIP_PATH, fname), "rb") as fdesc, open(
-        os.path.join(config.GEOIP_PATH, outname), "w", encoding="utf8"
-    ) as out:
+    with (
+        open(os.path.join(config.GEOIP_PATH, fname), "rb") as fdesc,
+        open(os.path.join(config.GEOIP_PATH, outname), "w", encoding="utf8") as out,
+    ):
         for line in fdesc:
             start, stop = (
                 utils.ip2int(elt)
@@ -84,20 +85,26 @@ def unzip_all(
         for filedesc in zdesc.infolist():
             if cond and not cond(filedesc):
                 continue
-            with zdesc.open(filedesc) as rdesc, open(
-                os.path.join(config.GEOIP_PATH, os.path.basename(filedesc.filename)),
-                "wb",
-            ) as wdesc:
+            with (
+                zdesc.open(filedesc) as rdesc,
+                open(
+                    os.path.join(
+                        config.GEOIP_PATH, os.path.basename(filedesc.filename)
+                    ),
+                    "wb",
+                ) as wdesc,
+            ):
                 copyfileobj(rdesc, wdesc)
 
 
 def gunzip(fname: str) -> None:
     if not fname.endswith(".gz"):
         raise Exception('filename should end with ".gz"')
     assert config.GEOIP_PATH is not None
-    with utils.open_file(os.path.join(config.GEOIP_PATH, fname)) as inp, open(
-        os.path.join(config.GEOIP_PATH, fname[:-3]), "wb"
-    ) as outp:
+    with (
+        utils.open_file(os.path.join(config.GEOIP_PATH, fname)) as inp,
+        open(os.path.join(config.GEOIP_PATH, fname[:-3]), "wb") as outp,
+    ):
         copyfileobj(inp, outp)
 
 

diff --git a/ivre/parser/weblog.py b/ivre/parser/weblog.py
@@ -53,11 +53,8 @@ def parse_line(self, line: bytes) -> Dict[str, Any]:
             "host": m.group("addr"),
             "ts": timestamp,
         }
-        # Python >= 3.8: if (useragent := m.group("useragent")) != "-":
-        useragent = m.group("useragent")
-        if useragent != "-":
+        if (useragent := m.group("useragent")) != "-":
             result["user-agent"] = useragent
-        # Python >= 3.8: if xff := m.group("x_forwarded_for"):
         xff = m.group("x_forwarded_for")
         if xff and xff != "-":
             result["x-forwarded-for"] = xff

diff --git a/ivre/passive.py b/ivre/passive.py
@@ -400,9 +400,7 @@ def _prepare_rec(spec, ignorenets, neverignore):
     elif spec["recontype"] == "SSL_CLIENT" and spec["source"] == "ja4":
         info = spec.setdefault("infos", {})
         value = spec["value"]
-        # Python >= 3.8: use :=
-        ja4_raw_match = JA4_RAW.search(value)
-        if ja4_raw_match is not None:
+        if (ja4_raw_match := JA4_RAW.search(value)) is not None:
             info.update(ja4_raw_match.groupdict())
             if info["ja4_c2_raw"] is None:
                 info["ja4_c2_raw"] = ""

diff --git a/ivre/utils.py b/ivre/utils.py
@@ -851,7 +851,7 @@ def recursive_filelisting(
 def serialize(obj: Any) -> str:
     """Return a JSON-compatible representation for `obj`"""
     regexp_types = (REGEXP_T, BsonRegex) if USE_BSON else REGEXP_T
-    if isinstance(obj, regexp_types):  # type: ignore
+    if isinstance(obj, regexp_types):
         return "/%s/%s" % (
             obj.pattern,
             "".join(x.lower() for x in "ILMSXU" if getattr(re, x) & obj.flags),
@@ -2529,7 +2529,7 @@ def download_if_newer(
 
     """
     if processor is None:
-        processor = shutil.copyfileobj  # type: ignore[assignment]
+        processor = shutil.copyfileobj
     assert processor is not None
     opener = build_opener()
     opener.addheaders = [("User-Agent", "IVRE/%s +https://ivre.rocks/" % VERSION)]

diff --git a/ivre/web/utils.py b/ivre/web/utils.py
@@ -812,9 +812,8 @@ def add_unused(neg, param, value):
             # ignore this parameter
             pass
         elif value is None:
-            if PORT.search(param) is not None:
-                # Python >= 3.8: if (match := PORT.search(param)) is not None
-                proto, port = PORT.search(param).groups()
+            if (match := PORT.search(param)) is not None:
+                proto, port = match.groups()
                 flt = dbase.flt_and(
                     flt, dbase.searchport(int(port), protocol=proto or "tcp", neg=neg)
                 )

diff --git a/pkg/runchecks b/pkg/runchecks
@@ -18,7 +18,7 @@
 
 # pip install -U -r requirements-linting.txt
 
-black -t py37 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre/ ./pkg/stubs/ && echo "black OK"
+black -t py311 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre/ ./pkg/stubs/ && echo "black OK"
 
 flake8 --ignore=E402,E501,F401 ./doc/conf.py && flake8 --ignore=E501,W503 ./setup.py ./bin/ivre && flake8 --ignore=E203,E402,E501,W503 ./tests/tests.py && flake8 --ignore=E203,E501,E704,W503 ./ivre/ && flake8 --ignore=E302,E305,E701,E704 ./pkg/stubs/*.pyi && echo "flake8 OK"
 

diff --git a/setup.py b/setup.py
@@ -149,10 +149,6 @@ def _write_pkg_file(self, file):
         "Intended Audience :: Telecommunications Industry",
         "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
         "Topic :: Scientific/Engineering :: Information Analysis",
@@ -161,7 +157,7 @@ def _write_pkg_file(self, file):
         "Topic :: System :: Networking :: Monitoring",
         "Topic :: System :: Software Distribution",
     ],
-    python_requires=">=3.7, <4",
+    python_requires=">=3.11, <4",
     install_requires=[
         "cryptography",
         "pymongo>=3.7",

diff --git a/tests/samples/results b/tests/samples/results
@@ -448,7 +448,7 @@ view_count_ssh_CiscoSSH_1_25 = 6
 view_count_ssh_OpenSSH_3_1p1 = 1
 view_count_ssh_OpenSSH_3_1p1_or_4_3 = 9
 view_count_ssh_OpenSSH_UNKNOWN = 0
-view_count_tags = 80
+view_count_tags = 72
 view_count_tags_honeypot = 7
 view_count_tags_honeypot_mushmush_102 = 3
 view_count_tags_tor = 10

diff --git a/web/static/doc/_images/graphviz-2f6369a26823883990bcb83b9ac3555fffef6cc9.png b/web/static/doc/_images/graphviz-2f6369a26823883990bcb83b9ac3555fffef6cc9.png
diff --git a/web/static/doc/_images/graphviz-2f6369a26823883990bcb83b9ac3555fffef6cc9.png.map b/web/static/doc/_images/graphviz-2f6369a26823883990bcb83b9ac3555fffef6cc9.png.map
@@ -1,2 +1,2 @@
-<map id="%3" name="%3">
+<map id="%1" name="%1">
 </map>
diff --git a/web/static/doc/_images/graphviz-895f16ae20541f9b2ee5370c2ec2d9ff25b1a4d9.png.map b/web/static/doc/_images/graphviz-895f16ae20541f9b2ee5370c2ec2d9ff25b1a4d9.png.map
@@ -1,2 +1,2 @@
-<map id="%3" name="%3">
+<map id="%1" name="%1">
 </map>
diff --git a/web/static/doc/_images/graphviz-f3a9d1551609259fb7dbeca90e2770cd757af72a.png.map b/web/static/doc/_images/graphviz-f3a9d1551609259fb7dbeca90e2770cd757af72a.png.map
@@ -1,2 +1,2 @@
-<map id="%3" name="%3">
+<map id="%1" name="%1">
 </map>
diff --git a/web/static/doc/_sources/dev/tests.rst.txt b/web/static/doc/_sources/dev/tests.rst.txt
@@ -87,6 +87,6 @@ GitHub actions
 
 Tests are run with several MongoDB and PostgreSQL versions, as well as
 TinyDB, SQLite and Elasticsearch for each pull requests. The tests run
-with Python 3.7 to 3.12.
+with Python 3.11 to 3.12.
 
 The configurations are in the `.github/workflows/*.yml` YAML files.
diff --git a/web/static/doc/_sources/install/installation.rst.txt b/web/static/doc/_sources/install/installation.rst.txt
@@ -40,7 +40,7 @@ your needs:
 Python
 ~~~~~~
 
-To install IVRE, you'll need `Python <http://www.python.org/>`__ 3.7
+To install IVRE, you'll need `Python <http://www.python.org/>`__ 3.11
 minimum, with the following modules:
 
 - `bottle <https://bottlepy.org/>`_.

diff --git a/web/static/doc/_static/basic.css b/web/static/doc/_static/basic.css
@@ -1,12 +1,5 @@
 /*
- * basic.css
- * ~~~~~~~~~
- *
  * Sphinx stylesheet -- basic theme.
- *
- * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS.
- * :license: BSD, see LICENSE for details.
- *
  */
 
 /* -- main layout ----------------------------------------------------------- */
@@ -115,15 +108,11 @@ img {
 /* -- search page ----------------------------------------------------------- */
 
 ul.search {
-    margin: 10px 0 0 20px;
-    padding: 0;
+    margin-top: 10px;
 }
 
 ul.search li {
-    padding: 5px 0 5px 20px;
-    background-image: url(file.png);
-    background-repeat: no-repeat;
-    background-position: 0 7px;
+    padding: 5px 0;
 }
 
 ul.search li a {

diff --git a/web/static/doc/_static/css/badge_only.css b/web/static/doc/_static/css/badge_only.css