Merge pull request ivre#1609 from p-l-/enh-httpx-headers-body

Active/httpx: support headers & body reported with `-irr` flag

.github/workflows/linting.yml

@@ -46,7 +46,7 @@ jobs:
 
     - run: mv ivre ivre_bak
 
-    - run: black -t py36 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
+    - run: black -t py37 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
 
     - run: bandit --severity-level high -r ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
 
@@ -65,7 +65,7 @@ jobs:
     - run: flake8 --ignore=E402,E501,F401 ./doc/conf.py
     - run: flake8 --ignore=E501,W503 ./setup.py ./bin/ivre
     - run: flake8 --ignore=E203,E402,E501,W503 ./tests/tests.py
-    - run: flake8 --ignore=E203,E501,W503 ./ivre_bak/
+    - run: flake8 --ignore=E203,E501,E704,W503 ./ivre_bak/
     - run: flake8 --ignore=E302,E305,E701,E704 ./pkg/stubs/*.pyi
 
     - run: rstcheck -r ./doc/

ivre/active/data.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2023 Pierre LALET <[email protected]>
+# Copyright 2011 - 2024 Pierre LALET <[email protected]>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -1077,7 +1077,9 @@ def handle_http_content(
     path: str = "/",
 ) -> None:
     title_m = _EXPR_TITLE.search(data)
-    if title_m is not None:
+    if title_m is not None and not any(
+        s["id"] == "http-title" for s in port.get("scripts", [])
+    ):
         title = nmap_encode_data(title_m.groups()[0])
         port.setdefault("scripts", []).append(
             {

ivre/db/__init__.py

@@ -1483,10 +1483,10 @@ def __migrate_schema_hosts_19_20(cls, doc):
             for script in port.get("scripts", []):
                 if script["id"] == "http-default-accounts":
                     if "http-default-accounts" in script:
-                        script[
-                            "http-default-accounts"
-                        ] = xmlnmap.change_http_default_accounts(
-                            script["http-default-accounts"]
+                        script["http-default-accounts"] = (
+                            xmlnmap.change_http_default_accounts(
+                                script["http-default-accounts"]
+                            )
                         )
                 elif script["id"].endswith("-nuclei") and script["id"] in script:
                     script["nuclei"] = script.pop(script["id"])
@@ -3143,6 +3143,7 @@ def store_scan_json_httpx(
             categories = []
         if tags is None:
             tags = []
+        http_hdr_split = re.compile(b"\r?\n")
         self.start_store_hosts()
         with utils.open_file(fname) as fdesc:
             for line in fdesc:
@@ -3191,6 +3192,90 @@ def store_scan_json_httpx(
                     )
                 if "technologies" in rec:
                     script["technologies"] = rec["technologies"]
+                if "raw_header" in rec:
+                    hdrs = rec["raw_header"].encode()
+                    hdrs_split = http_hdr_split.split(hdrs)
+                    if hdrs_split:
+                        hdr_output_list = [
+                            utils.nmap_encode_data(line) for line in hdrs_split
+                        ]
+                        # default values - TODO: handle specific path
+                        # with code from zgrabout
+                        method = "GET"
+                        path = "/"
+                        if "request" in rec:
+                            try:
+                                method, path, _ = rec["request"].split(None, 2)
+                            except ValueError:
+                                pass
+                        hdr_output_list.extend(["", f"(Request type: {method})"])
+                        structured = [
+                            {
+                                "name": "_status",
+                                "value": utils.nmap_encode_data(hdrs_split[0].strip()),
+                            }
+                        ]
+                        structured.extend(
+                            {
+                                "name": utils.nmap_encode_data(hdrname).lower(),
+                                "value": utils.nmap_encode_data(hdrval),
+                            }
+                            for hdrname, hdrval in (
+                                m.groups()
+                                for m in (
+                                    utils.RAW_HTTP_HEADER.search(part.strip())
+                                    for part in hdrs_split
+                                )
+                                if m
+                            )
+                        )
+                        port.setdefault("scripts", []).append(
+                            {
+                                "id": "http-headers",
+                                "output": "\n".join(hdr_output_list),
+                                "http-headers": structured,
+                                "masscan": {"raw": utils.encode_b64(hdrs).decode()},
+                            }
+                        )
+                        handle_http_headers(host, port, structured, path=path)
+                        raw_output = hdrs
+                        if "body" in rec:
+                            # usually, the whole answer should be that
+                            raw_output += b"\r\n\r\n" + rec["body"].encode()
+                        nmap_info = utils.match_nmap_svc_fp(
+                            output=raw_output,
+                            proto=port["protocol"],
+                            probe="GetRequest",
+                        )
+                        if nmap_info:
+                            try:
+                                del nmap_info["soft"]
+                            except KeyError:
+                                pass
+                            add_cpe_values(
+                                host,
+                                f"ports.port:{rec['port']}",
+                                nmap_info.pop("cpe", []),
+                            )
+                            host["cpes"] = list(host["cpes"].values())
+                            for cpe in host["cpes"]:
+                                cpe["origins"] = sorted(cpe["origins"])
+                            if not host["cpes"]:
+                                del host["cpes"]
+                            port.update(nmap_info)
+                            xmlnmap.add_service_hostname(
+                                nmap_info,
+                                host.setdefault("hostnames", []),
+                            )
+                if "body" in rec:
+                    body = rec["body"].encode()
+                    port.setdefault("scripts", []).append(
+                        {
+                            "id": "http-content",
+                            "output": utils.nmap_encode_data(body),
+                        }
+                    )
+                    handle_http_content(host, port, body)
                 if script:
                     output = []
                     if "technologies" in script:
@@ -3468,9 +3553,11 @@ def store_scan_json_shodan(
                             host.setdefault("hostnames", []),
                         )
                         banner = "".join(
-                            chr(d)
-                            if 32 <= d <= 126 or d in {9, 10, 13}
-                            else "\\x%02x" % d
+                            (
+                                chr(d)
+                                if 32 <= d <= 126 or d in {9, 10, 13}
+                                else "\\x%02x" % d
+                            )
                             for d in raw_output
                         )
                         port.setdefault("scripts", []).append(
@@ -3626,7 +3713,6 @@ def store_scan_json_dismap(
                         except ValueError:
                             hdrs = raw_output
                             body = None
-                        # TODO http-headers / http-content
                         hdrs_split = http_hdr_split.split(hdrs)
                         if hdrs_split:
                             hdr_output_list = [
@@ -3676,9 +3762,11 @@ def store_scan_json_dismap(
                             handle_http_content(host, port, body)
                     else:
                         banner = "".join(
-                            chr(d)
-                            if 32 <= d <= 126 or d in {9, 10, 13}
-                            else "\\x%02x" % d
+                            (
+                                chr(d)
+                                if 32 <= d <= 126 or d in {9, 10, 13}
+                                else "\\x%02x" % d
+                            )
                             for d in raw_output
                         )
                         port.setdefault("scripts", []).append(

ivre/db/elastic.py

@@ -1521,9 +1521,9 @@ def searchcert(
                         Q(
                             "match",
                             **{
-                                "ports.scripts.id": "ssl-cacert"
-                                if cacert
-                                else "ssl-cert"
+                                "ports.scripts.id": (
+                                    "ssl-cacert" if cacert else "ssl-cert"
+                                )
                             },
                         ),
                         Q(

ivre/db/http.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2023 Pierre LALET <[email protected]>
+# Copyright 2011 - 2024 Pierre LALET <[email protected]>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -234,8 +234,7 @@ def _get(self, spec, limit=None, skip=None, sort=None, fields=None):
             if not data:
                 break
             if limit is None:
-                for rec in data:
-                    yield rec
+                yield from data
             else:
                 for rec in data:
                     yield rec

ivre/db/maxmind.py

@@ -34,7 +34,6 @@
 
 
 class MaxMindFileIter:
-
     """Iterator for MaxMindFile"""
 
     def __init__(self, base):
@@ -93,7 +92,6 @@ def __next__(self):
 
 
 class EmptyMaxMindFile:
-
     """Stub to replace MaxMind databases parsers. Used when a file is
     missing to emit a warning message and return empty results.
 
@@ -108,7 +106,6 @@ def lookup(_):
 
 
 class MaxMindFile:
-
     """Parser for MaxMind databases.
 
     Code copied and adapted from