[pull] master from ivre:master

ivre/tags/__init__.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2023 Pierre LALET <[email protected]>
+# Copyright 2011 - 2024 Pierre LALET <[email protected]>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -54,7 +54,7 @@
 
 
 _TOR_NODES: Optional[Set[str]] = None
-_CDN_TABLE: Optional[Tuple[List[int], List[Optional[str]]]] = None
+_CDN_TABLE: Optional[Tuple[List[int], List[Optional[Tuple[str, str]]]]] = None
 _GOVCLOUD_TABLE: Optional[Tuple[List[int], List[Optional[List[str]]]]] = None
 _SCANNERS_TABLE: Optional[Tuple[List[int], List[Optional[str]]]] = None
 
@@ -204,13 +204,15 @@ def gen_addr_tags(addr: str) -> Generator[Tag, None, None]:
                     ],
                 ),
             )
-        cdn_name = _get_name(_CDN_TABLE, addr)
-        if cdn_name is not None:
+        cdn_type_name = _get_name(_CDN_TABLE, addr)
+        if cdn_type_name is not None:
+            cdn_type, cdn_name = cdn_type_name
             yield cast(
                 Tag,
                 dict(
                     TAG_CDN,
-                    info=[f"{cdn_name} as listed at <https://cdn.nuclei.sh/>"],
+                    value=cdn_type.upper(),
+                    info=[f"{cdn_name} as listed by cdncheck (projectdiscovery)"],
                 ),
             )
         govcloud_data = _get_name(_GOVCLOUD_TABLE, addr)

ivre/target.py

@@ -445,7 +445,11 @@ class TargetNmapPreScan(TargetZMapPreScan):
 
     @classmethod
     def _getaddr(cls, line):
-        addr = cls.match_addr.match(line)
+        try:
+            line_s = line.decode()
+        except UnicodeDecodeError:
+            return None
+        addr = cls.match_addr.match(line_s)
         if addr is not None:
             try:
                 return utils.ip2int(addr.groups()[0])

ivre/tools/getwebdata.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2023 Pierre LALET <[email protected]>
+# Copyright 2011 - 2024 Pierre LALET <[email protected]>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -20,7 +20,8 @@
 """Fetches IP addresses lists from public websites and creates
 data files to add tags to scan results. For now, the following lists are used:
 
-  - CDN providers, from <https://cdn.nuclei.sh/>
+  - CDN & Cloud providers, from
+    <https://raw.githubusercontent.com/projectdiscovery/cdncheck/main/sources_data.json>
 
   - (US) GovCloud IP ranges, from <https://github.com/daehee/govcloud>
 
@@ -45,12 +46,13 @@
 import os
 import re
 import socket
-from typing import BinaryIO, Callable, Generator, List, Tuple, cast
+from typing import BinaryIO, Callable, Generator, List, Tuple
 
 from ivre import config
 from ivre.data import govcloud
 from ivre.utils import (
     IPADDR,
+    NETADDR,
     download_if_newer,
     generic_ipaddr_extractor,
     generic_processor,
@@ -61,11 +63,11 @@
 
 def cdnjson2table(infd: BinaryIO, outfd: BinaryIO) -> None:
     table = make_range_tables(
-        [
-            net2range(net) + (cast(str, name),)
-            for name, nets in json.load(infd).items()
-            for net in nets
-        ]
+        net2range(net) + ((ntype, name),)
+        for ntype, name_nets in json.load(infd).items()
+        for name, nets in name_nets.items()
+        for net in nets
+        if NETADDR.search(net)  # TODO: handle domain names
     )
     outfd.write(b"[\n    (\n")
     outfd.writelines(f"        {elt[0]!r},\n".encode() for elt in table)
@@ -95,7 +97,7 @@ def dns_get_names(name: str) -> List[str]:
 assert config.DATA_PATH is not None
 URLS: List[Tuple[str, str, Callable[[BinaryIO, BinaryIO], None]]] = [
     (
-        "https://cdn.nuclei.sh/",
+        "https://raw.githubusercontent.com/projectdiscovery/cdncheck/main/sources_data.json",
         os.path.join(config.DATA_PATH, "cdn_nuclei.py"),
         cdnjson2table,
     ),

tests/samples/results

@@ -448,7 +448,7 @@ view_count_ssh_CiscoSSH_1_25 = 6
 view_count_ssh_OpenSSH_3_1p1 = 1
 view_count_ssh_OpenSSH_3_1p1_or_4_3 = 9
 view_count_ssh_OpenSSH_UNKNOWN = 0
-view_count_tags = 67
+view_count_tags = 80
 view_count_tags_honeypot = 7
 view_count_tags_honeypot_mushmush_102 = 3
 view_count_tags_tor = 10

tests/tests.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2023 Pierre LALET <[email protected]>
+# Copyright 2011 - 2024 Pierre LALET <[email protected]>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -3739,6 +3739,7 @@ def test_10_data(self):
     registered_country_name United States
     coordinates (37.751, -97.822)
     coordinates_accuracy_radius 1000
+    CDN: google as listed by cdncheck (projectdiscovery)
 """.splitlines()
             ),
         )
@@ -3759,6 +3760,13 @@ def test_10_data(self):
                 "registered_country_name": "United States",
                 "coordinates": [37.751, -97.822],
                 "coordinates_accuracy_radius": 1000,
+                "tags": [
+                    {
+                        "value": "CDN",
+                        "type": "info",
+                        "info": ["google as listed by cdncheck (projectdiscovery)"],
+                    }
+                ],
             },
         )