Merge remote-tracking branch 'upstream/pr/1751' into develop

CHANGELOG.md

@@ -11,8 +11,11 @@ CHANGELOG
   - New class `ClassificationTaxonomy` with fixed list of taxonomies and sanitiation
 - `intelmq.lib.bot`:
   - Handle `InvalidValue` exceptions upon message retrieval by dumping the message instead of repeating endlessly (#1765, PR#1766 by Filip Pokorný).
+  - Rewrite of the parameter loading and handling, getting rid of the `parameters` member (PR#1729 by Birger Schacht).
 - `intelmq.lib.exceptions`:
   - `InvalidValue`: Add optional parameter `object` (PR#1766 by Filip Pokorný).
+- `intelmq.lib.utils`:
+  - New function `list_all_bots` to list all available/installed bots as replacement for the BOTS file (#368, #552, #644, #757, #1069, #1750, PR#1751 by Sebastian Waldbauer).
 
 ### Development
 
@@ -36,6 +39,8 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
   - The type `vulnerable service` has been renamed to `vulnerable-system`.
 
 ### Bots
+- The parameters handling of numerous bots has been refactored (PR#1751, PR#1729, by Birger Schacht, Sebastian Wagner, Sebastian Waldbauer).
+
 #### Collectors
 - Remove `intelmq.bots.collectors.xmpp`: one of the dependencies of the bot was deprecated and according to a short survey on the IntelMQ
   users mailinglist, the bot is not used by anyone. (https://lists.cert.at/pipermail/intelmq-users/2020-October/000177.html, PR#1761, closes #1614)

intelmq/__init__.py

@@ -28,6 +28,5 @@
 HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf")
 PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf")
 RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf")
-BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS")
 STATE_FILE_PATH = path = os.path.abspath(os.path.join(VAR_STATE_PATH,
                                                       '../state.json'))

intelmq/bin/intelmqctl.py

@@ -22,7 +22,7 @@
 import pkg_resources
 from termstyle import green
 
-from intelmq import (BOTS_FILE, DEFAULT_LOGGING_LEVEL, DEFAULTS_CONF_FILE,  # noqa: F401
+from intelmq import (DEFAULT_LOGGING_LEVEL, DEFAULTS_CONF_FILE,  # noqa: F401
                      HARMONIZATION_CONF_FILE, PIPELINE_CONF_FILE,
                      RUNTIME_CONF_FILE, VAR_RUN_PATH, STATE_FILE_PATH,
                      DEFAULT_LOGGING_PATH, __version_info__,
@@ -865,6 +865,8 @@ def __init__(self, interactive: bool = False, return_type: str = "python", quiet
                                      help='Only show the total '
                                           'number of messages in queues. '
                                           'Only valid for listing queues.')
+            parser_list.add_argument('--configured', '-c', action='store_true',
+                                     help='Only show configured bots')
             parser_list.set_defaults(func=self.list)
 
             parser_clear = subparsers.add_parser('clear', help='Clear a queue')
@@ -1220,11 +1222,11 @@ def botnet_status(self, group=None):
                 retval = 1
         return retval, botnet_status
 
-    def list(self, kind=None, non_zero=False, count=False):
+    def list(self, kind=None, non_zero=False, count=False, configured=False):
         if kind == 'queues':
             return self.list_queues(non_zero=non_zero, count=count)
         elif kind == 'bots':
-            return self.list_bots(non_zero=non_zero)
+            return self.list_bots(non_zero=non_zero, configured=configured)
         elif kind == 'queues-and-status':
             q = self.list_queues()
             b = self.botnet_status()
@@ -1243,25 +1245,29 @@ def write_updated_runtime_config(self, filename=RUNTIME_CONF_FILE):
             self.abort('Can\'t update runtime configuration: Permission denied.')
         return True
 
-    def list_bots(self, non_zero=False):
+    def list_bots(self, non_zero=False, configured=False):
         """
-        Lists all configured bots from runtime.conf with bot id and
-        description.
+        Lists all (configured) bots from runtime.conf or generated on demand
+        with bot id/module and description and parameters.
 
         If description is not set, None is used instead.
         """
-        if RETURN_TYPE == 'text':
-            for bot_id in sorted(self.runtime_configuration.keys(), key=str.lower):
-                if non_zero and not self.runtime_configuration[bot_id].get('enabled'):
-                    continue
-                if QUIET:
-                    print(bot_id)
-                else:
-                    print("Bot ID: {}\nDescription: {}"
-                          "".format(bot_id, self.runtime_configuration[bot_id].get('description')))
-        return 0, [{'id': bot_id,
-                    'description': self.runtime_configuration[bot_id].get('description')}
-                   for bot_id in sorted(self.runtime_configuration.keys())]
+        if configured:
+            if RETURN_TYPE == 'text':
+                for bot_id in sorted(self.runtime_configuration.keys(), key=str.lower):
+                    if non_zero and not self.runtime_configuration[bot_id].get('enabled'):
+                        continue
+                    if QUIET:
+                        print(bot_id)
+                    else:
+                        print("Bot ID: {}\nDescription: {}"
+                              "".format(bot_id, self.runtime_configuration[bot_id].get('description')))
+            return 0, [{'id': bot_id,
+                        'description': self.runtime_configuration[bot_id].get('description')}
+                       for bot_id in sorted(self.runtime_configuration.keys())]
+        else:
+            val = utils.list_all_bots()
+            return 0, val
 
     def get_queues(self, with_internal_queues=False):
         """
@@ -1421,8 +1427,7 @@ def check(self, no_connections=False):
 
         # loading files and syntax check
         files = {DEFAULTS_CONF_FILE: None, PIPELINE_CONF_FILE: None,
-                 RUNTIME_CONF_FILE: None, BOTS_FILE: None,
-                 HARMONIZATION_CONF_FILE: None}
+                 RUNTIME_CONF_FILE: None, HARMONIZATION_CONF_FILE: None}
         check_logger.info('Reading configuration files.')
         for filename in files:
             try:
@@ -1555,7 +1560,7 @@ def check(self, no_connections=False):
             if bot_check:
                 for log_line in bot_check:
                     getattr(check_logger, log_line[0])("Bot %r: %s" % (bot_id, log_line[1]))
-        for group in files[BOTS_FILE].values():
+        for group in utils.list_all_bots():
             for bot_id, bot in group.items():
                 if subprocess.call(['which', bot['module']], stdout=subprocess.DEVNULL,
                                    stderr=subprocess.DEVNULL):
@@ -1874,7 +1879,7 @@ def debug(self, sections=None):
             variables = globals()
             if RETURN_TYPE == 'text':
                 print('Paths:')
-            for path in ('BOTS_FILE', 'DEFAULTS_CONF_FILE',
+            for path in ('DEFAULTS_CONF_FILE',
                          'HARMONIZATION_CONF_FILE', 'PIPELINE_CONF_FILE',
                          'RUNTIME_CONF_FILE', 'VAR_RUN_PATH', 'STATE_FILE_PATH',
                          'DEFAULT_LOGGING_PATH', '__file__',

intelmq/bin/intelmqsetup.py

@@ -44,7 +44,7 @@
 
 from termstyle import red
 from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
-                     VAR_STATE_PATH, BOTS_FILE, STATE_FILE_PATH)
+                     VAR_STATE_PATH, STATE_FILE_PATH)
 from intelmq.bin.intelmqctl import IntelMQController
 
 
@@ -161,13 +161,6 @@ def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
         if ownership:
             change_owner(destination_file, owner='intelmq', group='intelmq', log=log_ownership_change)
 
-    if Path(BOTS_FILE).is_symlink():
-        print('Skip writing BOTS file as it is a link.')
-    else:
-        print('Writing BOTS file.')
-        shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
-                    BOTS_FILE)
-
     if ownership:
         print('Setting intelmq as owner for it\'s directories.')
         for obj in (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,

intelmq/bots/collectors/amqp/collector_amqp.py

@@ -20,7 +20,6 @@ class AMQPCollectorBot(AMQPTopicOutputBot, CollectorBot):
     Collect data from an AMQP Server and fetch either intelmq or any other messages. Requires the pika python library.
     Inheriting from AMQPTopicOutputBot for connect_server method
     """
-    exchange: bool = False
     connection_attempts: int = 3
     connection_heartbeat: int = 3600
     connection_host: str = "127.0.0.1"  # TODO should be ipaddress

intelmq/bots/collectors/api/collector_api.py

@@ -28,9 +28,9 @@ class APICollectorBot(CollectorBot):
     """Collect data by exposing a HTTP API interface"""
     name: str = "API"
     port: int = 5000
+    __collector_empty_process: bool = True
     provider: str = "APICollector"
-    collector_empty_process: bool = True
-    is_multithreadable: bool = False
+    __is_multithreadable: bool = False
 
     def init(self):
         if IOLoop is None:

intelmq/bots/collectors/github_api/collector_github_contents_api.py

@@ -12,7 +12,7 @@
 import re
 
 from intelmq.lib.exceptions import InvalidArgument
-from intelmq.bots.collectors.github_api.collector_github_api import GithubAPICollectorBot
+from intelmq.bots.collectors.github_api._collector_github_api import GithubAPICollectorBot
 
 try:
     import requests

intelmq/bots/collectors/http/collector_http_stream.py

@@ -29,7 +29,7 @@
 
 class HTTPStreamCollectorBot(CollectorBot):
     "Open a streaming connection to the URL and process data per line"
-    sighup_delay = False
+    _sighup_delay: bool = False
     http_password: str = None
     http_url: str = "<insert url of feed>"
     http_username: str = None

intelmq/bots/collectors/mail/collector_mail_attach.py

@@ -10,7 +10,7 @@
 from intelmq.lib.utils import unzip
 from intelmq.lib.exceptions import InvalidArgument
 
-from .lib import MailCollectorBot
+from ._lib import MailCollectorBot
 
 
 class MailAttachCollectorBot(MailCollectorBot):

intelmq/bots/collectors/mail/collector_mail_body.py

@@ -2,12 +2,14 @@
 """
 Uses the common mail iteration method from the lib file.
 """
-from .lib import MailCollectorBot
+from typing import Union, Iterable
+
+from ._lib import MailCollectorBot
 
 
 class MailBodyCollectorBot(MailCollectorBot):
     "Monitor IMAP mailboxes and fetch mail bodies"
-    content_types = ['plain', 'html']
+    content_types: Union[bool, str, Iterable[str]] = ('plain', 'html')
     folder: str = "INBOX"
     mail_host: str = "<host>"
     mail_password: str = "<password>"

intelmq/bots/collectors/mail/collector_mail_url.py

@@ -8,7 +8,7 @@
 from intelmq.lib.splitreports import generate_reports
 from intelmq.lib.utils import create_request_session, file_name_from_response
 
-from .lib import MailCollectorBot
+from ._lib import MailCollectorBot
 from intelmq.lib.exceptions import MissingDependencyError
 
 try:
@@ -19,8 +19,6 @@
 
 class MailURLCollectorBot(MailCollectorBot):
     """Monitor IMAP mailboxes and fetch files from URLs contained in mail bodies"""
-    error_procedure = None
-
     chunk_replicate_header: bool = True
     chunk_size: int = None
     folder: str = "INBOX"

intelmq/bots/collectors/rsync/collector_rsync.py

@@ -14,24 +14,23 @@ class RsyncCollectorBot(CollectorBot):
     temp_directory: str = path.join(VAR_STATE_PATH, "rsync_collector")  # TODO: should be pathlib.Path
 
     def init(self):
-        self.rsync_data_directory = self.temp_directory
         try:
-            mkdir(self.rsync_data_directory)
+            mkdir(self.temp_directory)
         except FileExistsError:
             pass
 
     def process(self):
         self.logger.info("Updating file {}.".format(self.file))
         process = run(["rsync", path.join(self.rsync_path, self.file),
-                       self.rsync_data_directory],
+                       self.temp_directory],
                       stderr=PIPE)
         if process.returncode != 0:
             raise ValueError("Rsync on file {!r} failed with exitcode {} and stderr {!r}."
                              "".format(self.file,
                                        process.returncode,
                                        process.stderr))
         report = self.new_report()
-        with open(path.join(self.rsync_data_directory, self.file), "r") as rsync_file:
+        with open(path.join(self.temp_directory, self.file), "r") as rsync_file:
             report.add("raw", rsync_file.read())
             self.send_message(report)
 

intelmq/bots/collectors/rt/collector_rt.py

@@ -41,7 +41,7 @@ class RTCollectorBot(CollectorBot):
     url_regex: str = "https://dl.shadowserver.org/[a-zA-Z0-9?_-]*"  # TODO: type could be re?
     user: str = "intelmq"
 
-    parameter_mapping = {'search_owner': 'Owner',
+    PARAMETER_MAPPING = {'search_owner': 'Owner',
                          'search_queue': 'Queue',
                          'search_requestor': 'Requestor',
                          'search_status': 'Status',
@@ -100,7 +100,7 @@ def process(self):
         else:
             kwargs = {}
 
-        for parameter_name, rt_name in self.parameter_mapping.items():
+        for parameter_name, rt_name in self.PARAMETER_MAPPING.items():
             parameter_value = getattr(self, parameter_name, None)
             if parameter_value:
                 kwargs[rt_name] = parameter_value

intelmq/bots/collectors/stomp/collector.py

@@ -60,14 +60,13 @@ class StompCollectorBot(CollectorBot):
     exchange: str = ''
     port: int = 61614
     server: str = "n6stream.cert.pl"
-    http_verify_cert: bool = True
     ssl_ca_certificate: str = 'ca.pem'  # TODO pathlib.Path
     ssl_client_certificate: str = 'client.pem'  # TODO pathlib.Path
     ssl_client_certificate_key: str = 'client.key'  # TODO pathlib.Path
     heartbeat: int = 6000
 
-    collector_empty_process = True
-    conn = False  # define here so shutdown method can check for it
+    __collector_empty_process: bool = True
+    __conn = False  # define here so shutdown method can check for it
 
     def init(self):
         if stomp is None:
@@ -86,22 +85,22 @@ def init(self):
                 raise ValueError("Could not open file %r." % f)
 
         _host = [(self.server, self.port)]
-        self.conn = stomp.Connection(host_and_ports=_host, use_ssl=True,
-                                     ssl_key_file=self.ssl_cl_cert_key,
-                                     ssl_cert_file=self.ssl_cl_cert,
-                                     ssl_ca_certs=self.ssl_ca_cert,
-                                     heartbeats=(self.heartbeat,
-                                                 self.heartbeat))
-
-        self.conn.set_listener('', StompListener(self, self.conn, self.exchange))
-        connect_and_subscribe(self.conn, self.logger, self.exchange,
+        self.__conn = stomp.Connection(host_and_ports=_host, use_ssl=True,
+                                       ssl_key_file=self.ssl_cl_cert_key,
+                                       ssl_cert_file=self.ssl_cl_cert,
+                                       ssl_ca_certs=self.ssl_ca_cert,
+                                       heartbeats=(self.heartbeat,
+                                                   self.heartbeat))
+
+        self.__conn.set_listener('', StompListener(self, self.__conn, self.exchange))
+        connect_and_subscribe(self.__conn, self.logger, self.exchange,
                               start=stomp.__version__ < (4, 1, 20))
 
     def shutdown(self):
-        if not stomp or not self.conn:
+        if not stomp or not self.__conn:
             return
         try:
-            self.conn.disconnect()
+            self.__conn.disconnect()
         except stomp.exception.NotConnectedException:
             pass
 

intelmq/bots/experts/abusix/expert.py

@@ -4,9 +4,10 @@
 RIPE abuse contacts resolving through DNS TXT queries
 '''
 
-from intelmq.bots.experts.abusix.lib import Abusix
 from intelmq.lib.bot import Bot
 
+from ._lib import Abusix
+
 try:
     import querycontacts
 except ImportError:

intelmq/bots/experts/cymru_whois/expert.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 import json
 
-from intelmq.bots.experts.cymru_whois.lib import Cymru
 from intelmq.lib.bot import Bot
 from intelmq.lib.cache import Cache
 from intelmq.lib.harmonization import IPAddress
 
+from ._lib import Cymru
+
 CACHE_KEY = "%d_%s"
 
 

intelmq/bots/experts/domain_suffix/expert.py

@@ -19,15 +19,14 @@
 try:
     from publicsuffixlist import PublicSuffixList
 except ImportError:
-    from .lib import PublicSuffixList
+    from ._lib import PublicSuffixList
 
 
 ALLOWED_FIELDS = ['fqdn', 'reverse_dns']
 
 
 class DomainSuffixExpertBot(Bot):
     """Extract the domain suffix from a domain and save it in the the domain_suffix field. Requires a local file with valid domain suffixes"""
-    suffixes = {}
     field: str = None
     suffix_file: str = None  # TODO: should be pathlib.Path