refactor .py code for separation of concerns & OO abstractions (#8)

* refactor .py code for separation of concerns & OO abstractions

New abstractions added for IOC feed parsing, rule definition (and evaluation)
and the Event instances that are produced from evaluating rules.

I left RuleSet and Rule as partially defined, to keep things simple for now.
Event is basically schemaless as well, but the IOC feed parsing and how it
produces RuleSet instances is more well defined.  All of these are likely to
grow and evolve in upcoming commits and merges but this is a good place to
start.  It matches the features of the existing code and actually fixes a
couple of bugs.  Existing tests are refactored, more tests will be added soon.

* add tests for the rules module (add rule, merge rules, process endpoint)

* update pydoc for rules.py

Co-authored-by: Vinicius Fortuna <[email protected]>

* addresses review comments, mostly renaming, some restructuring

* rename *Scanner to *Monitor

* further refactoring of the monitor classes and related code.

also adds a process executor pool and runs the tshark monitors within that,
it will help propagate exceptions up to the main process.

* distinguish IOC rule types from ruleset, maintain ruleset as a simple struct

The iocs/ directory is now ioc_formats to make it clearer that it is only dealing
with the file formats (and config formats) of various IOC sources.  The iocs.py
module has been hoisted up to main, alongside event.py (to be populated more
explicitly in a follow-up when the Event schema has stabilized) and all rules-
related code is now in a matchers/ directory, to separate monitor (observation)
production and the logic needed for rule matching.

* add tests for IP matching and TLS connection matching, some typing

added types to constructors and methods that are called from other
modules

* some finishing cleanup, use datetime for timestamps and small naming change

config_example.json

@@ -1,29 +1,39 @@
 {
   "ioc_sources": [
     {
+      "format": "BAD_IP_LIST",
       "name": "EmergingThreats compromised IP addresses",
-      "url": "https://rules.emergingthreats.net/open/suricata-5.0/rules/compromised-ips.txt",
-      "format": "BAD_IP_LIST"
+      "url": "https://rules.emergingthreats.net/open/suricata-5.0/rules/compromised-ips.txt"
     },
     {
+      "format": "BAD_IP_LIST",
       "name": "3CORESec Poor Reputation IPs",
-      "url": "https://blacklist.3coresec.net/lists/et-open.txt",
-      "format": "BAD_IP_LIST"
+      "url": "https://blacklist.3coresec.net/lists/et-open.txt"
     },
     {
+      "format": "BAD_IP_LIST",
       "name": "CINSscore badguys list - cinsscore.com",
-      "url": "https://cinsscore.com/list/ci-badguys.txt",
-      "format": "BAD_IP_LIST"
+      "url": "https://cinsscore.com/list/ci-badguys.txt"
     },
     {
+      "format": "BAD_IP_LIST",
       "name": "blocklist.de suspicious IPs",
-      "url": "https://lists.blocklist.de/lists/all.txt",
-      "format": "BAD_IP_LIST"
+      "url": "https://lists.blocklist.de/lists/all.txt"
     },
     {
+      "format": "BAD_IP_LIST",
       "name": "TOR exit nodes (dan.me.uk)",
-      "url": "https://www.dan.me.uk/torlist/?exit",
-      "format": "BAD_IP_LIST"
+      "url": "https://www.dan.me.uk/torlist/?exit"
+    },
+    {
+      "format": "ALLOWED_SNI_PORT",
+      "name": "recognized unusual TLS (name, port) combinations",
+      "allow": [
+          ["mtalk.google.com", 5228],
+          ["proxy-safebrowsing.googleapis.com", 80],
+          ["courier.push.apple.com", 5223],
+          ["imap.gmail.com", 993]
+      ]
     }
   ]
 }

rids.service

@@ -4,7 +4,7 @@ Description=Remote Intrusion Detection System daemon
 [Service]
 User=root
 WorkingDirectory=/usr/local/sbin
-ExecStart=/usr/local/sbin/rids.py
+ExecStart="/usr/local/sbin/rids.py --eventlog_path=/var/rids/events.log --config_path=/etc/rids/config.json"
 Restart=always
 
 [Install]

tests/network_capture_test.py → rids/event.py

@@ -12,31 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-Tests for the network_capture library.
-"""
 
+class Event:
+  """Generalization of an Event sighting, i.e. suspicious activity or threat."""
 
-import io
-import pytest
+  def __init__(self, properties):
+    # The event properties have a very loose definition at this moment.
+    # TODO stabilize the schema, perhaps based on MISP or Dovecot event types.
+    self.properties = properties
 
-from rids import network_capture
-
-
-@pytest.fixture()
-def badips_map():
-  return {
-    '1.1.1.1': 'Source A',
-    '4.5.6.7': 'Best IOCs',
-    '100.12.34.56': 'Threatbusters',
-  }
-
-
-def test_good_ip_is_ok(badips_map):
-  detected = network_capture.detect_bad_ips('0\t127.0.0.1\t1.2.3.4', badips_map)
-  assert not detected
-
-
-def test_bad_ip_is_logged(badips_map):
-  detected = network_capture.detect_bad_ips('1\t4.5.6.7\t127.0.0.1', badips_map)
-  assert detected
+  def __str__(self):
+    return str(self.properties)

rids/ioc_formats/allowed_sni_port.py

@@ -0,0 +1,31 @@
+# Copyright 2022 Jigsaw Operations LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""IOC parser for allowing certain unusual (server name, port) sightings."""
+
+
+from rids.rules.ruleset import RuleSet
+from rids.rules.tls_matcher import TlsMatcher
+
+
+class AllowedEndpoints:
+  """Parses allowed (sni, port) pairs directly from an ioc_sources config."""
+  def __init__(self, config):
+    self.name = config['name']
+    self.allowlist = config['allow']
+
+  def provide_rules(self, ruleset: RuleSet):
+    """Add rules to the TlsMatcher according to the configured allowlist."""
+    for server_name, port in self.allowlist:
+      ruleset.tls_matcher.add_allowed_endpoint(server_name, port)

rids/ioc_formats/bad_ip_list.py

@@ -0,0 +1,57 @@
+# Copyright 2022 Jigsaw Operations LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""IOC feed parser for lists of compromised host IPs (newline separated)."""
+
+
+from datetime import datetime
+import ipaddress
+import urllib.request
+
+from rids.iocs import RuleSet
+from rids.rules.ip_matcher import IpRule
+from rids.rules.ruleset import RuleSet
+
+
+class BadIpAddresses:
+  """Parses newline-separated list of IP addresses into a RuleSet."""
+
+  def __init__(self, config):
+    self.source_url = config['url']
+    self.source_name = config.get('name', 'UNKNOWN')
+    self.msg = config.get('msg', 'contact with a potentially compromised host')
+
+  def provide_rules(self, ruleset: RuleSet):
+    """Add to the ruleset all the rules defined in this source."""
+    fetch_time = datetime.now()
+    with urllib.request.urlopen(self.source_url) as ioc_data:
+      for line in ioc_data.readlines():
+        line = line.decode('utf-8').strip()
+        if not line or line[0] == '#':
+          # skip empty lines and comments
+          continue
+        try:
+          bad_ip = ipaddress.ip_address(line)
+        except ValueError:
+          # Do nothing with badly-formatted addresses in these files.
+          continue
+
+        ip_rule = IpRule(
+            msg=self.msg,
+            name=self.source_name,
+            url=self.source_url,
+            fetched=fetch_time,
+            matches_ip=bad_ip,
+        )
+        ruleset.ip_matcher.add_ip_rule(ip_rule)

rids/iocs.py

@@ -0,0 +1,64 @@
+# Copyright 2022 Jigsaw Operations LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Combined IOC feed parser interface for the different formats of IOCs."""
+
+
+import collections
+from dataclasses import dataclass
+from types import Set
+
+from rids.ioc_formats import allowed_sni_port
+from rids.ioc_formats import bad_ip_list
+from rids.rules.ruleset import RuleSet
+
+
+_IOC_PARSERS = {
+  "BAD_IP_LIST": bad_ip_list.BadIpAddresses,
+  "ALLOWED_SNI_PORT": allowed_sni_port.AllowedEndpoints,
+}
+
+
+def fetch_iocs(rids_config):
+  """Parses the IOC indicated by `ioc_config` into a RuleSet instance.
+  
+  Args:
+    rids_config: dict{'ioc_sources': list[
+            dict{
+              'format': str,
+              ...other properties, format-dependent
+            }]}
+
+  Returns:
+    RuleSet representing all IOCs in the config.
+  """
+  ioc_sources = rids_config.get('ioc_sources', None)
+  ruleset = RuleSet()
+  if not ioc_sources:
+    return ruleset
+
+  for ioc_config in ioc_sources:
+    format = ioc_config.get('format', None)
+    if not format:
+      raise ValueError(
+        f'Expected a "format" specifier for ioc_sources config: {ioc_config}')
+
+    if format not in _IOC_PARSERS:
+      raise ValueError('Unrecognized IOC feed format "{format}"')
+
+    ioc_source = _IOC_PARSERS[format](ioc_config)
+    ioc_source.provide_rules(ruleset)
+
+  return ruleset
+

rids/monitors/ip_monitor.py

@@ -0,0 +1,67 @@
+# Copyright 2022 Jigsaw Operations LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""IP monitoring classes and functionality.
+
+IpPacketMonitor for obtaining remote IPs.
+IpPacket as the type for communicating to related rules.
+IpRule for representing the IOCs that packets should be checked against.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime
+import ipaddress
+from typing import Union
+from typing import Generator
+
+from rids.monitors import tshark
+
+
+IpAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address]
+
+
+@dataclass
+class IpPacket:
+  """Represents a remote IP endpoint and when it was seen."""
+  timestamp: datetime
+  ip_address: IpAddress
+
+
+class IpPacketMonitor:
+  """Continuously monitors the network traffic for remote IPs being contacted.
+
+  The monitor() function is an iterator that produces a dict{...} representing
+  the endpoints (specifically the `remote_ip` field and the packet's timestamp).
+  """
+
+  def __init__(self, host_ip: IpAddress):
+    self._host_ip = host_ip
+
+  def monitor(self) -> Generator[IpPacket, None, None]:
+    """Generator for observations of remote IP addresses.
+
+    The monitor is a blocking operation due to how process output is produced.
+    """
+    tshark_process = tshark.start_process(
+        capture_filter=f'ip and src ip == {self._host_ip}',
+        output_format='fields',
+        fields=['frame.time', 'ip.dst'])
+
+    for line in iter(tshark_process.stdout.readline, b''):
+      values = line.strip().split('\t')
+      ip_info = IpPacket(
+        timestamp=datetime.strptime(values[0], '%b %d, %Y %H:%M:%S %Z'),
+        remote_ip=ipaddress.ip_address(values[1]))
+      yield ip_info
+