diff --git a/config_example.json b/config_example.json index 145b937..2268dc8 100644 --- a/config_example.json +++ b/config_example.json @@ -1,29 +1,39 @@ { "ioc_sources": [ { + "format": "BAD_IP_LIST", "name": "EmergingThreats compromised IP addresses", - "url": "https://rules.emergingthreats.net/open/suricata-5.0/rules/compromised-ips.txt", - "format": "BAD_IP_LIST" + "url": "https://rules.emergingthreats.net/open/suricata-5.0/rules/compromised-ips.txt" }, { + "format": "BAD_IP_LIST", "name": "3CORESec Poor Reputation IPs", - "url": "https://blacklist.3coresec.net/lists/et-open.txt", - "format": "BAD_IP_LIST" + "url": "https://blacklist.3coresec.net/lists/et-open.txt" }, { + "format": "BAD_IP_LIST", "name": "CINSscore badguys list - cinsscore.com", - "url": "https://cinsscore.com/list/ci-badguys.txt", - "format": "BAD_IP_LIST" + "url": "https://cinsscore.com/list/ci-badguys.txt" }, { + "format": "BAD_IP_LIST", "name": "blocklist.de suspicious IPs", - "url": "https://lists.blocklist.de/lists/all.txt", - "format": "BAD_IP_LIST" + "url": "https://lists.blocklist.de/lists/all.txt" }, { + "format": "BAD_IP_LIST", "name": "TOR exit nodes (dan.me.uk)", - "url": "https://www.dan.me.uk/torlist/?exit", - "format": "BAD_IP_LIST" + "url": "https://www.dan.me.uk/torlist/?exit" + }, + { + "format": "ALLOWED_SNI_PORT", + "name": "recognized unusual TLS (name, port) combinations", + "allow": [ + ["mtalk.google.com", 5228], + ["proxy-safebrowsing.googleapis.com", 80], + ["courier.push.apple.com", 5223], + ["imap.gmail.com", 993] + ] } ] } diff --git a/rids.service b/rids.service index 170cbce..f250510 100644 --- a/rids.service +++ b/rids.service @@ -4,7 +4,7 @@ Description=Remote Intrusion Detection System daemon [Service] User=root WorkingDirectory=/usr/local/sbin -ExecStart=/usr/local/sbin/rids.py +ExecStart="/usr/local/sbin/rids.py --eventlog_path=/var/rids/events.log --config_path=/etc/rids/config.json" Restart=always [Install] diff --git a/tests/network_capture_test.py b/rids/event.py similarity index 52% rename from tests/network_capture_test.py rename to rids/event.py index 2de7147..d07a555 100644 --- a/tests/network_capture_test.py +++ b/rids/event.py @@ -12,31 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Tests for the network_capture library. -""" +class Event: + """Generalization of an Event sighting, i.e. suspicious activity or threat.""" -import io -import pytest + def __init__(self, properties): + # The event properties have a very loose definition at this moment. + # TODO stabilize the schema, perhaps based on MISP or Dovecot event types. + self.properties = properties -from rids import network_capture - - -@pytest.fixture() -def badips_map(): - return { - '1.1.1.1': 'Source A', - '4.5.6.7': 'Best IOCs', - '100.12.34.56': 'Threatbusters', - } - - -def test_good_ip_is_ok(badips_map): - detected = network_capture.detect_bad_ips('0\t127.0.0.1\t1.2.3.4', badips_map) - assert not detected - - -def test_bad_ip_is_logged(badips_map): - detected = network_capture.detect_bad_ips('1\t4.5.6.7\t127.0.0.1', badips_map) - assert detected + def __str__(self): + return str(self.properties) diff --git a/rids/ioc_formats/__init__.py b/rids/ioc_formats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rids/ioc_formats/allowed_sni_port.py b/rids/ioc_formats/allowed_sni_port.py new file mode 100644 index 0000000..a8aea67 --- /dev/null +++ b/rids/ioc_formats/allowed_sni_port.py @@ -0,0 +1,31 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""IOC parser for allowing certain unusual (server name, port) sightings.""" + + +from rids.rules.ruleset import RuleSet +from rids.rules.tls_matcher import TlsMatcher + + +class AllowedEndpoints: + """Parses allowed (sni, port) pairs directly from an ioc_sources config.""" + def __init__(self, config): + self.name = config['name'] + self.allowlist = config['allow'] + + def provide_rules(self, ruleset: RuleSet): + """Add rules to the TlsMatcher according to the configured allowlist.""" + for server_name, port in self.allowlist: + ruleset.tls_matcher.add_allowed_endpoint(server_name, port) \ No newline at end of file diff --git a/rids/ioc_formats/bad_ip_list.py b/rids/ioc_formats/bad_ip_list.py new file mode 100644 index 0000000..dcd5802 --- /dev/null +++ b/rids/ioc_formats/bad_ip_list.py @@ -0,0 +1,57 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""IOC feed parser for lists of compromised host IPs (newline separated).""" + + +from datetime import datetime +import ipaddress +import urllib.request + +from rids.iocs import RuleSet +from rids.rules.ip_matcher import IpRule +from rids.rules.ruleset import RuleSet + + +class BadIpAddresses: + """Parses newline-separated list of IP addresses into a RuleSet.""" + + def __init__(self, config): + self.source_url = config['url'] + self.source_name = config.get('name', 'UNKNOWN') + self.msg = config.get('msg', 'contact with a potentially compromised host') + + def provide_rules(self, ruleset: RuleSet): + """Add to the ruleset all the rules defined in this source.""" + fetch_time = datetime.now() + with urllib.request.urlopen(self.source_url) as ioc_data: + for line in ioc_data.readlines(): + line = line.decode('utf-8').strip() + if not line or line[0] == '#': + # skip empty lines and comments + continue + try: + bad_ip = ipaddress.ip_address(line) + except ValueError: + # Do nothing with badly-formatted addresses in these files. + continue + + ip_rule = IpRule( + msg=self.msg, + name=self.source_name, + url=self.source_url, + fetched=fetch_time, + matches_ip=bad_ip, + ) + ruleset.ip_matcher.add_ip_rule(ip_rule) diff --git a/rids/iocs.py b/rids/iocs.py new file mode 100644 index 0000000..15e663f --- /dev/null +++ b/rids/iocs.py @@ -0,0 +1,64 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Combined IOC feed parser interface for the different formats of IOCs.""" + + +import collections +from dataclasses import dataclass +from types import Set + +from rids.ioc_formats import allowed_sni_port +from rids.ioc_formats import bad_ip_list +from rids.rules.ruleset import RuleSet + + +_IOC_PARSERS = { + "BAD_IP_LIST": bad_ip_list.BadIpAddresses, + "ALLOWED_SNI_PORT": allowed_sni_port.AllowedEndpoints, +} + + +def fetch_iocs(rids_config): + """Parses the IOC indicated by `ioc_config` into a RuleSet instance. + + Args: + rids_config: dict{'ioc_sources': list[ + dict{ + 'format': str, + ...other properties, format-dependent + }]} + + Returns: + RuleSet representing all IOCs in the config. + """ + ioc_sources = rids_config.get('ioc_sources', None) + ruleset = RuleSet() + if not ioc_sources: + return ruleset + + for ioc_config in ioc_sources: + format = ioc_config.get('format', None) + if not format: + raise ValueError( + f'Expected a "format" specifier for ioc_sources config: {ioc_config}') + + if format not in _IOC_PARSERS: + raise ValueError('Unrecognized IOC feed format "{format}"') + + ioc_source = _IOC_PARSERS[format](ioc_config) + ioc_source.provide_rules(ruleset) + + return ruleset + diff --git a/rids/monitors/__init__.py b/rids/monitors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rids/monitors/ip_monitor.py b/rids/monitors/ip_monitor.py new file mode 100644 index 0000000..fc3791d --- /dev/null +++ b/rids/monitors/ip_monitor.py @@ -0,0 +1,67 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""IP monitoring classes and functionality. + +IpPacketMonitor for obtaining remote IPs. +IpPacket as the type for communicating to related rules. +IpRule for representing the IOCs that packets should be checked against. +""" + +from dataclasses import dataclass +from datetime import datetime +import ipaddress +from typing import Union +from typing import Generator + +from rids.monitors import tshark + + +IpAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + + +@dataclass +class IpPacket: + """Represents a remote IP endpoint and when it was seen.""" + timestamp: datetime + ip_address: IpAddress + + +class IpPacketMonitor: + """Continuously monitors the network traffic for remote IPs being contacted. + + The monitor() function is an iterator that produces a dict{...} representing + the endpoints (specifically the `remote_ip` field and the packet's timestamp). + """ + + def __init__(self, host_ip: IpAddress): + self._host_ip = host_ip + + def monitor(self) -> Generator[IpPacket, None, None]: + """Generator for observations of remote IP addresses. + + The monitor is a blocking operation due to how process output is produced. + """ + tshark_process = tshark.start_process( + capture_filter=f'ip and src ip == {self._host_ip}', + output_format='fields', + fields=['frame.time', 'ip.dst']) + + for line in iter(tshark_process.stdout.readline, b''): + values = line.strip().split('\t') + ip_info = IpPacket( + timestamp=datetime.strptime(values[0], '%b %d, %Y %H:%M:%S %Z'), + remote_ip=ipaddress.ip_address(values[1])) + yield ip_info + diff --git a/rids/monitors/tls_monitor.py b/rids/monitors/tls_monitor.py new file mode 100644 index 0000000..1fd525d --- /dev/null +++ b/rids/monitors/tls_monitor.py @@ -0,0 +1,92 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from dataclasses import dataclass +from datetime import datetime +import ipaddress +from typing import Generator +from typing import Union + +from rids.monitors import tshark + +IpAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + + +@dataclass +class TlsConnection: + """Represents salient properties of a TLS connection and when it was seen.""" + timestamp: datetime + remote_ip: IpAddress + remote_port: int + server_name: str + ja3: str + ja3_full: str + ja3s: str + ja3s_full: str + + +class TlsConnectionMonitor: + """Continuously monitors the network traffic for TLS handshakes. + + Use monitor() to generate TlsConnection instances from network traffic. + """ + def __init__(self, host_ip: IpAddress): + self._tls_streams = {} + self._host_ip = host_ip + + def monitor(self) -> Generator[TlsConnection, None, None]: + """Generator for observations of unusual TLS traffic. + + """ + tshark_process = tshark.start_process( + capture_filter='tcp and not (src port 443 or dst port 443)', + display_filter=( + f'(tls.handshake.type == 1 and ip.src == {self._host_ip})' + + f'or (tls.handshake.type == 2 and ip.dst == {self._host_ip})'), + output_format='fields', + fields=['frame.time', + 'tcp.stream', + 'tls.handshake.type', + 'ip.src', + 'tcp.srcport', + 'ip.dst', + 'tcp.dstport', + 'tls.handshake.extensions_server_name', + 'tls.handshake.ja3', + 'tls.handshake.ja3_full', + 'tls.handshake.ja3s', + 'tls.handshake.ja3s_full']) + + for line in iter(tshark_process.stdout.readline, b''): + values = line.split('\t') + stream_id = int(values[1]) + if int(values[2]) == 1: # client hello + tls_info = TlsConnection( + timestamp=datetime.strptime(values[0], '%b %d, %Y %H:%M:%S %Z'), + remote_ip=ipaddress.ip_address(values[5]), + remote_port=int(values[6]), + server_name=values[7], + ja3=values[8], + ja3_full=values[9]) + self._tls_streams[stream_id] = tls_info + + elif int(values[2]) == 2: # server hello + tls_info = self._tls_streams.get(stream_id, None) + if not tls_info: + continue + tls_info.ja3s = values[8] + tls_info.ja3s_full = values[9] + yield tls_info + del self._tls_streams[stream_id] diff --git a/rids/monitors/tshark.py b/rids/monitors/tshark.py new file mode 100644 index 0000000..8d1787c --- /dev/null +++ b/rids/monitors/tshark.py @@ -0,0 +1,58 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import subprocess + + +def start_process( + capture_filter: str = None, + display_filter: str = None, + fields: list[str] = [], + output_format: str = 'fields', + flush_every_packet: bool = True) -> subprocess.Popen: + """Starts tshark as a Popen process and pipes its stdout and stderr. + + For more on filter formats, see: + > man pcap-filter + https://www.tcpdump.org/manpages/pcap-filter.7.html + > man wireshark-filter + https://www.wireshark.org/docs/man-pages/wireshark-filter.html + + Args: + capture filter: pcap-style filter rule for packets to monitor + display filter: additional filters to apply (wireshark-filter format) + fields: list of field names as defined by wireshark + output_format: how to format the packet data being output + flush_every_packet: whether tshark should flush instead of buffering output + """ + if not len(fields): + raise ValueError("Must pass at least one field to tshark") + + command = ['tshark'] + if capture_filter: + command.extend(['-f', capture_filter]) + if display_filter: + command.extend(['-Y', display_filter]) + command.extend(['-T', output_format]) + for field in fields: + command.extend(['-e', field]) + if flush_every_packet: + command.append('-l') + + return subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) diff --git a/rids/network_capture.py b/rids/network_capture.py deleted file mode 100644 index 4636b16..0000000 --- a/rids/network_capture.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2022 Jigsaw Operations LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import ipaddress -import logging - - -allowed_sni_port = set([ - ('mtalk.google.com', 5228), - ('proxy-safebrowsing.googleapis.com', 80), - ('courier.push.apple.com', 5223), - ('imap.gmail.com', 993), -]) - - -# TODO: decouple detection and logging - -def detect_tls_events(input_stream): - unusual_tls_traffic = {} # map[int]dict stream_id -> connection_details - - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%m-%d %H:%M', - filename='/var/maldetector.log', - filemode='a') - - for line in iter(input_stream, b''): - values = line.split('\t') - if len(values) >= 8: - stream_id = int(values[1]) - if int(values[2]) == 1: # client hello - sni_and_port = (values[7], int(values[6])) - if sni_and_port not in allowed_sni_port: - unusual_tls_traffic[stream_id] = { - 'client_ts': values[0], - 'server_ip': values[5], - 'server_port': values[6], - 'server_name': values[7], - 'ja3': values[8], - 'ja3_full': values[9], - } - elif int(values[2]) == 2: # server hello - port = int(values[4]) - if port == 443: - continue - connection_details = unusual_tls_traffic.get(stream_id, None) - if not connection_details: - continue - connection_details['ja3s'] = values[8] - connection_details['ja3s_full'] = values[9] - logging.info('connection_details %s', connection_details) - del unusual_tls_traffic[stream_id] - - print(line.rstrip()) - - -def warn_about_ip_address(ip_address, bad_ips): - ip_address = str(ip_address) - if ip_address in bad_ips: - ioc_sources = bad_ips[ip_address] - logging.info('CONNECTING WITH BAD IP %s (found in %s)', - ip_address, ioc_sources) - return True - return False - - -def detect_bad_ips(dataline, bad_ips): - if not dataline: - return False - values = dataline.split('\t') - ip_from, ip_to = ipaddress.ip_address(values[1]), ipaddress.ip_address(values[2]) - if (warn_about_ip_address(ip_from, bad_ips) - or warn_about_ip_address(ip_to, bad_ips)): - return True - return False diff --git a/rids/rids.py b/rids/rids.py index 192670e..8a1888b 100755 --- a/rids/rids.py +++ b/rids/rids.py @@ -14,114 +14,146 @@ # See the License for the specific language governing permissions and # limitations under the License. - """ Main entry point for RIDS, a Remote Intrusion Detection System. """ - +import asyncio +from concurrent.futures import ThreadPoolExecutor +from functools import partial import ipaddress import json +import logging import os -import subprocess -import sys -import urllib.request +import urllib +import queue +from typing import Union from absl import app from absl import flags -from rids import network_capture +from rids import iocs +from rids.monitors.ip_monitor import IpPacketMonitor +from rids.monitors.tls_monitor import TlsConnectionMonitor +from rids.rules.ruleset import RuleSet FLAGS = flags.FLAGS flags.DEFINE_string('host_ip', None, - 'The IP of the host process, for ignoring direct requests') -flags.DEFINE_string('config_path', 'config.json', - 'Path where IOC configuration can be found; uses config.json by default') - - -def retrieve_bad_ips(): - bad_ips = {} - with open(FLAGS.config_path, 'r') as f: - config = json.load(f) - for source in config['ioc_sources']: - # Download a fresh version of each IOC source in the config. - with urllib.request.urlopen(source['url']) as ioc_data: - # TODO check format of ioc source, branch to different parsers - # For now, the only source format is newline-seprated bad IPv4 addresses - # TODO perform parsing in a separate function from config handling - for line in ioc_data.readlines(): - line = line.strip().decode('utf-8') - if not line: continue - bad_ip = str(ipaddress.ip_address(line)) - if bad_ip not in bad_ips: - bad_ips[bad_ip] = [source['name']] - else: - bad_ips[bad_ip].append(source['name']) - return bad_ips - - -def get_external_ip(): - with urllib.request.urlopen('https://ipinfo.io/ip') as my_ip: - return my_ip.read().decode("utf-8").strip() + 'The IP of the host process, helps determine whether an IP ' + 'is a client or a remote endpoint.') +flags.DEFINE_string('config_path', '/etc/rids/config.json', + 'file path indicating where IOC configuration can be found') +flags.DEFINE_string('eventlog_path', None, + 'file path indicating where to append new event logs') def main(argv): + """Main entry point of the app. Also bound to CLI `rids` by setuptools.""" if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') + eventlog_path = _get_eventlog_path() + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M', + filename=eventlog_path, + filemode='a') + + config = _load_config() + ruleset : RuleSet = iocs.fetch_iocs(config) + + host_ip = _get_host_ip() + print(f'Using {host_ip} as host IP address') + + event_queue = queue.Queue() + loop = asyncio.get_event_loop() + with ThreadPoolExecutor() as executor: + loop.run_in_executor( + executor, + partial(_inspect_tls_traffic, host_ip, ruleset, event_queue)) + loop.run_in_executor( + executor, + partial(_inspect_remote_ips, host_ip, ruleset, event_queue)) + + while True: + # We just log the suspicious events for now, but here is where we could + # do post-processing and/or share sightings of known / unknown threats. + event = event_queue.get() + logging.log(json.dumps(event)) + event_queue.task_done() + + +IpAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + +def _inspect_tls_traffic(host_ip: IpAddress, ruleset: RuleSet, q: queue.Queue): + """Worker function for thread that inspects client and server hellos. + + Args: + host_ip: the IP address associated with this server, for determining + client hellos and server hellos not interfacing directly with this host + ruleset: a RuleSet object to perform evaluation with + q: a queue.Queue for relaying events back to the main thread + """ + tls_connection = TlsConnectionMonitor(host_ip) + for tls_info in tls_connection.monitor(): + events = ruleset.tls_matcher.match_tls(tls_info) + for event in events: + q.put(event) + + +def _inspect_remote_ips(host_ip: IpAddress, ruleset: RuleSet, q: queue.Queue): + """Worker function for thread that inspects remote IP addresses. + + Args: + host_ip: the IP address associated with this server, for determining + which IP addresses are considered remote IPs + ruleset: a RuleSet object to perform evaluation with + q: a queue.Queue for relaying events back to the main thread + """ + remote_ip = IpPacketMonitor(host_ip) + for ip_info in remote_ip.monitor(): + events = ruleset.ip_matcher.match_ip(ip_info) + for event in events: + q.put(event) + + +def _get_host_ip() -> IpAddress: + """Retrieves the host IP address from its flag, else from a remote site. + + Returns: + ipaddress of the host running RIDS + """ host_ip = FLAGS.host_ip if not FLAGS.host_ip: - host_ip = get_external_ip() + with urllib.request.urlopen('https://ipinfo.io/ip') as my_ip: + host_ip = my_ip.read().decode("utf-8").strip() host_ip = ipaddress.ip_address(host_ip) - print('Using', host_ip, 'as host IP address') - - # Spawn tshark and read its output. This assumes the wireshark-dev/stable - # version of tshark has been installed. - oddtls_capture = subprocess.Popen([ - 'tshark', - '-f', 'tcp and not (src port 443 or dst port 443)', - '-Y', (f'(tls.handshake.type == 1 and ip.src == {host_ip})' + - f'or (tls.handshake.type == 2 and ip.dst == {host_ip})'), - '-Tfields', - # The order of the following fields determines the ordering in output - '-e', 'frame.time', - '-e', 'tcp.stream', - '-e', 'tls.handshake.type', - '-e', 'ip.src', - '-e', 'tcp.srcport', - '-e', 'ip.dst', - '-e', 'tcp.dstport', - '-e', 'tls.handshake.extensions_server_name', - '-e', 'tls.handshake.ja3', - '-e', 'tls.handshake.ja3_full', - '-e', 'tls.handshake.ja3s', - '-e', 'tls.handshake.ja3s_full', - '-l', - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - network_capture.detect_tls_events(oddtls_capture.stdout.readline) - - bad_ips = retrieve_bad_ips() - ip_capture = subprocess.Popen([ - 'tshark', - '-Tfields', - # The order of the following fields determines the ordering in output - '-e', 'frame.time', - '-e', 'ip.src', - '-e', 'ip.dst', - '-l', - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - - # TODO define a class in packet_filter instead of using a function - # filter = network_capture.Filter(); filter.add_bad_ips(bad_ips); ... - for line in iter(ip_capture.stdout.readline, b''): - network_capture.detect_bad_ips(line, bad_ips) + return host_ip + + +def _load_config() -> dict: + """Read and parse the config file contents. + + Returns: + dict of config contents + """ + config = {} + if FLAGS.config_path: + with open(FLAGS.config_path) as f: + config = json.load(f) + return config + + +def _get_eventlog_path() -> str: + """Determine where to save the event logs. + + If not specified in the flag, use the current working directory. + """ + path = FLAGS.eventlog_path + if not path: + path = os.getcwd() + return path if __name__ == '__main__': - app.run(main) + app.run(main) \ No newline at end of file diff --git a/rids/rules/__init__.py b/rids/rules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rids/rules/ip_matcher.py b/rids/rules/ip_matcher.py new file mode 100644 index 0000000..7175435 --- /dev/null +++ b/rids/rules/ip_matcher.py @@ -0,0 +1,76 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import datetime +from dataclasses import dataclass +import ipaddress +from typing import Union + +from rids.event import Event +from rids.monitors import ip_monitor + +IpAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + + +@dataclass +class IpRule: + """Represents a single rule that accommodates a variety of IOC rule types.""" + + msg: str + name: str + url: str + fetched: datetime.datetime + matches_ip: IpAddress + reference: str = None + + def __str__(self): + output = [ + f'[{self.matches_ip}] {self.msg}', + f'Found in [{self.name}] last fetched at {self.fetched}', + ] + + if self.reference: + output.append(self.reference) + return '\n'.join(output) + + +class IpMatcher: + """Index over IP address rules, compatible with both IPv4 and IPv6.""" + + def __init__(self): + self.ip_address_rules = collections.defaultdict(list) + + def add_ip_rule(self, ip_rule: IpRule) -> None: + """Add a single IP-based rule to this rule set.""" + self.ip_address_rules[str(ip_rule.matches_ip)].append(ip_rule) + + def match_ip(self, ip_packet: ip_monitor.IpPacket) -> Event: + """Process observations related to a remote IP address.""" + ip_str = str(ip_packet.ip_address) + events = [] + if ip_str in self.ip_address_rules: + for rule in self.ip_address_rules[ip_str]: + event = Event({ + 'timestamp': ip_packet.timestamp, + 'remote_ip': ip_packet.ip_address, + 'msg': rule.msg, + 'name': rule.name, + 'url': rule.url, + 'fetched': rule.fetched, + 'reference': rule.reference, + }) + events.append(event) + return events + diff --git a/rids/rules/ruleset.py b/rids/rules/ruleset.py new file mode 100644 index 0000000..ce12dd3 --- /dev/null +++ b/rids/rules/ruleset.py @@ -0,0 +1,23 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +from rids.rules.ip_matcher import IpMatcher +from rids.rules.tls_matcher import TlsMatcher + + +class RuleSet: + ip_matcher : IpMatcher = IpMatcher() + tls_matcher : TlsMatcher = TlsMatcher() diff --git a/rids/rules/tls_matcher.py b/rids/rules/tls_matcher.py new file mode 100644 index 0000000..97f9907 --- /dev/null +++ b/rids/rules/tls_matcher.py @@ -0,0 +1,51 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from rids.event import Event +from rids.monitors import tls_monitor + + +class TlsMatcher: + """Index over TLS-related rules to match against.""" + + def __init__(self): + self.allowed_sni_port = set() + + def add_allowed_endpoint(self, allowed_sni: str, expected_port: int) -> None: + """Add a single TLS connection-based rule to this rule set.""" + self.allowed_sni_port.add((allowed_sni, expected_port)) + + def match_tls(self, tls_connection: tls_monitor.TlsConnection) -> Event: + """Process observations of TLS client/server hellos. + + Returns: + list of Event detials, or an empty list if nothing matches. + """ + sni_and_port = (tls_connection.server_name, tls_connection.remote_port) + if sni_and_port in self.allowed_sni_port: + return [] + + event = Event({ + 'timestamp': tls_connection.timestamp, + 'remote_ip': tls_connection.remote_ip, + 'remote_port': tls_connection.remote_port, + 'server_name': tls_connection.server_name, + 'ja3': tls_connection.ja3, + 'ja3_full': tls_connection.ja3_full, + 'ja3s': tls_connection.ja3s, + 'ja3s_full': tls_connection.ja3s_full, + }) + return [event] + diff --git a/tests/ip_matcher_test.py b/tests/ip_matcher_test.py new file mode 100644 index 0000000..1678f3c --- /dev/null +++ b/tests/ip_matcher_test.py @@ -0,0 +1,59 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for IP address rules creation and matching.""" + +from datetime import datetime +import ipaddress +import pytest + +from rids.monitors import ip_monitor +from rids.rules.ip_matcher import IpMatcher +from rids.rules.ip_matcher import IpRule + + +@pytest.fixture() +def ip_matcher() -> IpMatcher: + ip_matcher = IpMatcher() + ip_matcher.add_ip_rule(IpRule( + msg='test', + name='TEST', + url=None, + fetched=datetime.now(), + matches_ip=ipaddress.ip_address('1.1.1.1') + )) + ip_matcher.add_ip_rule(IpRule( + msg='test', + name='TEST', + url=None, + fetched=datetime.now(), + matches_ip=ipaddress.ip_address('100.12.34.56') + )) + return ip_matcher + + +def test_good_ip_address(ip_matcher): + ip_packet = ip_monitor.IpPacket( + timestamp='Jan 10 13:37:42 EST 2023', + ip_address='3.5.7.9') + detected_events = ip_matcher.match_ip(ip_packet) + assert not detected_events + + +def test_bad_ip_address(ip_matcher): + ip_packet = ip_monitor.IpPacket( + timestamp='Jan 10 13:37:68 EST 2023', + ip_address='100.12.34.56') + detected_events = ip_matcher.match_ip(ip_packet) + assert detected_events diff --git a/tests/tls_matcher_test.py b/tests/tls_matcher_test.py new file mode 100644 index 0000000..3a0d07c --- /dev/null +++ b/tests/tls_matcher_test.py @@ -0,0 +1,71 @@ +# Copyright 2022 Jigsaw Operations LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for TLS connection rules creation and matching.""" + +from datetime import datetime +import ipaddress +import pytest + +from rids.ioc_formats.allowed_sni_port import AllowedEndpoints +from rids.rules.ruleset import RuleSet +from rids.rules.tls_matcher import TlsMatcher +from rids.monitors.tls_monitor import TlsConnection + + +@pytest.fixture() +def tls_matcher() -> TlsMatcher: + tls_matcher = TlsMatcher() + tls_matcher.add_allowed_endpoint('example.com', 421) + tls_matcher.add_allowed_endpoint('justchatting.com', 5223) + return tls_matcher + + +def _fake_connection(server_name: str, port: int): + tls_connection = TlsConnection( + timestamp=str(datetime.now()), + remote_ip=ipaddress.ip_address('101.23.45.67'), + remote_port=port, + server_name=server_name, + ja3='...', + ja3_full='...', + ja3s='...', + ja3s_full='...', + ) + return tls_connection + + +def test_ok_tls_connection(tls_matcher): + tls_connection = _fake_connection(server_name='justchatting.com', port=5223) + detected_events = tls_matcher.match_tls(tls_connection) + assert not detected_events + + +def test_suspicious_tls_connection(tls_matcher): + tls_connection = _fake_connection(server_name='notevil.com', port=444) + detected_events = tls_matcher.match_tls(tls_connection) + assert detected_events + + +def test_parse_allowed_sni_config(): + config = { + 'format': 'ALLOWED_SNI_PORT', + 'name': 'recognized unusual TLS (name, port) combinations', + 'allow': [ + ['mtalk.google.com', 5228], + ['courier.push.apple.com', 5223], + ] + } + parser = AllowedEndpoints(config) + parser.provide_rules(RuleSet()) \ No newline at end of file