Adding support for Recorded Future custom bundles

external-import/recorded-future/Makefile

@@ -0,0 +1,204 @@
+# The only place to bump the version
+VERSION := 1.0.2
+VERSION_FILE := src/rflib/_version.py
+BUILDID := $(if $(SVN_REVISION),$(SVN_REVISION),$(shell git rev-list --count HEAD))
+PCK_DIR := ps_opencti_notes$(VERSION)-$(BUILDID)
+PACKAGE := $(PCK_DIR).tar.gz
+BUILD_DIR := pkg_build
+BUILD_OUT := $(BUILD_DIR)/$(PCK_DIR)
+
+#####TODO: THIS SHOULD BE THE REVIEW
+###flake8 --ignore=E,W.
+##isort --profile black .
+##black .
+######
+# Commands
+SED := sed
+BLACK := black
+FLAKE8 := flake8 --ignore=E,W
+ISORT := isort --profile black .
+PYLINT := pylint
+PEPDOCSTYLE := pydocstyle
+PYTHON := python3
+RUN_UNIT_TESTS := $(PYTHON) -m pytest
+CWD := $(shell pwd)
+VIRTUALENV := $(shell command -v virtualenv 2> /dev/null)
+UNAME := $(shell uname)
+RSYNC := rsync -rav
+TAR := tar
+MV := mv
+
+# SED command arguments to replace 9.9.9-dev with the value of $(VERSION)
+NEW_VERSION_STRING:=
+ifeq ($(UNAME),Darwin)
+NEW_VERSION_STRING:= -i '' 's/9.9.9-dev/$(VERSION)/g' src/rflib/_version.py
+else
+NEW_VERSION_STRING:= -i 's/9.9.9-dev/$(VERSION)/g' src/rflib/_version.py
+endif
+
+# UNDO SED command arguments to replace value of $(VERSION) with 9.9.9-dev
+OLD_VERSION_STRING :=
+ifeq ($(UNAME),Darwin)
+OLD_VERSION_STRING := -i '' 's/$(VERSION)/9.9.9-dev/g' src/rflib/_version.py
+else
+OLD_VERSION_STRING := -i 's/$(VERSION)/9.9.9-dev/g' src/rflib/_version.py
+endif
+
+.PHONY: unittests, clear_cache, review, black, check_black, flake8, pylint, pydocstyle, test, clean, setup
+
+help:
+	@echo "Available targets:"
+	@echo " setup       - Setup dev environment (install venv, dependencies)"
+	@echo " test        - run review and unittests"
+	@echo " unittests   - run nosetests to perform unittests"
+	@echo " review      - run black_check, pylint and pydocstyle"
+	@echo "   review_style  - Code style review"
+	@echo "     check_black     - checks that black has been applied"
+	@echo "   pydocstyle      - review inline documentation"
+	@echo "   pylint          - pylint static analysis"
+	@echo " build       - prepares the files for packaging"
+	@echo " package     - run build & prepare the tar.gz file"
+	@echo "Miscellaneous targets:"
+	@echo " clean       - delete build and packaging artifacts"
+	@echo " clear_cache - remove python pycache and pytest_cache"
+	@echo " black       - formats source code with black"
+	@echo " flake8      - checks for flake8 code syntax"
+	@echo " isort       - runs isort"
+	@echo " format       - formats things according to black + isortstandard + flake9"
+
+##########################################
+#
+# Targets related to versioning (_version.py)
+#
+##########################################
+addVersion:
+	@echo "* Adding version $(VERSION) to $(VERSION_FILE)"
+	$(SED) $(NEW_VERSION_STRING)
+
+undoVersion:
+	@echo "* Removing version $(VERSION) from $(VERSION_FILE)"
+	$(SED) $(OLD_VERSION_STRING)
+
+
+##########################################
+#
+# Targets related to build
+#
+##########################################
+build: addVersion clear_cache build_dirs build_rsync undoVersion
+
+
+build_dirs:
+	@echo "* Creating build root directory"
+	mkdir -p $(BUILD_DIR)
+	@if [ -d $(BUILD_OUT) ]; then rm -rf $(BUILD_OUT); fi
+	@echo "* Creating build output directory $(BUILD_OUT)"
+	mkdir -p $(BUILD_OUT)
+	#mkdir -p $(BUILD_OUT)/deps
+
+build_rsync: build_dirs
+	@echo "* Copying files to build output directory"
+	$(RSYNC) src/ $(BUILD_OUT)/src --exclude='*.pyc' --exclude='config.yml'
+	$(RSYNC) Dockerfile $(BUILD_OUT)
+	$(RSYNC) docker-compose.yml $(BUILD_OUT)
+	$(RSYNC) README.md $(BUILD_OUT)
+	$(RSYNC) entrypoint.sh $(BUILD_OUT)
+
+##########################################
+#
+# Targets related to package
+#
+##########################################
+package: build package_gzip
+
+
+package_gzip:
+	@echo "* Packaging $(PACKAGE)"
+	$(TAR) -zcv --exclude='.DS_Store' --exclude "__MACOSX" --exclude "__pycache__" -C $(BUILD_DIR) -f $(PACKAGE) $(PCK_DIR)
+	@echo "* Removing build artifacts"
+	rm -rf $(BUILD_OUT)/*
+	@echo "* Moving package to build output directory"
+	$(MV) $(PACKAGE) $(BUILD_OUT)
+	@echo "* Packaging has compelted successfully"
+	@echo "* Package is ready for client delivery: $(BUILD_OUT)/$(PACKAGE)"
+
+##########################################
+#
+# Targets related to development environment setup
+#
+##########################################
+setup:
+	ifndef VIRTUALENV
+		@echo "virtualenv not found, using python3 -m venv venv"
+		$(PYTHON) -m venv venv
+	else
+		@echo "Creating virtual environment using virtualenv"
+		virtualenv venv
+	endif
+
+	. $(CWD)/venv/bin/activate && pip install -r requirements-dev.txt
+	@echo "Success! Development enviroment is ready"
+	@echo "Please activate the virtualenv with: source venv/bin/activate"
+
+
+##########################################
+#
+# Targets related to unittest
+#
+##########################################
+test: review unittests
+
+unittests:
+	@echo "* Starting unit tests"
+	$(RUN_UNIT_TESTS) --cov=src --cov-report html --cov-branch --cov-report term --junitxml=result.xml
+
+##########################################
+#
+# Targets related to code review
+#
+##########################################
+review: review_style pylint pydocstyle
+
+review_style: check_black
+
+format: flake8 isort black
+
+isort:
+	$(ISORT) src/rflib/*.py
+	$(ISORT) src/*.py
+	$(ISORT) tests/*.py
+
+black:
+	$(BLACK) src/rflib/*.py --exclude="static"
+	$(BLACK) src/*.py --exclude="static"
+	$(BLACK) tests/*.py --exclude="static"
+
+check_black:
+	$(BLACK) src/rflib/*.py --exclude="static" --check
+	$(BLACK) src/*.py --exclude="static" --check
+	$(BLACK) tests/*.py --exclude="static" --check
+
+flake8:
+	$(FLAKE8) src/rflib/*.py
+	$(FLAKE8) src/*.py
+	$(FLAKE8) tests/*.py
+
+pylint:
+	$(PYLINT) -E src/rflib/*.py
+	$(PYLINT) -E src/*.py
+	$(FLAKE8) tests/*.py
+
+pydocstyle:
+	$(PEPDOCSTYLE) src/rflib/*.py
+	$(PEPDOCSTYLE) src/*.py
+
+##########################################
+#
+# Misc targets
+#
+##########################################
+clear_cache:
+	@echo "* Removing __pycache__ and .pytest_cache"
+	rm -rf src/rflib/__pycache__
+	rm -rf tests/__pycache__
+	rm -rf .pytest_cache

external-import/recorded-future/README.md

@@ -110,6 +110,9 @@ Below are the parameters you'll need to set for Recorded Future connector:
 | Risk list threshold          | `risk_list_threshold`        | `RECORDED_FUTURE_RISK_LIST_THRESHOLD`       | `70`                                                  | No        | A threshold under which related indicators are not taken into account. Indicators from Risk Lists.                                                                                                                                                             |
 | Risk list related entities   | `risklist_related_entities`  | `RECORDED_FUTURE_RISKLIST_RELATED_ENTITIES` | `Malware,Hash,URL,Threat Actor,MitreAttackIdentifier` | Yes       | Related entities to an indicator from Risk List when it's imported. Required if pull_risk_list is True, possible values: Malware,Hash,URL,Threat Actor,MitreAttackIdentifier. Multiple related entities are allowed (separated by ',')                         |
 | Pull threat maps             | `pull_threat_maps`           | `RECORDED_FUTURE_PULL_THREAT_MAPS`          | `False`                                               | No        | A boolean flag of whether to pull entities from Threat Maps into OpenCTI.                                                                                                                                                                                      |
+| RF Custom Bundle Paths         | custom_bundle_paths      | `RECORDED_FUTURE_CUSTOM_BUNDLE_PATHS`      | /public/opencti/threat_actor_bundle.json                                                    | Yes       | Comma seperated Fusion file paths of STIX2 bundles to import from the Recorded Future API. Leaving this empty will import no bundles. Currently only one bundle contaiing threat actor profiles is availible.                                                                                                                                                              |
+| RF Custom Bundle interval         | custom_bundle_interval      | `RECORDED_CUSTOM_BUNDLE_INTERVAL`      | 24                                                    | Yes       | Interval in hours to reimport custom bundles specified in the preivous argument 1                                                                                                                                                               |
+                                                                                                                                                      |
 
 
 ## Deployment
@@ -242,7 +245,7 @@ Recorded Future comes equipped with five Recorded Future Risk Lists, which serve
 
 Subscribers who have API access can retrieve lists of entities that have been assigned risk scores by Recorded Future by utilizing the Connect API calls.
 
-Every item in a Risk List, whether it's an IP address, domain, or another element, comes with a risk score and the details that influenced that score. Additionally, having Fusion access enables the customization of Risk Lists.
+Every item in a Risk List, whether it's an IP address, domain, or another element, comes with a risk score and the details that influenced that score.
 
 Vulnerabilities are not handled by the connector.
 

external-import/recorded-future/docker-compose.yml

@@ -27,4 +27,6 @@ services:
       - RECORDED_FUTURE_RISK_LIST_THRESHOLD=70 #optional, can remove
       - RECORDED_FUTURE_RISKLIST_RELATED_ENTITIES='Malware,Hash,URL,Threat Actor,MitreAttackIdentifier' #required if RECORDED_FUTURE_PULL_RISK_LIST is True, possible values: Malware,Hash,URL,Threat Actor,MitreAttackIdentifier
       - RECORDED_FUTURE_PULL_THREAT_MAPS=False #optional, can remove
+      - RECORDED_FUTURE_CUSTOM_BUNDLE_PATHS=/public/opencti/threat_actor_bundle.json #optional
+      - RECORDED_FUTURE_CUSTOM_BUNDLE_INTERVAL=24 #in hours
     restart: always

external-import/recorded-future/requirements-dev.txt

@@ -0,0 +1,8 @@
+pytest==7.1.1
+pytest-vcr==1.0.2
+pytest-cov==3.0.0
+pytest-mock
+pylint==2.13.7
+pydocstyle==6.1.1
+black==22.3.0
+-r ./src/requirements.txt

external-import/recorded-future/src/config.yml.sample

@@ -29,6 +29,8 @@ rf:
   # Available choices: Malware,Hash,URL,Threat Actor,MitreAttackIdentifier
   risklist_related_entities: 'Malware,Threat Actor,MitreAttackIdentifier'
   pull_threat_maps: False # optional - Pull Threat Actors and Malware maps
+  custom_bundle_paths=/public/opencti/threat_actor_bundle.json #optional
+  custom_bundle_interval=24 #in hours
 
 alert:
   enable: False
@@ -39,4 +41,4 @@ alert:
   logo_abuse_rule_id: 'ChangeMe'
   initial_pull_off: 1
   download_path: 'images/'
-  logo_abuse_rule_id: 'ChangeMe'
+  logo_abuse_rule_id: 'ChangeMe'

external-import/recorded-future/src/main.py

@@ -18,6 +18,7 @@
 from pycti import OpenCTIConnectorHelper, get_config_variable
 from rflib import (
     APP_VERSION,
+    CustomBundles,
     RecordedFutureAlertConnector,
     RFClient,
     RiskList,
@@ -38,7 +39,9 @@ def __init__(self):
 
         # Extra config
         self.rf_token = get_config_variable(
-            "RECORDED_FUTURE_TOKEN", ["rf", "token"], config
+            "RECORDED_FUTURE_TOKEN",
+            ["rf", "token"],
+            config,
         )
         self.rf_initial_lookback = get_config_variable(
             "RECORDED_FUTURE_INITIAL_LOOKBACK",
@@ -99,6 +102,27 @@ def __init__(self):
             "RECORDED_FUTURE_PULL_THREAT_MAPS", ["rf", "pull_threat_maps"], config
         )
 
+        self.custom_bundle_paths = get_config_variable(
+            "RECORDED_FUTURE_CUSTOM_BUNDLE_PATHS",
+            ["rf", "custom_bundle_paths"],
+            config,
+        )
+        self.custom_bundle_interval = get_config_variable(
+            "RECORDED_FUTURE_CUSTOM_BUNDLE_INTERVAL",
+            ["rf", "custom_bundle_interval"],
+            config,
+        )
+        self.custom_bundle_paths = get_config_variable(
+            "RECORDED_FUTURE_CUSTOM_BUNDLE_PATHS",
+            ["rf", "custom_bundle_paths"],
+            config,
+        )
+        self.custom_bundle_interval = get_config_variable(
+            "RECORDED_FUTURE_CUSTOM_BUNDLE_INTERVAL",
+            ["rf", "custom_bundle_interval"],
+            config,
+        )
+
         risklist_related_entities_list = get_config_variable(
             "RECORDED_FUTURE_RISKLIST_RELATED_ENTITIES",
             ["rf", "risklist_related_entities"],
@@ -321,6 +345,18 @@ def all_processes(self):
         else:
             self.RF.helper.log_info("[ANALYST NOTES] Analyst notes fetching disabled")
 
+        if self.RF.custom_bundle_paths:
+            self.RF.helper.log_info("[CUSTOM BUNDLES] Starting bundles thread...")
+            self.CustomBundles = CustomBundles(
+                self.RF.helper,
+                self.RF.custom_bundle_interval,
+                self.RF.rfapi,
+                self.RF.custom_bundle_paths,
+            )
+            self.CustomBundles.start()
+        else:
+            self.RF.helper.log_info("[CUSTOM BUNDLES] Fetching custom bundles disabled")
+
     def run_all_processes(self):
         if self.RF.duration_period:
             self.RF.helper.schedule_iso(
@@ -339,6 +375,7 @@ def run_all_processes(self):
     try:
         RF_connector = RFConnector()
         RF_connector.run_all_processes()
+
     except Exception:
         traceback.print_exc()
         exit(1)

external-import/recorded-future/src/rflib/__init__.py

@@ -12,6 +12,7 @@
 
 from ._version import __version__ as APP_VERSION
 from .constants import RISK_LIST_TYPE_MAPPER
+from .custom_bundle import CustomBundles
 from .rf_alerts import RecordedFutureAlertConnector
 from .rf_client import RFClient
 from .rf_to_stix2 import (

external-import/recorded-future/src/rflib/custom_bundle.py

@@ -0,0 +1,43 @@
+import threading
+import time
+from datetime import datetime
+
+from urllib3.exceptions import HTTPError
+
+
+class CustomBundles(threading.Thread):
+    def __init__(
+        self,
+        helper,
+        interval,
+        rfapi,
+        paths,
+    ):
+        threading.Thread.__init__(self)
+        self.helper = helper
+        self.interval = interval
+        self.rfapi = rfapi
+        self.paths = paths
+
+    def run(self):
+        while True:
+            timestamp = int(time.time())
+            now = datetime.utcfromtimestamp(timestamp)
+            work_id = self.helper.api.work.initiate_work(
+                self.helper.connect_id,
+                "Recorded Future custom bundle import "
+                + now.strftime("%Y-%m-%d %H:%M:%S"),
+            )
+            for path in self.paths.split(",;"):
+                self.helper.log_info(f"[CUSTOM BUNDLES] Pulling bundle at {path}")
+                try:
+                    bundle = self.rfapi.get_fusion_file(path)
+                    self.helper.send_stix2_bundle(
+                        bundle,
+                        work_id=work_id,
+                    )
+                except (HTTPError, ValueError) as err:
+                    self.helper.log_error(err)
+                    continue
+            self.helper.set_state({"last_bundle_run": timestamp})
+            time.sleep(int(self.interval) * 3600)

external-import/recorded-future/src/rflib/rf_alerts.py

@@ -355,10 +355,12 @@ def run_for_time_period(self, trigger):
                     )
                 for entity in hit["entities"]:
                     if entity["type"] == "Image":
-                        image_presence, image_path, image_name = (
-                            self.api_recorded_future.get_image_and_save_temp_file(
-                                entity["name"]
-                            )
+                        (
+                            image_presence,
+                            image_path,
+                            image_name,
+                        ) = self.api_recorded_future.get_image_and_save_temp_file(
+                            entity["name"]
                         )
                         if image_presence:
                             stix_images.append(