From a47f9eed622a8e83809cce60ab29329fc7033396 Mon Sep 17 00:00:00 2001 From: DerekRushton Date: Wed, 29 Nov 2023 11:36:29 -0400 Subject: [PATCH] Initial To Stix mapping - Event and Transformers --- .../stix_translation/json/to_stix_map.json | 31 ++- .../stix_translation/json/to_stix_map.json | 206 +++++++++++++++++ .../stix_translation/query_constructor.py | 5 +- .../tanium/stix_translation/transformers.py | 212 +++++++++++++++++- .../tanium/stix_transmission/connector.py | 8 +- .../test_from_stix_to_query.py | 16 +- 6 files changed, 448 insertions(+), 30 deletions(-) diff --git a/stix_shifter_modules/alertflex/stix_translation/json/to_stix_map.json b/stix_shifter_modules/alertflex/stix_translation/json/to_stix_map.json index 2a5cbeefd..e62eccabe 100644 --- a/stix_shifter_modules/alertflex/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/alertflex/stix_translation/json/to_stix_map.json @@ -9,22 +9,21 @@ "transformer": "EpochToTimestamp" } ], - "srcip": - [ - { - "key": "ipv4-addr.value", - "object": "src_ip" - }, - { - "key": "ipv6-addr.value", - "object": "src_ip" - }, - { - "key": "network-traffic.src_ref", - "object": "nt", - "references": "src_ip" - } - ], + "srcip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], "dstip": [ { "key": "ipv4-addr.value", diff --git a/stix_shifter_modules/tanium/stix_translation/json/to_stix_map.json b/stix_shifter_modules/tanium/stix_translation/json/to_stix_map.json index 7a73a41bf..ed8a4805b 100644 --- a/stix_shifter_modules/tanium/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/tanium/stix_translation/json/to_stix_map.json @@ -1,2 +1,208 @@ { + "computerIpAddress": + [{ + "key": "ipv4-addr.value", + "object": "ip" + }, + { + "key": "x-oca-event.ip_refs", + "object": "x-oca-event", + "references": ["ip"] + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["ip"] + }], + "computerName": + [{ + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.host_ref", + "object": "x-oca-event", + "references": "asset" + }], + "state": + { + "key": "x-oca-event.outcome", + "object": "x-oca-event" + }, + "severity": + { + "key": "x-oca-event.severity", + "object": "x-oca-event" + }, + "details": [ + { + "key": "process", + "object": "process_all", + "transformer":"ProcessTransformer" + }, + { + "key": "process.pid", + "object": "process_some", + "transformer":"ProcessPidTransformer" + }, + { + "key": "process.created", + "object": "process_some", + "transformer":"ProcessCreatedTransformer" + }, + { + "key": "process.args", + "object": "process_some", + "transformer":"ProcessArgsTransformer" + }, + { + "key": "process.name", + "object": "process_some", + "transformer":"ProcessNameTransformer" + }, + { + "key": "process.cwd", + "object": "process_some", + "transformer":"ProcessCWDPathTransformer" + }, + { + "key": "user-account.user_id", + "object": "user", + "transformer":"ProcessUserIdTransformer" + }, + { + "key": "user-account.display_name", + "object": "user", + "transformer":"ProcessUserDisplayNameTransformer" + }, + { + "key": "user-account.is_service_account", + "object": "user", + "transformer":"ProcessUserDaemonTransformer" + }, + { + "key": "process.creator_user_ref", + "object": "process_some", + "references":"user" + }, + { + "key": "binary_ref.hashes", + "object": "processFile", + "transformer":"ProcessFileHashesTransformer" + }, + { + "key": "binary_ref.name", + "object": "processFile", + "transformer": "ProcessNameTransformer" + }, + { + "key": "parent_directory_ref.path", + "object": "processFileDirectory", + "transformer": "ProcessCWDPathTransformer" + }, + { + "key": "content_refs.issuer", + "object": "certificate", + "transformer": "ProcessFileCertificateIssuerTransformer" + }, + { + "key": "content_refs.subject", + "object": "certificate", + "transformer": "ProcessFileCertificateSubjectTransformer" + }, + { + "key": "binary_ref.parent_directory_ref", + "object": "processFile", + "references": "processFileDirectory" + }, + { + "key": "binary_ref.content_refs", + "object": "processFile", + "references": "certificate" + }, + { + "key": "process.binary_ref", + "object": "process_some", + "references": "processFile" + }, + { + "key": "x-oca-asset.process_ref", + "object": "x-oca-event", + "references": "process_some" + }, + { + "key": "x-oca-asset.file_ref", + "object": "x-oca-event", + "references": "processFile" + } +], + "matchType": + { + "key": "x-oca-event.category", + "object": "x-oca-event" + }, + "suppressedAt": + { + "key": "x-oca-event.end", + "object": "x-oca-event" + }, + "alertedAt": + { + "key": "x-oca-event.created", + "object": "x-oca-event" + }, + "createdAt": + { + "key":"x-oca-event.start", + "object":"x-oca-event" + }, + "intelDocs": + { + "id": + { + }, + "type": + { + "key":"x-oca-event.provider", + "object":"x-oca-event" + }, + "typeVersion": + { + }, + "md5": + { + }, + "name": + { + "key":"x-oca-event.action", + "object":"x-oca-event" + }, + "description": + { + "key":"x-oca-event.description", + "object":"x-oca-event" + }, + "size": + { + }, + "compiled": + { + }, + "isSchemaValid": + { + }, + "sourceId": + { + }, + "mitreAttack": + { + "techniques": + { + + } + }, + "status": + { + } + } } \ No newline at end of file diff --git a/stix_shifter_modules/tanium/stix_translation/query_constructor.py b/stix_shifter_modules/tanium/stix_translation/query_constructor.py index c53a7278d..e483b38ea 100644 --- a/stix_shifter_modules/tanium/stix_translation/query_constructor.py +++ b/stix_shifter_modules/tanium/stix_translation/query_constructor.py @@ -45,7 +45,7 @@ def _format_match(value) -> str: @staticmethod def _format_equality(value) -> str: - return '\'{}\''.format(value) + return '{}'.format(value) @staticmethod def _format_like(value) -> str: @@ -234,7 +234,6 @@ def translate_pattern(pattern: Pattern, data_model_mapping, options): print(f'{key}: {options[key]}') query = QueryStringPatternTranslator(pattern, data_model_mapping).translated - result_limit = f"&limit={options['result_limit']}" # Add space around START STOP qualifiers query = re.sub("START", "START ", query) @@ -246,4 +245,4 @@ def translate_pattern(pattern: Pattern, data_model_mapping, options): # A list is returned because some query languages require the STIX pattern to be split into multiple query strings. logger.info("The Query is " + query) - return ["%s%s" % (query, result_limit)] + return ["%s" % (query)] diff --git a/stix_shifter_modules/tanium/stix_translation/transformers.py b/stix_shifter_modules/tanium/stix_translation/transformers.py index e5ed096e2..103e2b93c 100644 --- a/stix_shifter_modules/tanium/stix_translation/transformers.py +++ b/stix_shifter_modules/tanium/stix_translation/transformers.py @@ -1,3 +1,6 @@ +import json +import pathlib +import shlex from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer from stix_shifter_utils.utils import logger @@ -6,13 +9,214 @@ # Implement custom transformer classes here. # The class name needs to be added to the module's to_stix_map.json -class SampleDataTransformer(ValueTransformer): +class ProcessTransformer(ValueTransformer): """A value transformer to convert to """ @staticmethod def transform(data): # Leave method name as is. try: - # add logic to transform data into desired format - return data + dataAsAJson = json.loads(data) + + #print (json.dumps(dataAsAJson, indent=4)) + + if(dataAsAJson["match"]['type'] == "process"): + process = dict() + process["type"] = dataAsAJson["match"]["type"] + process["pid"] = dataAsAJson["match"]["properties"]["pid"] + process["created"] = dataAsAJson["match"]["properties"]["start_time"] + + arguments_list = shlex.split(dataAsAJson["match"]["properties"]["args"]) + process["args"] = arguments_list + + converted_file = dataAsAJson["match"]["properties"]["name"].replace('\\', '/') + pathObject = pathlib.Path(converted_file) + process["name"] = pathObject.name + pathObject.suffix + process["cwd"] = pathObject.parent.as_posix() + + creator_user_ref = dict() + creator_user_ref["type"] = "user-account" + creator_user_ref["user_id"] = dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["user"]["user"]["user_id"] + creator_user_ref["display_name"] = dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["user"]["user"]["name"] + if (dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["user"]["user"]["domain"] is not None): + creator_user_ref["is_service_account"] = True + else: + creator_user_ref["is_service_account"] = False + + binary_ref = dict() + binary_ref["type"] = "file" + binary_ref["hashes"] = dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["file"]["file"]["hash"] + binary_ref["name"] = pathObject.name + pathObject.suffix + + parent_ref = dict() + parent_ref["type"] = "directory" + parent_ref["path"] = pathObject.parent.as_posix() + + binary_ref["parent_directory_ref"] = parent_ref + + certificate = dict() + certificate["type"] = "x509-certificate" + certificate["issuer"] = dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["file"]["file"]["signature_data"]["issuer"] + certificate["subject"] = dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["file"]["file"]["signature_data"]["subject"] + binary_ref["content_refs"] = [certificate] + + process["creator_user_ref"] = creator_user_ref + process["binary_ref"] = binary_ref + return process + else: + raise Exception("There was no process type") + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + + +class ProcessPidTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + + if(dataAsAJson["match"]['type'] == "process"): + return dataAsAJson["match"]["properties"]["pid"] + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessCreatedTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + + if(dataAsAJson["match"]['type'] == "process"): + return dataAsAJson["match"]["properties"]["start_time"] + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessArgsTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + + arguments_list = shlex.split(dataAsAJson["match"]["properties"]["args"]) + return arguments_list + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessNameTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + converted_file = dataAsAJson["match"]["properties"]["name"].replace('\\', '/') + pathObject = pathlib.Path(converted_file) + return pathObject.name + pathObject.suffix + + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessCWDPathTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + + converted_file = dataAsAJson["match"]["properties"]["name"].replace('\\', '/') + pathObject = pathlib.Path(converted_file) + return pathObject.parent.as_posix() + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessUserIdTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + return dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["user"]["user"]["user_id"] + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessUserDisplayNameTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + + return dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["user"]["user"]["name"] + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessUserDaemonTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + + if (dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["user"]["user"]["domain"] is not None): + return True + else: + return False + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessFileCertificateIssuerTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + return dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["file"]["file"]["signature_data"]["issuer"] + + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessFileCertificateSubjectTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + return dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["file"]["file"]["signature_data"]["subject"] + + except ValueError: + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) + +class ProcessFileHashesTransformer(ValueTransformer): + + @staticmethod + def transform(data): # Leave method name as is. + try: + dataAsAJson = json.loads(data) + return dataAsAJson["finding"]["whats"][0]["artifact_activity"]["acting_artifact"]["process"]["file"]["file"]["hash"] except ValueError: - LOGGER.error("Cannot convert data value {} to ".format(data)) \ No newline at end of file + LOGGER.error("Cannot convert data value {} to ".format(data)) + except Exception as err: + LOGGER.error(err) \ No newline at end of file diff --git a/stix_shifter_modules/tanium/stix_transmission/connector.py b/stix_shifter_modules/tanium/stix_transmission/connector.py index 6d250e232..1a8825d25 100644 --- a/stix_shifter_modules/tanium/stix_transmission/connector.py +++ b/stix_shifter_modules/tanium/stix_transmission/connector.py @@ -58,6 +58,7 @@ async def create_results_connection(self, query, offset, limit): self.return_obj["data"] = self.final_results self.return_obj['success'] = True self.return_obj['metadata'] = self.return_obj['metadata'] = {"next_offset": self.current_offset, "total_result_count": len(self.final_results)} + except Exception as err: self.logger.error('error when connecting to the Tanium datasource {}:'.format(self.return_obj["error"])) return self.return_obj @@ -65,9 +66,9 @@ async def create_results_connection(self, query, offset, limit): async def get_results(self, per_query_limit, query, current_offset): #Create initial query if(query != ""): - current_query = "%s%s&limit=%s&offset=%s" % (self._QUERY_ENDPOINT, query, per_query_limit, current_offset) + current_query = "%s%s&limit=%s&offset=%s&expand=intelDoc" % (self._QUERY_ENDPOINT, query, per_query_limit, current_offset) else: - current_query = "limit=%s&offset=%s" % (self._QUERY_ENDPOINT, per_query_limit, current_offset) + current_query = "limit=%s&offset=%s&expand=intelDoc" % (self._QUERY_ENDPOINT, per_query_limit, current_offset) response_data = await self.query_tanium_api(current_query) response_data_as_json = json.loads(response_data.content.decode('utf-8')) @@ -76,8 +77,7 @@ async def get_results(self, per_query_limit, query, current_offset): def _add_results_to_final_dataset(self, current_query_results): for batch_of_results in current_query_results: - for result in batch_of_results: - self.final_results.append(batch_of_results.get(result)) + self.final_results.append(batch_of_results) self.current_offset = self.current_offset + len(current_query_results) diff --git a/stix_shifter_modules/tanium/tests/stix_translation/test_from_stix_to_query.py b/stix_shifter_modules/tanium/tests/stix_translation/test_from_stix_to_query.py index a8fc27d28..9ffebcc31 100644 --- a/stix_shifter_modules/tanium/tests/stix_translation/test_from_stix_to_query.py +++ b/stix_shifter_modules/tanium/tests/stix_translation/test_from_stix_to_query.py @@ -1,7 +1,5 @@ from stix_shifter.stix_translation import stix_translation import unittest -import json -import os translation = stix_translation.StixTranslation() def _test_query_assertions(queryList, expectedQueryList): @@ -84,4 +82,16 @@ def test_action_query_all_query(self): f"&alertedAtFrom=2022-07-01T00:00:00.000Z" \ f"&alertedAtUntil=2024-07-27T00:05:00.000Z" \ f"&limit=10000"] - _test_stix_to_json(stix_pattern, expectedQueryList) \ No newline at end of file + _test_stix_to_json(stix_pattern, expectedQueryList) + + def test_get_observed_data_objects(self): + assert True + # result_bundle = json_to_stix_translator.convert_to_stix( + # data_source, map_data, [SAMPLE_DATA_DICT], get_module_transformers(MODULE), options) + # result_bundle_objects = result_bundle['objects'] + + # result_bundle_identity = result_bundle_objects[0] + # assert result_bundle_identity['type'] == data_source['type'] + # observed_data = result_bundle_objects[1] + + # assert 'objects' in observed_data \ No newline at end of file