diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 0f7714f16..7a8708fd7 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -43,16 +43,16 @@ }, "file": { "fields": { - "name": ["file.name", "dll.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword"], + "name": ["file.name", "dll.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword", "email.attachments.file.name"], "created": ["file.created", "file.ctime"], "modified": ["file.mtime"], "accessed": ["file.accessed"], "size": ["file.size"], "mime_type": ["file.mime_type"], - "hashes.MD5": ["file.hash.md5"], - "hashes.'SHA-1'": ["file.hash.sha1"], - "hashes.'SHA-256'": ["file.hash.sha256"], - "hashes.'SHA-512'": ["file.hash.sha512"], + "hashes.MD5": ["file.hash.md5", "email.attachments.file.hash.md5"], + "hashes.'SHA-1'": ["file.hash.sha1", "email.attachments.file.hash.sha1"], + "hashes.'SHA-256'": ["file.hash.sha256", "email.attachments.file.hash.256"], + "hashes.'SHA-512'": ["file.hash.sha512", "email.attachments.file.hash.512"], "parent_directory_ref.path": ["file.directory"], "x_attributes": ["file.attributes"], "x_extension": ["file.extension"], diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index d230671c7..a2a9c2e9a 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -49,10 +49,10 @@ "accessed": ["file.accessed"], "size": ["file.size"], "mime_type": ["file.mime_type"], - "hashes.MD5": ["file.hash.md5"], - "hashes.'SHA-1'": ["file.hash.sha1"], - "hashes.'SHA-256'": ["file.hash.sha256"], - "hashes.'SHA-512'": ["file.hash.sha512"], + "hashes.MD5": ["file.hash.md5", "email.attachments.file.hash.md5"], + "hashes.'SHA-1'": ["file.hash.sha1", "email.attachments.file.hash.sha1"], + "hashes.'SHA-256'": ["file.hash.sha256", "email.attachments.file.hash.256"], + "hashes.'SHA-512'": ["file.hash.sha512", "email.attachments.file.hash.512"], "parent_directory_ref.path": ["file.directory"], "x_attributes": ["file.attributes"], "x_extension": ["file.extension"], diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index a101785a7..6cb6098ad 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -2619,12 +2619,14 @@ "email": { "subject": { "key": "email-message.subject", + "unwrap": true, "object": "mail" }, "from": { "address": [ { "key": "email-addr.value", + "unwrap": true, "object": "emailfrom" }, { @@ -2666,7 +2668,30 @@ "file": { "name": { "key": "file.name", + "transformer": "UnlistIfList", "object": "attachment" + }, + "hash": { + "md5": { + "key": "file.hashes.MD5", + "transformer": "UnlistIfList", + "object": "attachment" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "transformer": "UnlistIfList", + "object": "attachment" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "transformer": "UnlistIfList", + "object": "attachment" + }, + "sha512": { + "key": "file.hashes.SHA-512", + "transformer": "UnlistIfList", + "object": "attachment" + } } } } diff --git a/stix_shifter_modules/elastic_ecs/stix_transmission/api_client.py b/stix_shifter_modules/elastic_ecs/stix_transmission/api_client.py index 168b46bc3..81c8fee22 100644 --- a/stix_shifter_modules/elastic_ecs/stix_transmission/api_client.py +++ b/stix_shifter_modules/elastic_ecs/stix_transmission/api_client.py @@ -66,7 +66,7 @@ async def search_pagination(self, query_expression, lastsortvalue=None, length=D "_source": { "includes": ["@timestamp", "source.*", "destination.*", "event.*", "client.*", "server.*", "observer.*", "host.*", "network.*", "process.*", "user.*", "file.*", "url.*", "registry.*", "dns.*", - "tags"] + "email.*", "tags"] }, "size": length, "query": { diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index cf34b9c32..22c2e07c6 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -403,6 +403,32 @@ } } +email_data2 = { + "email": { + "attachments": [ + { "file": { "name": [ "tabby.html" ] + , "mime_type": [ "text/html" ] + , "hash": { "md5": [ "5fbb1f4a0b7f798bb3f74e64bfeaeea2" ] + , "sha256": [ "29952af0823274dd686f8969968f07d618579dbb0159a8f8f9ca2a6e35dd9a98" ] + } + } + }, + { "file": { "name": [ "tabby.zip" ] + , "mime_type": [ "application/zip" ] + , "hash": { "md5": [ "187e31980d3368428c88d7a4e8da4572" ] + , "sha256": [ "08756f43e5536eaa1500ea10adb88e67816f5586a5dde1d2a4f77c814d8abc57" ] + } + } + } + ], + "subject": [ "Check out this picture of a cat!" ], + "from": { "address": [ "from@address.com" ] }, + "to": { "address": [ "to@address.com" ] }, + "cc": { "address": [ "cc@address.com" ] } + } +} + + class TestElasticEcsTransform(unittest.TestCase, object): @staticmethod def get_first(itr, constraint): @@ -881,8 +907,6 @@ def test_email(self): observed_data = translation_objects[1] stix_objects = observed_data.get("objects") assert len(stix_objects) == 8 - for k,v in stix_objects.items(): - print(f"{k}: {v}") emailmsg = stix_objects.get("2") assert ( emailmsg and emailmsg.get("type") == "email-message" and @@ -894,3 +918,37 @@ def test_email(self): assert to_emails == {"to1@address.com", "to2@address.com"} filenames = {stix_objects[str(ref)]["name"] for ref in range(len(stix_objects)) if stix_objects[str(ref)] and stix_objects[str(ref)]["type"] == "file"} assert filenames == {"tabby.zip", "tabby.html"} + + def test_email2(self): + result_bundle = run_in_thread(entry_point.translate_results, data_source, [email_data2]) + assert (result_bundle['type'] == 'bundle') + translation_objects = result_bundle.get('objects') + assert (translation_objects and len(translation_objects) == 2) + observed_data = translation_objects[1] + stix_objects = observed_data.get("objects") + assert len(stix_objects) == 6 + emailmsg = stix_objects.get("2") + assert ( + emailmsg and emailmsg.get("type") == "email-message" and + emailmsg.get("subject") == "Check out this picture of a cat!" + ) + from_email = stix_objects.get(emailmsg.get("from_ref")) + assert from_email and from_email.get("value") == "from@address.com" + to_emails = [stix_objects.get(ref).get("value") for ref in emailmsg.get("to_refs")] + assert to_emails == ["to@address.com"] + to_emails = [stix_objects.get(ref).get("value") for ref in emailmsg.get("cc_refs")] + assert to_emails == ["cc@address.com"] + file0 = stix_objects.get("0") + assert ( + file0 and file0.get("type") == "file" and + file0.get("name") == "tabby.html" and + file0.get("hashes").get("MD5") == "5fbb1f4a0b7f798bb3f74e64bfeaeea2" and + file0.get("hashes").get("SHA-256") == "29952af0823274dd686f8969968f07d618579dbb0159a8f8f9ca2a6e35dd9a98" + ) + file1 = stix_objects.get("1") + assert ( + file1 and file1.get("type") == "file" and + file1.get("name") == "tabby.zip" and + file1.get("hashes").get("MD5") == "187e31980d3368428c88d7a4e8da4572" and + file1.get("hashes").get("SHA-256") == "08756f43e5536eaa1500ea10adb88e67816f5586a5dde1d2a4f77c814d8abc57" + ) diff --git a/stix_shifter_utils/stix_translation/src/utils/transformers.py b/stix_shifter_utils/stix_translation/src/utils/transformers.py index 1228cc16e..41a24f9e3 100644 --- a/stix_shifter_utils/stix_translation/src/utils/transformers.py +++ b/stix_shifter_utils/stix_translation/src/utils/transformers.py @@ -331,6 +331,14 @@ class ValueToList(ValueTransformer): def transform(obj): return [obj] + +class UnlistIfList(ValueTransformer): + """A value transformer that returns the first item if it is a list""" + @staticmethod + def transform(obj): + return obj[0] if isinstance(obj, list) else obj + + class GraphIDToPID(ValueTransformer): """A value transformer that converts a single value into a list container the value"""