Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

second half of email.* mapping for elastic_ecs #1632

Merged
merged 1 commit into from
Jan 5, 2024
Merged

second half of email.* mapping for elastic_ecs #1632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,16 @@
},
"file": {
"fields": {
"name": ["file.name", "dll.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword"],
"name": ["file.name", "dll.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword", "email.attachments.file.name"],
"created": ["file.created", "file.ctime"],
"modified": ["file.mtime"],
"accessed": ["file.accessed"],
"size": ["file.size"],
"mime_type": ["file.mime_type"],
"hashes.MD5": ["file.hash.md5"],
"hashes.'SHA-1'": ["file.hash.sha1"],
"hashes.'SHA-256'": ["file.hash.sha256"],
"hashes.'SHA-512'": ["file.hash.sha512"],
"hashes.MD5": ["file.hash.md5", "email.attachments.file.hash.md5"],
"hashes.'SHA-1'": ["file.hash.sha1", "email.attachments.file.hash.sha1"],
"hashes.'SHA-256'": ["file.hash.sha256", "email.attachments.file.hash.256"],
"hashes.'SHA-512'": ["file.hash.sha512", "email.attachments.file.hash.512"],
"parent_directory_ref.path": ["file.directory"],
"x_attributes": ["file.attributes"],
"x_extension": ["file.extension"],
Expand Down
8 changes: 4 additions & 4 deletions stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@
"accessed": ["file.accessed"],
"size": ["file.size"],
"mime_type": ["file.mime_type"],
"hashes.MD5": ["file.hash.md5"],
"hashes.'SHA-1'": ["file.hash.sha1"],
"hashes.'SHA-256'": ["file.hash.sha256"],
"hashes.'SHA-512'": ["file.hash.sha512"],
"hashes.MD5": ["file.hash.md5", "email.attachments.file.hash.md5"],
"hashes.'SHA-1'": ["file.hash.sha1", "email.attachments.file.hash.sha1"],
"hashes.'SHA-256'": ["file.hash.sha256", "email.attachments.file.hash.256"],
"hashes.'SHA-512'": ["file.hash.sha512", "email.attachments.file.hash.512"],
"parent_directory_ref.path": ["file.directory"],
"x_attributes": ["file.attributes"],
"x_extension": ["file.extension"],
Expand Down
25 changes: 25 additions & 0 deletions stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2619,12 +2619,14 @@
"email": {
"subject": {
"key": "email-message.subject",
"unwrap": true,
"object": "mail"
},
"from": {
"address": [
{
"key": "email-addr.value",
"unwrap": true,
"object": "emailfrom"
},
{
Expand Down Expand Up @@ -2666,7 +2668,30 @@
"file": {
"name": {
"key": "file.name",
"transformer": "UnlistIfList",
"object": "attachment"
},
"hash": {
"md5": {
"key": "file.hashes.MD5",
"transformer": "UnlistIfList",
"object": "attachment"
},
"sha1": {
"key": "file.hashes.SHA-1",
"transformer": "UnlistIfList",
"object": "attachment"
},
"sha256": {
"key": "file.hashes.SHA-256",
"transformer": "UnlistIfList",
"object": "attachment"
},
"sha512": {
"key": "file.hashes.SHA-512",
"transformer": "UnlistIfList",
"object": "attachment"
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion stix_shifter_modules/elastic_ecs/stix_transmission/api_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ async def search_pagination(self, query_expression, lastsortvalue=None, length=D
"_source": {
"includes": ["@timestamp", "source.*", "destination.*", "event.*", "client.*", "server.*", "observer.*",
"host.*", "network.*", "process.*", "user.*", "file.*", "url.*", "registry.*", "dns.*",
"tags"]
"email.*", "tags"]
},
"size": length,
"query": {
Expand Down
62 changes: 60 additions & 2 deletions stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,32 @@
}
}

email_data2 = {
"email": {
"attachments": [
{ "file": { "name": [ "tabby.html" ]
, "mime_type": [ "text/html" ]
, "hash": { "md5": [ "5fbb1f4a0b7f798bb3f74e64bfeaeea2" ]
, "sha256": [ "29952af0823274dd686f8969968f07d618579dbb0159a8f8f9ca2a6e35dd9a98" ]
}
}
},
{ "file": { "name": [ "tabby.zip" ]
, "mime_type": [ "application/zip" ]
, "hash": { "md5": [ "187e31980d3368428c88d7a4e8da4572" ]
, "sha256": [ "08756f43e5536eaa1500ea10adb88e67816f5586a5dde1d2a4f77c814d8abc57" ]
}
}
}
],
"subject": [ "Check out this picture of a cat!" ],
"from": { "address": [ "[email protected]" ] },
"to": { "address": [ "[email protected]" ] },
"cc": { "address": [ "[email protected]" ] }
}
}


class TestElasticEcsTransform(unittest.TestCase, object):
@staticmethod
def get_first(itr, constraint):
Expand Down Expand Up @@ -881,8 +907,6 @@ def test_email(self):
observed_data = translation_objects[1]
stix_objects = observed_data.get("objects")
assert len(stix_objects) == 8
for k,v in stix_objects.items():
print(f"{k}: {v}")
emailmsg = stix_objects.get("2")
assert (
emailmsg and emailmsg.get("type") == "email-message" and
Expand All @@ -894,3 +918,37 @@ def test_email(self):
assert to_emails == {"[email protected]", "[email protected]"}
filenames = {stix_objects[str(ref)]["name"] for ref in range(len(stix_objects)) if stix_objects[str(ref)] and stix_objects[str(ref)]["type"] == "file"}
assert filenames == {"tabby.zip", "tabby.html"}

def test_email2(self):
result_bundle = run_in_thread(entry_point.translate_results, data_source, [email_data2])
assert (result_bundle['type'] == 'bundle')
translation_objects = result_bundle.get('objects')
assert (translation_objects and len(translation_objects) == 2)
observed_data = translation_objects[1]
stix_objects = observed_data.get("objects")
assert len(stix_objects) == 6
emailmsg = stix_objects.get("2")
assert (
emailmsg and emailmsg.get("type") == "email-message" and
emailmsg.get("subject") == "Check out this picture of a cat!"
)
from_email = stix_objects.get(emailmsg.get("from_ref"))
assert from_email and from_email.get("value") == "[email protected]"
to_emails = [stix_objects.get(ref).get("value") for ref in emailmsg.get("to_refs")]
assert to_emails == ["[email protected]"]
to_emails = [stix_objects.get(ref).get("value") for ref in emailmsg.get("cc_refs")]
assert to_emails == ["[email protected]"]
file0 = stix_objects.get("0")
assert (
file0 and file0.get("type") == "file" and
file0.get("name") == "tabby.html" and
file0.get("hashes").get("MD5") == "5fbb1f4a0b7f798bb3f74e64bfeaeea2" and
file0.get("hashes").get("SHA-256") == "29952af0823274dd686f8969968f07d618579dbb0159a8f8f9ca2a6e35dd9a98"
)
file1 = stix_objects.get("1")
assert (
file1 and file1.get("type") == "file" and
file1.get("name") == "tabby.zip" and
file1.get("hashes").get("MD5") == "187e31980d3368428c88d7a4e8da4572" and
file1.get("hashes").get("SHA-256") == "08756f43e5536eaa1500ea10adb88e67816f5586a5dde1d2a4f77c814d8abc57"
)
8 changes: 8 additions & 0 deletions stix_shifter_utils/stix_translation/src/utils/transformers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,14 @@ class ValueToList(ValueTransformer):
def transform(obj):
return [obj]


class UnlistIfList(ValueTransformer):
"""A value transformer that returns the first item if it is a list"""
@staticmethod
def transform(obj):
return obj[0] if isinstance(obj, list) else obj


class GraphIDToPID(ValueTransformer):
"""A value transformer that converts a single value into a list container the value"""

Expand Down