diff --git a/stix_shifter_modules/splunk/README.md b/stix_shifter_modules/splunk/README.md index d1274b4dd..edd7d0bcc 100644 --- a/stix_shifter_modules/splunk/README.md +++ b/stix_shifter_modules/splunk/README.md @@ -1,80 +1,676 @@ -# Splunk +# Splunk Enterprise ## Supported STIX Mappings See the [table of mappings](splunk_supported_stix.md) for the STIX objects and operators supported by this connector. +**Table of Contents** + +- [Splunk API Endpoints](#splunk-api-endpoints) +- [Pattern expression with STIX attributes - Single Observation](#single-observation) +- [Pattern expression with STIX attributes - Multiple Observation](#multiple-observation) +- [Pattern expression with STIX attributes - Execute Query](#stix-execute-query) +- [References](#references) + +### Splunk API Endpoints + + |Connector Method|Splunk API Endpoint| Method + | ---- | ------ | -----| + |Ping Endpoint |https://:/services/|GET + |Query Endpoint |https://:/services/search/jobs|POST + |Result Endpoint|https://:/services/search/v2/jobs//results|GET + ### Format for calling stix-shifter from the command line +``` +python main.py `` `` `` `` `` +``` +### Pattern expression with STIX attributes + +### Single Observation + +#### STIX Translate query +```shell +translate splunk query "{}" "[ipv4-addr:value = '1.1.1.1'] START t'2023-01-01T11:00:00.000Z' STOP t'2023-03-08T11:00:00.003Z'" +``` +#### STIX Translate query - output +```json +{ + "queries": [ + "search ((src_ip = \"1.1.1.1\") OR (dest_ip = \"1.1.1.1\")) earliest=\"01/01/2023:11:00:00\" latest=\"03/08/2023:11:00:00\" | head 10000 | fields src_ip, src_port, src_mac, dest_ip, dest_port, dest_mac, file_hash, user, url, protocol, host, source, severity, process, process_id, process_name, process_exec, process_path, process_hash, process_guid, parent_process, parent_process_id, parent_process_name, parent_process_exec, description, signature, signature_id, query, answer, transport, bytes_in, bytes_out, packets_in, packets_out, direction, name, message_type, query_count, query_type, record_type, reply_code, reply_code_id, vendor_product, duration, transaction_id, action, file_access_time, file_acl, registry_hive, registry_path, registry_key_name, registry_value_data, registry_value_name, registry_value_text, registry_value_type, status, ssl_version, ssl_serial, ssl_issuer, ssl_subject, ssl_signature_algorithm, ssl_publickey_algorithm, ssl_start_time, ssl_end_time, ssl_is_valid, ssl_issuer_common_name, ssl_subject_common_name, ssl_name, ssl_publickey, ssl_issuer_email, ssl_subject_email, ssl_issuer_email_domain, ssl_subject_email_domain, ssl_issuer_organization, ssl_subject_organization, recipient, subject, file_hash, file_name, file_size, recipient_domain, src_user_domain, internal_message_id, message_id, message_info, app, authentication_method, authentication_service, dest, src, src_user, user_name, user_id, user_type, user_agent, http_method, http_referrer, http_user_agent, uri_path, uri_query, os, dvc, id, msft, cve, cvss, mskb, type, eventtype, event_id, mitre_technique_id, mem_used, original_file_name, file_create_time, file_modify_time" + ] +} +``` +#### STIX Transmit ping + +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +ping +``` + +#### STIX Transmit ping - output +```json +{ + "success": true +} +``` +#### STIX Transmit query + +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +query +"search ((src_ip = \"1.1.1.1\") OR (dest_ip = \"1.1.1.1\")) earliest=\"01/01/2023:11:00:00\" latest=\"03/08/2023:11:00:00\" | head 10000 | fields src_ip, src_port, src_mac, dest_ip, dest_port, dest_mac, file_hash, user, url, protocol, host, source, severity, process, process_id, process_name, process_exec, process_path, process_hash, process_guid, parent_process, parent_process_id, parent_process_name, parent_process_exec, description, signature, signature_id, query, answer, transport, bytes_in, bytes_out, packets_in, packets_out, direction, name, message_type, query_count, query_type, record_type, reply_code, reply_codeid, vendor_product, duration, transaction_id, action, file_access_time, file_acl, registry_hive, registry_path, registry_key_name, registry_value_data, registry_value_name, registry_value_text, registry_value_type, status, ssl_version, ssl_serial, ssl_issuer, ssl_subject, ssl_signature_algorithm, ssl_publickey_algorithm, ssl_start_time, ssl_end_time, ssl_is_valid, ssl_issuer_common_name, ssl_subject_common_name, ssl_name, ssl_publickey, ssl_issuer_email, ssl_subject_email, ssl_issuer_email_domain, ssl_subject_email_domain, ssl_issuer_organization, ssl_subject_organization, recipient, subject, file_hash, file_name, file_size, recipient_domain, src_user_domain, internal_message_id, message_id, message_info, app, authentication_method, authentication_service, dest, src, src_user, user_name, user_id, user_type, user_agent, http_method, http_referrer, http_user_agent, uri_path, uri_query, os, dvc, id, msft, cve, cvss, mskb, type, eventtype, event_id, mitre_technique_id, mem_used, original_file_name, original_file_name, file_create_time, file_modify_time" +``` + +#### STIX Transmit query - output +```json +{ + "success": true, + "search_id": "xxxxx" +} +``` +#### STIX Transmit status + +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +status +"xxxxxx" +``` + +#### STIX Transmit status - output +```json +{ + "success": true, + "status": "RUNNING", + "progress": 3 +} +``` +#### STIX Transmit results + +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +results +"xxxxxx" +0 +1 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "src_ip": "1.1.1.1", + "src_port": null, + "dest_ip": "1.1.1.1", + "dest_port": null, + "user": "xxxx", + "protocol": "ip", + "host": "xxxx", + "source": "xxxx", + "process_id": "532", + "process_name": "xxxx", + "process_exec": null, + "process_guid": "xxxxx", + "signature": "Network connection", + "signature_id": "3", + "transport": "tcp", + "direction": "inbound", + "vendor_product": "Microsoft Sysmon", + "action": "allowed", + "app": "C:\\Windows\\System32\\svchost.exe", + "dest": null, + "src": null, + "user_id": "'S-1-5-18'", + "dvc": "xxxxx", + "id": "9383", + "eventtype": [ + "corelight_idx", + "endpoint_services_processes", + "ms-sysmon-network", + "windows_event_signature" + ], + "event_id": "9383", + "_bkt": "main~9~FF7D676A-5C1A-492E-9A6D-4653BFD400A6", + "_cd": "9:31677", + "_eventtype_color": "none", + "_indextime": "1676460270", + "_raw": "xxxx", + "_serial": "0", + "_si": [ + "xxxxxx", + "main" + ], + "_sourcetype": "XmlWinEventLog", + "_time": "2023-01-11T15:29:45.000+00:00", + "name": null, + "query_count": null, + "query_type": null, + "networkdata": { + "src_port": "60318", + "dest_port": "3389", + "protocol": "ip", + "transport": "tcp", + "direction": "inbound" + }, + "file_hash": null, + "file_hashes": {} + } + ] +} +``` + -python stix_shifter.py `` `` `` `` +#### STIX Translate results -(Note the identity object is only used when converting from AQL to STIX, but due to positional arguments, an empty hash will need to be passed in when converting from STIX patterns to AQL. Keyword arguments should be implemented to overcome this). +```shell +translate +splunk +results +"{\"type\": \"identity\", \"id\": \"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\", \"name\": \"splunk\", \"identity_class\": \"events\", \"created\": \"2022-03-22T13:22:50.336Z\", \"modified\": \"2022-03-22T13:22:50.336Z\"}" +"[ { \"src_ip\": \"1.1.1.1\", \"src_port\": null, \"dest_ip\": \"1.1.1.1\", \"dest_port\": null, \"user\": \"NETWORK SERVICE\", \"protocol\": \"ip\", \"host\": \"xxxxx\", \"source\": \"XmlWinEventLog:Microsoft-Windows-Sysmon/Operational\", \"process_id\": \"532\", \"process_name\": \"svchost.exe\", \"process_exec\": null, \"process_guid\": \"xxxxx\", \"signature\": \"Network connection\", \"signature_id\": \"3\", \"transport\": \"tcp\", \"direction\": \"inbound\", \"vendor_product\": \"Microsoft Sysmon\", \"action\": \"allowed\", \"app\": \"C:\\Windows\\System32\\svchost.exe\", \"dest\": null, \"src\": null, \"user_id\": \"'S-1-5-18'\", \"dvc\": \"xxxxx\", \"id\": \"9383\", \"eventtype\": [ \"corelight_idx\", \"endpoint_services_processes\", \"ms-sysmon-network\", \"windows_event_signature\" ], \"event_id\": \"9383\", \"_bkt\": \"main~9~FF7D676A-5C1A-492E-9A6D-4653BFD400A6\", \"_cd\": \"9:31677\", \"_eventtype_color\": \"none\", \"_indextime\": \"1676460270\", \"_raw\": \"354300x80000000000000009383Microsoft-Windows-Sysmon/Operationalxxxxxtechnique_id=T1021,technique_name=Remote Services2023-01-11 15:29:43.732xxxxxx532C:\\Windows\\System32\\svchost.exeNT AUTHORITY\\NETWORK SERVICEtcpfalsefalse198.235.24.155-60318-false172.31.29.79-3389-\", \"_serial\": \"0\", \"_si\": [ \"xxxxx\", \"main\" ], \"_sourcetype\": \"XmlWinEventLog\", \"_time\": \"2023-01-11T15:29:45.000+00:00\", \"name\": null, \"query_count\": null, \"query_type\": null, \"networkdata\": { \"src_port\": \"60318\", \"dest_port\": \"3389\", \"protocol\": \"ip\", \"transport\": \"tcp\", \"direction\": \"inbound\" }, \"file_hash\": null, \"file_hashes\": {} } ]" +``` + +#### STIX Translate results - output +```json +{ + "type": "bundle", + "id": "bundle--e41614dc-a4c3-49b7-a95b-467cf75ef2d9", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "splunk", + "identity_class": "events", + "created": "2022-03-22T13:22:50.336Z", + "modified": "2022-03-22T13:22:50.336Z" + }, + { + "id": "observed-data--4d3a2d6c-c8f2-4f83-a36e-cbf0565968aa", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-05-08T13:12:54.984Z", + "modified": "2023-05-08T13:12:54.984Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "1": { + "type": "network-traffic", + "src_ref": "0", + "dst_ref": "4", + "src_port": 60318, + "dst_port": 3389, + "protocols": [ + "ip", + "tcp" + ], + "x_direction": "inbound" + }, + "2": { + "type": "x-oca-event", + "ip_refs": [ + "0", + "4" + ], + "user_ref": "5", + "host_ref": "6", + "module": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "process_ref": "8", + "action": "Network connection", + "code": "3", + "provider": "Microsoft Sysmon", + "outcome": "allowed", + "x_application": "C:\\Windows\\System32\\svchost.exe", + "x_event_id": "9383", + "original_ref": "10", + "created": "2023-01-11T15:29:45.000Z", + "network_ref": "1" + }, + "4": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "5": { + "type": "user-account", + "user_id": "NETWORK SERVICE", + "account_login": "'S-1-5-18'" + }, + "6": { + "type": "x-oca-asset", + "hostname": "xxxxx" + }, + "7": { + "type": "x-splunk-data", + "log_source": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "event_type": [ + "corelight_idx", + "endpoint_services_processes", + "ms-sysmon-network", + "windows_event_signature" + ], + "log_source_type": "XmlWinEventLog" + }, + "8": { + "type": "process", + "pid": 532, + "name": "svchost.exe", + "binary_ref": "9", + "x_unique_id": "xxxxx", + "opened_connection_refs": [ + "1" + ] + }, + "9": { + "type": "file", + "name": "svchost.exe" + }, + "10": { + "type": "artifact", + "payload_bin": "PEV2ZW50IHhtbG5zPSdodHRwOi8vc2NoZW1hcy5taWNyb3NvZnQuY29tL3dpbi8yMDA0LzA4L2V2ZW50cy9ldmVudCc+PFN5c3RlbT48UHJvdmlkZXIgTmFtZT0nTWljcm9zb2Z0LVdpbmRvd3MtU3lzbW9uJyBHdWlkPSd7NTc3MDM4NWYtYzIyYS00M2UwLWJmNGMtMDZmNTY5OGZmYmQ5fScvPjxFdmVudElEPjM8L0V2ZW50SUQ+PFZlcnNpb24+NTwvVmVyc2lvbj48TGV2ZWw+NDwvTGV2ZWw+PFRhc2s+MzwvVGFzaz48T3Bjb2RlPjA8L09wY29kZT48S2V5d29yZHM+MHg4MDAwMDAwMDAwMDAwMDAwPC9LZXl3b3Jkcz48VGltZUNyZWF0ZWQgU3lzdGVtVGltZT0nMjAyMy0wMS0xMVQxNToyOTo0NS44MzIyODc0WicvPjxFdmVudFJlY29yZElEPjkzODM8L0V2ZW50UmVjb3JkSUQ+PENvcnJlbGF0aW9uLz48RXhlY3V0aW9uIFByb2Nlc3NJRD0nMTg4OCcgVGhyZWFkSUQ9JzI5NTYnLz48Q2hhbm5lbD5NaWNyb3NvZnQtV2luZG93cy1TeXNtb24vT3BlcmF0aW9uYWw8L0NoYW5uZWw+PENvbXB1dGVyPkVDMkFNQVotTzRLQ1VLMzwvQ29tcHV0ZXI+PFNlY3VyaXR5IFVzZXJJRD0nUy0xLTUtMTgnLz48L1N5c3RlbT48RXZlbnREYXRhPjxEYXRhIE5hbWU9J1J1bGVOYW1lJz50ZWNobmlxdWVfaWQ9VDEwMjEsdGVjaG5pcXVlX25hbWU9UmVtb3RlIFNlcnZpY2VzPC9EYXRhPjxEYXRhIE5hbWU9J1V0Y1RpbWUnPjIwMjMtMDEtMTEgMTU6Mjk6NDMuNzMyPC9EYXRhPjxEYXRhIE5hbWU9J1Byb2Nlc3NHdWlkJz57N2E5N2I4NWYtNDU1MC02M2JlLTEzMDAtMDAwMDAwMDA3ZDAwfTwvRGF0YT48RGF0YSBOYW1lPSdQcm9jZXNzSWQnPjUzMjwvRGF0YT48RGF0YSBOYW1lPSdJbWFnZSc+QzpcV2luZG93c1xTeXN0ZW0zMlxzdmNob3N0LmV4ZTwvRGF0YT48RGF0YSBOYW1lPSdVc2VyJz5OVCBBVVRIT1JJVFlcTkVUV09SSyBTRVJWSUNFPC9EYXRhPjxEYXRhIE5hbWU9J1Byb3RvY29sJz50Y3A8L0RhdGE+PERhdGEgTmFtZT0nSW5pdGlhdGVkJz5mYWxzZTwvRGF0YT48RGF0YSBOYW1lPSdTb3VyY2VJc0lwdjYnPmZhbHNlPC9EYXRhPjxEYXRhIE5hbWU9J1NvdXJjZUlwJz4xOTguMjM1LjI0LjE1NTwvRGF0YT48RGF0YSBOYW1lPSdTb3VyY2VIb3N0bmFtZSc+LTwvRGF0YT48RGF0YSBOYW1lPSdTb3VyY2VQb3J0Jz42MDMxODwvRGF0YT48RGF0YSBOYW1lPSdTb3VyY2VQb3J0TmFtZSc+LTwvRGF0YT48RGF0YSBOYW1lPSdEZXN0aW5hdGlvbklzSXB2Nic+ZmFsc2U8L0RhdGE+PERhdGEgTmFtZT0nRGVzdGluYXRpb25JcCc+MTcyLjMxLjI5Ljc5PC9EYXRhPjxEYXRhIE5hbWU9J0Rlc3RpbmF0aW9uSG9zdG5hbWUnPi08L0RhdGE+PERhdGEgTmFtZT0nRGVzdGluYXRpb25Qb3J0Jz4zMzg5PC9EYXRhPjxEYXRhIE5hbWU9J0Rlc3RpbmF0aW9uUG9ydE5hbWUnPi08L0RhdGE+PC9FdmVudERhdGE+PC9FdmVudD4=", + "mime_type": "text/plain" + } + }, + "first_observed": "2023-01-11T15:29:45.000Z", + "last_observed": "2023-01-11T15:29:45.000Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Multiple Observation + +#### STIX Translate query +```shell +translate +splunk +query +"{}" +"[network-traffic:src_ref.value = '1.1.1.1'] AND [network-traffic:dst_ref.value ='2.2.2.2'] START t'2023-01-01T11:00:00.000Z' STOP t'2023-02-28T11:00:00.003Z'" +``` + +#### STIX Translate query - output -## Converting from Splunk CIM to STIX Cyber Observable Object +```json +{ + "queries": [ + "search (src_ip = \"1.1.1.1\") OR (dest_ip = \"2.2.2.2\") earliest=\"01/01/2023:11:00:00\" latest=\"02/28/2023:11:00:00\" | head 10000 | fields src_ip, src_port, src_mac, dest_ip, dest_port, dest_mac, file_hash, user, url, protocol, host, source, severity, process, process_id, process_name, process_exec, process_path, process_hash, process_guid, parent_process, parent_process_id, parent_process_name, parent_process_exec, description, signature, signature_id, query, answer, transport, bytes_in, bytes_out, packets_in, packets_out, direction, name, message_type, query_count, query_type, record_type, reply_code, reply_codeid, vendor_product, duration, transaction_id, action, file_access_time, file_acl, registry_hive, registry_path, registry_key_name, registry_value_data, registry_value_name, registry_value_text, registry_value_type, status, ssl_version, ssl_serial, ssl_issuer, ssl_subject, ssl_signature_algorithm, ssl_publickey_algorithm, ssl_start_time, ssl_end_time, ssl_is_valid, ssl_issuer_common_name, ssl_subject_common_name, ssl_name, ssl_publickey, ssl_issuer_email, ssl_subject_email, ssl_issuer_email_domain, ssl_subject_email_domain, ssl_issuer_organization, ssl_subject_organization, recipient, subject, file_hash, file_name, file_size, recipient_domain, src_user_domain, internal_message_id, message_id, message_info, app, authentication_method, authentication_service, dest, src, src_user, user_name, user_id, user_type, user_agent, http_method, http_referrer, http_user_agent, uri_path, uri_query, os, dvc, id, msft, cve, cvss, mskb, type, eventtype, event_id, mitre_technique_id, mem_used, original_file_name, original_file_name, file_create_time, file_modify_time" + ] +} +``` -Splunk data to Stix mapping is defined in `to_stix_map.json` which is located in the Splunk module. +#### STIX Transmit query + +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +query +"search (src_ip = \"1.1.1.1\") OR (dest_ip = \"2.2.2.2\") earliest=\"01/01/2023:11:00:00\" latest=\"02/28/2023:11:00:00\" | head 10000 | fields src_ip, src_port, src_mac, dest_ip, dest_port, dest_mac, file_hash, user, url, protocol, host, source, severity, process, process_id, process_name, process_exec, process_path, process_hash, process_guid, parent_process, parent_process_id, parent_process_name, parent_process_exec, description, signature, signature_id, query, answer, transport, bytes_in, bytes_out, packets_in, packets_out, direction, name, message_type, query_count, query_type, record_type, reply_code, reply_codeid, vendor_product, duration, transaction_id, action, file_access_time, file_acl, registry_hive, registry_path, registry_key_name, registry_value_data, registry_value_name, registry_value_text, registry_value_type, status, ssl_version, ssl_serial, ssl_issuer, ssl_subject, ssl_signature_algorithm, ssl_publickey_algorithm, ssl_start_time, ssl_end_time, ssl_is_valid, ssl_issuer_common_name, ssl_subject_common_name, ssl_name, ssl_publickey, ssl_issuer_email, ssl_subject_email, ssl_issuer_email_domain, ssl_subject_email_domain, ssl_issuer_organization, ssl_subject_organization, recipient, subject, file_hash, file_name, file_size, recipient_domain, src_user_domain, internal_message_id, message_id, message_info, app, authentication_method, authentication_service, dest, src, src_user, user_name, user_id, user_type, user_agent, http_method, http_referrer, http_user_agent, uri_path, uri_query, os, dvc, id, msft, cve, cvss, mskb, type, eventtype, event_id, mitre_technique_id, mem_used, original_file_name, original_file_name, file_create_time, file_modify_time" +``` -As Splunk uses common field names across multiple Splunk CIM objects, the unique CIM compliant event `tag` field located in the `` object clarifies which objects can potentially be created. +#### STIX Transmit query - output -This example Splunk data (based on the `Change Analysis` CIM): +```json +{ + "success": true, + "search_id": "xxxxx" +} +``` +#### STIX Transmit status -`python main.py translate "splunk" "results" '{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "Splunk", "identity_class": "events"}' '[{"tag": "change", "_time": "2018-08-21T15:11:55.000+00:00", "event_count": 1, "bytes": "300", "user": "ibm_user", "object_path": "hkey_local_machine\\system\\bar\\foo", "file_path": "C:\\Users\\someuser\\sample.dll", "file_create_time": "2018-08-15T15:11:55.676+00:00", "file_modify_time": "2018-08-15T18:10:30.456+00:00", "file_hash": "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", "file_name": "sample.dll", "file_size": 25536}]'` +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +status +"xxxxxx" +``` -Will return the following valid STIX Cyber Observable Object: +#### STIX Transmit status - output ```json -{ - "type": "bundle", - "id": "bundle--26a18dba-427e-44b3-82cf-ed6a666ddccd", - "objects": [ - { - "type": "identity", - "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", - "name": "Splunk", - "identity_class": "events" - }, - { - "id": "observed-data--00693dee-0690-4211-bf6a-364fae53df18", - "type": "observed-data", - "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", - "objects": { - "0": { - "type": "user-account", - "account_login": "ibm_user", - "user_id": "ibm_user" - }, - "1": { - "type": "windows-registry-key", - "creator_user_ref": "0", - "key": "hkey_local_machine\\system\\bar\\foo" - }, - "2": { - "type": "directory", - "path": "C:\\Users\\someuser\\sample.dll", - "created": "2018-08-15T15:11:55.676Z", - "modified": "2018-08-15T18:10:30.456Z" - }, - "3": { - "type": "file", - "parent_directory_ref": "2", - "created": "2018-08-15T15:11:55.676Z", - "modified": "2018-08-15T18:10:30.456Z", - "hashes": { - "SHA-256": "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" - }, - "name": "sample.dll", - "size": 25536 - } - }, - "created": "2018-08-21T15:11:55.000Z", - "modified": "2018-08-21T15:11:55.000Z", - "first_observed": "2018-08-21T15:11:55.000Z", - "last_observed": "2018-08-21T15:11:55.000Z", - "number_observed": 1, - "x_splunk_spl": { - "bytes": "300", - "user": "ibm_user" - } - } - ] -} +{ + "success": true, + "status": "RUNNING", + "progress": 3 +} ``` +#### STIX Transmit results + +```shell +transmit +splunk +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +results +"xxxxxx" +0 +1 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "src_ip": "xxxx", + "src_port": null, + "dest_ip": "xxxx", + "dest_port": null, + "user": "NETWORK SERVICE", + "protocol": "ip", + "host": "xxxxx", + "source": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "process_id": "1028", + "process_name": "svchost.exe", + "process_exec": null, + "process_guid": "xxxx", + "signature": "Network connection", + "signature_id": "3", + "transport": "tcp", + "direction": "inbound", + "vendor_product": "Microsoft Sysmon", + "action": "allowed", + "app": "C:\\Windows\\System32\\svchost.exe", + "dest": null, + "src": null, + "user_id": "'S-1-5-18'", + "dvc": "xxxxx", + "id": "13220", + "eventtype": [ + "corelight_idx", + "endpoint_services_processes", + "ms-sysmon-network", + "windows_event_signature" + ], + "event_id": "13220", + "_bkt": "main~9~FF7D676A-5C1A-492E-9A6D-4653BFD400A6", + "_cd": "9:193619", + "_eventtype_color": "none", + "_indextime": "1676461141", + "_raw": "xxxx", + "_serial": "0", + "_si": [ + "xxxxxx", + "main" + ], + "_sourcetype": "XmlWinEventLog", + "_time": "2023-02-15T11:38:26.000+00:00", + "name": null, + "query_count": null, + "query_type": null, + "networkdata": { + "src_port": "61111", + "dest_port": "3389", + "protocol": "ip", + "transport": "tcp", + "direction": "inbound" + }, + "file_hash": null, + "file_hashes": {} + } + ] +} +``` + + +#### STIX Translate results + +```shell +translate +splunk +results +"{\"type\": \"identity\", \"id\": \"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\", \"name\": \"splunk\", \"identity_class\": \"events\", \"created\": \"2022-03-22T13:22:50.336Z\", \"modified\": \"2022-03-22T13:22:50.336Z\"}" +"[{ \"src_ip\": \"1.1.1.1\", \"src_port\": null, \"dest_ip\": \"2.2.2.2\", \"dest_port\": null, \"user\": \"NETWORK SERVICE\", \"protocol\": \"ip\", \"host\": \"xxxxx\", \"source\": \"XmlWinEventLog:Microsoft-Windows-Sysmon/Operational\", \"process_id\": \"1028\", \"process_name\": \"svchost.exe\", \"process_exec\": null, \"process_guid\": \"{7a97b85f-7dba-63ec-1300-000000007e00}\", \"signature\": \"Network connection\", \"signature_id\": \"3\", \"transport\": \"tcp\", \"direction\": \"inbound\", \"vendor_product\": \"Microsoft Sysmon\", \"action\": \"allowed\", \"app\": \"C:\\Windows\\System32\\svchost.exe\", \"dest\": null, \"src\": null, \"user_id\": \"'S-1-5-18'\", \"dvc\": \"xxxxx\", \"id\": \"13220\", \"eventtype\": [ \"corelight_idx\", \"endpoint_services_processes\", \"ms-sysmon-network\", \"windows_event_signature\" ], \"event_id\": \"13220\", \"_bkt\": \"main~9~FF7D676A-5C1A-492E-9A6D-4653BFD400A6\", \"_cd\": \"9:193619\", \"_eventtype_color\": \"none\", \"_indextime\": \"1676461141\", \"_raw\": \"354300x800000000000000013220Microsoft-Windows-Sysmon/Operationalxxxxxtechnique_id=T1021,technique_name=Remote Services2023-02-15 11:38:24.751xxxxx1028C:\\Windows\\System32\\svchost.exeNT AUTHORITY\\NETWORK SERVICEtcpfalsefalsexxxxx-61111-falsexxxxx-3389-\", \"_serial\": \"0\", \"_si\": [ \"xxxxx\", \"main\" ], \"_sourcetype\": \"XmlWinEventLog\", \"_time\": \"2023-02-15T11:38:26.000+00:00\", \"name\": null, \"query_count\": null, \"query_type\": null, \"networkdata\": { \"src_port\": \"61111\", \"dest_port\": \"3389\", \"protocol\": \"ip\", \"transport\": \"tcp\", \"direction\": \"inbound\" }, \"file_hash\": null, \"file_hashes\": {} }]" +``` + +#### STIX Translate results - output +```json +{ + "type": "bundle", + "id": "bundle--e8d34045-57ea-498c-b1fd-1644ce5c387c", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "splunk", + "identity_class": "events", + "created": "2022-03-22T13:22:50.336Z", + "modified": "2022-03-22T13:22:50.336Z" + }, + { + "id": "observed-data--0c6e731b-38fe-4b76-a8a4-583fcd92535b", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-05-08T13:37:00.218Z", + "modified": "2023-05-08T13:37:00.218Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "1": { + "type": "network-traffic", + "src_ref": "0", + "dst_ref": "4", + "src_port": 61111, + "dst_port": 3389, + "protocols": [ + "ip", + "tcp" + ], + "x_direction": "inbound" + }, + "2": { + "type": "x-oca-event", + "ip_refs": [ + "0", + "4" + ], + "user_ref": "5", + "host_ref": "6", + "module": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "process_ref": "8", + "action": "Network connection", + "code": "3", + "provider": "Microsoft Sysmon", + "outcome": "allowed", + "x_application": "C:\\Windows\\System32\\svchost.exe", + "x_event_id": "13220", + "original_ref": "10", + "created": "2023-02-15T11:38:26.000Z", + "network_ref": "1" + }, + "4": { + "type": "ipv4-addr", + "value": "2.2.2.2" + }, + "5": { + "type": "user-account", + "user_id": "NETWORK SERVICE", + "account_login": "'S-1-5-18'" + }, + "6": { + "type": "x-oca-asset", + "hostname": "1.1.1.1" + }, + "7": { + "type": "x-splunk-data", + "log_source": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "event_type": [ + "corelight_idx", + "endpoint_services_processes", + "ms-sysmon-network", + "windows_event_signature" + ], + "log_source_type": "XmlWinEventLog" + }, + "8": { + "type": "process", + "pid": 1028, + "name": "svchost.exe", + "binary_ref": "9", + "x_unique_id": "xxxxxx", + "opened_connection_refs": [ + "1" + ] + }, + "9": { + "type": "file", + "name": "svchost.exe" + }, + "10": { + "type": "artifact", + "payload_bin": "PEV2ZW50IHhtbG5zPSdodHRwOi8vc2NoZW1hcy5taWNyb3NvZnQuY29tL3dpbi8yMDA0LzA4L2V2ZW50cy9ldmVudCc+PFN5c3RlbT48UHJvdmlkZXIgTmFtZT0nTWljcm9zb2Z0LVdpbmRvd3MtU3lzbW9uJyBHdWlkPSd7NTc3MDM4NWYtYzIyYS00M2UwLWJmNGMtMDZmNTY5OGZmYmQ5fScvPjxFdmVudElEPjM8L0V2ZW50SUQ+PFZlcnNpb24+NTwvVmVyc2lvbj48TGV2ZWw+NDwvTGV2ZWw+PFRhc2s+MzwvVGFzaz48T3Bjb2RlPjA8L09wY29kZT48S2V5d29yZHM+MHg4MDAwMDAwMDAwMDAwMDAwPC9LZXl3b3Jkcz48VGltZUNyZWF0ZWQgU3lzdGVtVGltZT0nMjAyMy0wMi0xNVQxMTozODoyNi40MDU0MjE2WicvPjxFdmVudFJlY29yZElEPjEzMjIwPC9FdmVudFJlY29yZElEPjxDb3JyZWxhdGlvbi8+PEV4ZWN1dGlvbiBQcm9jZXNzSUQ9JzE1MjgnIFRocmVhZElEPSczMDMyJy8+PENoYW5uZWw+TWljcm9zb2Z0LVdpbmRvd3MtU3lzbW9uL09wZXJhdGlvbmFsPC9DaGFubmVsPjxDb21wdXRlcj5FQzJBTUFaLU80S0NVSzM8L0NvbXB1dGVyPjxTZWN1cml0eSBVc2VySUQ9J1MtMS01LTE4Jy8+PC9TeXN0ZW0+PEV2ZW50RGF0YT48RGF0YSBOYW1lPSdSdWxlTmFtZSc+dGVjaG5pcXVlX2lkPVQxMDIxLHRlY2huaXF1ZV9uYW1lPVJlbW90ZSBTZXJ2aWNlczwvRGF0YT48RGF0YSBOYW1lPSdVdGNUaW1lJz4yMDIzLTAyLTE1IDExOjM4OjI0Ljc1MTwvRGF0YT48RGF0YSBOYW1lPSdQcm9jZXNzR3VpZCc+ezdhOTdiODVmLTdkYmEtNjNlYy0xMzAwLTAwMDAwMDAwN2UwMH08L0RhdGE+PERhdGEgTmFtZT0nUHJvY2Vzc0lkJz4xMDI4PC9EYXRhPjxEYXRhIE5hbWU9J0ltYWdlJz5DOlxXaW5kb3dzXFN5c3RlbTMyXHN2Y2hvc3QuZXhlPC9EYXRhPjxEYXRhIE5hbWU9J1VzZXInPk5UIEFVVEhPUklUWVxORVRXT1JLIFNFUlZJQ0U8L0RhdGE+PERhdGEgTmFtZT0nUHJvdG9jb2wnPnRjcDwvRGF0YT48RGF0YSBOYW1lPSdJbml0aWF0ZWQnPmZhbHNlPC9EYXRhPjxEYXRhIE5hbWU9J1NvdXJjZUlzSXB2Nic+ZmFsc2U8L0RhdGE+PERhdGEgTmFtZT0nU291cmNlSXAnPjU5LjIuNTIuMTIyPC9EYXRhPjxEYXRhIE5hbWU9J1NvdXJjZUhvc3RuYW1lJz4tPC9EYXRhPjxEYXRhIE5hbWU9J1NvdXJjZVBvcnQnPjYxMTExPC9EYXRhPjxEYXRhIE5hbWU9J1NvdXJjZVBvcnROYW1lJz4tPC9EYXRhPjxEYXRhIE5hbWU9J0Rlc3RpbmF0aW9uSXNJcHY2Jz5mYWxzZTwvRGF0YT48RGF0YSBOYW1lPSdEZXN0aW5hdGlvbklwJz4xNzIuMzEuMjkuNzk8L0RhdGE+PERhdGEgTmFtZT0nRGVzdGluYXRpb25Ib3N0bmFtZSc+LTwvRGF0YT48RGF0YSBOYW1lPSdEZXN0aW5hdGlvblBvcnQnPjMzODk8L0RhdGE+PERhdGEgTmFtZT0nRGVzdGluYXRpb25Qb3J0TmFtZSc+LTwvRGF0YT48L0V2ZW50RGF0YT48L0V2ZW50Pg==", + "mime_type": "text/plain" + } + }, + "first_observed": "2023-02-15T11:38:26.000Z", + "last_observed": "2023-02-15T11:38:26.000Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` +#### STIX Execute query +```shell +execute +splunk +splunk +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\", \"name\":\"splunk\",\"identity_class\":\"events\", \"created\": \"2022-01-22T13:22:50.336Z\", \"modified\": \"2023-03-22T13:22:50.336Z\"}" +"{\"host\":\"xxxx\",\"port\": xxxx,\"selfSignedCert\": \"-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\": \"xxxx\", \"password\": \"xxxx\"}}" +"[process:name = 'cmd.exe' AND process:pid = 100] START t'2023-01-01T11:00:00.000Z' STOP t'2023-02-28T11:00:00.003Z'" +``` +#### STIX Execute query - output +```json +{ + "type": "bundle", + "id": "bundle--6afd74ac-8ea5-4989-9b5d-5a8c8f4fd955", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "splunk", + "identity_class": "events", + "created": "2022-01-22T13:22:50.336Z", + "modified": "2023-03-22T13:22:50.336Z" + }, + { + "id": "observed-data--1dca7286-c0f3-496b-9474-0c803fd02aa3", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-05-09T07:33:49.320Z", + "modified": "2023-05-09T07:33:49.320Z", + "objects": { + "0": { + "type": "user-account", + "user_id": "SYSTEM", + "account_login": "'S-1-5-18'" + }, + "1": { + "type": "x-oca-event", + "user_ref": "0", + "host_ref": "2", + "module": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "process_ref": "4", + "parent_process_ref": "8", + "action": "Process creation", + "code": "1", + "provider": "Microsoft Sysmon", + "outcome": "allowed", + "x_dest": "xxxxx", + "x_event_id": "13153", + "original_ref": "10", + "created": "2023-02-15T11:25:49.000Z" + }, + "2": { + "type": "x-oca-asset", + "hostname": "xxxxx", + "x_operating_system": "Microsoft Windows" + }, + "3": { + "type": "x-splunk-data", + "log_source": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "event_type": [ + "corelight_idx", + "endpoint_services_processes", + "ms-sysmon-process", + "windows_event_signature" + ], + "log_source_type": "XmlWinEventLog" + }, + "4": { + "type": "process", + "command_line": "btool server list general --no-log", + "pid": 5248, + "name": "btool.exe", + "binary_ref": "5", + "x_unique_id": "{7a97b85f-c13d-63ec-1103-000000007e00}", + "parent_ref": "8", + "x_original_file_name": "btool.exe" + }, + "5": { + "type": "file", + "name": "btool.exe", + "parent_directory_ref": "6" + }, + "6": { + "type": "directory", + "path": "C:\\SplunkForwarder\\bin" + }, + "7": { + "type": "file", + "hashes": { + "SHA-1": "6C0F0DBCBC1442960CFEE43105BDF2CED4A7133E", + "MD5": "95439F1985B1CE750FF6693E7FB0F4BA", + "SHA-256": "303113F43A6187007B250230D34499082CCB42BB7920754DDC4E5302294A0D2D", + "x_IMPHASH": "D7E7C7FB1023AE6F5D81B244992E7451" + } + }, + "8": { + "type": "process", + "pid": 100, + "name": "cmd.exe", + "binary_ref": "9" + }, + "9": { + "type": "file", + "name": "cmd.exe" + }, + "10": { + "type": "artifact", + "payload_bin": "PEV2ZW50IHhtbG5zPSdodHRwOi8vc2NoZW1hcy5taWNyb3NvZnQuY29tL3dpbi8yMDA0LzA4L2V2ZW50cy9ldmVudCc+PFN5c3RlbT48UHJvdmlkZXIgTmFtZT0nTWljcm9zb2Z0LVdpbmRvd3MtU3lzbW9uJyBHdWlkPSd7NTc3MDM4NWYtYzIyYS00M2UwLWJmNGMtMDZmNTY5OGZmYmQ5fScvPjxFdmVudElEPjE8L0V2ZW50SUQ+PFZlcnNpb24+NTwvVmVyc2lvbj48TGV2ZWw+NDwvTGV2ZWw+PFRhc2s+MTwvVGFzaz48T3Bjb2RlPjA8L09wY29kZT48S2V5d29yZHM+MHg4MDAwMDAwMDAwMDAwMDAwPC9LZXl3b3Jkcz48VGltZUNyZWF0ZWQgU3lzdGVtVGltZT0nMjAyMy0wMi0xNVQxMToyNTo0OS42NDQ0NzMxWicvPjxFdmVudFJlY29yZElEPjEzMTUzPC9FdmVudFJlY29yZElEPjxDb3JyZWxhdGlvbi8+PEV4ZWN1dGlvbiBQcm9jZXNzSUQ9JzE1MjgnIFRocmVhZElEPSczMDI0Jy8+PENoYW5uZWw+TWljcm9zb2Z0LVdpbmRvd3MtU3lzbW9uL09wZXJhdGlvbmFsPC9DaGFubmVsPjxDb21wdXRlcj5FQzJBTUFaLU80S0NVSzM8L0NvbXB1dGVyPjxTZWN1cml0eSBVc2VySUQ9J1MtMS01LTE4Jy8+PC9TeXN0ZW0+PEV2ZW50RGF0YT48RGF0YSBOYW1lPSdSdWxlTmFtZSc+dGVjaG5pcXVlX2lkPVQxMDU5LHRlY2huaXF1ZV9uYW1lPUNvbW1hbmQtTGluZSBJbnRlcmZhY2U8L0RhdGE+PERhdGEgTmFtZT0nVXRjVGltZSc+MjAyMy0wMi0xNSAxMToyNTo0OS42NDA8L0RhdGE+PERhdGEgTmFtZT0nUHJvY2Vzc0d1aWQnPns3YTk3Yjg1Zi1jMTNkLTYzZWMtMTEwMy0wMDAwMDAwMDdlMDB9PC9EYXRhPjxEYXRhIE5hbWU9J1Byb2Nlc3NJZCc+NTI0ODwvRGF0YT48RGF0YSBOYW1lPSdJbWFnZSc+QzpcU3BsdW5rRm9yd2FyZGVyXGJpblxidG9vbC5leGU8L0RhdGE+PERhdGEgTmFtZT0nRmlsZVZlcnNpb24nPjkuMC4zPC9EYXRhPjxEYXRhIE5hbWU9J0Rlc2NyaXB0aW9uJz5idG9vbDwvRGF0YT48RGF0YSBOYW1lPSdQcm9kdWN0Jz5zcGx1bmsgQXBwbGljYXRpb248L0RhdGE+PERhdGEgTmFtZT0nQ29tcGFueSc+U3BsdW5rIEluYy48L0RhdGE+PERhdGEgTmFtZT0nT3JpZ2luYWxGaWxlTmFtZSc+YnRvb2wuZXhlPC9EYXRhPjxEYXRhIE5hbWU9J0NvbW1hbmRMaW5lJz5idG9vbCAgc2VydmVyIGxpc3QgZ2VuZXJhbCAtLW5vLWxvZzwvRGF0YT48RGF0YSBOYW1lPSdDdXJyZW50RGlyZWN0b3J5Jz5DOlxXaW5kb3dzXHN5c3RlbTMyXDwvRGF0YT48RGF0YSBOYW1lPSdVc2VyJz5OVCBBVVRIT1JJVFlcU1lTVEVNPC9EYXRhPjxEYXRhIE5hbWU9J0xvZ29uR3VpZCc+ezdhOTdiODVmLTdkYjgtNjNlYy1lNzAzLTAwMDAwMDAwMDAwMH08L0RhdGE+PERhdGEgTmFtZT0nTG9nb25JZCc+MHgzZTc8L0RhdGE+PERhdGEgTmFtZT0nVGVybWluYWxTZXNzaW9uSWQnPjA8L0RhdGE+PERhdGEgTmFtZT0nSW50ZWdyaXR5TGV2ZWwnPlN5c3RlbTwvRGF0YT48RGF0YSBOYW1lPSdIYXNoZXMnPlNIQTE9NkMwRjBEQkNCQzE0NDI5NjBDRkVFNDMxMDVCREYyQ0VENEE3MTMzRSxNRDU9OTU0MzlGMTk4NUIxQ0U3NTBGRjY2OTNFN0ZCMEY0QkEsU0hBMjU2PTMwMzExM0Y0M0E2MTg3MDA3QjI1MDIzMEQzNDQ5OTA4MkNDQjQyQkI3OTIwNzU0RERDNEU1MzAyMjk0QTBEMkQsSU1QSEFTSD1EN0U3QzdGQjEwMjNBRTZGNUQ4MUIyNDQ5OTJFNzQ1MTwvRGF0YT48RGF0YSBOYW1lPSdQYXJlbnRQcm9jZXNzR3VpZCc+ezdhOTdiODVmLWMxM2QtNjNlYy0xMDAzLTAwMDAwMDAwN2UwMH08L0RhdGE+PERhdGEgTmFtZT0nUGFyZW50UHJvY2Vzc0lkJz4xMDA8L0RhdGE+PERhdGEgTmFtZT0nUGFyZW50SW1hZ2UnPkM6XFdpbmRvd3NcU3lzdGVtMzJcY21kLmV4ZTwvRGF0YT48RGF0YSBOYW1lPSdQYXJlbnRDb21tYW5kTGluZSc+QzpcV2luZG93c1xzeXN0ZW0zMlxjbWQuZXhlIC9jIGJ0b29sIHNlcnZlciBsaXN0IGdlbmVyYWwgLS1uby1sb2c8L0RhdGE+PERhdGEgTmFtZT0nUGFyZW50VXNlcic+TlQgQVVUSE9SSVRZXFNZU1RFTTwvRGF0YT48L0V2ZW50RGF0YT48L0V2ZW50Pg==", + "mime_type": "text/plain" + } + }, + "first_observed": "2023-01-11T15:29:45.000Z", + "last_observed": "2023-01-11T15:29:45.000Z", + "number_observed": 1 + } + ], +"spec_version": "2.0" +} +``` + +### References +- [Splunk](https://docs.splunk.com/Documentation/Splunk) \ No newline at end of file diff --git a/stix_shifter_modules/splunk/splunk_supported_stix.md b/stix_shifter_modules/splunk/splunk_supported_stix.md index 7e4cdf5cf..53897ca2f 100644 --- a/stix_shifter_modules/splunk/splunk_supported_stix.md +++ b/stix_shifter_modules/splunk/splunk_supported_stix.md @@ -1,4 +1,4 @@ -##### Updated on 05/15/23 +##### Updated on 05/18/23 ## Splunk Enterprise Security ### Results STIX Domain Objects * Identity @@ -9,97 +9,143 @@ | STIX Operator | Data Source Operator | |--|--| +| AND (Comparision) | AND | +| OR (Comparision) | OR | +| = | = | +| != | != | +| LIKE | like({field}, {value}) | +| MATCHES | match({field}, {value}) | | > | > | | >= | >= | | < | < | | <= | <= | -| = | = | -| != | != | -| LIKE | encoders.like | -| IN | encoders.set | -| MATCHES | encoders.matches | -| AND (Comparision) | AND | -| OR (Comparision) | OR | -| ISSUBSET | encoders.subset | -| AND (Observation) | {expr1} OR {expr2} | -| OR (Observation) | {expr1} OR {expr2} | +| IN | IN | +| OR (Observation) | OR | +| AND (Observation) | OR | +| ISSUBSET | cidrmatch({field}, {value}) | | FOLLOWEDBY | latest=[search {expr2} | append [makeresults 1 | eval _time=0] | head 1 | return $_time] | where {expr1} | |
| | ### Searchable STIX objects and properties | STIX Object and Property | Mapped Data Source Fields | |--|--| +| **ipv4-addr**:value | src_ip, dest_ip | +| **ipv4-addr**:resolves_to_refs[*].value | src_mac, dest_mac | +| **ipv6-addr**:value | src_ip, dest_ip | +| **ipv6-addr**:resolves_to_refs[*].value | src_mac, dest_mac | +| **mac-addr**:value | src_mac, dest_mac | +| **network-traffic**:dst_port | dest_port | +| **network-traffic**:src_port | src_port | +| **network-traffic**:protocols[*] | protocol, transport | +| **network-traffic**:src_ref.value | src_ip | +| **network-traffic**:dst_ref.value | dest_ip | +| **network-traffic**:dst_ref.value | dest_ip | +| **network-traffic**:dst_byte_count | bytes_in | +| **network-traffic**:src_byte_count | bytes_out | +| **network-traffic**:dst_packets | packets_in | +| **network-traffic**:src_packets | packets_out | +| **network-traffic**:x_direction | direction | +| **network-traffic**:extensions.'dns-ext'.name | name | +| **network-traffic**:extensions.'dns-ext'.question.name_ref.value | query | +| **network-traffic**:extensions.'dns-ext'.resolved_ip_refs[*].value | answer | +| **network-traffic**:extensions.'dns-ext'.message_type | message_type | +| **network-traffic**:extensions.'dns-ext'.query_count | query_count | +| **network-traffic**:extensions.'dns-ext'.query_type | query_type | +| **network-traffic**:extensions.'dns-ext'.record_type | record_type | +| **network-traffic**:extensions.'dns-ext'.reply_code | reply_code | +| **network-traffic**:extensions.'dns-ext'.reply_code_id | reply_code_id | +| **network-traffic**:extensions.'dns-ext'.transaction_id | transaction_id | +| **network-traffic**:extensions.'http-request-ext'.request_method | http_method | +| **network-traffic**:extensions.'http-request-ext'.request_value | uri_path | +| **network-traffic**:extensions.'http-request-ext'.request_header.Referer | http_referrer | +| **network-traffic**:extensions.'http-request-ext'.request_header.'User-Agent' | http_user_agent | +| **network-traffic**:extensions.'http-request-ext'.x_uri_query | uri_query | +| **domain-name**:value | query, recipient_domain, src_user_domain, ssl_issuer_email_domain, ssl_subject_email_domain | +| **url**:value | url | +| **process**:command_line | process, parent_process | +| **process**:pid | process_id, parent_process_id | +| **process**:name | process_name, parent_process_name | +| **process**:cwd | process_current_directory | +| **process**:binary_ref.name | process_name, parent_process_name | +| **process**:x_original_file_name | original_file_name | +| **process**:x_memory_used | mem_used | +| **process**:x_unique_id | process_guid, parent_process_guid | +| **file**:name | file_name, process_name, parent_process_name, process_exec, parent_process_exec | +| **file**:size | file_size | +| **file**:hashes.MD5 | file_hash | +| **file**:hashes.'SHA-1' | file_hash | +| **file**:hashes.'SHA-256' | file_hash | +| **file**:parent_directory_ref.path | file_path, process_path, parent_process_path | +| **file**:created | file_create_time | +| **file**:modified | file_modify_time | +| **file**:accessed | file_access_time | +| **file**:x_acl | file_acl | +| **directory**:path | file_path, process_path, parent_process_path | +| **user-account**:user_id | user | +| **user-account**:account_login | user_id | +| **user-account**:x_user_name | user_name | +| **windows-registry-key**:key | registry_key_name | +| **windows-registry-key**:values[*].name | registry_value_name | +| **windows-registry-key**:values[*].data | registry_value_data | +| **windows-registry-key**:x_hive | registry_hive | +| **windows-registry-key**:x_path | registry_path | +| **windows-registry-key**:x_value_text | registry_value_text | +| **x-oca-asset**:hostname | host | +| **x-oca-asset**:x_operating_system | os | | **x-oca-event**:code | signature_id | | **x-oca-event**:action | signature | -| **x-oca-event**:outcome | result | +| **x-oca-event**:outcome | action | | **x-oca-event**:module | source | | **x-oca-event**:created | _time | -| **x-oca-event**:process_ref.command_line | process | +| **x-oca-event**:duration | duration | +| **x-oca-event**:provider | vendor_product | +| **x-oca-event**:severity | severity | +| **x-oca-event**:file_ref.name | file_name | | **x-oca-event**:process_ref.binary_ref.name | process_exec | -| **x-oca-event**:process_ref.parent_ref.command_line | parent_process | -| **x-oca-event**:process_ref.creator_user_ref.user_id | process_user | | **x-oca-event**:process_ref.name | process_name | -| **x-oca-event**:process_ref.pid | process_id | -| **x-oca-event**:parent_process_ref.command_line | parent_process | -| **x-oca-event**:parent_process_ref.binary_ref.name | parent_process_exec | | **x-oca-event**:parent_process_ref.pid | parent_process_id | | **x-oca-event**:parent_process_ref.name | parent_process_name | -| **x-oca-event**:domain_ref.value | url, url_domain | -| **x-oca-event**:file_ref.name | file_name | +| **x-oca-event**:domain_ref.value | query | | **x-oca-event**:host_ref.hostname | host | -| **x-oca-event**:host_ref.ip_refs[*].value | src_ip | -| **x-oca-event**:registry_ref.key | ObjectName, RegistryKey | +| **x-oca-event**:ip_refs[*].value | src_ip, dest_ip | +| **x-oca-event**:registry_ref.key | registry_key_name | | **x-oca-event**:user_ref.user_id | user | | **x-oca-event**:url_ref.value | url | -| **x-oca-asset**:hostname | host | -| **directory**:path | file_path, process_path, parent_process_path | -| **directory**:created | file_create_time | -| **directory**:modified | file_modify_time | -| **domain-name**:value | host, url | +| **x-oca-event**:network_ref.src_port | src_port | +| **x-oca-event**:network_ref.dst_port | dest_port | +| **x-oca-event**:x_dest | dest | +| **x-oca-event**:x_src | src | +| **x-oca-event**:x_application | app | +| **x-oca-event**:x_status | status | +| **x-oca-event**:x_event_id | event_id | | **x-readable-payload**:value | _raw | -| **email-addr**:value | src_user, recipient | -| **email-message**:body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.MD5 | file_hash | -| **email-message**:body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-1' | file_hash | -| **email-message**:body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-256' | file_hash | -| **email-message**:body_multipart.[*].'mime-part-type'.body_raw_ref.name | file_name | -| **email-message**:body_multipart.[*].'mime-part-type'.body_raw_ref.size | file_size | -| **email-message**:to_refs.[*].value | recipient | -| **email-message**:cc_refs.[*].value | recipient | -| **email-message**:bcc_refs.[*].value | recipient | +| **email-addr**:value | src_user, recipient, ssl_issuer_email, ssl_subject_email | +| **email-addr**:x_recipient_domain_ref.value | recipient_domain | +| **email-message**:to_refs[*].value | recipient | | **email-message**:subject | subject | -| **email-message**:sender_ref.value | src_user | | **email-message**:from_ref.value | src_user | -| **file**:hashes.MD5 | file_hash | -| **file**:hashes.'SHA-1' | file_hash | -| **file**:hashes.'SHA-256' | file_hash | -| **file**:name | file_name | -| **file**:created | file_create_time | -| **file**:modified | file_modify_time | -| **file**:parent_directory_ref.path | file_path | -| **file**:size | file_size | -| **ipv4-addr**:value | src_ip, dest_ip | -| **ipv6-addr**:value | src_ipv6, dest_ipv6 | -| **mac-addr**:value | src_mac, dest_mac | -| **network-traffic**:src_ref.value | src_ip | -| **network-traffic**:src_port | src_port | -| **network-traffic**:dst_ref.value | dest_ip | -| **network-traffic**:dst_port | dest_port | -| **network-traffic**:protocols[*] | protocol | -| **network-traffic**:start | earliest | -| **network-traffic**:end | latest | -| **process**:name | process_name, parent_process_name | -| **process**:command_line | process, parent_process | -| **process**:pid | process_id, parent_process_id | -| **process**:creator_user_ref.account_login | user | -| **process**:creator_user_ref.user_login | user | -| **process**:binary_ref.parent_directory_ref.path | process_path | -| **process**:binary_ref.name | process_exec | -| **process**:x_unique_id | process_guid | -| **url**:value | url | -| **user-account**:user_id | user | -| **user-account**:account_login | user | -| **windows-registry-key**:key | object | -| **windows-registry-key**:values[*] | result | -| **windows-registry-key**:creator_user_ref.account_login | user | +| **email-message**:x_url_ref.value | url | +| **email-message**:x_internal_message_id | internal_message_id | +| **email-message**:x_message_id | message_id | +| **x-ibm-ttp-tagging**:extensions.'mitre-attack-ext'.technique_id | mitre_technique_id | +| **x-splunk-vulnerability**:msft | msft | +| **x-splunk-vulnerability**:cve | cve | +| **x-splunk-vulnerability**:cvss | cvss | +| **x-splunk-vulnerability**:mskb | mskb | +| **x-splunk-authentication**:user_type | user_type | +| **x-splunk-authentication**:user_agent | user_agent | +| **x-splunk-authentication**:method | authentication_method | +| **x-splunk-authentication**:service | authentication_service | +| **x-splunk-data**:log_source | source | +| **x-splunk-data**:log_source_type | _sourcetype | +| **x-splunk-data**:event_type | eventtype | +| **x-ibm-finding**:severity | severity | +| **x-ibm-finding**:finding_type | type | +| **x-ibm-finding**:name | signature | +| **x-ibm-finding**:alert_id | id | +| **x-ibm-finding**:description | description | +| **x-ibm-finding**:src_ip_ref.value | src_ip | +| **x-ibm-finding**:dst_ip_ref.value | dest_ip | +| **x-ibm-finding**:ttp_tagging_refs[*].extensions.'mitre-attack-ext'.technique_id | mitre_technique_id | | **x509-certificate**:hashes.'SHA-256' | ssl_hash | | **x509-certificate**:hashes.'SHA-1' | ssl_hash | | **x509-certificate**:version | ssl_version | @@ -108,151 +154,218 @@ | **x509-certificate**:issuer | ssl_issuer | | **x509-certificate**:subject | ssl_subject | | **x509-certificate**:subject_public_key_algorithm | ssl_publickey_algorithm | -| **x-splunk**:log_source | source | -| **x-splunk**:log_source_type | _sourcetype | -| **x-splunk**:direction | Direction | -| **x-splunk**:event_id | EventID | -| **x-splunk**:event_name | EventName | -| **x-splunk**:mitre_tactic_id | TacticId | -| **x-splunk**:mitre_tactic | Tactic | -| **x-splunk**:mitre_technique_id | TechniqueId | -| **x-splunk**:mitre_technique | Technique | -| **x-ibm-finding**:name | ss_name | -| **x-ibm-finding**:src_device | DeviceType | -| **x-ibm-finding**:severity | severity | +| **x509-certificate**:validity_not_before | ssl_start_time | +| **x509-certificate**:validity_not_after | ssl_end_time | +| **x509-certificate**:x_ssl_is_valid | ssl_is_valid | +| **x509-certificate**:x_ssl_issuer_common_name | ssl_issuer_common_name | +| **x509-certificate**:x_ssl_subject_common_name | ssl_subject_common_name | +| **x509-certificate**:x_ssl_name | ssl_name | +| **x509-certificate**:x_ssl_publickey | ssl_publickey | +| **x509-certificate**:x_ssl_issuer_email_ref.value | ssl_issuer_email | +| **x509-certificate**:x_ssl_subject_email_ref.value | ssl_subject_email | +| **x509-certificate**:x_ssl_issuer_domain_ref.value | ssl_issuer_email_domain | +| **x509-certificate**:x_ssl_subject_domain_ref.value | ssl_subject_email_domain | +| **x509-certificate**:x_ssl_issuer_organization | ssl_issuer_organization | +| **x509-certificate**:x_ssl_subject_organization | ssl_subject_organization | |
| | ### Supported STIX Objects and Properties for Query Results | STIX Object | STIX Property | Data Source Field | |--|--|--| -| artifact | payload_bin | _raw | -| artifact | mime_type | mime_type_raw | -|
| | | | directory | path | process_path | | directory | path | parent_process_path | | directory | path | file_path | -| directory | created | file_create_time | -| directory | modified | file_modify_time | |
| | | -| domain-name | value | url | | domain-name | value | query | +| domain-name | value | ssl_issuer_email_domain | +| domain-name | value | ssl_subject_email_domain | +| domain-name | value | recipient_domain | +| domain-name | value | src_user_domain | |
| | | -| email-addr | value | src_user | -|
| | | -| email-message | sender_ref | src_user | -| email-message | from_ref | src_user | -| email-message | subject | subject | -| email-message | is_multipart | is_multipart | -|
| | | +| file | name | file_name | +| file | name | process_name | | file | name | process_exec | +| file | name | parent_process_name | +| file | name | parent_process_exec | +| file | size | file_size | | file | hashes | process_hash | +| file | hashes.MD5 | file_md5 | +| file | hashes.SHA-1 | file_sha1 | +| file | hashes.SHA-256 | file_sha256 | | file | parent_directory_ref | process_path | -| file | name | parent_process_exec | | file | parent_directory_ref | parent_process_path | | file | parent_directory_ref | file_path | | file | created | file_create_time | | file | modified | file_modify_time | -| file | hashes.UNKNOWN | file_hash | -| file | name | file_name | -| file | size | file_size | +| file | accessed | file_access_time | +| file | x_acl | file_acl | |
| | | | ipv4-addr | value | dest_ip | | ipv4-addr | value | src_ip | +| ipv4-addr | value | answer | | ipv4-addr | resolves_to_refs | src_mac | | ipv4-addr | resolves_to_refs | dest_mac | -| ipv4-addr | value | answer | |
| | | | ipv6-addr | value | dest_ip | +| ipv6-addr | value | answer | | ipv6-addr | value | src_ip | -| ipv6-addr | resolves_to_refs | src_mac | -| ipv6-addr | resolves_to_refs | dest_mac | |
| | | | mac-addr | value | src_mac | | mac-addr | value | dest_mac | |
| | | -| network-traffic | dst_ref | dest_ip | | network-traffic | src_ref | src_ip | +| network-traffic | protocols | protocol | +| network-traffic | protocols | transport | +| network-traffic | dst_ref | dest_ip | | network-traffic | dst_port | dest_port | | network-traffic | src_port | src_port | -| network-traffic | protocols | protocol | -| network-traffic | extensions.dns-ext.question.domain_ref | query | +| network-traffic | x_direction | direction | +| network-traffic | dst_byte_count | bytes_in | +| network-traffic | src_byte_count | bytes_out | +| network-traffic | dst_packets | packets_in | +| network-traffic | src_packets | packets_out | +| network-traffic | extensions.dns-ext.name | name | +| network-traffic | extensions.dns-ext.message_type | message_type | +| network-traffic | extensions.dns-ext.query_count | query_count | +| network-traffic | extensions.dns-ext.query_type | query_type | +| network-traffic | extensions.dns-ext.record_type | record_type | +| network-traffic | extensions.dns-ext.reply_code | reply_code | +| network-traffic | extensions.dns-ext.reply_code_id | reply_code_id | +| network-traffic | extensions.dns-ext.question.name_ref | query | | network-traffic | extensions.dns-ext.resolved_ip_refs | answer | +| network-traffic | extensions.dns-ext.transaction_id | transaction_id | +| network-traffic | extensions.http-request-ext.request_method | http_method | +| network-traffic | extensions.http-request-ext.request_header.Referer | http_referrer | +| network-traffic | extensions.http-request-ext.request_header.User-Agent | http_user_agent | +| network-traffic | extensions.http-request-ext.request_value | uri_path | +| network-traffic | extensions.http-request-ext.x_uri_query | uri_query | |
| | | -| process | opened_connection_refs | dest_ip | -| process | opened_connection_refs | src_ip | -| process | opened_connection_refs | dest_port | -| process | opened_connection_refs | src_port | -| process | opened_connection_refs | protocol | -| process | creator_user_ref | process_user | -| process | pid | process_id | -| process | name | process_name | -| process | command_line | process | +| process | binary_ref | process_name | | process | binary_ref | process_exec | +| process | binary_ref | parent_process_exec | | process | x_unique_id | process_guid | +| process | x_unique_id | parent_process_guid | +| process | cwd | process_current_directory | +| process | command_line | process | +| process | command_line | parent_process | +| process | x_original_file_name | original_file_name | +| process | x_memory_used | mem_used | +| process | pid | process_id | | process | pid | parent_process_id | +| process | name | process_name | | process | parent_ref | parent_process_id | | process | name | parent_process_name | +| process | opened_connection_refs | protocol | +| process | opened_connection_refs | transport | | process | parent_ref | parent_process_name | -| process | command_line | parent_process | -| process | binary_ref | parent_process_exec | -| process | opened_connection_refs | query | -| process | opened_connection_refs | answer | |
| | | | url | value | url | |
| | | -| user-account | account_login | user | | user-account | user_id | user | +| user-account | x_user_name | user_name | +| user-account | account_login | user_id | |
| | | -| windows-registry-key | key | object_path | -|
| | | -| x-ibm-finding | name | ss_name | -| x-ibm-finding | src_device | DeviceType | -| x-ibm-finding | severity | severity | +| windows-registry-key | key | registry_key_name | +| windows-registry-key | values | registry_value | +| windows-registry-key | x_hive | registry_hive | +| windows-registry-key | x_path | registry_path | +| windows-registry-key | x_value_text | registry_value_text | |
| | | +| x-oca-asset | x_operating_system | os | | x-oca-asset | hostname | host | |
| | | | x-oca-event | original_ref | _raw | | x-oca-event | ip_refs | dest_ip | -| x-oca-event | network_ref | dest_ip | | x-oca-event | ip_refs | src_ip | -| x-oca-event | network_ref | src_ip | | x-oca-event | network_ref | dest_port | | x-oca-event | network_ref | src_port | -| x-oca-event | network_ref | protocol | -| x-oca-event | created | _time | +| x-oca-event | network_ref | transport | +| x-oca-event | domain_ref | query | | x-oca-event | user_ref | user | | x-oca-event | process_ref | process_id | | x-oca-event | process_ref | process_name | -| x-oca-event | file_ref | process_exec | +| x-oca-event | process_ref | process | | x-oca-event | parent_process_ref | parent_process_id | | x-oca-event | parent_process_ref | parent_process_name | +| x-oca-event | process_ref | parent_process | +| x-oca-event | parent_process_ref | parent_process | | x-oca-event | file_ref | file_name | -| x-oca-event | registry_ref | object_path | +| x-oca-event | registry_ref | registry_key_name | +| x-oca-event | url_ref | url | | x-oca-event | host_ref | host | | x-oca-event | module | source | -| x-oca-event | action | description | +| x-oca-event | x_status | status | +| x-oca-event | provider | vendor_product | +| x-oca-event | outcome | action | +| x-oca-event | duration | duration | +| x-oca-event | severity | severity | +| x-oca-event | created | _time | | x-oca-event | action | signature | | x-oca-event | code | signature_id | -| x-oca-event | outcome | result | -| x-oca-event | domain_ref | query | -| x-oca-event | network_ref | query | -| x-oca-event | network_ref | answer | +| x-oca-event | x_application | app | +| x-oca-event | x_dest | dest | +| x-oca-event | x_src | src | +| x-oca-event | x_event_id | event_id | +|
| | | +| x-splunk-authentication | method | authentication_method | +| x-splunk-authentication | service | authentication_service | +| x-splunk-authentication | user_agent | user_agent | +| x-splunk-authentication | user_type | user_type | +|
| | | +| x-splunk-vulnerability | msft | msft | +| x-splunk-vulnerability | cve | cve | +| x-splunk-vulnerability | cvss | cvss | +| x-splunk-vulnerability | mskb | mskb | |
| | | -| x-splunk | direction | Direction | -| x-splunk | event_id | EventID | -| x-splunk | mitre_tactic_id | TacticId | -| x-splunk | mitre_tactic | Tactic | -| x-splunk | mitre_technique_id | TechniqueId | -| x-splunk | mitre_technique | Technique | -| x-splunk | event_name | EventName | +| x-ibm-ttp-tagging | extensions.mitre-attack-ext.technique_id | mitre_technique_id | +|
| | | +| artifact | payload_bin | _raw | +|
| | | +| x-splunk-data | log_source_type | _sourcetype | +| x-splunk-data | event_type | eventtype | +| x-splunk-data | log_source | source | +|
| | | +| x-ibm-finding | dst_ip_ref | dest_ip | +| x-ibm-finding | src_ip_ref | src_ip | +| x-ibm-finding | finding_type | finding_type | +| x-ibm-finding | severity | alert_severity | +| x-ibm-finding | name | alert_signature | +| x-ibm-finding | alert_id | alert_id | +| x-ibm-finding | description | alert_description | +| x-ibm-finding | ttp_tagging_refs | mitre_technique_id | +|
| | | +| email-addr | value | src_user | +| email-addr | value | recipient | +| email-addr | value | ssl_issuer_email | +| email-addr | value | ssl_subject_email | +| email-addr | x_recipient_domain_ref | recipient_domain | +| email-addr | x_src_user_domain_ref | src_user_domain | +|
| | | +| email-message | from_ref | src_user | +| email-message | to_refs | recipient | +| email-message | subject | subject | +| email-message | is_multipart | is_multipart | +| email-message | x_internal_message_id | internal_message_id | +| email-message | x_message_id | message_id | +| email-message | x_message_info | message_info | |
| | | | x509-certificate | hashes.SHA-256 | ssl_hash | | x509-certificate | version | ssl_version | | x509-certificate | serial_number | ssl_serial | | x509-certificate | signature_algorithm | ssl_signature_algorithm | -| x509-certificate | issuer | ssl_issuer | +| x509-certificate | ssl_issuer | issuer | | x509-certificate | subject | ssl_subject | | x509-certificate | subject_public_key_algorithm | ssl_publickey_algorithm | -|
| | | -| x_splunk_spl | user | user | -| x_splunk_spl | bytes | bytes | -|
| | | +| x509-certificate | validity_not_before | ssl_start_time | +| x509-certificate | validity_not_after | ssl_end_time | +| x509-certificate | x_ssl_is_valid | ssl_is_valid | +| x509-certificate | x_ssl_issuer_common_name | ssl_issuer_common_name | +| x509-certificate | x_ssl_subject_common_name | ssl_subject_common_name | +| x509-certificate | x_ssl_name | ssl_name | +| x509-certificate | x_ssl_publickey | ssl_publickey | +| x509-certificate | x_ssl_issuer_domain_ref | ssl_issuer_email_domain | +| x509-certificate | x_ssl_subject_domain_ref | ssl_subject_email_domain | +| x509-certificate | x_ssl_issuer_organization | ssl_issuer_organization | +| x509-certificate | x_ssl_subject_organization | ssl_subject_organization | +| x509-certificate | x_ssl_issuer_email_ref | ssl_issuer_email | +| x509-certificate | x_ssl_subject_email_ref | ssl_subject_email | +|
| | | \ No newline at end of file diff --git a/stix_shifter_modules/splunk/stix_translation/json/from_stix_map.json b/stix_shifter_modules/splunk/stix_translation/json/from_stix_map.json index 6cfa76ae1..6f9c82c13 100644 --- a/stix_shifter_modules/splunk/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/splunk/stix_translation/json/from_stix_map.json @@ -1,198 +1,551 @@ { - "x-oca-event": { - "cim_type": "alert", - "fields": { - "code": ["signature_id"], - "action": ["signature"], - "outcome": ["result"], - "module": ["source"], - "created": ["_time"], - "process_ref.command_line": ["process"], - "process_ref.binary_ref.name": ["process_exec"], - "process_ref.parent_ref.command_line": ["parent_process"], - "process_ref.creator_user_ref.user_id": ["process_user"], - "process_ref.name": ["process_name"], - "process_ref.pid": ["process_id"], - "parent_process_ref.command_line": ["parent_process"], - "parent_process_ref.binary_ref.name": ["parent_process_exec"], - "parent_process_ref.pid": ["parent_process_id"], - "parent_process_ref.name": ["parent_process_name"], - "domain_ref.value": ["url", "url_domain"], - "file_ref.name": ["file_name"], - "host_ref.hostname": ["host"], - "host_ref.ip_refs[*].value": ["src_ip"], - "registry_ref.key": ["ObjectName", "RegistryKey"], - "user_ref.user_id": ["user"], - "url_ref.value": ["url"] - } - }, - "x-oca-asset": { - "cim_type": "host", - "fields": { - "hostname": "host" - } - }, - "directory": { - "cim_type": "endpoint", - "fields": { - "path": ["file_path", "process_path", "parent_process_path"], - "created": "file_create_time", - "modified": "file_modify_time" - } - }, - "domain-name": { - "cim_type": "flow", - "fields": { - "value": ["host", "url"] - } - }, - "x-readable-payload": { - "cim_type": "", - "fields": { - "value": "_raw" - } - }, - "email-addr": { - "cim_type": "email", - "fields": { - "value": ["src_user", "recipient"] - } - }, - "email-message": { - "cim_type": "email", - "fields": { - "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.MD5": "file_hash", - "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-1'": "file_hash", - "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-256'": "file_hash", - "body_multipart.[*].'mime-part-type'.body_raw_ref.name": "file_name", - "body_multipart.[*].'mime-part-type'.body_raw_ref.size": "file_size", - "to_refs.[*].value": "recipient", - "cc_refs.[*].value": "recipient", - "bcc_refs.[*].value": "recipient", - "subject": "subject", - "sender_ref.value": "src_user", - "from_ref.value": "src_user" - } - }, - "file": { - "cim_type": "endpoint", - "fields": { - "hashes.MD5": "file_hash", - "hashes.'SHA-1'": "file_hash", - "hashes.'SHA-256'": "file_hash", - "name": "file_name", - "created": "file_create_time", - "modified": "file_modify_time", - "parent_directory_ref.path": "file_path", - "size": "file_size" - } - }, - "ipv4-addr": { - "cim_type": "flow", - "fields": { - "value": [ - "src_ip", - "dest_ip" - ] - } - }, - "ipv6-addr": { - "cim_type": "flow", - "fields": { - "value": [ - "src_ipv6", - "dest_ipv6" - ] - } - }, - "mac-addr": { - "cim_type": "flow", - "fields": { - "value": [ - "src_mac", - "dest_mac" - ] - } - }, - "network-traffic": { - "cim_type": "network", - "fields": { - "src_ref.value": ["src_ip"], - "src_port": "src_port", - "dst_ref.value": ["dest_ip"], - "dst_port": "dest_port", - "protocols[*]": "protocol", - "start": "earliest", - "end": "latest" - } - }, - "process": { - "cim_type": "process", - "fields": { - "name": ["process_name", "parent_process_name"], - "command_line": ["process", "parent_process"], - "pid": ["process_id", "parent_process_id"], - "creator_user_ref.account_login": "user", - "creator_user_ref.user_login": "user", - "binary_ref.parent_directory_ref.path": "process_path", - "binary_ref.name": "process_exec", - "x_unique_id": "process_guid" - } - }, - "url": { - "cim_type": "web", - "fields": { - "value": "url" - } - }, - "user-account": { - "cim_type": "authentication", - "fields": { - "user_id": "user", - "account_login": "user" - } - }, - "windows-registry-key": { - "cim_type": "endpoint", - "fields": { - "key": "object", - "values[*]": "result", - "creator_user_ref.account_login": "user" - } - }, - "x509-certificate": { - "cim_type": "certificate", - "fields": { - "hashes.'SHA-256'": "ssl_hash", - "hashes.'SHA-1'": "ssl_hash", - "version": "ssl_version", - "serial_number": "ssl_serial", - "signature_algorithm": "ssl_signature_algorithm", - "issuer": "ssl_issuer", - "subject": "ssl_subject", - "subject_public_key_algorithm": "ssl_publickey_algorithm" - } - }, - "x-splunk": { - "cim_type": "endpoint", - "fields": { - "log_source": ["source"], - "log_source_type": ["_sourcetype"], - "direction": ["Direction"], - "event_id": ["EventID"], - "event_name": ["EventName"], - "mitre_tactic_id": ["TacticId"], - "mitre_tactic": ["Tactic"], - "mitre_technique_id": ["TechniqueId"], - "mitre_technique": ["Technique"] - } - }, - "x-ibm-finding": { - "cim_type": "endpoint", - "fields": { - "name": ["ss_name"], - "src_device": ["DeviceType"], - "severity": ["severity"] - } - } -} + "x-oca-event": { + "cim_type": "endpoint", + "fields": { + "code": [ + "signature_id" + ], + "action": [ + "signature" + ], + "outcome": [ + "action" + ], + "module": [ + "source" + ], + "created": [ + "_time" + ], + "duration": [ + "duration" + ], + "provider": [ + "vendor_product" + ], + "severity": [ + "severity" + ], + "file_ref.name": [ + "file_name" + ], + "process_ref.binary_ref.name": [ + "process_exec" + ], + "process_ref.name": [ + "process_name" + ], + "process_ref.pid": [ + "process_id" + ], + "parent_process_ref.pid": [ + "parent_process_id" + ], + "parent_process_ref.name": [ + "parent_process_name" + ], + "domain_ref.value": [ + "query" + ], + "host_ref.hostname": [ + "host" + ], + "ip_refs[*].value": [ + "src_ip", + "dest_ip" + ], + "registry_ref.key": [ + "registry_key_name" + ], + "user_ref.user_id": [ + "user" + ], + "url_ref.value": [ + "url" + ], + "network_ref.src_port": [ + "src_port" + ], + "network_ref.dst_port": [ + "dest_port" + ], + "x_dest": [ + "dest" + ], + "x_src": [ + "src" + ], + "x_application": [ + "app" + ], + "x_status": [ + "status" + ], + "x_event_id": [ + "event_id" + ] + } + }, + "x-oca-asset": { + "cim_type": "host", + "fields": { + "hostname": [ + "host" + ], + "x_operating_system": [ + "os" + ] + } + }, + "directory": { + "cim_type": "endpoint", + "fields": { + "path": [ + "file_path", + "process_path", + "parent_process_path" + ] + } + }, + "domain-name": { + "cim_type": "flow", + "fields": { + "value": [ + "query", + "recipient_domain", + "src_user_domain", + "ssl_issuer_email_domain", + "ssl_subject_email_domain" + ] + } + }, + "x-readable-payload": { + "cim_type": "", + "fields": { + "value": [ + "_raw" + ] + } + }, + "email-addr": { + "cim_type": "email", + "fields": { + "value": [ + "src_user", + "recipient", + "ssl_issuer_email", + "ssl_subject_email" + ], + "x_recipient_domain_ref.value": [ + "recipient_domain" + ], + "x_src_user_domain_ref.value": [ + "src_user_domain" + ] + } + }, + "email-message": { + "cim_type": "email", + "fields": { + "to_refs[*].value": [ + "recipient" + ], + "subject": [ + "subject" + ], + "from_ref.value": [ + "src_user" + ], + "x_url_ref.value": [ + "url" + ], + "x_internal_message_id": [ + "internal_message_id" + ], + "x_message_id": [ + "message_id" + ] + } + }, + "file": { + "cim_type": "endpoint", + "fields": { + "hashes.MD5": [ + "file_hash" + ], + "hashes.'SHA-1'": [ + "file_hash" + ], + "hashes.'SHA-256'": [ + "file_hash" + ], + "name": [ + "file_name", + "process_name", + "parent_process_name", + "process_exec", + "parent_process_exec" + ], + "created": [ + "file_create_time" + ], + "modified": [ + "file_modify_time" + ], + "accessed": [ + "file_access_time" + ], + "parent_directory_ref.path": [ + "file_path", + "process_path", + "parent_process_path" + ], + "size": [ + "file_size" + ], + "x_acl": [ + "file_acl" + ] + } + }, + "ipv4-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_ip", + "dest_ip" + ], + "resolves_to_refs[*].value": [ + "src_mac", + "dest_mac" + ] + } + }, + "ipv6-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_ip", + "dest_ip" + ], + "resolves_to_refs[*].value": [ + "src_mac", + "dest_mac" + ] + } + }, + "mac-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_mac", + "dest_mac" + ] + } + }, + "network-traffic": { + "cim_type": "network", + "fields": { + "src_ref.value": [ + "src_ip" + ], + "src_port": [ + "src_port" + ], + "dst_ref.value": [ + "dest_ip" + ], + "dst_port": [ + "dest_port" + ], + "protocols[*]": [ + "protocol", + "transport" + ], + "dst_byte_count": [ + "bytes_in" + ], + "src_byte_count": [ + "bytes_out" + ], + "dst_packets": [ + "packets_in" + ], + "src_packets": [ + "packets_out" + ], + "x_direction": [ + "direction" + ], + "extensions.'dns-ext'.name": [ + "name" + ], + "extensions.'dns-ext'.question.name_ref.value": [ + "query" + ], + "extensions.'dns-ext'.resolved_ip_refs[*].value": [ + "answer" + ], + "extensions.'dns-ext'.message_type": [ + "message_type" + ], + "extensions.'dns-ext'.query_count": [ + "query_count" + ], + "extensions.'dns-ext'.query_type": [ + "query_type" + ], + "extensions.'dns-ext'.record_type": [ + "record_type" + ], + "extensions.'dns-ext'.reply_code": [ + "reply_code" + ], + "extensions.'dns-ext'.reply_code_id": [ + "reply_code_id" + ], + "extensions.'dns-ext'.transaction_id": [ + "transaction_id" + ], + "extensions.'http-request-ext'.request_method": [ + "http_method" + ], + "extensions.'http-request-ext'.request_value": [ + "uri_path" + ], + "extensions.'http-request-ext'.request_header.Referer": [ + "http_referrer" + ], + "extensions.'http-request-ext'.request_header.'User-Agent'": [ + "http_user_agent" + ], + "extensions.'http-request-ext'.x_uri_query": [ + "uri_query" + ] + } + }, + "process": { + "cim_type": "process", + "fields": { + "name": [ + "process_name", + "parent_process_name" + ], + "command_line": [ + "process", + "parent_process" + ], + "pid": [ + "process_id", + "parent_process_id" + ], + "cwd": [ + "process_current_directory" + ], + "binary_ref.name": [ + "process_name", + "parent_process_name" + ], + "x_unique_id": [ + "process_guid", + "parent_process_guid" + ], + "x_original_file_name": [ + "original_file_name" + ], + "x_memory_used": [ + "mem_used" + ] + } + }, + "url": { + "cim_type": "web", + "fields": { + "value": [ + "url" + ] + } + }, + "user-account": { + "cim_type": "authentication", + "fields": { + "user_id": [ + "user" + ], + "account_login": [ + "user_id" + ], + "x_user_name": [ + "user_name" + ] + } + }, + "windows-registry-key": { + "cim_type": "endpoint", + "fields": { + "key": [ + "registry_key_name" + ], + "values[*].name": [ + "registry_value_name" + ], + "values[*].data": [ + "registry_value_data" + ], + "x_hive": [ + "registry_hive" + ], + "x_path": [ + "registry_path" + ], + "x_value_text": [ + "registry_value_text" + ] + } + }, + "x509-certificate": { + "cim_type": "certificate", + "fields": { + "hashes.'SHA-256'": [ + "ssl_hash" + ], + "hashes.'SHA-1'": [ + "ssl_hash" + ], + "version": [ + "ssl_version" + ], + "serial_number": [ + "ssl_serial" + ], + "signature_algorithm": [ + "ssl_signature_algorithm" + ], + "issuer": [ + "ssl_issuer" + ], + "subject": [ + "ssl_subject" + ], + "subject_public_key_algorithm": [ + "ssl_publickey_algorithm" + ], + "validity_not_before": [ + "ssl_start_time" + ], + "validity_not_after": [ + "ssl_end_time" + ], + "x_ssl_is_valid": [ + "ssl_is_valid" + ], + "x_ssl_issuer_common_name": [ + "ssl_issuer_common_name" + ], + "x_ssl_subject_common_name": [ + "ssl_subject_common_name" + ], + "x_ssl_name": [ + "ssl_name" + ], + "x_ssl_publickey": [ + "ssl_publickey" + ], + "x_ssl_issuer_email_ref.value": [ + "ssl_issuer_email" + ], + "x_ssl_subject_email_ref.value": [ + "ssl_subject_email" + ], + "x_ssl_issuer_domain_ref.value": [ + "ssl_issuer_email_domain" + ], + "x_ssl_subject_domain_ref.value": [ + "ssl_subject_email_domain" + ], + "x_ssl_issuer_organization": [ + "ssl_issuer_organization" + ], + "x_ssl_subject_organization": [ + "ssl_subject_organization" + ] + } + }, + "x-ibm-finding": { + "cim_type": "alert", + "fields": { + "severity": [ + "severity" + ], + "finding_type": [ + "type" + ], + "name": [ + "signature" + ], + "alert_id": [ + "id" + ], + "description": [ + "description" + ], + "src_ip_ref.value": [ + "src_ip" + ], + "dst_ip_ref.value": [ + "dest_ip" + ], + "ttp_tagging_refs[*].extensions.'mitre-attack-ext'.technique_id": [ + "mitre_technique_id" + ] + } + }, + "x-splunk-data": { + "cim_type": "endpoint", + "fields": { + "log_source": [ + "source" + ], + "log_source_type": [ + "_sourcetype" + ], + "event_type": [ + "eventtype" + ] + } + }, + "x-splunk-authentication": { + "cim_type": "authentication", + "fields": { + "user_type": [ + "user_type" + ], + "user_agent": [ + "user_agent" + ], + "method": [ + "authentication_method" + ], + "service": [ + "authentication_service" + ] + } + }, + "x-splunk-vulnerability": { + "cim_type": "vulnerability", + "fields": { + "msft": "msft", + "cve": "cve", + "cvss": "cvss", + "mskb": "mskb" + } + }, + "x-ibm-ttp-tagging": { + "cim_type": "alert", + "fields": { + "extensions.'mitre-attack-ext'.technique_id": [ + "mitre_technique_id" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/splunk/stix_translation/json/select_fields.json b/stix_shifter_modules/splunk/stix_translation/json/select_fields.json index ed851d842..92b57b859 100644 --- a/stix_shifter_modules/splunk/stix_translation/json/select_fields.json +++ b/stix_shifter_modules/splunk/stix_translation/json/select_fields.json @@ -3,27 +3,16 @@ "src_ip", "src_port", "src_mac", - "src_ipv6", "dest_ip", "dest_port", "dest_mac", - "dest_ipv6", "file_hash", "user", "url", "protocol", "host", "source", - "DeviceType", - "Direction", "severity", - "EventID", - "EventName", - "ss_name", - "TacticId", - "Tactic", - "TechniqueId", - "Technique", "process", "process_id", "process_name", @@ -36,10 +25,95 @@ "parent_process_name", "parent_process_exec", "description", - "result", "signature", "signature_id", "query", - "answer" + "answer", + "transport", + "bytes_in", + "bytes_out", + "packets_in", + "packets_out", + "direction", + "name", + "message_type", + "query_count", + "query_type", + "record_type", + "reply_code", + "reply_code_id", + "vendor_product", + "duration", + "transaction_id", + "action", + "file_access_time", + "file_acl", + "registry_hive", + "registry_path", + "registry_key_name", + "registry_value_data", + "registry_value_name", + "registry_value_text", + "registry_value_type", + "status", + "ssl_version", + "ssl_serial", + "ssl_issuer", + "ssl_subject", + "ssl_signature_algorithm", + "ssl_publickey_algorithm", + "ssl_start_time", + "ssl_end_time", + "ssl_is_valid", + "ssl_issuer_common_name", + "ssl_subject_common_name", + "ssl_name", + "ssl_publickey", + "ssl_issuer_email", + "ssl_subject_email", + "ssl_issuer_email_domain", + "ssl_subject_email_domain", + "ssl_issuer_organization", + "ssl_subject_organization", + "recipient", + "subject", + "file_hash", + "file_name", + "file_size", + "recipient_domain", + "src_user_domain", + "internal_message_id", + "message_id", + "message_info", + "app", + "authentication_method", + "authentication_service", + "dest", + "src", + "src_user", + "user_name", + "user_id", + "user_type", + "user_agent", + "http_method", + "http_referrer", + "http_user_agent", + "uri_path", + "uri_query", + "os", + "dvc", + "id", + "msft", + "cve", + "cvss", + "mskb", + "type", + "eventtype", + "event_id", + "mitre_technique_id", + "mem_used", + "original_file_name", + "file_create_time", + "file_modify_time" ] } \ No newline at end of file diff --git a/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json index 10537fdbc..345895333 100644 --- a/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json +++ b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json @@ -1,198 +1,548 @@ { - "x-oca-event": { - "cim_type": "alert", - "fields": { - "code": ["signature_id"], - "action": ["signature"], - "outcome": ["result"], - "module": ["source"], - "created": ["_time"], - "process_ref.command_line": ["process"], - "process_ref.image_ref.name": ["process_exec"], - "process_ref.parent_ref.command_line": ["parent_process"], - "process_ref.creator_user_ref.user_id": ["process_user"], - "process_ref.name": ["process_name"], - "process_ref.pid": ["process_id"], - "parent_process_ref.command_line": ["parent_process"], - "parent_process_ref.image_ref.name": ["parent_process_exec"], - "parent_process_ref.pid": ["parent_process_id"], - "parent_process_ref.name": ["parent_process_name"], - "domain_ref.value": ["url", "url_domain"], - "file_ref.name": ["file_name"], - "host_ref.hostname": ["host"], - "host_ref.ip_refs[*].value": ["src_ip"], - "registry_ref.key": ["ObjectName", "RegistryKey"], - "user_ref.user_id": ["user"], - "url_ref.value": ["url"] - } - }, - "x-oca-asset": { - "cim_type": "host", - "fields": { - "hostname": "host" - } - }, - "directory": { - "cim_type": "endpoint", - "fields": { - "path": ["file_path", "process_path", "parent_process_path"], - "ctime": "file_create_time", - "mtime": "file_modify_time" - } - }, - "domain-name": { - "cim_type": "flow", - "fields": { - "value": ["host", "url"] - } - }, - "x-readable-payload": { - "cim_type": "", - "fields": { - "value": "_raw" - } - }, - "email-addr": { - "cim_type": "email", - "fields": { - "value": ["src_user", "recipient"] - } - }, - "email-message": { - "cim_type": "email", - "fields": { - "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.MD5": "file_hash", - "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-1'": "file_hash", - "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-256'": "file_hash", - "body_multipart.[*].'mime-part-type'.body_raw_ref.name": "file_name", - "body_multipart.[*].'mime-part-type'.body_raw_ref.size": "file_size", - "to_refs.[*].value": "recipient", - "cc_refs.[*].value": "recipient", - "bcc_refs.[*].value": "recipient", - "subject": "subject", - "sender_ref.value": "src_user", - "from_ref.value": "src_user" - } - }, - "file": { - "cim_type": "endpoint", - "fields": { - "hashes.MD5": "file_hash", - "hashes.'SHA-1'": "file_hash", - "hashes.'SHA-256'": "file_hash", - "name": "file_name", - "ctime": "file_create_time", - "mtime": "file_modify_time", - "parent_directory_ref.path": "file_path", - "size": "file_size" - } - }, - "ipv4-addr": { - "cim_type": "flow", - "fields": { - "value": [ - "src_ip", - "dest_ip" - ] - } - }, - "ipv6-addr": { - "cim_type": "flow", - "fields": { - "value": [ - "src_ipv6", - "dest_ipv6" - ] - } - }, - "mac-addr": { - "cim_type": "flow", - "fields": { - "value": [ - "src_mac", - "dest_mac" - ] - } - }, - "network-traffic": { - "cim_type": "network", - "fields": { - "src_ref.value": ["src_ip"], - "src_port": "src_port", - "dst_ref.value": ["dest_ip"], - "dst_port": "dest_port", - "protocols[*]": "protocol", - "start": "earliest", - "end": "latest" - } - }, - "process": { - "cim_type": "process", - "fields": { - "x_name": ["process_name", "parent_process_name"], - "command_line": ["process", "parent_process"], - "pid": ["process_id", "parent_process_id"], - "creator_user_ref.account_login": "user", - "creator_user_ref.user_login": "user", - "image_ref.parent_directory_ref.path": "process_path", - "image_ref.name": "process_exec", - "x_unique_id": "process_guid" - } - }, - "url": { - "cim_type": "web", - "fields": { - "value": "url" - } - }, - "user-account": { - "cim_type": "authentication", - "fields": { - "user_id": "user", - "account_login": "user" - } - }, - "windows-registry-key": { - "cim_type": "endpoint", - "fields": { - "key": "object", - "values[*]": "result", - "creator_user_ref.account_login": "user" - } - }, - "x509-certificate": { - "cim_type": "certificate", - "fields": { - "hashes.'SHA-256'": "ssl_hash", - "hashes.'SHA-1'": "ssl_hash", - "version": "ssl_version", - "serial_number": "ssl_serial", - "signature_algorithm": "ssl_signature_algorithm", - "issuer": "ssl_issuer", - "subject": "ssl_subject", - "subject_public_key_algorithm": "ssl_publickey_algorithm" - } - }, - "x-splunk": { - "cim_type": "endpoint", - "fields": { - "log_source": ["source"], - "log_source_type": ["_sourcetype"], - "direction": ["Direction"], - "event_id": ["EventID"], - "event_name": ["EventName"], - "mitre_tactic_id": ["TacticId"], - "mitre_tactic": ["Tactic"], - "mitre_technique_id": ["TechniqueId"], - "mitre_technique": ["Technique"] - } - }, - "x-ibm-finding": { - "cim_type": "endpoint", - "fields": { - "name": ["ss_name"], - "src_device": ["DeviceType"], - "severity": ["severity"] - } - } -} + "x-oca-event": { + "cim_type": "endpoint", + "fields": { + "code": [ + "signature_id" + ], + "action": [ + "signature" + ], + "outcome": [ + "action" + ], + "module": [ + "source" + ], + "created": [ + "_time" + ], + "duration": [ + "duration" + ], + "provider": [ + "vendor_product" + ], + "severity": [ + "severity" + ], + "file_ref.name": [ + "file_name" + ], + "process_ref.binary_ref.name": [ + "process_exec" + ], + "process_ref.name": [ + "process_name" + ], + "process_ref.pid": [ + "process_id" + ], + "parent_process_ref.pid": [ + "parent_process_id" + ], + "parent_process_ref.name": [ + "parent_process_name" + ], + "domain_ref.value": [ + "query" + ], + "host_ref.hostname": [ + "host" + ], + "ip_refs[*].value": [ + "src_ip", + "dest_ip" + ], + "registry_ref.key": [ + "registry_key_name" + ], + "user_ref.user_id": [ + "user" + ], + "url_ref.value": [ + "url" + ], + "network_ref.src_port": [ + "src_port" + ], + "network_ref.dst_port": [ + "dest_port" + ], + "x_dest": [ + "dest" + ], + "x_src": [ + "src" + ], + "x_application": [ + "app" + ], + "x_status": [ + "status" + ], + "x_event_id": [ + "event_id" + ] + } + }, + "x-oca-asset": { + "cim_type": "host", + "fields": { + "hostname": [ + "host" + ], + "x_operating_system": [ + "os" + ] + } + }, + "directory": { + "cim_type": "endpoint", + "fields": { + "path": [ + "file_path", + "process_path", + "parent_process_path" + ], + "ctime": "file_create_time", + "mtime": "file_modify_time", + "atime": "file_access_time" + } + }, + "domain-name": { + "cim_type": "flow", + "fields": { + "value": [ + "query", + "recipient_domain", + "src_user_domain", + "ssl_issuer_email_domain", + "ssl_subject_email_domain" + ] + } + }, + "x-readable-payload": { + "cim_type": "", + "fields": { + "value": [ + "_raw" + ] + } + }, + "email-addr": { + "cim_type": "email", + "fields": { + "value": [ + "src_user", + "recipient", + "ssl_issuer_email", + "ssl_subject_email" + ], + "x_recipient_domain_ref.value": [ + "recipient_domain" + ], + "x_src_user_domain_ref.value": [ + "src_user_domain" + ] + } + }, + "email-message": { + "cim_type": "email", + "fields": { + "to_refs[*].value": [ + "recipient" + ], + "subject": [ + "subject" + ], + "from_ref.value": [ + "src_user" + ], + "x_url_ref.value": [ + "url" + ], + "x_internal_message_id": [ + "internal_message_id" + ], + "x_message_id": [ + "message_id" + ] + } + }, + "file": { + "cim_type": "endpoint", + "fields": { + "hashes.MD5": [ + "file_hash" + ], + "hashes.'SHA-1'": [ + "file_hash" + ], + "hashes.'SHA-256'": [ + "file_hash" + ], + "name": [ + "file_name", + "process_name", + "parent_process_name", + "process_exec", + "parent_process_exec" + ], + "ctime": "file_create_time", + "mtime": "file_modify_time", + "atime": "file_access_time", + "parent_directory_ref.path": [ + "file_path", + "process_path", + "parent_process_path" + ], + "size": [ + "file_size" + ], + "x_acl": [ + "file_acl" + ] + } + }, + "ipv4-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_ip", + "dest_ip" + ], + "resolves_to_refs[*].value": [ + "src_mac", + "dest_mac" + ] + } + }, + "ipv6-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_ip", + "dest_ip" + ], + "resolves_to_refs[*].value": [ + "src_mac", + "dest_mac" + ] + } + }, + "mac-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_mac", + "dest_mac" + ] + } + }, + "network-traffic": { + "cim_type": "network", + "fields": { + "src_ref.value": [ + "src_ip" + ], + "src_port": [ + "src_port" + ], + "dst_ref.value": [ + "dest_ip" + ], + "dst_port": [ + "dest_port" + ], + "protocols[*]": [ + "protocol", + "transport" + ], + "dst_byte_count": [ + "bytes_in" + ], + "src_byte_count": [ + "bytes_out" + ], + "dst_packets": [ + "packets_in" + ], + "src_packets": [ + "packets_out" + ], + "x_direction": [ + "direction" + ], + "extensions.'dns-ext'.name": [ + "name" + ], + "extensions.'dns-ext'.question.name_ref.value": [ + "query" + ], + "extensions.'dns-ext'.resolved_ip_refs[*].value": [ + "answer" + ], + "extensions.'dns-ext'.message_type": [ + "message_type" + ], + "extensions.'dns-ext'.query_count": [ + "query_count" + ], + "extensions.'dns-ext'.query_type": [ + "query_type" + ], + "extensions.'dns-ext'.record_type": [ + "record_type" + ], + "extensions.'dns-ext'.reply_code": [ + "reply_code" + ], + "extensions.'dns-ext'.reply_code_id": [ + "reply_code_id" + ], + "extensions.'dns-ext'.transaction_id": [ + "transaction_id" + ], + "extensions.'http-request-ext'.request_method": [ + "http_method" + ], + "extensions.'http-request-ext'.request_value": [ + "uri_path" + ], + "extensions.'http-request-ext'.request_header.Referer": [ + "http_referrer" + ], + "extensions.'http-request-ext'.request_header.'User-Agent'": [ + "http_user_agent" + ], + "extensions.'http-request-ext'.x_uri_query": [ + "uri_query" + ] + } + }, + "process": { + "cim_type": "process", + "fields": { + "x_name": [ + "process_name", + "parent_process_name" + ], + "command_line": [ + "process", + "parent_process" + ], + "pid": [ + "process_id", + "parent_process_id" + ], + "cwd": [ + "process_current_directory" + ], + "binary_ref.name": [ + "process_name", + "parent_process_name" + ], + "x_unique_id": [ + "process_guid", + "parent_process_guid" + ], + "x_original_file_name": [ + "original_file_name" + ], + "x_memory_used": [ + "mem_used" + ] + } + }, + "url": { + "cim_type": "web", + "fields": { + "value": [ + "url" + ] + } + }, + "user-account": { + "cim_type": "authentication", + "fields": { + "user_id": [ + "user" + ], + "account_login": [ + "user_id" + ], + "x_user_name": [ + "user_name" + ] + } + }, + "windows-registry-key": { + "cim_type": "endpoint", + "fields": { + "key": [ + "registry_key_name" + ], + "values[*].name": [ + "registry_value_name" + ], + "values[*].data": [ + "registry_value_data" + ], + "x_hive": [ + "registry_hive" + ], + "x_path": [ + "registry_path" + ], + "x_value_text": [ + "registry_value_text" + ] + } + }, + "x509-certificate": { + "cim_type": "certificate", + "fields": { + "hashes.'SHA-256'": [ + "ssl_hash" + ], + "hashes.'SHA-1'": [ + "ssl_hash" + ], + "version": [ + "ssl_version" + ], + "serial_number": [ + "ssl_serial" + ], + "signature_algorithm": [ + "ssl_signature_algorithm" + ], + "issuer": [ + "ssl_issuer" + ], + "subject": [ + "ssl_subject" + ], + "subject_public_key_algorithm": [ + "ssl_publickey_algorithm" + ], + "validity_not_before": [ + "ssl_start_time" + ], + "validity_not_after": [ + "ssl_end_time" + ], + "x_ssl_is_valid": [ + "ssl_is_valid" + ], + "x_ssl_issuer_common_name": [ + "ssl_issuer_common_name" + ], + "x_ssl_subject_common_name": [ + "ssl_subject_common_name" + ], + "x_ssl_name": [ + "ssl_name" + ], + "x_ssl_publickey": [ + "ssl_publickey" + ], + "x_ssl_issuer_email_ref.value": [ + "ssl_issuer_email" + ], + "x_ssl_subject_email_ref.value": [ + "ssl_subject_email" + ], + "x_ssl_issuer_domain_ref.value": [ + "ssl_issuer_email_domain" + ], + "x_ssl_subject_domain_ref.value": [ + "ssl_subject_email_domain" + ], + "x_ssl_issuer_organization": [ + "ssl_issuer_organization" + ], + "x_ssl_subject_organization": [ + "ssl_subject_organization" + ] + } + }, + "x-ibm-finding": { + "cim_type": "alert", + "fields": { + "severity": [ + "severity" + ], + "finding_type": [ + "type" + ], + "name": [ + "signature" + ], + "alert_id": [ + "id" + ], + "description": [ + "description" + ], + "src_ip_ref.value": [ + "src_ip" + ], + "dst_ip_ref.value": [ + "dest_ip" + ], + "ttp_tagging_refs[*].extensions.'mitre-attack-ext'.technique_id": [ + "mitre_technique_id" + ] + } + }, + "x-splunk-data": { + "cim_type": "endpoint", + "fields": { + "log_source": [ + "source" + ], + "log_source_type": [ + "_sourcetype" + ], + "event_type": [ + "eventtype" + ] + } + }, + "x-splunk-authentication": { + "cim_type": "authentication", + "fields": { + "user_type": [ + "user_type" + ], + "user_agent": [ + "user_agent" + ], + "method": [ + "authentication_method" + ], + "service": [ + "authentication_service" + ] + } + }, + "x-splunk-vulnerability": { + "cim_type": "vulnerability", + "fields": { + "msft": "msft", + "cve": "cve", + "cvss": "cvss", + "mskb": "mskb" + } + }, + "x-ibm-ttp-tagging": { + "cim_type": "alert", + "fields": { + "extensions.'mitre-attack-ext'.technique_id": [ + "mitre_technique_id" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json index f3ed10675..1ad2f0ff0 100644 --- a/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json @@ -22,15 +22,19 @@ "dest_ip": [ { "key": "ipv4-addr.value", - "object": "dst_ip" + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" }, { "key": "ipv6-addr.value", - "object": "dst_ip" + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" }, { "key": "network-traffic.dst_ref", - "object": "network-traffic", + "object": "network_traffic", "references": "dst_ip" }, { @@ -42,30 +46,27 @@ "group": true }, { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, - { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "x-ibm-finding.dst_ip_ref", + "object": "ibm_finding", + "references": "dst_ip" } ], "src_ip": [ { "key": "ipv4-addr.value", - "object": "src_ip" + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv4List" }, { "key": "ipv6-addr.value", - "object": "src_ip" + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv6List" }, { "key": "network-traffic.src_ref", - "object": "network-traffic", + "object": "network_traffic", "references": "src_ip" }, { @@ -75,112 +76,253 @@ "src_ip" ], "group": true + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "ibm_finding", + "references": "src_ip" + } + ], + "networkdata": { + "dest_port": [ + { + "key": "network-traffic.dst_port", + "object": "network_traffic", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + } + ], + "src_port": [ + { + "key": "network-traffic.src_port", + "object": "network_traffic", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + } + ], + "protocol": [ + { + "key": "network-traffic.protocols", + "object": "network_traffic", + "group": true, + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "network_traffic" + ] + } + ], + "transport": [ + { + "key": "network-traffic.protocols", + "object": "network_traffic", + "group": true, + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "network_traffic" + ] + } + ], + "direction": [ + { + "key": "network-traffic.x_direction", + "object": "network_traffic" + } + ] + }, + "bytes_in": [ + { + "key": "network-traffic.dst_byte_count", + "object": "network_traffic", + "transformer": "ToInteger" + } + ], + "bytes_out": [ + { + "key": "network-traffic.src_byte_count", + "object": "network_traffic", + "transformer": "ToInteger" + } + ], + "packets_in": [ + { + "key": "network-traffic.dst_packets", + "object": "network_traffic" + } + ], + "packets_out": [ + { + "key": "network-traffic.src_packets", + "object": "network_traffic" + } + ], + "name": [ + { + "key": "network-traffic.extensions.dns-ext.name", + "object": "network_traffic" + } + ], + "message_type": [ + { + "key": "network-traffic.extensions.dns-ext.message_type", + "object": "network_traffic" + } + ], + "query_count": [ + { + "key": "network-traffic.extensions.dns-ext.query_count", + "object": "network_traffic" + } + ], + "query_type": [ + { + "key": "network-traffic.extensions.dns-ext.query_type", + "object": "network_traffic" + } + ], + "record_type": [ + { + "key": "network-traffic.extensions.dns-ext.record_type", + "object": "network_traffic" + } + ], + "reply_code": [ + { + "key": "network-traffic.extensions.dns-ext.reply_code", + "object": "network_traffic" + } + ], + "reply_code_id": [ + { + "key": "network-traffic.extensions.dns-ext.reply_code_id", + "object": "network_traffic" + } + ], + "query": [ + { + "key": "domain-name.value", + "object": "dns_domain" }, { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, - { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] - } - ], - "dest_port": [ - { - "key": "network-traffic.dst_port", - "object": "network-traffic", - "transformer": "ToInteger" - }, - { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" + "key": "network-traffic.extensions.dns-ext.question.name_ref", + "object": "network_traffic", + "references": "dns_domain" }, { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] - } - ], - "src_port": [ - { - "key": "network-traffic.src_port", - "object": "network-traffic", - "transformer": "ToInteger" - }, - { - "key": "x-oca-event.network_ref", + "key": "x-oca-event.domain_ref", "object": "event", - "references": "network-traffic" + "references": "dns_domain" }, { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "network-traffic.dst_ref", + "object": "network_traffic", + "references": "dns_domain" } ], - "protocol": [ + "answer": [ { - "key": "network-traffic.protocols", - "object": "network-traffic", - "transformer": "ToLowercaseArray" + "key": "ipv4-addr.value", + "object": "resolved_ip", + "unwrap": true, + "transformer": "FilterIPv4List" }, { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" + "key": "ipv6-addr.value", + "object": "resolved_ip", + "unwrap": true, + "transformer": "FilterIPv6List" }, { - "key": "process.opened_connection_refs", - "object": "process", + "key": "network-traffic.extensions.dns-ext.resolved_ip_refs", + "object": "network_traffic", "references": [ - "network-traffic" + "resolved_ip" ] } ], - "_time": [ - { - "key": "first_observed", - "transformer": "SplunkToTimestamp" - }, - { - "key": "last_observed", - "transformer": "SplunkToTimestamp" - }, - { - "key": "x-oca-event.created", - "transformer": "SplunkToTimestamp", - "object": "event" + "transaction_id": [ + { + "key": "network-traffic.extensions.dns-ext.transaction_id", + "object": "network_traffic" + } + ], + "http_method": [ + { + "key": "network-traffic.extensions.http-request-ext.request_method", + "object": "network_traffic" + } + ], + "http_referrer": [ + { + "key": "network-traffic.extensions.http-request-ext.request_header.Referer", + "object": "network_traffic" + } + ], + "http_user_agent": [ + { + "key": "network-traffic.extensions.http-request-ext.request_header.User-Agent", + "object": "network_traffic" + } + ], + "uri_path": [ + { + "key": "network-traffic.extensions.http-request-ext.request_value", + "object": "network_traffic" + } + ], + "uri_query": [ + { + "key": "network-traffic.extensions.http-request-ext.x_uri_query", + "object": "network_traffic" } ], "user": [ - { - "key": "user-account.account_login", - "object": "authentication" - }, { "key": "user-account.user_id", - "object": "authentication" + "object": "user", + "unwrap": true }, { "key": "x-oca-event.user_ref", "object": "event", - "references": "authentication" + "references": "user" } ], - "process_user": [ + "user_id": [ { - "key": "process.creator_user_ref", - "object": "process", - "references": "authentication" + "key": "user-account.account_login", + "object": "user" + } + ], + "user_name": [ + { + "key": "user-account.x_user_name", + "object": "user" } ], "ssl_hash": [ @@ -213,10 +355,71 @@ "key": "x509-certificate.subject_public_key_algorithm", "object": "certificate" }, + "ssl_start_time": { + "key": "x509-certificate.validity_not_before", + "object": "certificate", + "transformer": "EpochToTimestamp" + }, + "ssl_end_time": { + "key": "x509-certificate.validity_not_after", + "object": "certificate", + "transformer": "EpochToTimestamp" + }, + "ssl_is_valid": { + "key": "x509-certificate.x_ssl_is_valid", + "object": "certificate" + }, + "ssl_issuer_common_name": { + "key": "x509-certificate.x_ssl_issuer_common_name", + "object": "certificate" + }, + "ssl_subject_common_name": { + "key": "x509-certificate.x_ssl_subject_common_name", + "object": "certificate" + }, + "ssl_name": { + "key": "x509-certificate.x_ssl_name", + "object": "certificate" + }, + "ssl_publickey": { + "key": "x509-certificate.x_ssl_publickey", + "object": "certificate" + }, + "ssl_issuer_email_domain": [ + { + "key": "domain-name.value", + "object": "ssl_email_domain" + }, + { + "key": "x509-certificate.x_ssl_issuer_domain_ref", + "object": "certificate", + "references": "ssl_email_domain" + } + ], + "ssl_subject_email_domain": [ + { + "key": "domain-name.value", + "object": "ssl_subject_domain" + }, + { + "key": "x509-certificate.x_ssl_subject_domain_ref", + "object": "certificate", + "references": "ssl_subject_domain" + } + ], + "ssl_issuer_organization": { + "key": "x509-certificate.x_ssl_issuer_organization", + "object": "certificate" + }, + "ssl_subject_organization": { + "key": "x509-certificate.x_ssl_subject_organization", + "object": "certificate" + }, "process_id": [ { "key": "process.pid", - "object": "process" + "object": "process", + "transformer": "ConvertHexAndStringToInteger" }, { "key": "x-oca-event.process_ref", @@ -227,6 +430,28 @@ "process_name": [ { "key": "process.x_name", + "object": "process", + "transformer": "CheckProcessName" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + }, + { + "key": "file.name", + "object": "process_file", + "transformer": "CheckProcessName" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "process": [ + { + "key": "process.command_line", "object": "process" }, { @@ -235,24 +460,15 @@ "references": "process" } ], - "process": { - "key": "process.command_line", - "object": "process" - }, "process_exec": [ { "key": "file.name", "object": "process-exec" }, { - "key": "process.object_ref", + "key": "process.binary_ref", "object": "process", "references": "process-exec" - }, - { - "key": "x-oca-event.file_ref", - "object": "event", - "references": "process-exec" } ], "process_hash": { @@ -264,21 +480,31 @@ "key": "process.x_unique_id", "object": "process" }, + "parent_process_guid": { + "key": "process.x_unique_id", + "object": "parent-process" + }, + "process_current_directory": { + "key": "process.cwd", + "object": "process" + }, "process_path": [ { "key": "directory.path", - "object": "process-path" + "object": "process_directory", + "transformer": "CheckProcessPath" }, { "key": "file.parent_directory_ref", - "object": "process-exec", - "references": "process-path" + "object": "process_file", + "references": "process_directory" } ], "parent_process_id": [ { "key": "process.pid", - "object": "parent-process" + "object": "parent-process", + "transformer": "ConvertHexAndStringToInteger" }, { "key": "x-oca-event.parent_process_ref", @@ -305,19 +531,40 @@ "key": "process.parent_ref", "object": "process", "references": "parent-process" + }, + { + "key": "file.name", + "object": "parent_process_file" + }, + { + "key": "process.binary_ref", + "object": "parent-process", + "references": "parent_process_file" + } + ], + "parent_process": [ + { + "key": "process.command_line", + "object": "parent-process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "parent-process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent-process" } ], - "parent_process": { - "key": "process.command_line", - "object": "parent-process" - }, "parent_process_exec": [ { "key": "file.name", "object": "parent-process-exec" }, { - "key": "process.object_ref", + "key": "process.binary_ref", "object": "parent-process", "references": "parent-process-exec" } @@ -325,52 +572,59 @@ "parent_process_path": [ { "key": "directory.path", - "object": "parent-process-path" + "object": "parent-directory" }, { "key": "file.parent_directory_ref", - "object": "parent-process-exec", - "references": "parent-process-path" + "object": "parent_process_file", + "references": "parent-directory" } ], "file_path": [ { "key": "directory.path", - "object": "directory" + "object": "file_directory" }, { "key": "file.parent_directory_ref", "object": "file", - "references": "directory" + "references": "file_directory" } ], "file_create_time": [ - { - "key": "directory.ctime", - "object": "directory", - "transformer": "SplunkToTimestamp" - }, { "key": "file.ctime", "object": "file", - "transformer": "SplunkToTimestamp" + "transformer": "FormatTimestamp" } ], "file_modify_time": [ - { - "key": "directory.mtime", - "object": "directory", - "transformer": "SplunkToTimestamp" - }, { "key": "file.mtime", "object": "file", - "transformer": "SplunkToTimestamp" + "transformer": "FormatTimestamp" } ], - "file_hash": { - "key": "file.hashes.UNKNOWN", - "object": "file" + "file_access_time": [ + { + "key": "file.atime", + "object": "file", + "transformer": "FormatTimestamp" + } + ], + "file_hashes": { + "file_sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "file_sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "file_md5": { + "key": "file.hashes.MD5", + "object": "file" + } }, "file_name": [ { @@ -385,6 +639,11 @@ ], "file_size": { "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "file_acl": { + "key": "file.x_acl", "object": "file" }, "src_mac": [ @@ -399,13 +658,6 @@ "references": [ "src_mac" ] - }, - { - "key": "ipv6-addr.resolves_to_refs", - "object": "src_ip", - "references": [ - "src_mac" - ] } ], "dest_mac": [ @@ -420,19 +672,13 @@ "references": [ "dst_mac" ] - }, - { - "key": "ipv6-addr.resolves_to_refs", - "object": "dst_ip", - "references": [ - "dst_mac" - ] } ], - "object_path": [ + "registry_key_name": [ { "key": "windows-registry-key.key", - "object": "windows-registry-key" + "object": "windows-registry-key", + "transformer": "RegistryValueName" }, { "key": "x-oca-event.registry_ref", @@ -440,30 +686,92 @@ "references": "windows-registry-key" } ], + "registry_value": { + "key": "windows-registry-key.values", + "object": "windows-registry-key", + "transformer": "FormatToStixRegistryValue" + }, + "registry_hive": [ + { + "key": "windows-registry-key.x_hive", + "object": "windows-registry-key" + } + ], + "registry_path": [ + { + "key": "windows-registry-key.x_path", + "object": "windows-registry-key" + } + ], + "registry_value_text": [ + { + "key": "windows-registry-key.x_value_text", + "object": "windows-registry-key" + } + ], "url": [ { "key": "url.value", - "object": "web" + "object": "url" }, { - "key": "domain-name.value", - "transformer": "ToDomainName" + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" } ], "src_user": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "unwrap": true }, { - "key": "email-message.sender_ref", + "key": "email-message.from_ref", "object": "email-message", "references": "email-addr" + } + ], + "recipient": [ + { + "key": "email-addr.value", + "object": "email-recipient", + "unwrap": true }, { - "key": "email-message.from_ref", + "key": "email-message.to_refs", "object": "email-message", - "references": "email-addr" + "references": [ + "email-recipient" + ] + } + ], + "ssl_issuer_email": [ + { + "key": "email-addr.value", + "object": "issuer_email", + "unwrap": true + }, + { + "key": "x509-certificate.x_ssl_issuer_email_ref", + "object": "certificate", + "references": [ + "issuer_email" + ] + } + ], + "ssl_subject_email":[ + { + "key": "email-addr.value", + "object": "subject_email", + "unwrap": true + }, + { + "key": "x509-certificate.x_ssl_subject_email_ref", + "object": "certificate", + "references": [ + "subject_email" + ] } ], "subject": { @@ -472,8 +780,7 @@ }, "is_multipart": { "key": "email-message.is_multipart", - "object": "email-message", - "transformer": "StringToBool" + "object": "email-message" }, "host": [ { @@ -486,136 +793,247 @@ "references": "host" } ], - "Direction": [ + "recipient_domain": [ + { + "key": "domain-name.value", + "object": "recipient_domain" + }, + { + "key": "email-addr.x_recipient_domain_ref", + "object": "email-recipient", + "references": "recipient_domain" + } + ], + "src_user_domain": [ + { + "key": "domain-name.value", + "object": "src_domain" + }, { - "key": "x-splunk.direction", - "object": "x-splunk" + "key": "email-addr.x_src_user_domain_ref", + "object": "email-addr", + "references": "src_domain" } ], - "EventID": [ + "internal_message_id": [ { - "key": "x-splunk.event_id", - "object": "x-splunk" + "key": "email-message.x_internal_message_id", + "object": "email-message" } ], - "TacticId": [ + "message_id": [ { - "key": "x-splunk.mitre_tactic_id", - "object": "x-splunk" + "key": "email-message.x_message_id", + "object": "email-message" } ], - "Tactic": [ + "message_info": [ { - "key": "x-splunk.mitre_tactic", - "object": "x-splunk" + "key": "email-message.x_message_info", + "object": "email-message" } ], - "TechniqueId": [ + "os": [ { - "key": "x-splunk.mitre_technique_id", - "object": "x-splunk" + "key": "x-oca-asset.x_operating_system", + "object": "host" } ], - "Technique": [ + "original_file_name": [ { - "key": "x-splunk.mitre_technique", - "object": "x-splunk" + "key": "process.x_original_file_name", + "object": "process" } ], - "EventName": [ + "mem_used": [ { - "key": "x-splunk.event_name", - "object": "x-splunk" + "key": "process.x_memory_used", + "object": "process" } ], - "ss_name": { - "key": "x-ibm-finding.name", - "object": "x-ibm-finding" + "finding_type": { + "key": "x-ibm-finding.finding_type", + "object": "ibm_finding" }, - "DeviceType": [ + "alert_severity": [ { - "key": "x-ibm-finding.src_device", - "object": "x-ibm-finding" + "key": "x-ibm-finding.x_severity", + "object": "ibm_finding", + "transformer": "SeverityToScore" } ], - "severity": [ + "alert_signature": [ { - "key": "x-ibm-finding.severity", - "object": "x-ibm-finding" + "key": "x-ibm-finding.name", + "object": "ibm_finding" } ], - "source": { - "key": "x-oca-event.module", - "object": "event" - }, - "description": { - "key": "x-oca-event.action", + "alert_id": [ + { + "key": "x-ibm-finding.alert_id", + "object": "ibm_finding" + } + ], + "alert_description": [ + { + "key": "x-ibm-finding.description", + "object": "ibm_finding" + } + ], +"source": [ + { + "key": "x-oca-event.module", + "object": "event" +}, +{ + "key": "x-splunk-data.log_source", + "object": "x_splunk" +} +], +"_sourcetype": { + "key": "x-splunk-data.log_source_type", + "object": "x_splunk" +}, +"eventtype": { + "key": "x-splunk-data.event_type", + "object": "x_splunk" +}, +"status": [ + { + "key": "x-oca-event.x_status", "object": "event" - }, + } + ], + "vendor_product": [ + { + "key": "x-oca-event.provider", + "object": "event" + } + ], + "action": [ + { + "key": "x-oca-event.outcome", + "object": "event" + } + ], + "duration": [ + { + "key": "x-oca-event.duration", + "object": "event", + "transformer": "ToInteger" + } + ], + "severity": [ + { + "key": "x-oca-event.x_severity", + "object": "event", + "transformer": "SeverityToScore" + } + ], + "_time": [ + { + "key": "first_observed", + "transformer": "SplunkToTimestamp" + }, + { + "key": "last_observed", + "transformer": "SplunkToTimestamp" + }, + { + "key": "x-oca-event.x_created", + "transformer": "SplunkToTimestamp", + "object": "event" + } + ], "signature": { - "key": "x-oca-event.action", + "key": "x-oca-event.x_action", "object": "event" }, "signature_id": { - "key": "x-oca-event.code", + "key": "x-oca-event.x_code", "object": "event" }, - "result": { - "key": "x-oca-event.outcome", - "object": "event" - }, - "query": [ + "app": [ { - "key": "domain-name.value", - "object": "domain" - }, + "key": "x-oca-event.x_application", + "object": "event" + } + ], + "dest": [ { - "key": "network-traffic.extensions.dns-ext.question.domain_ref", - "object": "network-traffic", - "references": "domain" - }, + "key": "x-oca-event.x_dest", + "object": "event" + } + ], + "src": [ { - "key": "x-oca-event.domain_ref", - "object": "event", - "references": "domain" - }, + "key": "x-oca-event.x_src", + "object": "event" + } + ], + "event_id": [ { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, + "key": "x-oca-event.x_event_id", + "object": "event" + } + ], + "authentication_method": [ { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "x-splunk-authentication.method", + "object": "auth" } ], - "answer": [ + "authentication_service": [ { - "key": "ipv4-addr.value", - "object": "resolved-ip" - }, + "key": "x-splunk-authentication.service", + "object": "auth" + } + ], + "user_agent": [ { - "key": "network-traffic.extensions.dns-ext.resolved_ip_refs", - "object": "network-traffic", - "references": [ - "resolved-ip" - ], - "group": true - }, + "key": "x-splunk-authentication.user_agent", + "object": "auth" + } + ], + "user_type": [ { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" + "key": "x-splunk-authentication.user_type", + "object": "auth" + } + ], + "msft": [ + { + "key": "x-splunk-vulnerability.msft", + "object": "vulnerability" + } + ], + "cve": [ + { + "key": "x-splunk-vulnerability.cve", + "object": "vulnerability" + } + ], + "cvss": [ + { + "key": "x-splunk-vulnerability.cvss", + "object": "vulnerability" + } + ], + "mskb": [ + { + "key": "x-splunk-vulnerability.mskb", + "object": "vulnerability" + } + ], + "mitre_technique_id": [ + { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_id", + "object": "ttp_tag" }, { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "ibm_finding", + "references": ["ttp_tag"] } ] } diff --git a/stix_shifter_modules/splunk/stix_translation/json/to_stix_map.json b/stix_shifter_modules/splunk/stix_translation/json/to_stix_map.json index 6fc55b579..df978999e 100644 --- a/stix_shifter_modules/splunk/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/splunk/stix_translation/json/to_stix_map.json @@ -22,15 +22,19 @@ "dest_ip": [ { "key": "ipv4-addr.value", - "object": "dst_ip" + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" }, { "key": "ipv6-addr.value", - "object": "dst_ip" + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" }, { "key": "network-traffic.dst_ref", - "object": "network-traffic", + "object": "network_traffic", "references": "dst_ip" }, { @@ -42,30 +46,27 @@ "group": true }, { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, - { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "x-ibm-finding.dst_ip_ref", + "object": "ibm_finding", + "references": "dst_ip" } ], "src_ip": [ { "key": "ipv4-addr.value", - "object": "src_ip" + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv4List" }, { "key": "ipv6-addr.value", - "object": "src_ip" + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv6List" }, { "key": "network-traffic.src_ref", - "object": "network-traffic", + "object": "network_traffic", "references": "src_ip" }, { @@ -77,110 +78,251 @@ "group": true }, { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, + "key": "x-ibm-finding.src_ip_ref", + "object": "ibm_finding", + "references": "src_ip" + } + ], + "networkdata": { + "dest_port": [ + { + "key": "network-traffic.dst_port", + "object": "network_traffic", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + } + ], + "src_port": [ + { + "key": "network-traffic.src_port", + "object": "network_traffic", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + } + ], + "protocol": [ + { + "key": "network-traffic.protocols", + "object": "network_traffic", + "group": true, + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "network_traffic" + ] + } + ], + "transport": [ + { + "key": "network-traffic.protocols", + "object": "network_traffic", + "group": true, + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network_traffic" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "network_traffic" + ] + } + ], + "direction": [ + { + "key": "network-traffic.x_direction", + "object": "network_traffic" + } + ] + }, + "bytes_in": [ { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "network-traffic.dst_byte_count", + "object": "network_traffic", + "transformer": "ToInteger" } ], - "dest_port": [ + "bytes_out": [ { - "key": "network-traffic.dst_port", - "object": "network-traffic", + "key": "network-traffic.src_byte_count", + "object": "network_traffic", "transformer": "ToInteger" - }, + } + ], + "packets_in": [ { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, + "key": "network-traffic.dst_packets", + "object": "network_traffic" + } + ], + "packets_out": [ { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "network-traffic.src_packets", + "object": "network_traffic" } ], - "src_port": [ + "name": [ { - "key": "network-traffic.src_port", - "object": "network-traffic", - "transformer": "ToInteger" + "key": "network-traffic.extensions.dns-ext.name", + "object": "network_traffic" + } + ], + "message_type": [ + { + "key": "network-traffic.extensions.dns-ext.message_type", + "object": "network_traffic" + } + ], + "query_count": [ + { + "key": "network-traffic.extensions.dns-ext.query_count", + "object": "network_traffic" + } + ], + "query_type": [ + { + "key": "network-traffic.extensions.dns-ext.query_type", + "object": "network_traffic" + } + ], + "record_type": [ + { + "key": "network-traffic.extensions.dns-ext.record_type", + "object": "network_traffic" + } + ], + "reply_code": [ + { + "key": "network-traffic.extensions.dns-ext.reply_code", + "object": "network_traffic" + } + ], + "reply_code_id": [ + { + "key": "network-traffic.extensions.dns-ext.reply_code_id", + "object": "network_traffic" + } + ], + "query": [ + { + "key": "domain-name.value", + "object": "dns_domain" + }, + { + "key": "network-traffic.extensions.dns-ext.question.name_ref", + "object": "network_traffic", + "references": "dns_domain" }, { - "key": "x-oca-event.network_ref", + "key": "x-oca-event.domain_ref", "object": "event", - "references": "network-traffic" + "references": "dns_domain" }, { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "network-traffic.dst_ref", + "object": "network_traffic", + "references": "dns_domain" } ], - "protocol": [ + "answer": [ { - "key": "network-traffic.protocols", - "object": "network-traffic", - "transformer": "ToLowercaseArray" + "key": "ipv4-addr.value", + "object": "resolved_ip", + "unwrap": true, + "transformer": "FilterIPv4List" }, { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" + "key": "ipv6-addr.value", + "object": "resolved_ip", + "unwrap": true, + "transformer": "FilterIPv6List" }, { - "key": "process.opened_connection_refs", - "object": "process", + "key": "network-traffic.extensions.dns-ext.resolved_ip_refs", + "object": "network_traffic", "references": [ - "network-traffic" + "resolved_ip" ] } ], - "_time": [ + "transaction_id": [ { - "key": "first_observed", - "transformer": "SplunkToTimestamp" - }, + "key": "network-traffic.extensions.dns-ext.transaction_id", + "object": "network_traffic" + } + ], + "http_method": [ { - "key": "last_observed", - "transformer": "SplunkToTimestamp" - }, + "key": "network-traffic.extensions.http-request-ext.request_method", + "object": "network_traffic" + } + ], + "http_referrer": [ { - "key": "x-oca-event.created", - "transformer": "SplunkToTimestamp", - "object": "event" + "key": "network-traffic.extensions.http-request-ext.request_header.Referer", + "object": "network_traffic" } ], - "user": [ + "http_user_agent": [ { - "key": "user-account.account_login", - "object": "authentication" - }, + "key": "network-traffic.extensions.http-request-ext.request_header.User-Agent", + "object": "network_traffic" + } + ], + "uri_path": [ + { + "key": "network-traffic.extensions.http-request-ext.request_value", + "object": "network_traffic" + } + ], + "uri_query": [ + { + "key": "network-traffic.extensions.http-request-ext.x_uri_query", + "object": "network_traffic" + } + ], + "user": [ { "key": "user-account.user_id", - "object": "authentication" + "object": "user", + "unwrap": true }, { "key": "x-oca-event.user_ref", "object": "event", - "references": "authentication" + "references": "user" } ], - "process_user": [ + "user_id": [ { - "key": "process.creator_user_ref", - "object": "process", - "references": "authentication" + "key": "user-account.account_login", + "object": "user" + } + ], + "user_name": [ + { + "key": "user-account.x_user_name", + "object": "user" } ], "ssl_hash": [ @@ -213,10 +355,71 @@ "key": "x509-certificate.subject_public_key_algorithm", "object": "certificate" }, + "ssl_start_time": { + "key": "x509-certificate.validity_not_before", + "object": "certificate", + "transformer": "EpochToTimestamp" + }, + "ssl_end_time": { + "key": "x509-certificate.validity_not_after", + "object": "certificate", + "transformer": "EpochToTimestamp" + }, + "ssl_is_valid": { + "key": "x509-certificate.x_ssl_is_valid", + "object": "certificate" + }, + "ssl_issuer_common_name": { + "key": "x509-certificate.x_ssl_issuer_common_name", + "object": "certificate" + }, + "ssl_subject_common_name": { + "key": "x509-certificate.x_ssl_subject_common_name", + "object": "certificate" + }, + "ssl_name": { + "key": "x509-certificate.x_ssl_name", + "object": "certificate" + }, + "ssl_publickey": { + "key": "x509-certificate.x_ssl_publickey", + "object": "certificate" + }, + "ssl_issuer_email_domain": [ + { + "key": "domain-name.value", + "object": "ssl_email_domain" + }, + { + "key": "x509-certificate.x_ssl_issuer_domain_ref", + "object": "certificate", + "references": "ssl_email_domain" + } + ], + "ssl_subject_email_domain": [ + { + "key": "domain-name.value", + "object": "ssl_subject_domain" + }, + { + "key": "x509-certificate.x_ssl_subject_domain_ref", + "object": "certificate", + "references": "ssl_subject_domain" + } + ], + "ssl_issuer_organization": { + "key": "x509-certificate.x_ssl_issuer_organization", + "object": "certificate" + }, + "ssl_subject_organization": { + "key": "x509-certificate.x_ssl_subject_organization", + "object": "certificate" + }, "process_id": [ { "key": "process.pid", - "object": "process" + "object": "process", + "transformer": "ConvertHexAndStringToInteger" }, { "key": "x-oca-event.process_ref", @@ -227,6 +430,28 @@ "process_name": [ { "key": "process.name", + "object": "process", + "transformer": "CheckProcessName" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + }, + { + "key": "file.name", + "object": "process_file", + "transformer": "CheckProcessName" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "process": [ + { + "key": "process.command_line", "object": "process" }, { @@ -235,10 +460,6 @@ "references": "process" } ], - "process": { - "key": "process.command_line", - "object": "process" - }, "process_exec": [ { "key": "file.name", @@ -248,11 +469,6 @@ "key": "process.binary_ref", "object": "process", "references": "process-exec" - }, - { - "key": "x-oca-event.file_ref", - "object": "event", - "references": "process-exec" } ], "process_hash": { @@ -264,21 +480,31 @@ "key": "process.x_unique_id", "object": "process" }, + "parent_process_guid": { + "key": "process.x_unique_id", + "object": "parent-process" + }, + "process_current_directory": { + "key": "process.cwd", + "object": "process" + }, "process_path": [ { "key": "directory.path", - "object": "process-path" + "object": "process_directory", + "transformer": "CheckProcessPath" }, { "key": "file.parent_directory_ref", - "object": "process-exec", - "references": "process-path" + "object": "process_file", + "references": "process_directory" } ], "parent_process_id": [ { "key": "process.pid", - "object": "parent-process" + "object": "parent-process", + "transformer": "ConvertHexAndStringToInteger" }, { "key": "x-oca-event.parent_process_ref", @@ -305,12 +531,33 @@ "key": "process.parent_ref", "object": "process", "references": "parent-process" + }, + { + "key": "file.name", + "object": "parent_process_file" + }, + { + "key": "process.binary_ref", + "object": "parent-process", + "references": "parent_process_file" + } + ], + "parent_process": [ + { + "key": "process.command_line", + "object": "parent-process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "parent-process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent-process" } ], - "parent_process": { - "key": "process.command_line", - "object": "parent-process" - }, "parent_process_exec": [ { "key": "file.name", @@ -325,52 +572,59 @@ "parent_process_path": [ { "key": "directory.path", - "object": "parent-process-path" + "object": "parent-directory" }, { "key": "file.parent_directory_ref", - "object": "parent-process-exec", - "references": "parent-process-path" + "object": "parent_process_file", + "references": "parent-directory" } ], "file_path": [ { "key": "directory.path", - "object": "directory" + "object": "file_directory" }, { "key": "file.parent_directory_ref", "object": "file", - "references": "directory" + "references": "file_directory" } ], "file_create_time": [ - { - "key": "directory.created", - "object": "directory", - "transformer": "SplunkToTimestamp" - }, { "key": "file.created", "object": "file", - "transformer": "SplunkToTimestamp" + "transformer": "FormatTimestamp" } ], "file_modify_time": [ - { - "key": "directory.modified", - "object": "directory", - "transformer": "SplunkToTimestamp" - }, { "key": "file.modified", "object": "file", - "transformer": "SplunkToTimestamp" + "transformer": "FormatTimestamp" } ], - "file_hash": { - "key": "file.hashes.UNKNOWN", - "object": "file" + "file_access_time": [ + { + "key": "file.accessed", + "object": "file", + "transformer": "FormatTimestamp" + } + ], + "file_hashes": { + "file_sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "file_sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "file_md5": { + "key": "file.hashes.MD5", + "object": "file" + } }, "file_name": [ { @@ -385,6 +639,11 @@ ], "file_size": { "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "file_acl": { + "key": "file.x_acl", "object": "file" }, "src_mac": [ @@ -399,13 +658,6 @@ "references": [ "src_mac" ] - }, - { - "key": "ipv6-addr.resolves_to_refs", - "object": "src_ip", - "references": [ - "src_mac" - ] } ], "dest_mac": [ @@ -420,19 +672,13 @@ "references": [ "dst_mac" ] - }, - { - "key": "ipv6-addr.resolves_to_refs", - "object": "dst_ip", - "references": [ - "dst_mac" - ] } ], - "object_path": [ + "registry_key_name": [ { "key": "windows-registry-key.key", - "object": "windows-registry-key" + "object": "windows-registry-key", + "transformer": "RegistryValueName" }, { "key": "x-oca-event.registry_ref", @@ -440,30 +686,92 @@ "references": "windows-registry-key" } ], + "registry_value": { + "key": "windows-registry-key.values", + "object": "windows-registry-key", + "transformer": "FormatToStixRegistryValue" + }, + "registry_hive": [ + { + "key": "windows-registry-key.x_hive", + "object": "windows-registry-key" + } + ], + "registry_path": [ + { + "key": "windows-registry-key.x_path", + "object": "windows-registry-key" + } + ], + "registry_value_text": [ + { + "key": "windows-registry-key.x_value_text", + "object": "windows-registry-key" + } + ], "url": [ { "key": "url.value", - "object": "web" + "object": "url" }, { - "key": "domain-name.value", - "transformer": "ToDomainName" + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" } ], "src_user": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "unwrap": true }, { - "key": "email-message.sender_ref", + "key": "email-message.from_ref", "object": "email-message", "references": "email-addr" + } + ], + "recipient": [ + { + "key": "email-addr.value", + "object": "email-recipient", + "unwrap": true }, { - "key": "email-message.from_ref", + "key": "email-message.to_refs", "object": "email-message", - "references": "email-addr" + "references": [ + "email-recipient" + ] + } + ], + "ssl_issuer_email": [ + { + "key": "email-addr.value", + "object": "issuer_email", + "unwrap": true + }, + { + "key": "x509-certificate.x_ssl_issuer_email_ref", + "object": "certificate", + "references": [ + "issuer_email" + ] + } + ], + "ssl_subject_email":[ + { + "key": "email-addr.value", + "object": "subject_email", + "unwrap": true + }, + { + "key": "x509-certificate.x_ssl_subject_email_ref", + "object": "certificate", + "references": [ + "subject_email" + ] } ], "subject": { @@ -472,8 +780,7 @@ }, "is_multipart": { "key": "email-message.is_multipart", - "object": "email-message", - "transformer": "StringToBool" + "object": "email-message" }, "host": [ { @@ -486,72 +793,158 @@ "references": "host" } ], - "Direction": [ + "recipient_domain": [ + { + "key": "domain-name.value", + "object": "recipient_domain" + }, { - "key": "x-splunk.direction", - "object": "x-splunk" + "key": "email-addr.x_recipient_domain_ref", + "object": "email-recipient", + "references": "recipient_domain" } ], - "EventID": [ + "src_user_domain": [ { - "key": "x-splunk.event_id", - "object": "x-splunk" + "key": "domain-name.value", + "object": "src_domain" + }, + { + "key": "email-addr.x_src_user_domain_ref", + "object": "email-addr", + "references": "src_domain" } ], - "TacticId": [ + "internal_message_id": [ { - "key": "x-splunk.mitre_tactic_id", - "object": "x-splunk" + "key": "email-message.x_internal_message_id", + "object": "email-message" } ], - "Tactic": [ + "message_id": [ { - "key": "x-splunk.mitre_tactic", - "object": "x-splunk" + "key": "email-message.x_message_id", + "object": "email-message" } ], - "TechniqueId": [ + "message_info": [ { - "key": "x-splunk.mitre_technique_id", - "object": "x-splunk" + "key": "email-message.x_message_info", + "object": "email-message" } ], - "Technique": [ + "os": [ { - "key": "x-splunk.mitre_technique", - "object": "x-splunk" + "key": "x-oca-asset.x_operating_system", + "object": "host" } ], - "EventName": [ + "original_file_name": [ { - "key": "x-splunk.event_name", - "object": "x-splunk" + "key": "process.x_original_file_name", + "object": "process" } ], - "ss_name": { - "key": "x-ibm-finding.name", - "object": "x-ibm-finding" - }, - "DeviceType": [ + "mem_used": [ { - "key": "x-ibm-finding.src_device", - "object": "x-ibm-finding" + "key": "process.x_memory_used", + "object": "process" } ], - "severity": [ + "finding_type": { + "key": "x-ibm-finding.finding_type", + "object": "ibm_finding" + }, + "alert_severity": [ { "key": "x-ibm-finding.severity", - "object": "x-ibm-finding" + "object": "ibm_finding", + "transformer": "SeverityToScore" + } + ], + "alert_signature": [ + { + "key": "x-ibm-finding.name", + "object": "ibm_finding" } ], - "source": { + "alert_id": [ + { + "key": "x-ibm-finding.alert_id", + "object": "ibm_finding" + } + ], + "alert_description": [ + { + "key": "x-ibm-finding.description", + "object": "ibm_finding" + } + ], + "source": [ + { "key": "x-oca-event.module", "object": "event" + }, + { + "key": "x-splunk-data.log_source", + "object": "x_splunk" + } + ], + "_sourcetype": { + "key": "x-splunk-data.log_source_type", + "object": "x_splunk" }, - "description": { - "key": "x-oca-event.action", - "object": "event" + "eventtype": { + "key": "x-splunk-data.event_type", + "object": "x_splunk" }, + "status": [ + { + "key": "x-oca-event.x_status", + "object": "event" + } + ], + "vendor_product": [ + { + "key": "x-oca-event.provider", + "object": "event" + } + ], + "action": [ + { + "key": "x-oca-event.outcome", + "object": "event" + } + ], + "duration": [ + { + "key": "x-oca-event.duration", + "object": "event", + "transformer": "ToInteger" + } + ], + "severity": [ + { + "key": "x-oca-event.severity", + "object": "event", + "transformer": "SeverityToScore" + } + ], + "_time": [ + { + "key": "first_observed", + "transformer": "SplunkToTimestamp" + }, + { + "key": "last_observed", + "transformer": "SplunkToTimestamp" + }, + { + "key": "x-oca-event.created", + "transformer": "SplunkToTimestamp", + "object": "event" + } + ], "signature": { "key": "x-oca-event.action", "object": "event" @@ -560,62 +953,87 @@ "key": "x-oca-event.code", "object": "event" }, - "result": { - "key": "x-oca-event.outcome", - "object": "event" - }, - "query": [ + "app": [ { - "key": "domain-name.value", - "object": "domain" - }, + "key": "x-oca-event.x_application", + "object": "event" + } + ], + "dest": [ { - "key": "network-traffic.extensions.dns-ext.question.domain_ref", - "object": "network-traffic", - "references": "domain" - }, + "key": "x-oca-event.x_dest", + "object": "event" + } + ], + "src": [ { - "key": "x-oca-event.domain_ref", - "object": "event", - "references": "domain" - }, + "key": "x-oca-event.x_src", + "object": "event" + } + ], + "event_id": [ { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" - }, + "key": "x-oca-event.x_event_id", + "object": "event" + } + ], + "authentication_method": [ { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "x-splunk-authentication.method", + "object": "auth" } ], - "answer": [ + "authentication_service": [ { - "key": "ipv4-addr.value", - "object": "resolved-ip" - }, + "key": "x-splunk-authentication.service", + "object": "auth" + } + ], + "user_agent": [ { - "key": "network-traffic.extensions.dns-ext.resolved_ip_refs", - "object": "network-traffic", - "references": [ - "resolved-ip" - ], - "group": true - }, + "key": "x-splunk-authentication.user_agent", + "object": "auth" + } + ], + "user_type": [ { - "key": "x-oca-event.network_ref", - "object": "event", - "references": "network-traffic" + "key": "x-splunk-authentication.user_type", + "object": "auth" + } + ], + "msft": [ + { + "key": "x-splunk-vulnerability.msft", + "object": "vulnerability" + } + ], + "cve": [ + { + "key": "x-splunk-vulnerability.cve", + "object": "vulnerability" + } + ], + "cvss": [ + { + "key": "x-splunk-vulnerability.cvss", + "object": "vulnerability" + } + ], + "mskb": [ + { + "key": "x-splunk-vulnerability.mskb", + "object": "vulnerability" + } + ], + "mitre_technique_id": [ + { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_id", + "object": "ttp_tag" }, { - "key": "process.opened_connection_refs", - "object": "process", - "references": [ - "network-traffic" - ] + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "ibm_finding", + "references": ["ttp_tag"] } ] } diff --git a/stix_shifter_modules/splunk/stix_translation/query_constructor.py b/stix_shifter_modules/splunk/stix_translation/query_constructor.py index 4d89b1f63..95733360f 100644 --- a/stix_shifter_modules/splunk/stix_translation/query_constructor.py +++ b/stix_shifter_modules/splunk/stix_translation/query_constructor.py @@ -157,12 +157,9 @@ def translate(self, expression): field_mapping = self.dmm.map_field(stix_object, stix_path) # This scopes the query to the object object_scoping = self.object_scoper(object_mapping) - + if stix_object == "x-readable-payload" and stix_path == "value": - return "_raw=*{}*".format(expression.value) - # Special case where we want the risk finding - if stix_object == 'x-ibm-finding' and stix_path == 'name': - return f'index=_audit ss_name="{expression.value}" action=alert_fired' + return "_raw=*{}*".format(expression.value) # Check if mapping has multiple fields if isinstance(field_mapping, list): comparison_string = "" @@ -190,9 +187,32 @@ def translate(self, expression): self.translate(expression.expr2) ) + def _field_severity(self, value): + """ + check for severity and convert input value to + informational,low,medium,high,critical + param value + return value(str) + """ + value = int(value) + if 1 <= value <= 20: + value = "informational" + elif 21 <= value <= 40: + value = "low" + elif 41 <= value <= 60: + value = "medium" + elif 61 <= value <= 80: + value = "high" + elif 81 <= value <= 100: + value = "critical" + else: + raise NotImplementedError('only 1-100 integer values are supported with severity field') + return value + def _build_comparison(self, expression, object_scoping, field_mapping): comparator = self._lookup_comparison_operator(self, expression.comparator) - + if field_mapping == 'severity': + expression.value = self._field_severity(expression.value) if isinstance(comparator, str): if comparator == "encoders.like": comparison = encoders.like(field_mapping, expression.value) diff --git a/stix_shifter_modules/splunk/stix_translation/results_translator.py b/stix_shifter_modules/splunk/stix_translation/results_translator.py index 1bdf06d52..fd7b7ad01 100644 --- a/stix_shifter_modules/splunk/stix_translation/results_translator.py +++ b/stix_shifter_modules/splunk/stix_translation/results_translator.py @@ -1,16 +1,13 @@ -import json - from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix -from .splunk_utils import hash_type_lookup + class ResultsTranslator(JSONToStix): - - def __init__(self, options, dialect, base_file_path=None, callback=None): - super().__init__(options, dialect, base_file_path, hash_type_lookup) + def __init__(self, options, dialect, base_file_path=None, callback=None): + super().__init__(options, dialect, base_file_path) - def translate_results(self, data_source, data): - for result in data: - if result.get('_raw'): - result['mime_type_raw'] = 'text/plain' + def translate_results(self, data_source, data): + for result in data: + if result.get('_raw'): + result['mime_type_raw'] = 'text/plain' - return super().translate_results(data_source, data) \ No newline at end of file + return super().translate_results(data_source, data) diff --git a/stix_shifter_modules/splunk/stix_translation/transformers.py b/stix_shifter_modules/splunk/stix_translation/transformers.py index 714757e14..0c4730e34 100644 --- a/stix_shifter_modules/splunk/stix_translation/transformers.py +++ b/stix_shifter_modules/splunk/stix_translation/transformers.py @@ -1,9 +1,13 @@ -from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer import re +import ntpath +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +from stix_shifter_utils.utils import logger + +LOGGER = logger.set_logger(__name__) + class SplunkToTimestamp(ValueTransformer): """A value transformer for converting Splunk timestamp to regular timestamp""" - @staticmethod def transform(splunkTime): return splunkTime[:-6] + 'Z' @@ -34,6 +38,12 @@ def get_pair_of_hash(hash_raw): :return: """ if hash_raw: + if "SHA1" in hash_raw: + hash_raw = hash_raw.replace("SHA1","SHA-1") + elif "SHA256" in hash_raw: + hash_raw = hash_raw.replace("SHA256", "SHA-256") + elif "IMPHASH" in hash_raw: + hash_raw = hash_raw.replace("IMPHASH", "x_IMPHASH") splitted = hash_raw.split("=") if len(splitted) != 2: raise ValueError("hash should be in the format of ") @@ -50,16 +60,120 @@ def get_pair_of_hash(hash_raw): class SplunkMacFormatChange(ValueTransformer): """ A value transformer for converting MAC value into stix format(using : separator) """ - @staticmethod def transform(macvalue): """correcting mac address presentation, it should be 6 octate separated by only colon (:) not by any other special character """ macvalue = re.sub("[^A-Fa-f0-9]", "", macvalue) maclength = len(macvalue) - if (maclength<12): + if (maclength < 12): for i in range(maclength, 12): macvalue = "0" + macvalue value = ':'.join([macvalue[i:i + 2] for i in range(0, len(macvalue), 2)]) return value.lower() + + +class ConvertHexAndStringToInteger(ValueTransformer): + """ converts hexadecimal and string values to integer """ + @staticmethod + def transform(pidvalue): + if isinstance(pidvalue, int): + return pidvalue + else: + if pidvalue.startswith('0x'): + return int(pidvalue, 16) + else: + return int(pidvalue) + + +class SeverityToScore(ValueTransformer): + """value transformer to convert severity string value to integer value on a scale of 1-100""" + @staticmethod + def transform(severity): + severity_string_to_integer = {"informational": 20, + "low": 40, + "medium": 60, + "high": 80, + "critical": 100 + } + try: + return severity_string_to_integer[severity] + + except KeyError: + LOGGER.error("Cannot convert string to severity scale value") + + +class CheckProcessName(ValueTransformer): + """ Check process name, if it contains path then remove it.""" + @staticmethod + def transform(value): + try: + _, file_name = ntpath.split(value) + return file_name + except ValueError: + LOGGER.error("Cannot convert input to file name string") + + +class CheckProcessPath(ValueTransformer): + """ Check process path, if it contains filename then remove it.""" + @staticmethod + def transform(value): + try: + file_path, _ = ntpath.split(value) + return file_path + except ValueError: + LOGGER.error("Cannot convert input to file path string") + + +class RegistryValueName(ValueTransformer): + """changing Hive into expanded form""" + @staticmethod + def transform(registry): + reghive_dict = {'HKLM':'HKEY_LOCAL_MACHINE', 'hklm': 'hkey_local_machine', + 'HKCU': 'HKEY_CURRENT_USER', 'hkcu': 'hkey_current_user', + 'HKCR': 'HKEY_CLASSES_ROOT', 'hkcr': 'hkey_classes_root', + 'HKU': 'HKEY_USERS', 'hku': 'hkey_users', + 'HKCC': 'HKEY_CURRENT_CONFIG', 'hkcc': 'hkey_current_config', + 'HKPD': 'HKEY_PERFORMANCE_DATA', 'hkpd': 'hkey_performance_data'} + + registry = str(registry) + data = registry.split('\\') + expand_key = reghive_dict.get(data[0]) + if expand_key is not None: + replaced_key = registry.replace(data[0], expand_key) + return replaced_key + return registry + + +class FormatToStixRegistryValue(ValueTransformer): + """A value transformer to convert Registry value to windows-registry-key.value STIX""" + + @staticmethod + def transform(obj): + + try: + stix_mapping = {"registryValueName": "name", "registryValueData": "data"} + converted_value = [] + #for each_value in obj: + registryvalue_dict = {} + for key, value in obj.items(): + registryvalue_dict.update({stix_mapping[key]: value}) + converted_value.append(registryvalue_dict) + return converted_value + + except ValueError: + LOGGER.error("Cannot convert root value to Stix formatted windows registry value") + +class FormatTimestamp(ValueTransformer): + """A value transformer for converting timestamp to regular timestamp""" + @staticmethod + def transform(splunkTime): + try: + if splunkTime[10] == " " and len(splunkTime) > 10: + splunkTime = splunkTime.replace(" ", "T") + if splunkTime[-1] != "Z" or splunkTime[-1] != "z": + splunkTime = splunkTime + "Z" + except ValueError: + LOGGER.error("Error while converting timestamp") + return splunkTime diff --git a/stix_shifter_modules/splunk/stix_transmission/README.md b/stix_shifter_modules/splunk/stix_transmission/README.md deleted file mode 100644 index cf7559f8c..000000000 --- a/stix_shifter_modules/splunk/stix_transmission/README.md +++ /dev/null @@ -1,17 +0,0 @@ -#### Transmission - used to communicate with datasource (QRadar/Splunk). Functionalities: query , status , results , ping, - -Running Query in Splunk: -``` -python main.py transmit splunk '{"host": "", "port": , "selfSignedCert": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----"}' '{"auth": {"username": "","password": ""}}' query 'search (url = "example.com") OR ((src_mac = "00-00-5E-00-53-00") OR (dest_mac = "00-00-5E-00-53-00")) earliest="-5minutes" | head 10000| fields src_ip, src_port, src_mac, src_ipv6, dest_ip, dest_port, dest_mac, dest_ipv6, file_hash, user, url, protocol' -``` - -`{'success': True, 'search_id': '1547476760.2132'}` - ---------------- - -Getting Results for the query: -``` -python main.py transmit splunk '{"host": "", "port": , "selfSignedCert": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----"}' '{"auth": {"username": "","password": ""}}' results 1547476760.2132 0 10 -``` - -`{'success': True, 'data': [{}]}` diff --git a/stix_shifter_modules/splunk/stix_transmission/api_client.py b/stix_shifter_modules/splunk/stix_transmission/api_client.py index b1e74ea1a..6a00a7411 100644 --- a/stix_shifter_modules/splunk/stix_transmission/api_client.py +++ b/stix_shifter_modules/splunk/stix_transmission/api_client.py @@ -1,5 +1,6 @@ -from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync import json +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync + class APIClient(): # API METHODS @@ -14,12 +15,12 @@ def __init__(self, connection, configuration): # This version of the Splunk APIClient is designed to function with # Splunk Enterprise version >= 6.5.0 and <= 7.1.2 - # http://docs.splunk.com/Documentation/Splunk/7.1.2/RESTREF/RESTprolog - + # http://docs.splunk.com/Documentation/Splunk/7.1.2/RESTREF/RESTprolog . + self.output_mode = 'json' self.endpoint_start = 'services/' self.authenticated = False - headers = dict() + headers = {} self.client = RestApiClientAsync(connection.get('host'), connection.get('port'), headers, @@ -30,11 +31,13 @@ def __init__(self, connection, configuration): self.timeout = connection['options'].get('timeout') async def authenticate(self): + """ method to authenticate """ if not self.authenticated: await self.set_splunk_auth_token(self.auth, self.headers) self.authenticated = True - + async def set_splunk_auth_token(self, auth, headers): + """ method to set splunk auth token """ data = {'username': auth['username'], 'password': auth['password'], 'output_mode': 'json'} endpoint = self.endpoint_start + 'auth/login' try: @@ -44,15 +47,22 @@ async def set_splunk_auth_token(self, auth, headers): except KeyError as e: raise Exception('Authentication error occured while getting auth token: ' + str(e)) - async def ping_box(self): + """ + ping or check the system status + """ await self.authenticate() - endpoint = self.endpoint_start + 'server/status' + endpoint = self.endpoint_start data = {'output_mode': self.output_mode} return await self.client.call_api(endpoint, 'GET', data=data, timeout=self.timeout) - + async def create_search(self, query_expression): - # sends a POST request to + """ + init query + :param data source query + :return:queryId + """ + # sends a POST request to # https://:/services/search/jobs await self.authenticate() endpoint = self.endpoint_start + "search/jobs" @@ -60,28 +70,42 @@ async def create_search(self, query_expression): return await self.client.call_api(endpoint, 'POST', data=data, timeout=self.timeout) async def get_search(self, search_id): + """ + get query status + :param queryId: + :return: information about the search job and its properties + """ # sends a GET request to # https://:/services/search/jobs/ # returns information about the search job and its properties. await self.authenticate() - endpoint = self.endpoint_start + 'search/jobs/' + search_id - data = {'output_mode': self.output_mode} + endpoint = self.endpoint_start + 'search/jobs/' + search_id + data = {'output_mode': self.output_mode} return await self.client.call_api(endpoint, 'GET', data=data, timeout=self.timeout) async def get_search_results(self, search_id, offset, count): + """ + Get results from Data Source + :param query: Data Source search_id,offset,count + :return: Response Object + """ # sends a GET request to # https://:/services/search/jobs//results # returns results associated with the search job. await self.authenticate() - endpoint = self.endpoint_start + "search/jobs/" + search_id + '/results' + endpoint = self.endpoint_start + "search/v2/jobs/" + search_id + '/results' data = {'output_mode': self.output_mode} if ((offset is not None) and (count is not None)): data['offset'] = str(offset) data['count'] = str(count) # response object body should contain information pertaining to search. return await self.client.call_api(endpoint, 'GET', urldata=data, timeout=self.timeout) - + async def delete_search(self, search_id): + """ + :param search_id: + :return:dict + """ # sends a DELETE request to # https://:/services/search/jobs/ # cancels and deletes search created earlier. diff --git a/stix_shifter_modules/splunk/stix_transmission/delete_connector.py b/stix_shifter_modules/splunk/stix_transmission/delete_connector.py index da9adc8b0..fdfd445e2 100644 --- a/stix_shifter_modules/splunk/stix_transmission/delete_connector.py +++ b/stix_shifter_modules/splunk/stix_transmission/delete_connector.py @@ -1,27 +1,60 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector import json +from asyncio.exceptions import TimeoutError +from aiohttp.client_exceptions import ClientConnectionError +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger class DeleteConnector(BaseDeleteConnector): + """Delete connector class""" def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) self.connector = __name__.split('.')[1] - + async def delete_query_connection(self, search_id): - # Grab the response, extract the response code, and convert it to readable json - response = await self.api_client.delete_search(search_id) - response_code = response.code - response_dict = json.load(response) + """ + Delete query response + :param search_id: + :return: + """ + return_obj = {} + response_dict = {} + try: + # Grab the response, extract the response code, and convert it to readable json + response = await self.api_client.delete_search(search_id) + response_code = response.code + response_dict = json.load(response) + response_text = response.content - # Construct a response object - return_obj = dict() - if response_code == 200: - return_obj['success'] = True - else: - self.logger.error('Fill Error: {}'.format(response_dict)) - ErrorResponder.fill_error(return_obj, response_dict, ['messages',0,'text'], connector=self.connector) + if response_code == 200: + return_obj['success'] = True + else: + response_dict['type'] = str(response_code) + if response_code == 404: + response_dict['type'] = "Unknown_sid" + self.logger.error('Fill Error: {}'.format(response_dict)) + response_dict['message'] = response_text + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except ClientConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, + ['message'], connector=self.connector) + except TimeoutError as ex: + response_dict['type'] = "Timeout" + response_dict['messages'] = "TimeoutError" + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except Exception as ex: + if 'Authentication error' in str(ex): + response_dict['type'] = "AuthenticationError" + elif 'timeout_error' in str(ex): + response_dict['type'] = "Timeout" + response_dict['messages'] = str(ex) + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) return return_obj diff --git a/stix_shifter_modules/splunk/stix_transmission/error_mapper.py b/stix_shifter_modules/splunk/stix_transmission/error_mapper.py index 330b28198..55c0271e4 100644 --- a/stix_shifter_modules/splunk/stix_transmission/error_mapper.py +++ b/stix_shifter_modules/splunk/stix_transmission/error_mapper.py @@ -3,32 +3,35 @@ from stix_shifter_utils.utils import logger error_mapping = { - 'Unknown sid.': ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, - 'Unable to parse the search': ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR + 'Unknown_sid': ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, + 'Unable_to_parse_search': ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 'Timeout': ErrorCode.TRANSMISSION_CONNECT, + "ConnectionError": ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + "AuthenticationError": ErrorCode.TRANSMISSION_AUTH_CREDENTIALS } class ErrorMapper(): + """ + Set Error Code + """ logger = logger.set_logger(__name__) DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR @staticmethod def set_error_code(json_data, return_obj, connector=None): - message_text = None + """Set Error Code""" + err_type = None try: - message_text = json_data['messages'][0]['text'] - except Exception as e: - ErrorMapper.logger.error("failed to find the message_0_text in: " + str(json_data)) - raise e - - error_code = ErrorMapper.DEFAULT_ERROR - ErrorMapper.logger.error('error code message: ' + message_text) + err_type = json_data['type'] + except Exception: + pass - for k,v in error_mapping.items(): - if k in message_text: - error_code = v - break - - if error_code == ErrorMapper.DEFAULT_ERROR: - ErrorMapper.logger.error("failed to map: "+ str(json_data)) + error_type = ErrorMapper.DEFAULT_ERROR - ErrorMapperBase.set_error_code(return_obj, error_code, connector=connector) + if err_type in error_mapping: + error_type = error_mapping.get(err_type) + + if error_type == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: %s", str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_type, connector=connector) diff --git a/stix_shifter_modules/splunk/stix_transmission/ping_connector.py b/stix_shifter_modules/splunk/stix_transmission/ping_connector.py index 22aea4b06..3c3bdb567 100644 --- a/stix_shifter_modules/splunk/stix_transmission/ping_connector.py +++ b/stix_shifter_modules/splunk/stix_transmission/ping_connector.py @@ -1,22 +1,54 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector import json +from asyncio.exceptions import TimeoutError +from aiohttp.client_exceptions import ClientConnectionError +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector from stix_shifter_utils.utils.error_response import ErrorResponder + class PingConnector(BasePingConnector): + """Ping connector class """ def __init__(self, api_client): self.api_client = api_client self.connector = __name__.split('.')[1] - + async def ping_connection(self): - response = await self.api_client.ping_box() - response_code = response.code + """ + Ping the endpoint + :return: dict + """ + return_obj = {} + response_dict = {} + try: + response = await self.api_client.ping_box() + response_code = response.code + response_text = response.content + + response_dict = json.loads(response.read()) - response_dict = json.loads(response.read()) - - return_obj = dict() + if len(response_dict) > 0 and response_code == 200: + return_obj['success'] = True + else: + response_dict['type'] = str(response_code) + response_dict['message'] = response_text + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except ClientConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, + ['message'], connector=self.connector) + except TimeoutError as ex: + response_dict['type'] = "Timeout" + response_dict['messages'] = "TimeoutError" + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except Exception as ex: + if 'Authentication error' in str(ex): + response_dict['type'] = "AuthenticationError" + elif 'timeout_error' in str(ex): + response_dict['type'] = "Timeout" + response_dict['messages'] = str(ex) + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) - if len(response_dict) > 0 and response_code == 200: - return_obj['success'] = True - else: - ErrorResponder.fill_error(return_obj, response_dict, ['messages',0,'text'], connector=self.connector) return return_obj diff --git a/stix_shifter_modules/splunk/stix_transmission/query_connector.py b/stix_shifter_modules/splunk/stix_transmission/query_connector.py index c2a76cc17..3a173fb36 100644 --- a/stix_shifter_modules/splunk/stix_transmission/query_connector.py +++ b/stix_shifter_modules/splunk/stix_transmission/query_connector.py @@ -1,25 +1,59 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_connector import BaseQueryConnector import json +from asyncio.exceptions import TimeoutError +from aiohttp.client_exceptions import ClientConnectionError +from stix_shifter_utils.modules.base.stix_transmission.base_connector import BaseQueryConnector from stix_shifter_utils.utils.error_response import ErrorResponder class QueryConnector(BaseQueryConnector): + """ Query connector base class """ def __init__(self, api_client): self.api_client = api_client self.connector = __name__.split('.')[1] async def create_query_connection(self, query): - # Grab the response, extract the response code, and convert it to readable json - response = await self.api_client.create_search(query) - response_code = response.code - response_dict = json.loads(response.read()) - + """ + init query + :param query + :return:search id + """ # Construct a response object - return_obj = dict() - - if response_code == 201: - return_obj['success'] = True - return_obj['search_id'] = response_dict['sid'] - else: - ErrorResponder.fill_error(return_obj, response_dict, ['messages', 0, 'text'], connector=self.connector) + return_obj = {} + response_dict = {} + try: + # Grab the response, extract the response code, and convert it to readable json + response = await self.api_client.create_search(query) + response_code = response.code + response_dict = json.loads(response.read()) + response_text = response.content + + if response_code == 201: + return_obj['success'] = True + return_obj['search_id'] = response_dict['sid'] + else: + response_dict['type'] = str(response_code) + if response_code == 400: + response_dict['type'] = "Unable_to_parse_search" + response_dict['message'] = response_text + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except ClientConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, + ['message'], connector=self.connector) + except TimeoutError as ex: + response_dict['type'] = "Timeout" + response_dict['messages'] = "TimeoutError" + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except Exception as ex: + if 'Authentication error' in str(ex): + response_dict['type'] = "AuthenticationError" + elif 'timeout_error' in str(ex): + response_dict['type'] = "Timeout" + response_dict['messages'] = str(ex) + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/splunk/stix_transmission/results_connector.py b/stix_shifter_modules/splunk/stix_transmission/results_connector.py index 32efb70fa..2daea6429 100644 --- a/stix_shifter_modules/splunk/stix_transmission/results_connector.py +++ b/stix_shifter_modules/splunk/stix_transmission/results_connector.py @@ -1,29 +1,214 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector -from .api_client import APIClient +import re import json +from asyncio.exceptions import TimeoutError +from aiohttp.client_exceptions import ClientConnectionError +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder class ResultsConnector(BaseJsonResultsConnector): + """ResultsConnector class""" def __init__(self, api_client): self.api_client = api_client self.connector = __name__.split('.')[1] async def create_results_connection(self, search_id, offset, length): + """ + Fetching the results using search_id, offset and length + :param search_id: Data Source search_id + :param offset: Offset + :param length: Length + :return: dict + """ # Grab the response, extract the response code, and convert it to readable json - response = await self.api_client.get_search_results(search_id, offset, length) - response_code = response.code - response_dict = json.load(response) - - # Construct a response object - return_obj = dict() - if response_code == 200: - if "results" in response_dict: - results = response_dict['results'] + return_obj = {} + response_dict = {} + try: + response = await self.api_client.get_search_results(search_id, offset, length) + response_code = response.code + response_dict = json.load(response) + response_text = response.content + + # Construct a response object + return_obj = {} + if response_code == 200: + if "results" in response_dict: + results = response_dict['results'] + else: + results = [] + if results != []: + results = ResultsConnector.check_data(results) + else: + results = [] + return_obj['success'] = True + return_obj['data'] = results else: - results = [] - return_obj['success'] = True - return_obj['data'] = results - else: - ErrorResponder.fill_error(return_obj, response_dict, ['messages', 0, 'text'], connector=self.connector) + response_dict['type'] = str(response_code) + if response_code == 404: + response_dict['type'] = "Unknown_sid" + response_dict['message'] = response_text + ErrorResponder.fill_error(return_obj, response_dict, ['messages'], + connector=self.connector) + except ClientConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except TimeoutError as ex: + response_dict['type'] = "Timeout" + response_dict['messages'] = "TimeoutError" + ErrorResponder.fill_error(return_obj, response_dict, ['messages'], connector=self.connector) + except Exception as ex: + if 'Authentication error' in str(ex): + response_dict['type'] = "AuthenticationError" + elif 'timeout_error' in str(ex): + response_dict['type'] = "Timeout" + response_dict['messages'] = str(ex) + ErrorResponder.fill_error(return_obj, response_dict, ['messages'], connector=self.connector) + return return_obj + + @staticmethod + def check_data(res_data): + """ + check data and update the api response as required + :param item: list of dictionary items + :return: list + """ + for item in res_data: + # check if value is '-', set it None + for key, value in item.items(): + if item.get(key) is not None and value == "-": + item[key] = None + # set name and query count None as query (dns) is none + if item.get("query") is None: + item["name"] = None + item["query_count"] = None + item["query_type"] = None + # create custom network data + if (item.get("src_ip") is not None or item.get("src_port") is not None) and ( + item.get("dest_port") is not None or item.get("dest_ip") is not None): + item["networkdata"] = ResultsConnector.network_traffic_check(item) + + # set subject as None if it is not an email event + if (item.get("subject") is not None and item.get("recipient") is None): + item["subject"] = None + # set process_exec as none if process_name and process_exec are same + ResultsConnector.process_data_check(item) + + # check email data + ResultsConnector.email_data_check(item) + + # if file name is none set file_hash also none + if item.get("file_name") is None: + item["file_hash"] = None + + # create registry value as dictionary object + if item.get("registry_value_name") is not None: + item["registry_value"] = ResultsConnector.registry_value_check(item) + + # ibm finding object change for alert + if item.get("severity") is not None: + if item.get("type") is not None and item.get("type") == "alert": + ResultsConnector.alert_check_ibm_finding(item) + + # extracts SHA1,SHA256 and MD5 values from file_hash + item["file_hashes"] = ResultsConnector.filehash_check(item) + + return res_data + + @staticmethod + def process_data_check(item_in): + """check process data""" + if (item_in.get("process_name") is not None and item_in.get("process_exec") is not None): + if item_in["process_name"] == item_in["process_exec"]: + item_in["process_exec"] = None + if (item_in.get("parent_process_name") is not None and item_in.get("parent_process_exec") is not None): + if item_in["parent_process_name"] == item_in["parent_process_exec"]: + item_in["parent_process_exec"] = None + + @staticmethod + def email_data_check(item_in): + """check email data and set is_multipart required field""" + if item_in.get("src_user") is not None and item_in.get("src_user_domain") is None: + item_in["src_user"] = None + # set is_multipart with false as default if there is email message property + if (item_in.get("subject") is not None or item_in.get("recipient") is not None or item_in.get( + "src_user") is not None) and item_in.get("is_multipart") is None: + item_in["is_multipart"] = False + + @staticmethod + def registry_value_check(item_in): + """check and create valid network traffic event data""" + registry_value_dict = {} + registry_value_dict.update({"registryValueName": item_in["registry_value_name"]}) + if item_in.get("registry_value_data") is not None: + registry_value_dict.update({"registryValueData": item_in["registry_value_data"]}) + return registry_value_dict + + @staticmethod + def alert_check_ibm_finding(item_in): + """check alert event and create custom field""" + item_in["finding_type"] = item_in.get("type") + item_in["alert_severity"] = item_in.get("severity") + item_in["alert_signature"] = item_in.get("signature") + item_in["alert_id"] = item_in.get("id") + item_in["alert_description"] = item_in.get("description") + item_in["severity"] = None + + @staticmethod + def network_traffic_check(item_in): + """check and create valid network traffic event data""" + + # set dest as None if it is same as dest_ip + if item_in.get("dest_ip") is not None and item_in.get("dest") is not None: + if item_in["dest_ip"] == item_in["dest"]: + item_in["dest"] = None + # set src as None if it is same as src_ip + if item_in.get("src_ip") is not None and item_in.get("src") is not None: + if item_in["src_ip"] == item_in["src"]: + item_in["src"] = None + + network_dict = {} + if item_in.get("src_port") is not None: + network_dict.update({"src_port": item_in["src_port"]}) + item_in["src_port"] = None + if item_in.get("dest_port") is not None: + network_dict.update({"dest_port": item_in["dest_port"]}) + item_in["dest_port"] = None + if item_in.get("protocol") is None and item_in.get("transport") is None: + network_dict.update({"protocol": "tcp"}) + if item_in.get("protocol") is not None: + network_dict.update({"protocol": item_in["protocol"]}) + if item_in.get("transport") is not None: + network_dict.update({"transport": item_in["transport"]}) + if item_in.get("direction") is not None: + network_dict.update({"direction": item_in["direction"]}) + + return network_dict + + @staticmethod + def filehash_check(item_in): + """file_hash value check""" + hashdict = {} + if item_in.get('file_hash') is not None and ',' in item_in.get('file_hash'): + values = item_in.get('file_hash').split(',') + del item_in["file_hash"] + for value in values: + if 'SHA1' in value: + hashdict.update({"file_sha1": value.split('=')[1]}) + elif 'SHA256' in value: + hashdict.update({"file_sha256": value.split('=')[1]}) + elif 'MD5' in value: + hashdict.update({"file_md5": value.split('=')[1]}) + elif item_in.get('file_hash') is not None: + if re.compile("^[a-f0-9]{32}$").match(item_in["file_hash"]) is not None: + hashdict.update({"file_md5": item_in['file_hash']}) + elif re.compile(r'\b[0-9a-f]{40}\b').match(item_in["file_hash"]) is not None: + hashdict.update({"file_sha1": item_in['file_hash']}) + elif re.compile("[A-Fa-f0-9]{64}").match(item_in["file_hash"]) is not None: + hashdict.update({"file_sha256": item_in['file_hash']}) + else: + hashdict.update({"unknown": item_in['file_hash']}) + del item_in['file_hash'] + + return hashdict diff --git a/stix_shifter_modules/splunk/stix_transmission/status_connector.py b/stix_shifter_modules/splunk/stix_transmission/status_connector.py index bc126b8dc..b05e89c86 100644 --- a/stix_shifter_modules/splunk/stix_transmission/status_connector.py +++ b/stix_shifter_modules/splunk/stix_transmission/status_connector.py @@ -1,48 +1,86 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector, Status -from .api_client import APIClient import json import math from enum import Enum +from asyncio.exceptions import TimeoutError +from aiohttp.client_exceptions import ClientConnectionError +from stix_shifter_utils.modules.base.stix_transmission.\ + base_status_connector import BaseStatusConnector, Status from stix_shifter_utils.utils.error_response import ErrorResponder + class StatusSplunk(Enum): + """query status values""" COMPLETED = 'DONE' ERROR = 'FAILED' RUNNING = 'RUNNING' + class StatusConnector(BaseStatusConnector): + """check query status class""" def __init__(self, api_client): self.api_client = api_client self.connector = __name__.split('.')[1] async def create_status_connection(self, search_id): + """ + get query status + :param search_id + :return: data response + """ # Grab the response, extract the response code, and convert it to readable json - response = await self.api_client.get_search(search_id) - response_code = response.code - response_dict = json.load(response) - - status, progress = '', '' - - if 'entry' in response_dict and isinstance(response_dict['entry'], list): - content = response_dict['entry'][0]['content'] - progress = math.ceil(content['doneProgress'] * 100) # convert 0-1.0 scale to 0-100 - status = content['dispatchState'] - - if status == StatusSplunk.COMPLETED.value: - status = Status.COMPLETED.value - elif status == StatusSplunk.ERROR.value: - status = Status.ERROR.value - elif content['isFinalized'] is True: - status = Status.CANCELED.value + + return_obj = {} + response_dict = {} + try: + response = await self.api_client.get_search(search_id) + response_code = response.code + response_dict = json.load(response) + response_text = response.content + + status, progress = '', '' + + if 'entry' in response_dict and isinstance(response_dict['entry'], list): + content = response_dict['entry'][0]['content'] + progress = math.ceil(content['doneProgress'] * 100) # convert 0-1.0 scale to 0-100 + status = content['dispatchState'] + + if status == StatusSplunk.COMPLETED.value: + status = Status.COMPLETED.value + elif status == StatusSplunk.ERROR.value: + status = Status.ERROR.value + elif content['isFinalized'] is True: + status = Status.CANCELED.value + else: + status = Status.RUNNING.value + + if response_code == 200: + return_obj['success'] = True + return_obj['status'] = status + return_obj['progress'] = progress else: - status = Status.RUNNING.value - - # Construct a response object - return_obj = dict() - if response_code == 200: - return_obj['success'] = True - return_obj['status'] = status - return_obj['progress'] = progress - else: - ErrorResponder.fill_error(return_obj, response_dict, ['messages',0,'text'], connector=self.connector) + response_dict['type'] = str(response_code) + if response_code == 404: + response_dict['type'] = "Unknown_sid" + response_dict['message'] = response_text + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except ClientConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, + ['message'], connector=self.connector) + except TimeoutError as ex: + response_dict['type'] = "Timeout" + response_dict['messages'] = "TimeoutError" + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + except Exception as ex: + if 'Authentication error' in str(ex): + response_dict['type'] = "AuthenticationError" + elif 'timeout_error' in str(ex): + response_dict['type'] = "Timeout" + response_dict['messages'] = str(ex) + ErrorResponder.fill_error(return_obj, response_dict, + ['messages'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_json_to_stix.py b/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_json_to_stix.py index 4a5e66749..963503fd5 100644 --- a/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_json_to_stix.py +++ b/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_json_to_stix.py @@ -2,11 +2,9 @@ import logging import unittest from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers from stix_shifter.stix_translation import stix_translation from stix_shifter_modules.splunk.entry_point import EntryPoint -from stix2validator import validate_instance -from stix_shifter_modules.splunk.stix_translation.splunk_utils import hash_type_lookup -from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers MODULE = "splunk" logging.basicConfig(level=logging.DEBUG) @@ -23,16 +21,21 @@ options = {} sample_splunk_data_x_oca = { - "src_ip": "123.123.123.123", + "networkdata": {"src_ip": "123.123.123.123", + "src_port": "56109", + "dest_ip": "1.1.1.1", + "dest_port": "9389", + "protocol": "ip" + }, "src_mac": "12-5d-6a-52.78", - "src_port": "56109", - "dest_ip": "1.1.1.1", - "dest_port": "9389", + "alert_severity": "high", "user": "SYSTEM", - "protocol": "-", "process_id": "912", "process_name": "powershell.exe", "process_exec": "powershell.exe", + "registry_key_name": "HKLM\\System\\CurrentControlSet\\Services\\bam\\State\\User" + "Settings\\S-1-5-21-190217562-876380155-933912998-500\\Dev" + "ice\\HarddiskVolume1\\Windows\\System32\\cmd.exe", "process_path": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", "host": "WIN01", "source": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", @@ -71,103 +74,109 @@ def get_nth_of_type(itr, typ, n): return None if len(of_type) <= n else of_type[n] def test_common_prop(self): + """to test common stix object properties""" data = {"_time": "2018-08-21T15:11:55.000+00:00", "event_count": 5} result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], get_module_transformers(MODULE), options) - assert(result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] - assert(result_bundle_identity['type'] == data_source['type']) - assert(result_bundle_identity['id'] == data_source['id']) - assert(result_bundle_identity['name'] == data_source['name']) - assert(result_bundle_identity['identity_class'] - == data_source['identity_class']) + assert result_bundle_identity['type'] == data_source['type'] + assert result_bundle_identity['id'] == data_source['id'] + assert result_bundle_identity['name'] == data_source['name'] + assert result_bundle_identity['identity_class'] == data_source['identity_class'] observed_data = result_bundle_objects[1] - assert(observed_data['id'] is not None) - assert(observed_data['type'] == "observed-data") - assert(observed_data['created_by_ref'] == result_bundle_identity['id']) + assert observed_data['id'] is not None + assert observed_data['type'] == "observed-data" + assert observed_data['created_by_ref'] == result_bundle_identity['id'] - assert(observed_data['number_observed'] == 5) - assert(observed_data['created'] is not None) - assert(observed_data['modified'] is not None) - assert(observed_data['first_observed'] is not None) - assert(observed_data['last_observed'] is not None) + assert observed_data['number_observed'] == 5 + assert observed_data['created'] is not None + assert observed_data['modified'] is not None + assert observed_data['first_observed'] is not None + assert observed_data['last_observed'] is not None def test_change_cim_to_stix(self): + """to test different stix object properties""" count = 1 time = "2018-08-21T15:11:55.000+00:00" file_bytes = "300" user = "ibm_user" - objPath = "hkey_local_machine\\system\\bar\\foo" - filePath = "C:\\Users\\someuser\\sample.dll" + obj_path = "HKCU\\System\\CurrentControlSet\\Services\\Tcpip\\Parame" \ + "ters\\Interfaces\\{d7f2a7dc-ea23-44ef-8ebe-84bad862a9d8}" + file_path = "C:\\Users\\someuser\\sample.dll" create_time = "2018-08-15T15:11:55.676+00:00" modify_time = "2018-08-15T18:10:30.456+00:00" - file_hash = "41a26255d16d121dc525a6445144b895" + file_hashes = { + "file_md5": "41a26255d16d121dc525a6445144b895", + } file_name = "sample.dll" file_size = 25536 data = { "event_count": count, "_time": time, "user": user, - "bytes": file_bytes, "object_path": objPath, "file_path": filePath, + "bytes": file_bytes, "registry_key_name": obj_path, "file_path": file_path, "file_create_time": create_time, "file_modify_time": modify_time, - "file_hash": file_hash, "file_size": file_size, "file_name": file_name + "file_hashes": file_hashes, "file_size": file_size, "file_name": file_name } result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options, callback=hash_type_lookup) + data_source, map_data, [data], get_module_transformers(MODULE), + options) - assert(result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] # Test objects in Stix observable data model after transform wrk_obj = TestTransform.get_first_of_type(objects.values(), 'windows-registry-key') - assert(wrk_obj is not None) - assert(wrk_obj.keys() == {'type', 'key'}) - assert(wrk_obj['key'] == "hkey_local_machine\\system\\bar\\foo") + assert wrk_obj is not None + assert wrk_obj.keys() == {'type', 'key'} + assert wrk_obj['key'] == "HKEY_CURRENT_USER\\System\\CurrentControlSet\\Services" \ + "\\Tcpip\\Parameters\\Interfaces" \ + "\\{d7f2a7dc-ea23-44ef-8ebe-84bad862a9d8}" user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') - assert(user_obj is not None), 'user-account object type not found' - assert(user_obj.keys() == {'type', 'account_login', 'user_id'}) - assert(user_obj['account_login'] == "ibm_user") - assert(user_obj['user_id'] == "ibm_user") + assert user_obj is not None, 'user-account object type not found' + assert user_obj.keys() == {'type', 'user_id'} + assert user_obj['user_id'] == "ibm_user" file_obj = TestTransform.get_first_of_type(objects.values(), 'file') - assert(file_obj is not None), 'file object type not found' - assert(file_obj.keys() == {'type', 'parent_directory_ref', 'created', 'modified', 'hashes', 'name', 'size'}) + assert file_obj is not None, 'file object type not found' + assert file_obj.keys() == {'type', 'parent_directory_ref', 'created', + 'modified', 'hashes', 'name', 'size'} - assert(file_obj['created'] == "2018-08-15T15:11:55.676Z") - assert(file_obj['modified'] == "2018-08-15T18:10:30.456Z") - assert(file_obj['name'] == "sample.dll") - assert(file_obj['size'] == 25536) - assert (file_obj['hashes']['MD5'] == "41a26255d16d121dc525a6445144b895") + assert file_obj['created'] == "2018-08-15T15:11:55.676+00:00Z" + assert file_obj['modified'] == "2018-08-15T18:10:30.456+00:00Z" + assert file_obj['name'] == "sample.dll" + assert file_obj['size'] == 25536 + assert file_obj['hashes']['MD5'] == "41a26255d16d121dc525a6445144b895" dir_ref = file_obj['parent_directory_ref'] - assert(dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found" + assert dir_ref in objects, f"parent_directory_ref with key " \ + f"{file_obj['parent_directory_ref']} not found" dir_obj = objects[dir_ref] - assert(dir_obj is not None), 'directory object type not found' - assert(dir_obj.keys() == {'type', 'path', 'created', 'modified'}) - assert(dir_obj['path'] == "C:\\Users\\someuser\\sample.dll") - assert (dir_obj['created'] == "2018-08-15T15:11:55.676Z") - assert (dir_obj['modified'] == "2018-08-15T18:10:30.456Z") - - assert (objects.keys() == set(map(str, range(0, 5)))) + assert dir_obj is not None, 'directory object type not found' + assert dir_obj.keys() == {'type', 'path'} + assert dir_obj['path'] == "C:\\Users\\someuser\\sample.dll" + assert objects.keys() == set(map(str, range(0, 5))) def test_certificate_cim_to_stix(self): + """to test x509-certificate stix object properties""" count = 1 time = "2018-08-21T15:11:55.000+00:00" serial = "1234" @@ -175,7 +184,8 @@ def test_certificate_cim_to_stix(self): sig_algorithm = "md5WithRSAEncryption" key_algorithm = "rsaEncryption" issuer = "C=US, ST=California, O=www.example.com, OU=new, CN=new" - subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/emailAddress=doe@example.com" + subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, " \ + "CN=www.example.com/emailAddress=doe@example.com" ssl_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" data = { @@ -188,153 +198,132 @@ def test_certificate_cim_to_stix(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], get_module_transformers(MODULE), options) - assert(result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] # Test objects in Stix observable data model after transform cert_obj = TestTransform.get_first_of_type(objects.values(), 'x509-certificate') - assert(cert_obj is not None), 'x509-certificate object type not found' - assert(cert_obj.keys() == {'type', 'serial_number', 'version', "signature_algorithm", "subject_public_key_algorithm", "issuer", "subject", "hashes"}) - assert(cert_obj['serial_number'] == "1234") - assert(cert_obj['version'] == "1") - assert(cert_obj['signature_algorithm'] == "md5WithRSAEncryption") - assert(cert_obj['issuer'] == "C=US, ST=California, O=www.example.com, OU=new, CN=new") - assert(cert_obj['subject'] == "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/emailAddress=doe@example.com") - assert(cert_obj['subject_public_key_algorithm'] == "rsaEncryption") - assert(cert_obj['hashes']['SHA-256'] == "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f") - assert(objects.keys() == set(map(str, range(0, 2)))) + assert cert_obj is not None, 'x509-certificate object type not found' + assert cert_obj.keys() == {'type', 'serial_number', 'version', "signature_algorithm", + "subject_public_key_algorithm", "issuer", "subject", "hashes"} + assert cert_obj['serial_number'] == "1234" + assert cert_obj['version'] == "1" + assert cert_obj['signature_algorithm'] == "md5WithRSAEncryption" + assert cert_obj['issuer'] == "C=US, ST=California, O=www.example.com, OU=new, CN=new" + assert cert_obj['subject'] == "C=US, ST=Maryland, L=Baltimore, O=John Doe, " \ + "OU=ExampleCorp, CN=www.example.com/emailAddress=doe@example.com" + assert cert_obj['subject_public_key_algorithm'] == "rsaEncryption" + assert cert_obj['hashes']['SHA-256'] == "aec070645fe53ee3b3763059376134f058cc337" \ + "247c978add178b6ccdfb0019f" + assert objects.keys() == set(map(str, range(0, 2))) def test_process_cim_to_stix(self): + """to test process stix object properties""" count = 1 time = "2018-08-21T15:11:55.000+00:00" user = "test_user" - pid = 0 + pid = "0x64" name = "test_process" - filePath = "C:\\Users\\someuser\\sample.dll" - create_time = "2018-08-15T15:11:55.676+00:00" - modify_time = "2018-08-15T18:10:30.456+00:00" - #file_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" - file_hash = "MD5=5A0B0E6F407C89916515328F318842A1,SHA256=8FC86B75926043F048971696BC7A407615C9A03D9B1BFACC54785C8903B82A91,IMPHASH=406DD24835F1447987FB607C78597252" + file_path = "C:\\Users\\someuser\\sample.dll" + file_hash = "MD5=5A0B0E6F407C89916515328F318842A1,SHA256=8FC86B75926043F048971696" \ + "BC7A407615C9A03D9B1BFACC54785C8903B82A91,IMPHASH=406DD24835F1447987FB607C78597252" file_name = "sample.dll" - file_size = 25536 process_guid = "7a97b85f-b34c-63be-c00d-000000007d00" data = { "event_count": count, "_time": time, "user": user, - "process_name": name, "process_id": pid, "process_path": filePath, + "process_name": name, "process_id": pid, "process_path": file_path, "process_hash": file_hash, "process_exec": file_name, "process_guid": process_guid } result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options, callback=hash_type_lookup) + data_source, map_data, [data], get_module_transformers(MODULE), + options) - assert(result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] # Test objects in Stix observable data model after transform event_obj = TestTransform.get_first_of_type(objects.values(), 'x-oca-event') - assert (event_obj is not None), 'event object type not found' - self.assertEqual(event_obj.keys(), {'type', 'process_ref', 'created', 'user_ref', 'file_ref'}) + assert event_obj is not None, 'event object type not found' + self.assertEqual(event_obj.keys(), {'type', 'process_ref', 'created', 'user_ref'}) proc_obj = TestTransform.get_first_of_type(objects.values(), 'process') - assert (proc_obj is not None), 'process object type not found' - assert (proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref', 'x_unique_id'}) - assert (proc_obj['name'] == "test_process") - assert (proc_obj['pid'] == 0) - assert (proc_obj['x_unique_id'] == '7a97b85f-b34c-63be-c00d-000000007d00') + assert proc_obj is not None, 'process object type not found' + assert proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref', 'x_unique_id'} + assert proc_obj['name'] == "test_process" + assert proc_obj['pid'] == 100 + assert proc_obj['x_unique_id'] == '7a97b85f-b34c-63be-c00d-000000007d00' user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') + assert user_obj is not None, 'user-account object type not found' + assert user_obj.keys() == {'type', 'user_id'} + assert user_obj['user_id'] == "test_user" - assert (user_obj is not None), 'user-account object type not found' - assert (user_obj.keys() == {'type', 'account_login', 'user_id'}) - assert (user_obj['account_login'] == "test_user") - assert (user_obj['user_id'] == "test_user") - - bin_ref = proc_obj['binary_ref'] - assert (bin_ref in objects), f"binary_ref with key {proc_obj['binary_ref']} not found" - file_obj = objects[bin_ref] - - assert (file_obj is not None), 'file object type not found' - assert (file_obj.keys() == {'type', 'parent_directory_ref', 'name', 'hashes'}) - assert (file_obj['name'] == "sample.dll") - assert (file_obj['hashes']['MD5'] == '5A0B0E6F407C89916515328F318842A1') - assert (file_obj['hashes']['SHA256'] == '8FC86B75926043F048971696BC7A407615C9A03D9B1BFACC54785C8903B82A91') - assert (file_obj['hashes']['IMPHASH'] == '406DD24835F1447987FB607C78597252') + file_obj = TestTransform.get_first_of_type(objects.values(), 'file') + assert file_obj is not None, 'file object type not found' + assert file_obj.keys() == {'type', 'name', 'parent_directory_ref'} + assert file_obj['name'] == "test_process" dir_ref = file_obj['parent_directory_ref'] - assert (dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found" dir_obj = objects[dir_ref] - - assert(dir_obj is not None), 'directory object type not found' - assert(dir_obj.keys() == {'type', 'path'}) - assert(dir_obj['path'] == "C:\\Users\\someuser\\sample.dll") - - - assert (objects.keys() == set(map(str, range(0, 5)))) + assert dir_obj is not None, 'directory object type not found' + assert dir_obj.keys() == {'type', 'path'} + assert dir_obj['path'] == "C:\\Users\\someuser" + assert objects.keys() == set(map(str, range(0, 6))) def test_network_cim_to_stix(self): + """to test network stix object properties""" count = 2 time = "2018-08-21T15:11:55.000+00:00" user = "ibm_user" - dest_ip = "127.0.0.1" - dest_port = "8090" - src_ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - src_port = "8080" - transport = "http" + networkdata = { + "dest_ip": "127.0.0.1", + "dest_port": 8090, + "src_ip": "198.235.24.155", + "src_port": 8080, + "transport": "http" + } data = {"event_count": count, "_time": time, "user": user, - "dest_ip": dest_ip, "dest_port": dest_port, "src_ip": src_ip, - "src_port": src_port, "protocol": transport + "networkdata": networkdata } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], get_module_transformers(MODULE), options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] nt_obj = TestTransform.get_first_of_type(objects.values(), 'network-traffic') - assert(nt_obj is not None), 'network-traffic object type not found' - assert(nt_obj.keys() == {'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'}) - assert(nt_obj['src_port'] == 8080) - assert(nt_obj['dst_port'] == 8090) - assert(nt_obj['protocols'] == ['http']) - - ip_ref = nt_obj['dst_ref'] - assert(ip_ref in objects), f"dst_ref with key {nt_obj['dst_ref']} not found" - ip_obj = objects[ip_ref] - assert(ip_obj.keys() == {'type', 'value'}) - assert(ip_obj['type'] == 'ipv4-addr') - assert(ip_obj['value'] == dest_ip) - - ip_ref = nt_obj['src_ref'] - assert(ip_ref in objects), f"src_ref with key {nt_obj['src_ref']} not found" - ip_obj = objects[ip_ref] - assert(ip_obj.keys() == {'type', 'value'}) - assert(ip_obj['type'] == 'ipv6-addr') - assert(ip_obj['value'] == src_ip) + assert nt_obj is not None, 'network-traffic object type not found' + assert nt_obj.keys() == {'type', 'src_port', 'dst_port', 'protocols'} + assert nt_obj['src_port'] == 8080 + assert nt_obj['dst_port'] == 8090 + assert nt_obj['protocols'] == ['http'] def test_email_cim_to_stix(self): + """to test email-message stix object properties""" count = 3 time = "2018-08-21T15:11:55.000+00:00" src_user = "Jane_Doe@ibm.com" @@ -348,7 +337,7 @@ def test_email_cim_to_stix(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], get_module_transformers(MODULE), options) - assert(result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] @@ -356,34 +345,25 @@ def test_email_cim_to_stix(self): # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] msg_obj = TestTransform.get_first_of_type(objects.values(), 'email-message') - assert(msg_obj is not None), 'email-message object type not found' - assert(msg_obj.keys() == {'type', 'subject', 'sender_ref', 'from_ref', 'is_multipart'}) - assert(msg_obj['subject'] == "Test Subject") - assert(msg_obj['is_multipart'] == False) - - sender_ref = msg_obj['sender_ref'] - assert(sender_ref in objects), f"sender_ref with key {msg_obj['sender_ref']} not found" - - addr_obj = objects[sender_ref] - assert(addr_obj.keys() == {'type', 'value'}) - assert(addr_obj['type'] == 'email-addr') - assert(addr_obj['value'] == src_user) + assert msg_obj is not None, 'email-message object type not found' + assert msg_obj.keys() == {'type', 'subject', 'from_ref', 'is_multipart'} + assert msg_obj['subject'] == "Test Subject" + assert msg_obj['is_multipart'] == 'False' from_ref = msg_obj['from_ref'] - assert(sender_ref in objects), f"from_ref with key {msg_obj['from_ref']} not found" - addr_obj = objects[from_ref] - assert(addr_obj.keys() == {'type', 'value'}) - assert(addr_obj['type'] == 'email-addr') - assert(addr_obj['value'] == src_user) + assert addr_obj.keys() == {'type', 'value'} + assert addr_obj['type'] == 'email-addr' + assert addr_obj['value'] == src_user def test_custom_mapping(self): - - data_source = json.loads("{\"type\": \"identity\", \"id\": \"identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3\", \"name\": \"Splunk\", \"identity_class\": \"events\"}") + """to test custom mapping properties""" + data_source = json.loads( + "{\"type\": \"identity\", \"id\": \"identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3\", \"name\": \"Splunk\", \"identity_class\": \"events\"}") data = json.loads("[{\"tag\":\"network\", \"src_ip\": \"127.0.0.1\"}]") options = { @@ -406,14 +386,18 @@ def test_custom_mapping(self): "key": "ipv4-addr.value", "object": "src_ip" }, - { - "key": "ipv6-addr.value", - "object": "src_ip" - }, { "key": "network-traffic.src_ref", "object": "network-traffic", "references": "src_ip" + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "src_ip" + ], + "group": True } ] } @@ -427,21 +411,29 @@ def test_custom_mapping(self): result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] curr_obj = TestTransform.get_first_of_type(objects.values(), 'ipv4-addr') - assert(curr_obj is not None), 'ipv4-addr object type not found' - assert(curr_obj.keys() == {'type', 'value'}) - assert(curr_obj['value'] == "127.0.0.1") + assert curr_obj is not None, 'ipv4-addr object type not found' + assert curr_obj.keys() == {'type', 'value'} + assert curr_obj['value'] == "127.0.0.1" def test_cim_to_stix_no_tags(self): - - data = {"src_ip": "169.250.0.1", "src_port": "1220", "src_mac": "aa:bb:cc:dd:11:22", - "dest_ip": "127.0.0.1", "dest_port": "1120", "dest_mac": "ee:dd:bb:aa:cc:11", - "file_hash": "cf23df2207d99a74fbe169e3eba035e633b65d94", + """to test cim to stix no tag properties""" + data = {"networkdata": {"src_ip": "169.250.0.1", + "src_port": "1220", + "dest_ip": "127.0.0.1", + "dest_port": "1120", + "protocol": "tcp" + }, + "dest_mac": "ee:dd:bb:aa:cc:11", + "src_mac": "aa:bb:cc:dd:11:22", + "file_hashes": { + "file_sha1": "cf23df2207d99a74fbe169e3eba035e633b65d94" + }, "user": "sname", "url": "https://wally.fireeye.com/malware_analysis/analyses?maid=1", - "protocol": "tcp", "_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "44:12606114", + "_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "44:12606114", "_indextime": "1546960685", "_raw": "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|" "malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1" @@ -458,63 +450,41 @@ def test_cim_to_stix_no_tags(self): } result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options, callback=hash_type_lookup) + data_source, map_data, [data], get_module_transformers(MODULE), options) - assert(result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # somehow breaking the stix validation # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) - assert('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] nt_obj = TestTransform.get_first_of_type(objects.values(), 'network-traffic') - assert(nt_obj is not None), 'network-traffic object type not found' - assert(nt_obj.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'}) - assert(nt_obj['src_port'] == 1220) - assert(nt_obj['dst_port'] == 1120) - assert(nt_obj['protocols'] == ['tcp']) - - ip_ref = nt_obj['dst_ref'] - assert(ip_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found" - ip_obj = objects[ip_ref] - assert(ip_obj.keys() == {'type', 'value', 'resolves_to_refs'}) - assert(ip_obj['type'] == 'ipv4-addr') - assert(ip_obj['value'] == '127.0.0.1') - assert (isinstance(ip_obj['resolves_to_refs'], list) and isinstance(ip_obj['resolves_to_refs'][0], str)) - - ip_ref = nt_obj['src_ref'] - assert(ip_ref in objects), "src_ref with key {nt_obj['src_ref']} not found" - ip_obj = objects[ip_ref] - assert(ip_obj.keys() == {'type', 'value', 'resolves_to_refs'}) - assert(ip_obj['type'] == 'ipv4-addr') - assert(ip_obj['value'] == '169.250.0.1') - assert (isinstance(ip_obj['resolves_to_refs'], list) and isinstance(ip_obj['resolves_to_refs'][0], str)) + assert nt_obj is not None, 'network-traffic object type not found' + assert nt_obj.keys() == {'type', 'src_port', 'dst_port', 'protocols'} + assert nt_obj['src_port'] == 1220 + assert nt_obj['dst_port'] == 1120 + assert nt_obj['protocols'] == ['tcp'] file_obj = TestTransform.get_first_of_type(objects.values(), 'file') - assert (file_obj is not None), 'file object type not found' - assert (file_obj.keys() == {'type', 'hashes'}) - assert (file_obj['hashes']['SHA-1'] == "cf23df2207d99a74fbe169e3eba035e633b65d94") + assert file_obj is not None, 'file object type not found' + assert file_obj.keys() == {'type', 'hashes'} + assert file_obj['hashes']['SHA-1'] == "cf23df2207d99a74fbe169e3eba035e633b65d94" user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') - assert (user_obj is not None), 'user object type not found' - assert (user_obj.keys() == {'type', 'account_login', 'user_id'}) - assert (user_obj['account_login'] == "sname") - assert (user_obj['user_id'] == "sname") + assert user_obj is not None, 'user object type not found' + assert user_obj.keys() == {'type', 'user_id'} + assert user_obj['user_id'] == "sname" url_obj = TestTransform.get_first_of_type(objects.values(), 'url') - assert (url_obj is not None), 'url object type not found' - assert (url_obj.keys() == {'type', 'value'}) - assert (url_obj['value'] == "https://wally.fireeye.com/malware_analysis/analyses?maid=1") - - domain_obj = TestTransform.get_first_of_type(objects.values(), 'domain-name') - assert (domain_obj is not None), 'domain object type not found' - assert (domain_obj.keys() == {'type', 'value'}) - assert (domain_obj['value'] == "wally.fireeye.com") + assert url_obj is not None, 'url object type not found' + assert url_obj.keys() == {'type', 'value'} + assert url_obj['value'] == "https://wally.fireeye.com/malware_analysis/analyses?maid=1" payload_obj = TestTransform.get_first_of_type(objects.values(), 'artifact') - assert (payload_obj is not None), 'payload object type not found' - assert (payload_obj.keys() == {'type', 'payload_bin'}) + assert payload_obj is not None, 'payload object type not found' + assert payload_obj.keys() == {'type', 'payload_bin'} payload = 'SmFuIDA4IDIwMTkgMTU6MTg6MDQgMTkyLjE2OC4zMy4xMzEgZmVub3RpZnktMi5hbGVydDogQ0VGOjB8RmlyZUV5ZXxNQV' \ 'N8Ni4yLjAuNzQyOTh8TU98bWFsd2FyZS1vYmplY3R8NHxydD1KYW4gMDggMjAxOSAxNToxODowNCBaIHNyYz0xNjkuMjUw' \ 'LjAuMSBkcHQ9MTEyMCBkc3Q9MTI3LjAuMC4xIHNwdD0xMjIwIHNtYWM9QUE6QkI6Q0M6REQ6MTE6MjIgZG1hYz1FRTpERD' \ @@ -525,9 +495,10 @@ def test_cim_to_stix_no_tags(self): 'IuMTAxLjEwMSBjbjFMYWJlbD12bGFuIGNuMT0wIGV4dGVybmFsSWQ9MSBjczRMYWJlbD1saW5rIGNzND1odHRwczovL3dh' \ 'bGx5LmZpcmVleWUuY29tL21hbHdhcmVfYW5hbHlzaXMvYW5hbHlzZXM/bWFpZD0xIGNzMkxhYmVsPWFub21hbHkgY3MyPW' \ '1pc2MtYW5vbWFseSBjczFMYWJlbD1zbmFtZSBjczE9RkVfVVBYO1Ryb2phbi5QV1MuT25saW5lR2FtZXM=' - assert (payload_obj['payload_bin'] == payload) + assert payload_obj['payload_bin'] == payload def test_event_cim_to_stix(self): + """to test cim event properties""" data = { "_time": "2018-08-21T15:11:55.000+00:00", "process_name": "powershell.exe", @@ -536,7 +507,7 @@ def test_event_cim_to_stix(self): "process_guid": "7a97b85f-b34c-63be-c00d-000000007d00", "process_path": "C:\\Windows\\SysWOW64\\WindowsPowerShell\\v1.0", "parent_process_name": "cmd.exe", - "parent_process_id": 1, + "parent_process_id": "1", "answer": "1.2.3.4", "query": "test-domain.com", "host": "test_host", @@ -544,16 +515,17 @@ def test_event_cim_to_stix(self): } result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options, callback=hash_type_lookup) + data_source, map_data, [data], get_module_transformers(MODULE), + options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid is True) - assert ('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] object_vals = objects.values() @@ -561,81 +533,68 @@ def test_event_cim_to_stix(self): proc_obj = TestTransform.get_nth_of_type(object_vals, 'process', 0) parent_proc_obj = TestTransform.get_nth_of_type(object_vals, 'process', 1) - assert (proc_obj is not None and parent_proc_obj is not None), 'process object type not found' - assert ('parent_ref' in proc_obj or 'parent_ref' in parent_proc_obj) + assert proc_obj is not None and parent_proc_obj is not None, 'process object type not found' + assert 'parent_ref' in proc_obj or 'parent_ref' in parent_proc_obj if 'parent_ref' in parent_proc_obj: proc_obj, parent_proc_obj = parent_proc_obj, proc_obj - assert (proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref', 'parent_ref', 'opened_connection_refs', 'x_unique_id'}) - assert (proc_obj['name'] == "powershell.exe") - assert (proc_obj['pid'] == 2) - assert (proc_obj['x_unique_id'] == '7a97b85f-b34c-63be-c00d-000000007d00') - assert (objects[proc_obj['parent_ref']] == parent_proc_obj) - assert (parent_proc_obj['name'] == 'cmd.exe') - assert (parent_proc_obj['pid'] == 1) + assert proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref', 'parent_ref', 'x_unique_id'} + assert proc_obj['name'] == "powershell.exe" + assert proc_obj['pid'] == 2 + assert proc_obj['x_unique_id'] == '7a97b85f-b34c-63be-c00d-000000007d00' + assert objects[proc_obj['parent_ref']] == parent_proc_obj + assert parent_proc_obj['name'] == 'cmd.exe' + assert parent_proc_obj['pid'] == 1 user_obj = TestTransform.get_first_of_type(object_vals, 'user-account') - assert (user_obj is not None), 'user-account object type not found' - assert (user_obj.keys() == {'type', 'account_login', 'user_id'}) - assert (user_obj['account_login'] == "test_user") - assert (user_obj['user_id'] == "test_user") - - bin_ref = proc_obj['binary_ref'] - assert (bin_ref in objects), f"binary_ref with key {proc_obj['binary_ref']} not found" - file_obj = objects[bin_ref] - - assert (file_obj is not None), 'file object type not found' - assert (file_obj.keys() == {'type', 'parent_directory_ref', 'name'}) - assert (file_obj['name'] == "powershell.exe") - dir_ref = file_obj['parent_directory_ref'] - assert (dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found" - dir_obj = objects[dir_ref] - - assert (dir_obj is not None), 'directory object type not found' - assert (dir_obj.keys() == {'type', 'path'}) - assert (dir_obj['path'] == "C:\\Windows\\SysWOW64\\WindowsPowerShell\\v1.0") + assert user_obj is not None, 'user-account object type not found' + assert user_obj.keys() == {'type', 'user_id'} + assert user_obj['user_id'] == "test_user" host_obj = TestTransform.get_first_of_type(object_vals, 'x-oca-asset') - assert (host_obj is not None) - assert (host_obj["hostname"] == "test_host") + assert host_obj is not None + assert host_obj["hostname"] == "test_host" ip_obj = TestTransform.get_first_of_type(object_vals, 'ipv4-addr') - assert (ip_obj is not None) - assert (ip_obj['value'] == '1.2.3.4') + assert ip_obj is not None + assert ip_obj['value'] == '1.2.3.4' domain_obj = TestTransform.get_first_of_type(object_vals, 'domain-name') - assert (domain_obj is not None) - assert (domain_obj['value'] == "test-domain.com") + assert domain_obj is not None + assert domain_obj['value'] == "test-domain.com" - nt_obj = TestTransform.get_first_of_type(object_vals, 'network-traffic') - assert (nt_obj is not None) - assert (objects[nt_obj["extensions"]["dns-ext"]["question"]["domain_ref"]] == domain_obj) - assert (objects[nt_obj["extensions"]["dns-ext"]["resolved_ip_refs"][0]] == ip_obj) + directory_obj = TestTransform.get_first_of_type(object_vals, 'directory') + assert directory_obj is not None + assert directory_obj['path'] == "C:\\Windows\\SysWOW64\\WindowsPowerShell" event_obj = TestTransform.get_first_of_type(object_vals, 'x-oca-event') - assert (event_obj is not None), 'event object type not found' + assert event_obj is not None, 'event object type not found' self.assertEqual(event_obj.keys(), - {'user_ref', 'file_ref', 'parent_process_ref', 'network_ref', 'created', 'host_ref', + {'user_ref', 'parent_process_ref', + 'created', 'host_ref', 'domain_ref', 'type', 'process_ref'}) - for key, obj in [('process_ref', proc_obj), ('user_ref', user_obj), ('parent_process_ref', parent_proc_obj), + for key, obj in [('process_ref', proc_obj), ('user_ref', user_obj), + ('parent_process_ref', parent_proc_obj), ('domain_ref', domain_obj), ('host_ref', host_obj)]: - assert (objects[event_obj[key]] == obj) - assert (objects.keys() == set(map(str, range(0, 10)))) + assert objects[event_obj[key]] == obj + + assert objects.keys() == set(map(str, range(0, 12))) def test_x_oca_event_fields(self): + """to test x-oca-event stix object properties""" result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), options, - callback=hash_type_lookup) + data_source, map_data, [sample_splunk_data_x_oca], + get_module_transformers(MODULE), options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid is True) - assert ('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] object_vals = objects.values() @@ -645,11 +604,12 @@ def test_x_oca_event_fields(self): self.assertEqual(x_oca_event['code'], '3') def test_x_oca_event_network_ref(self): + """to test x-oca-event network-ref stix object properties""" result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), options, - callback=hash_type_lookup) + data_source, map_data, [sample_splunk_data_x_oca], + get_module_transformers(MODULE), options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] @@ -668,22 +628,15 @@ def test_x_oca_event_network_ref(self): self.assertIsNotNone(network_ref_obj) self.assertEqual(56109, network_ref_obj['src_port']) self.assertEqual(9389, network_ref_obj['dst_port']) - - src_ip_obj = objects[network_ref_obj['src_ref']] - dst_ip_obj = objects[network_ref_obj['dst_ref']] - - self.assertEqual("ipv4-addr", src_ip_obj['type']) - self.assertEqual("ipv4-addr", dst_ip_obj['type']) - - self.assertEqual(src_ip_obj['value'], '123.123.123.123') - self.assertEqual(dst_ip_obj['value'], '1.1.1.1') + self.assertEqual(['ip'], network_ref_obj['protocols']) def test_x_oca_event_file_ref(self): + """to test x-oca-event file-ref stix object properties""" result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), options, - callback=hash_type_lookup) + data_source, map_data, [sample_splunk_data_x_oca], + get_module_transformers(MODULE), options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] @@ -696,51 +649,88 @@ def test_x_oca_event_file_ref(self): x_oca_event = TestTransform.get_first_of_type(object_vals, 'x-oca-event') - file_ref = x_oca_event.get("file_ref") - self.assertIsNotNone(file_ref) - file_ref_obj = objects.get(file_ref) - self.assertIsNotNone(file_ref_obj) - self.assertEqual("file", file_ref_obj['type']) - self.assertEqual("powershell.exe", file_ref_obj['name']) - parent_directory_obj = objects[file_ref_obj['parent_directory_ref']] - self.assertEqual("directory", parent_directory_obj['type']) - self.assertEqual("C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", parent_directory_obj['path']) + host_ref = x_oca_event.get("host_ref") + self.assertIsNotNone(host_ref) + host_ref_obj = objects.get(host_ref) + self.assertIsNotNone(host_ref_obj) + self.assertEqual("x-oca-asset", host_ref_obj['type']) + self.assertEqual("WIN01", host_ref_obj['hostname']) def test_x_oca_asset(self): + """to test x-oca-asset stix object properties""" result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), options, - callback=hash_type_lookup) + data_source, map_data, [sample_splunk_data_x_oca], + get_module_transformers(MODULE), options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid is True) - assert ('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] object_vals = objects.values() x_oca_asset = TestTransform.get_first_of_type(object_vals, 'x-oca-asset') self.assertEqual(x_oca_asset['hostname'], 'WIN01') - def test_mac_addr(self): + """to test mac-address stix object properties""" result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), options, - callback=hash_type_lookup) + data_source, map_data, [sample_splunk_data_x_oca], + get_module_transformers(MODULE), options) - assert (result_bundle['type'] == 'bundle') + assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] - assert ('objects' in observed_data) + assert 'objects' in observed_data objects = observed_data['objects'] object_vals = objects.values() mac = TestTransform.get_first_of_type(object_vals, 'mac-addr') self.assertEqual(mac['value'], '00:12:5d:6a:52:78') + def test_x_ibm_finding(self): + """to test x-ibm-finding stix object properties""" + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), + options) + + assert result_bundle['type'] == 'bundle' + result_bundle_objects = result_bundle['objects'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + object_vals = objects.values() + + severity = TestTransform.get_first_of_type(object_vals, 'x-ibm-finding') + self.assertEqual(severity['severity'], 80) + + def test_windows_registry_key(self): + """to test windows-registry-key stix object properties""" + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [sample_splunk_data_x_oca], get_module_transformers(MODULE), + options) + + assert result_bundle['type'] == 'bundle' + result_bundle_objects = result_bundle['objects'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + object_vals = objects.values() + + windows_registry = TestTransform.get_first_of_type(object_vals, 'windows-registry-key') + self.assertEqual(windows_registry['key'], 'HKEY_LOCAL_MACHINE\\System\\CurrentControl' + 'Set\\Services\\bam\\State\\User' + 'Settings\\S-1-5-21-190217562-876380155-933912998-' + '500\\Device\\HarddiskVolume1\\Windows\\System' + '32\\cmd.exe') + + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_stix_to_spl.py b/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_stix_to_spl.py index 96291a11e..3c24a76fc 100644 --- a/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_stix_to_spl.py +++ b/stix_shifter_modules/splunk/tests/stix_translation/test_splunk_stix_to_spl.py @@ -1,8 +1,8 @@ +import unittest +import random from stix_shifter.stix_translation import stix_translation from stix_shifter_utils.utils.error_response import ErrorCode -import unittest -import random DEFAULT_LIMIT = 10000 DEFAULT_TIMERANGE = 5 @@ -32,27 +32,16 @@ "src_ip", "src_port", "src_mac", - "src_ipv6", "dest_ip", "dest_port", "dest_mac", - "dest_ipv6", "file_hash", "user", "url", "protocol", "host", "source", - "DeviceType", - "Direction", "severity", - "EventID", - "EventName", - "ss_name", - "TacticId", - "Tactic", - "TechniqueId", - "Technique", "process", "process_id", "process_name", @@ -65,11 +54,96 @@ "parent_process_name", "parent_process_exec", "description", - "result", "signature", "signature_id", "query", - "answer" + "answer", + "transport", + "bytes_in", + "bytes_out", + "packets_in", + "packets_out", + "direction", + "name", + "message_type", + "query_count", + "query_type", + "record_type", + "reply_code", + "reply_code_id", + "vendor_product", + "duration", + "transaction_id", + "action", + "file_access_time", + "file_acl", + "registry_hive", + "registry_path", + "registry_key_name", + "registry_value_data", + "registry_value_name", + "registry_value_text", + "registry_value_type", + "status", + "ssl_version", + "ssl_serial", + "ssl_issuer", + "ssl_subject", + "ssl_signature_algorithm", + "ssl_publickey_algorithm", + "ssl_start_time", + "ssl_end_time", + "ssl_is_valid", + "ssl_issuer_common_name", + "ssl_subject_common_name", + "ssl_name", + "ssl_publickey", + "ssl_issuer_email", + "ssl_subject_email", + "ssl_issuer_email_domain", + "ssl_subject_email_domain", + "ssl_issuer_organization", + "ssl_subject_organization", + "recipient", + "subject", + "file_hash", + "file_name", + "file_size", + "recipient_domain", + "src_user_domain", + "internal_message_id", + "message_id", + "message_info", + "app", + "authentication_method", + "authentication_service", + "dest", + "src", + "src_user", + "user_name", + "user_id", + "user_type", + "user_agent", + "http_method", + "http_referrer", + "http_user_agent", + "uri_path", + "uri_query", + "os", + "dvc", + "id", + "msft", + "cve", + "cvss", + "mskb", + "type", + "eventtype", + "event_id", + "mitre_technique_id", + "mem_used", + "original_file_name", + "file_create_time", + "file_modify_time" ]) @@ -80,175 +154,355 @@ def _test_query_assertions(query, queries): class TestStixToSpl(unittest.TestCase, object): def test_ipv4_query(self): + """ test to check ipv4 stix pattern to native data source query """ stix_pattern = "[ipv4-addr:value = '192.168.122.83' OR ipv4-addr:value = '192.168.122.84']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (((src_ip = "192.168.122.84") OR (dest_ip = "192.168.122.84")) OR ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83"))) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (((src_ip = "192.168.122.84") OR (dest_ip = "192.168.122.84")) OR ' \ + f'((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83"))) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_ipv6_query(self): + """ test to check ipv6 stix pattern to native data source query """ stix_pattern = "[ipv6-addr:value = 'fe80::8c3b:a720:dc5c:2abf%19']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((src_ipv6 = "fe80::8c3b:a720:dc5c:2abf%19") OR (dest_ipv6 = "fe80::8c3b:a720:dc5c:2abf%19")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((src_ip = "fe80::8c3b:a720:dc5c:2abf%19") ' \ + f'OR (dest_ip = "fe80::8c3b:a720:dc5c:2abf%19")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_traffic_query(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port = 3389]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port = 3389) earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_query_greater_than(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port > 3389]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port > 3389) earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_query_not_equals(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port != 3389]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port != 3389) earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_query_less_than(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port < 3389]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port < 3389) earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_query_lessthan_or_equals(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port <= 3389]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port <= 3389) earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_query_greaterthan_or_equals(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port >= 3389]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port >= 3389) earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_network_query_in_operator(self): + """ test to check network traffic stix pattern to native data source query """ + stix_pattern = r"[network-traffic:dst_port IN (80,3389)]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (dest_port IN (80, 3389)) earliest=\"-5minutes\" | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_url_query(self): + """ test to check url stix pattern to native data source query """ stix_pattern = "[url:value = 'http://www.testaddress.com']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (url = "http://www.testaddress.com") earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (url = "http://www.testaddress.com") earliest="-5minutes" | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_not_equal_operator(self): + """ test not equal operator stix pattern to native data source query """ stix_pattern = "[url:value != 'http://www.testaddress.com']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (url != "http://www.testaddress.com") earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (url != "http://www.testaddress.com") earliest="-5minutes" | ' \ + f'head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_followedby_operator(self): + """ test followedby operator stix pattern to native data source query """ + stix_pattern = r"[file:name MATCHES '^x.\\..*$'] FOLLOWEDBY " \ + r"[file:name = 'y1.exe'] START t'2022-01-19T11:00:00.000Z' " \ + r"STOP t'2023-02-28T11:00:00.003Z'" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search |eval latest=[search ((file_name = "y1.exe") ' \ + f'OR (process_name = "y1.exe") OR (parent_process_name = "y1.exe") ' \ + f'OR (process_exec = "y1.exe") OR (parent_process_exec = "y1.exe")) ' \ + f'earliest="01/19/2022:11:00:00" latest="02/28/2023:11:00:00" | ' \ + f'append [makeresults 1 | eval _time=0] | head 1 | ' \ + f'return $_time] | where ((match(file_name, "^x.\\..*$")) ' \ + f'OR (match(process_name, "^x.\\..*$")) ' \ + f'OR (match(parent_process_name, "^x.\\..*$")) ' \ + f'OR (match(process_exec, "^x.\\..*$")) ' \ + f'OR (match(parent_process_exec, "^x.\\..*$"))) | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_NOT_operator(self): + """ test NOT operator stix pattern to native data source query """ stix_pattern = "[url:value NOT = 'http://www.testaddress.com']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (NOT (url = "http://www.testaddress.com")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (NOT (url = "http://www.testaddress.com")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_mac_address_query(self): + """ test mac address stix pattern to native data source query """ stix_pattern = "[mac-addr:value = '00-00-5E-00-53-00']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((src_mac = "00-00-5E-00-53-00") OR (dest_mac = "00-00-5E-00-53-00")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((src_mac = "00-00-5E-00-53-00") OR (dest_mac = "00-00-5E-00-53-00")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_domain_query(self): + """ test domain-name stix pattern to native data source query """ stix_pattern = "[domain-name:value = 'example.com']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((host = "example.com") OR (url = "example.com")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((query = "example.com") ' \ + f'OR (recipient_domain = "example.com") ' \ + f'OR (src_user_domain = "example.com") ' \ + f'OR (ssl_issuer_email_domain = "example.com") ' \ + f'OR (ssl_subject_email_domain = "example.com")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_query_from_multiple_observation_expressions_joined_by_AND(self): - stix_pattern = "[domain-name:value = 'example.com'] AND [mac-addr:value = '00-00-5E-00-53-00']" + """ test multiple observation stix pattern to native data source query """ + stix_pattern = "[domain-name:value = 'example.com'] AND " \ + "[mac-addr:value = '00-00-5E-00-53-00']" query = translation.translate('splunk', 'query', '{}', stix_pattern) # Expect the STIX AND to convert to an SPL OR. - queries = f'search ((host = "example.com") OR (url = "example.com")) OR ((src_mac = "00-00-5E-00-53-00") OR (dest_mac = "00-00-5E-00-53-00")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((query = "example.com") ' \ + f'OR (recipient_domain = "example.com") ' \ + f'OR (src_user_domain = "example.com") ' \ + f'OR (ssl_issuer_email_domain = "example.com") ' \ + f'OR (ssl_subject_email_domain = "example.com")) ' \ + f'OR ((src_mac = "00-00-5E-00-53-00") ' \ + f'OR (dest_mac = "00-00-5E-00-53-00")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_query_from_multiple_comparison_expressions_joined_by_AND(self): - stix_pattern = "[domain-name:value = 'example.com' AND mac-addr:value = '00-00-5E-00-53-00']" + """ test multiple comparison stix pattern to native data source query """ + stix_pattern = "[domain-name:value = 'example.com' AND " \ + "mac-addr:value = '00-00-5E-00-53-00']" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + # Expect the STIX AND to convert to an AQL AND. + queries = f'search (((src_mac = "00-00-5E-00-53-00") ' \ + f'OR (dest_mac = "00-00-5E-00-53-00")) ' \ + f'AND ((query = "example.com") ' \ + f'OR (recipient_domain = "example.com") ' \ + f'OR (src_user_domain = "example.com") ' \ + f'OR (ssl_issuer_email_domain = "example.com") ' \ + f'OR (ssl_subject_email_domain = "example.com"))) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_query_from_morethan_two_comparison_expressions_joined_by_and(self): + """ test multiple more than two comparison stix pattern to native data source query """ + stix_pattern = r"[domain-name:value = 'example.com' AND " \ + r"mac-addr:value = '00-00-5E-00-53-00' AND " \ + r"ipv4-addr:value = '192.168.122.84']" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + # Expect the STIX AND to convert to an AQL AND. + queries = f'search (((src_ip = "192.168.122.84") OR (dest_ip = "192.168.122.84")) ' \ + f'AND (((src_mac = "00-00-5E-00-53-00") ' \ + f'OR (dest_mac = "00-00-5E-00-53-00")) ' \ + f'AND ((query = "example.com") ' \ + f'OR (recipient_domain = "example.com") ' \ + f'OR (src_user_domain = "example.com") ' \ + f'OR (ssl_issuer_email_domain = "example.com") ' \ + f'OR (ssl_subject_email_domain = "example.com")))) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_multiple_observation_query(self): + """ test multiple observation stix pattern to native data source query """ + stix_pattern = r"([file: hashes.MD5 = '0bbd4d92d3a0178463ef6e0ad46c986a' " \ + r"AND file:name = 'log' AND x-oca-event:code = '4998'] " \ + r"AND [file:created = 'PE'] " \ + r"AND [ x-oca-asset:hostname = '21.6.6.1200' " \ + r"AND network-traffic:src_port > 80 ]) " \ + r"START t'2022-01-19T11:00:00.000Z' STOP t'2023-03-09T11:00:00.003Z'" query = translation.translate('splunk', 'query', '{}', stix_pattern) # Expect the STIX AND to convert to an AQL AND. - queries = f'search (((src_mac = "00-00-5E-00-53-00") OR (dest_mac = "00-00-5E-00-53-00")) AND ((host = "example.com") OR (url = "example.com"))) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((signature_id = "4998") AND (((file_name = "log") ' \ + f'OR (process_name = "log") OR (parent_process_name = "log") ' \ + f'OR (process_exec = "log") OR (parent_process_exec = "log")) ' \ + f'AND (file_hash = "0bbd4d92d3a0178463ef6e0ad46c986a"))) ' \ + f'OR (file_create_time = "PE") OR ((src_port > 80) ' \ + f'AND (host = "21.6.6.1200")) earliest="01/19/2022:11:00:00" ' \ + f'latest="03/09/2023:11:00:00" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_file_query(self): + """ test file stix pattern to native data source query """ stix_pattern = "[file:name = 'some_file.exe']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (file_name = "some_file.exe") earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((file_name = "some_file.exe") ' \ + f'OR (process_name = "some_file.exe") ' \ + f'OR (parent_process_name = "some_file.exe") ' \ + f'OR (process_exec = "some_file.exe") ' \ + f'OR (parent_process_exec = "some_file.exe")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_file_hash_query(self): + """ test file stix pattern to native data source query """ stix_pattern = "[file:hashes.'SHA-1' = '5e5a7065f1b551eb3632fb189ce1baefd23158aa']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (file_hash = \"5e5a7065f1b551eb3632fb189ce1baefd23158aa\") earliest=\"-5minutes\" | head 10000 | fields {fields}' - _test_query_assertions(query, queries) - - def test_risk_finding(self): - stix_pattern = "[x-ibm-finding:name = 'sample_alert']" - query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search index=_audit ss_name="sample_alert" action=alert_fired earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (file_hash = \"5e5a7065f1b551eb3632fb189ce1baefd23158aa\") ' \ + f'earliest=\"-5minutes\" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_dst_ref_queries(self): + """ test network-traffic stix pattern to native data source query """ stix_pattern = "[network-traffic:dst_ref.value = '192.168.122.83']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (dest_ip = "192.168.122.83") earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (dest_ip = "192.168.122.83") earliest="-5minutes" | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_port_queries(self): + """ test network-traffic stix pattern to native data source query """ stix_pattern = "[network-traffic:src_port = 12345 OR network-traffic:dst_port = 23456]" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((dest_port = 23456) OR (src_port = 12345)) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((dest_port = 23456) OR (src_port = 12345)) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_unmapped_attribute_handling_with_OR(self): stix_pattern = "[url:value = 'http://www.testaddress.com' OR unmapped:attribute = 'something']" translated_query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (url = "http://www.testaddress.com") earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search (url = "http://www.testaddress.com") earliest="-5minutes" | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(translated_query, queries) def test_unmapped_attribute_handling_with_AND(self): stix_pattern = "[url:value = 'http://www.testaddress.com' AND unmapped:attribute = 'something']" result = translation.translate('splunk', 'query', '{}', stix_pattern) - assert result['success'] == False + assert result['success'] is False assert ErrorCode.TRANSLATION_MAPPING_ERROR.value == result['code'] assert "data mapping error : Unable to map the following STIX objects and properties: " \ "['unmapped:attribute'] to data source fields" in result['error'] def test_invalid_stix_pattern(self): + """ test invalid stix pattern to native data source query """ stix_pattern = "[not_a_valid_pattern]" result = translation.translate('splunk', 'query', '{}', stix_pattern, {'validate_pattern': 'true'}) - assert False == result['success'] + assert False is result['success'] assert ErrorCode.TRANSLATION_STIX_VALIDATION.value == result['code'] assert stix_pattern[1:-1] in result['error'] def test_network_traffic_protocols(self): + """ test network-traffic protocol """ for key, value in protocols.items(): # Test for both upper and lower case protocols in the STIX pattern if random.randint(0, 1) == 0: key = key.upper() stix_pattern = "[network-traffic:protocols[*] = '" + key + "']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = 'search (protocol = "' + key + '") earliest="{}" | head {} | fields {}'.format(default_time_range_spl, + queries = 'search ((protocol = "' + key + '") OR (transport = "' + key + '")) earliest="{}" | head {} | fields {}'.format(default_time_range_spl, DEFAULT_LIMIT, fields) _test_query_assertions(query, queries) def test_network_traffic_start_stop(self): - stix_pattern = "[network-traffic:start = '2018-06-14T08:36:24.000Z' OR network-traffic:end = '2018-06-14T08:36:24.000Z']" + """test stix pattern with start stop qualifier to native data source query""" + stix_pattern = "[network-traffic:dst_packets = 400 " \ + "OR network-traffic:src_packets = 500]" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((latest = "2018-06-14T08:36:24.000Z") OR (earliest = "2018-06-14T08:36:24.000Z")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((packets_out = 500) OR (packets_in = 400)) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_start_stop_qualifiers(self): - stix_pattern = "[network-traffic:src_port = 37020] START t'2016-06-01T01:30:00.000Z' STOP t'2016-06-01T02:20:00.000Z' OR [ipv4-addr:value = '192.168.122.83'] START t'2016-06-01T03:55:00.000Z' STOP t'2016-06-01T04:30:00.000Z'" + """test stix pattern with start stop qualifier to native data source query""" + stix_pattern = "[network-traffic:src_port = 37020] START t'2016-06-01T01:30:00.000Z' " \ + "STOP t'2016-06-01T02:20:00.000Z' OR [ipv4-addr:value = '192.168.122.83'] " \ + "START t'2016-06-01T03:55:00.000Z' STOP t'2016-06-01T04:30:00.000Z'" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (src_port = 37020) earliest="06/01/2016:01:30:00" latest="06/01/2016:02:20:00" OR ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83")) earliest="06/01/2016:03:55:00" latest="06/01/2016:04:30:00" | head 10000 | fields {fields}' + queries = f'search (src_port = 37020) earliest="06/01/2016:01:30:00" ' \ + f'latest="06/01/2016:02:20:00" OR ((src_ip = "192.168.122.83") ' \ + f'OR (dest_ip = "192.168.122.83")) earliest="06/01/2016:03:55:00" ' \ + f'latest="06/01/2016:04:30:00" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_start_stop_qualifiers_one_time(self): - stix_pattern = "[network-traffic:src_port = 37020] START t'2016-06-01T01:30:00.000Z' STOP t'2016-06-01T02:20:00.000Z'" + """test stix pattern with start stop qualifier to native data source query""" + stix_pattern = "[network-traffic:src_port = 37020] START t'2016-06-01T01:30:00.000Z' " \ + "STOP t'2016-06-01T02:20:00.000Z'" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (src_port = 37020) earliest="06/01/2016:01:30:00" latest="06/01/2016:02:20:00" | head 10000 | fields {fields}' + queries = f'search (src_port = 37020) earliest="06/01/2016:01:30:00" ' \ + f'latest="06/01/2016:02:20:00" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_start_stop_qualifiers_seconds(self): - stix_pattern = "[network-traffic:src_port = 37020] START t'2016-06-01T01:30:00Z' STOP t'2016-06-01T02:20:00Z'" + """test stix pattern with start stop qualifier to native data source query""" + stix_pattern = "[network-traffic:src_port = 37020] START t'2016-06-01T01:30:00Z' " \ + "STOP t'2016-06-01T02:20:00Z'" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search (src_port = 37020) earliest="06/01/2016:01:30:00" latest="06/01/2016:02:20:00" | head 10000 | fields {fields}' + queries = f'search (src_port = 37020) earliest="06/01/2016:01:30:00" ' \ + f'latest="06/01/2016:02:20:00" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_issubset_operator(self): + """test issubset operator stix pattern to native data source query""" stix_pattern = "[ipv4-addr:value ISSUBSET '198.51.100.0/24']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search earliest="-5minutes" | where ((cidrmatch("198.51.100.0/24", src_ip)) OR (cidrmatch("198.51.100.0/24", dest_ip))) | head 10000 | fields {fields}' + queries = f'search earliest="-5minutes" | ' \ + f'where ((cidrmatch("198.51.100.0/24", src_ip)) ' \ + f'OR (cidrmatch("198.51.100.0/24", dest_ip))) | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_custom_time_limit_and_result_count(self): + """test custom time limit and result count native data source query""" stix_pattern = "[ipv4-addr:value = '192.168.122.83']" options = {"time_range": 25, "result_limit": 5000} query = translation.translate('splunk', 'query', '{}', stix_pattern, options) - queries = f'search ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83")) earliest="-25minutes" | head 5000 | fields {fields}' + queries = f'search ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83")) ' \ + f'earliest="-25minutes" | head 5000 | fields {fields}' _test_query_assertions(query, queries) def test_custom_index(self): + """test custom index native data source query""" stix_pattern = "[ipv4-addr:value = '192.168.122.83']" options = {"index": "my_index"} query = translation.translate('splunk', 'query', '{}', stix_pattern, options) - queries = f'search index="my_index" ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search index="my_index" ((src_ip = "192.168.122.83") ' \ + f'OR (dest_ip = "192.168.122.83")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_custom_indices(self): + """test custom indices native data source query""" stix_pattern = "[ipv4-addr:value = '192.168.122.83']" options = {"index": "i1, i2"} query = translation.translate('splunk', 'query', '{}', stix_pattern, options) - queries = f'search index="i1" OR index="i2" ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search index="i1" OR index="i2" ((src_ip = "192.168.122.83") ' \ + f'OR (dest_ip = "192.168.122.83")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_custom_mapping(self): + """test custom mapping native data source query""" stix_pattern = "[ipv4-addr:value = '192.168.122.83' AND mac-addr:value = '00-00-5E-00-53-00']" options = { @@ -280,10 +534,13 @@ def test_custom_mapping(self): } query = translation.translate('splunk', 'query', '{}', stix_pattern, options) - queries = f'search ((mac = "00-00-5E-00-53-00") AND ((src_ip = "192.168.122.83") OR (dest_ip = "192.168.122.83"))) earliest="-15minutes" | head 1000 | fields src_ip, src_port' + queries = f'search ((mac = "00-00-5E-00-53-00") AND ((src_ip = "192.168.122.83") ' \ + f'OR (dest_ip = "192.168.122.83"))) earliest="-15minutes" | ' \ + f'head 1000 | fields src_ip, src_port' _test_query_assertions(query, queries) def test_free_search(self): + """test free search native data source query""" stix_pattern = "[x-readable-payload:value = 'malware']" options = {"time_range": 25, "result_limit": 5000} query = translation.translate('splunk', 'query', '{}', stix_pattern, options) @@ -291,72 +548,192 @@ def test_free_search(self): _test_query_assertions(query, queries) def test_event_query(self): + """test event native data source query""" stix_pattern = "[x-oca-event:code = 1]" query = translation.translate('splunk', 'query', '{}', stix_pattern) queries = f'search (signature_id = 1) earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_x_oca_asset_by_hostname(self): + """test x-oca-asset native data source query""" stix_pattern = "[x-oca-asset:hostname = 'omer']" result_query = translation.translate('splunk', 'query', '{}', stix_pattern) expected_query = f'search (host = "omer") earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(result_query, expected_query) def test_x_oca_event(self): - stix_pattern = "[(x-oca-event:action = 'DNS Query' OR x-oca-event:code = 22) AND (x-oca-event:module = 'XmlWinEventLog:Microsoft-Windows-Sysmon/Operational')]" + """test x-oca-event native data source query""" + stix_pattern = "[(x-oca-event:action = 'DNS Query' OR x-oca-event:code = 22) " \ + "AND (x-oca-event:module = 'XmlWinEventLog:Microsoft-Windows-Sysmon/Operational')]" result_query = translation.translate('splunk', 'query', '{}', stix_pattern) - expected_queries = f'search ((source = "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational") AND ((signature_id = 22) OR (signature = "DNS Query"))) earliest="-5minutes" | head 10000 | fields {fields}' + expected_queries = f'search ((source = "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational") ' \ + f'AND ((signature_id = 22) OR (signature = "DNS Query"))) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(result_query, expected_queries) def test_x_oca_event_missing_vals(self): - stix_pattern = "[(x-oca-event:action = '' OR x-oca-event:code = '') AND (x-oca-event:module = '')]" + """test x-oca-event native data source query""" + stix_pattern = "[(x-oca-event:action = '' OR x-oca-event:code = '') " \ + "AND (x-oca-event:module = '')]" result_query = translation.translate('splunk', 'query', '{}', stix_pattern) print(result_query) - expected_queries = f'search ((source = "") AND ((signature_id = "") OR (signature = ""))) earliest="-5minutes" | head 10000 | fields {fields}' + expected_queries = f'search ((source = "") AND ((signature_id = "") ' \ + f'OR (signature = ""))) earliest="-5minutes" | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(result_query, expected_queries) def test_proc_command_line_query(self): + """test process command line native data source query""" stix_pattern = "[process:command_line = 'wmic.exe process call create calc']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((process = "wmic.exe process call create calc") OR (parent_process = "wmic.exe process call create calc")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((process = "wmic.exe process call create calc") ' \ + f'OR (parent_process = "wmic.exe process call create calc")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_proc_name_query(self): + """test process name native data source query""" stix_pattern = "[process:name = 'wmic.exe']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((process_name = "wmic.exe") OR (parent_process_name = "wmic.exe")) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((process_name = "wmic.exe") OR ' \ + f'(parent_process_name = "wmic.exe")) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_ipv4_query_in_operator(self): + """test ipv4 native data source query""" stix_pattern = "[ipv4-addr:value IN ('192.168.122.83', '192.168.122.84')]" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search ((src_ip IN ("192.168.122.83", "192.168.122.84")) OR (dest_ip IN ("192.168.122.83", "192.168.122.84"))) earliest="-5minutes" | head 10000 | fields {fields}' + queries = f'search ((src_ip IN ("192.168.122.83", "192.168.122.84")) ' \ + f'OR (dest_ip IN ("192.168.122.83", "192.168.122.84"))) ' \ + f'earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_like(self): + """test like operator native data source query""" stix_pattern = "[file:name LIKE 'x_.%']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search earliest="-5minutes" | where (like(file_name, "x_.%")) | head 10000 | fields {fields}' + queries = f'search earliest="-5minutes" | where ((like(file_name, "x_.%")) ' \ + f'OR (like(process_name, "x_.%")) ' \ + f'OR (like(parent_process_name, "x_.%")) ' \ + f'OR (like(process_exec, "x_.%")) ' \ + f'OR (like(parent_process_exec, "x_.%"))) | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_like_or_equal(self): + """test like operator native data source query""" stix_pattern = "[file:name LIKE 'x_.%' OR file:name = 'y1.exe']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search earliest="-5minutes" | where ((file_name = "y1.exe") OR (like(file_name, "x_.%"))) | head 10000 | fields {fields}' + queries = f'search earliest="-5minutes" | where (((file_name = "y1.exe") ' \ + f'OR (process_name = "y1.exe") ' \ + f'OR (parent_process_name = "y1.exe") ' \ + f'OR (process_exec = "y1.exe") ' \ + f'OR (parent_process_exec = "y1.exe")) ' \ + f'OR ((like(file_name, "x_.%")) ' \ + f'OR (like(process_name, "x_.%")) ' \ + f'OR (like(parent_process_name, "x_.%")) ' \ + f'OR (like(process_exec, "x_.%")) ' \ + f'OR (like(parent_process_exec, "x_.%")))) | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_match(self): + """test match operator native data source query""" stix_pattern = r"[file:name MATCHES '^x.\\..*$']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search earliest="-5minutes" | where (match(file_name, "^x.\\..*$")) | head 10000 | fields {fields}' + queries = f'search earliest="-5minutes" | where ((match(file_name, "^x.\\..*$")) ' \ + f'OR (match(process_name, "^x.\\..*$")) ' \ + f'OR (match(parent_process_name, "^x.\\..*$")) ' \ + f'OR (match(process_exec, "^x.\\..*$")) ' \ + f'OR (match(parent_process_exec, "^x.\\..*$"))) | ' \ + f'head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_multiple_comparison_with_like_and_match(self): + """test multiple comparison match and like operator native data source query""" + stix_pattern = r"[file:name MATCHES '^x.\\..*$' OR file:name = 'y1.exe' " \ + r"AND file:name LIKE 'x_.%' ] START t'2022-01-19T11:00:00.000Z' " \ + r"STOP t'2023-02-28T11:00:00.003Z'" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search earliest="01/19/2022:11:00:00" latest="02/28/2023:11:00:00" | ' \ + f'where ((((like(file_name, "x_.%")) ' \ + f'OR (like(process_name, "x_.%")) ' \ + f'OR (like(parent_process_name, "x_.%")) ' \ + f'OR (like(process_exec, "x_.%")) ' \ + f'OR (like(parent_process_exec, "x_.%"))) ' \ + f'AND ((file_name = "y1.exe") OR (process_name = "y1.exe") ' \ + f'OR (parent_process_name = "y1.exe") ' \ + f'OR (process_exec = "y1.exe") ' \ + f'OR (parent_process_exec = "y1.exe"))) ' \ + f'OR ((match(file_name, "^x.\\..*$")) ' \ + f'OR (match(process_name, "^x.\\..*$")) ' \ + f'OR (match(parent_process_name, "^x.\\..*$")) ' \ + f'OR (match(process_exec, "^x.\\..*$")) ' \ + f'OR (match(parent_process_exec, "^x.\\..*$")))) | ' \ + f'head 10000 | fields {fields}' _test_query_assertions(query, queries) def test_match_or_equal(self): + """test multiple comparison match and OR operator native data source query""" stix_pattern = r"[file:name MATCHES '^x.\\..*$' OR file:name = 'y1.exe']" query = translation.translate('splunk', 'query', '{}', stix_pattern) - queries = f'search earliest="-5minutes" | where ((file_name = "y1.exe") OR (match(file_name, "^x.\\..*$"))) | head 10000 | fields {fields}' + queries = f'search earliest="-5minutes" | where (((file_name = "y1.exe") ' \ + f'OR (process_name = "y1.exe") OR (parent_process_name = "y1.exe") ' \ + f'OR (process_exec = "y1.exe") ' \ + f'OR (parent_process_exec = "y1.exe")) ' \ + f'OR ((match(file_name, "^x.\\..*$")) ' \ + f'OR (match(process_name, "^x.\\..*$")) ' \ + f'OR (match(parent_process_name, "^x.\\..*$")) ' \ + f'OR (match(process_exec, "^x.\\..*$")) ' \ + f'OR (match(parent_process_exec, "^x.\\..*$")))) | ' \ + f'head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_severity_informational(self): + """test severity native data source query""" + stix_pattern = r"[x-ibm-finding:severity = 15]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (severity = "informational") earliest="-5minutes" | ' \ + f'head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_severity_low(self): + """test severity native data source query""" + stix_pattern = r"[x-ibm-finding:severity = 25]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (severity = "low") earliest="-5minutes" | head 10000 | fields {fields}' _test_query_assertions(query, queries) + def test_severity_medium(self): + """test severity native data source query""" + stix_pattern = r"[x-ibm-finding:severity = 45]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (severity = "medium") earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_severity_high(self): + """test severity native data source query""" + stix_pattern = r"[x-ibm-finding:severity = 65]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (severity = "high") earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_severity_critical(self): + """test severity native data source query""" + stix_pattern = r"[x-ibm-finding:severity = 85]" + query = translation.translate('splunk', 'query', '{}', stix_pattern) + queries = f'search (severity = "critical") earliest="-5minutes" | head 10000 | fields {fields}' + _test_query_assertions(query, queries) + + def test_out_of_range_severity(self): + """test severity native data source query""" + stix_pattern = r"[x-ibm-finding:severity = 105]" + result = translation.translate('splunk', 'query', '{}', stix_pattern) + assert result['success'] is False + assert 'splunk connector error => wrong parameter : only 1-100 integer ' \ + 'values are supported with severity field' in result['error'] + if __name__ == '__main__': unittest.main() diff --git a/stix_shifter_modules/splunk/tests/stix_transmission/api_response/result_by_sid.json b/stix_shifter_modules/splunk/tests/stix_transmission/api_response/result_by_sid.json index 2d6046b9f..f7da78fd2 100644 --- a/stix_shifter_modules/splunk/tests/stix_transmission/api_response/result_by_sid.json +++ b/stix_shifter_modules/splunk/tests/stix_transmission/api_response/result_by_sid.json @@ -67,7 +67,11 @@ "_bkt": "shifter~3~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "3:101", "_eventtype_color": "none", + "file_name": "svchost.exe", + "file_hash": "922547e866c89b8f677312df0ccec8ee", "_indextime": "1535991010", + "registry_value_name": "Security", + "registry_value_data": "Binary Data", "_raw": "{\n\t\"log_type\": \"network\", \n\t\"bytes\": \"300\", \n\t\"dest_ip\": \"127.0.0.1\", \n\t\"dest_port\": \"80\", \n\t\"src_ip\": \"2001:0db8:85a3:0000:0000:8a2e:0370:7334\", \n\t\"src_port\": \"80\", \n\t\"transport\": \"http\", \n\t\"user\": \"IBM\", \n\t\"event_count\": 1\n}", "_serial": "0", "_si": [ @@ -79,13 +83,114 @@ "bytes": "300", "dest_ip": "127.0.0.1", "dest_port": "80", + "protocol": "ip", + "bytes_in": "442", + "bytes_out": "5073", + "direction": "inbound", + "name": "Key file operation", "event_count": "1", "log_type": "network", "src_ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "src_port": "80", "transport": "http", "user": "IBM" - } + }, + { + "src_ip": "-", + "src_port": "-", + "dest_port": "-", + "dest_ip": "-", + "registry_value_name": "Security", + "registry_value_data": "Binary Data", + "file_name": "svchost.exe", + "file_hash": "addf120b430021c36c232c99ef8d926aea2acd6b", + "user": "SYSTEM", + "host": "EC2AMAZ-AMMDNSQ", + "source": "XmlWinEventLog:Security", + "EventID": "4624", + "process": "C:\\Windows\\System32\\services.exe", + "process_id": "0x338", + "process_name": "C:\\Windows\\System32\\services.exe", + "process_path": "C:\\Windows\\System32\\services.exe", + "signature": "An account was successfully logged on", + "signature_id": "4624", + "name": "An account was successfully logged on", + "vendor_product": "Microsoft Windows", + "action": "success", + "_bkt": "main~63~FF7D676A-5C1A-492E-9A6D-4653BFD400A6", + "_cd": "63:13139", + "_indextime": "1681210813", + "_raw": "4624201254400x8020000000000000137635SecurityEC2AMAZ-AMMDNSQNT AUTHORITY\\SYSTEMEC2AMAZ-AMMDNSQ$WORKGROUP0x3e7NT AUTHORITY\\SYSTEMSYSTEMNT AUTHORITY0x3e75Advapi Negotiate-{00000000-0000-0000-0000-000000000000}--00x338C:\\Windows\\System32\\services.exe--%%1833---%%18430x0%%1842", + "_serial": "0", + "_si": ["EC2AMAZ-AMMDNSQ", "main"], + "_sourcetype": "XmlWinEventLog", + "_time": "2023-04-11T10:59:26.000+00:00" + }, + { + "src_ip": "172.67.87.4", + "src_port": "765", + "dest_port": "654", + "dest_ip": "172.67.87.6", + "packets_in": 60, + "packets_out": 80, + "query_count": 1, + "reply_code_id": 9003, + "http_method": "GET", + "http_referrer": 5502, + "http_user_agent": 0, + "status": 0, + "uri_path": 80, + "uri_query": "status", + "registry_value_name": "Security", + "registry_value_data": "Binary Data", + "message_type": "abcdd", + "query_type": "abbdbf", + "record_type": "internal", + "reply_code": "12rd", + "file_name": "svchost.exe", + "file_hash": "SHA1=C1DC24E3E12E3B365D9ED916B408849453F65332,MD5=F15289CC62D3D2F6659792161646AE95,SHA256=D1EDBCAF690D605EB7B3517FF860A93A1247EEE7DE43BE30E505199D7DFFF288,IMPHASH=AE9F3216A47DDC6E9BD93CD2E9FF5C45", + "user": "Administrator", + "host": "EC2AMAZ-O4KCUK3", + "source": "XmlWinEventLog:Microsoft-Windows-Sysmon/Operational", + "EventID": "26", + "process_id": "1752", + "process_name": "cleanmgr.exe", + "process_exec": "cleanmgr.exe", + "process_path": "C:\\Windows\\system32\\cleanmgr.exe", + "process_guid": "{7a97b85f-ee4a-63ec-d907-000000007e00}", + "signature": "FileDeleteDetected (File Delete logged)", + "signature_id": "26", + "vendor_product": "Microsoft Sysmon", + "action": "deleted", + "_bkt": "main~9~FF7D676A-5C1A-492E-9A6D-4653BFD400A6", + "_cd": "9:217242", + "_indextime": "1676471935", + "_raw": "26542600x800000000000000013715Microsoft-Windows-Sysmon/OperationalEC2AMAZ-O4KCUK3-2023-02-15 14:38:46.058{7a97b85f-ee4a-63ec-d907-000000007e00}1752EC2AMAZ-O4KCUK3\\AdministratorC:\\Windows\\system32\\cleanmgr.exeC:\\Users\\Administrator\\AppData\\Local\\Temp\\412FA113-329C-401A-927B-EA6C4C2F6B1F\\TransmogProvider.dllSHA1=C1DC24E3E12E3B365D9ED916B408849453F65332,MD5=F15289CC62D3D2F6659792161646AE95,SHA256=D1EDBCAF690D605EB7B3517FF860A93A1247EEE7DE43BE30E505199D7DFFF288,IMPHASH=AE9F3216A47DDC6E9BD93CD2E9FF5C45true", + "_serial": "5", + "_si": ["EC2AMAZ-AMMDNSQ", "main"], + "_sourcetype": "XmlWinEventLog", + "_time": "2023-02-15T14:38:46.000+00:00" + }, + { + "file_name": "svchost.exe", + "file_hash": "a364cf9a45f65a5114c47d716d4f88954e4998a6097144de662b9493db791a29", + "host": "EC2AMAZ-AMMDNSQ", + "registry_value_name": "Security", + "registry_value_data": "Binary Data", + "source": "pp-message4.log", + "process": "agent", + "vendor_product": "Proofpoint Inc", + "duration": "0.008932", + "_bkt": "main~40~FF7D676A-5C1A-492E-9A6D-4653BFD400A6", + "_cd": "40:21125", + "_indextime": "1678465846", + "_raw": "26542600x800000000000000013718Microsoft-Windows-Sysmon/OperationalEC2AMAZ-O4KCUK3-2023-02-15 14:38:46.108{7a97b85f-ee4a-63ec-d907-000000007e00}1752EC2AMAZ-O4KCUK3\\AdministratorC:\\Windows\\system32\\cleanmgr.exeC:\\Users\\Administrator\\AppData\\Local\\Temp\\412FA113-329C-401A-927B-EA6C4C2F6B1F\\WimProvider.dllSHA1=0C8EF379305A61D52BDDDB9188D23B3DE68C29A8,MD5=944CC6B1A962000BC604963F2345EF79,SHA256=5D7BB0C11210F397B489E8F75011673AFBAA6B678423D7CED3708603C6017ECD,IMPHASH=2712C5FEFC99EF050B68C1EBE951D48Dtrue", + "_serial": "0", + "_si": ["EC2AMAZ-AMMDNSQ", "main"], + "_sourcetype": "pps_messagelog", + "_subsecond": ".000", + "_time": "2023-03-08T09:50:19.000+00:00" + } ], "highlighted": {} } \ No newline at end of file diff --git a/stix_shifter_modules/splunk/tests/stix_transmission/test_class.py b/stix_shifter_modules/splunk/tests/stix_transmission/test_class.py index d35dc2c18..475d3f364 100644 --- a/stix_shifter_modules/splunk/tests/stix_transmission/test_class.py +++ b/stix_shifter_modules/splunk/tests/stix_transmission/test_class.py @@ -1,22 +1,26 @@ -from stix_shifter_modules.splunk.entry_point import EntryPoint -from unittest.mock import patch import unittest import os +from unittest.mock import patch +from aiohttp.client_exceptions import ClientConnectionError +from stix_shifter_modules.splunk.entry_point import EntryPoint from stix_shifter.stix_transmission import stix_transmission from stix_shifter.stix_transmission.stix_transmission import run_in_thread -from tests.utils.async_utils import get_mock_response from stix_shifter_utils.utils.error_response import ErrorCode +from tests.utils.async_utils import get_mock_response +from asyncio.exceptions import TimeoutError class TestSplunkConnection(unittest.TestCase, object): + """ class for test splunk connection""" + def test_is_async(self): - + """check for synchronous or asynchronous""" config = { "auth": { "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 @@ -29,21 +33,22 @@ def test_is_async(self): @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.ping_box') def test_ping_endpoint(self, mock_ping_response): + """test to check ping_data_source function""" mocked_return_value = '["mock", "placeholder"]' mock_ping_response.return_value = get_mock_response(200, mocked_return_value) - + config = { "auth": { "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 } - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) ping_response = transmission.ping() assert ping_response is not None @@ -51,6 +56,7 @@ def test_ping_endpoint(self, mock_ping_response): @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.ping_box') def test_ping_endpoint_exception(self, mock_ping_response): + """test to check ping_data exception""" # mocked_return_value = '["mock", "placeholder"]' # mock_ping_response.return_value = get_mock_response(200, mocked_return_value) mock_ping_response.side_effect = Exception('exception') @@ -65,7 +71,7 @@ def test_ping_endpoint_exception(self, mock_ping_response): "port": 8080 } - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) ping_response = transmission.ping() assert ping_response is not None @@ -74,6 +80,7 @@ def test_ping_endpoint_exception(self, mock_ping_response): @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.create_search') def test_query_response(self, mock_query_response): + """test to check query response""" mocked_return_value = '{"sid":"1536672851.4012"}' mock_query_response.return_value = get_mock_response(201, mocked_return_value) @@ -82,14 +89,14 @@ def test_query_response(self, mock_query_response): "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 } query = 'search eventtype=network_traffic | fields + tag| spath' - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) query_response = transmission.query(query) assert query_response is not None @@ -99,6 +106,7 @@ def test_query_response(self, mock_query_response): @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.create_search') def test_query_response_exception(self, mock_query_response): + """test to check query response exception""" # mocked_return_value = '{"sid":"1536672851.4012"}' # mock_query_response.return_value = get_mock_response(201, mocked_return_value) mock_query_response.side_effect = Exception('exception') @@ -116,19 +124,21 @@ def test_query_response_exception(self, mock_query_response): query = 'search eventtype=network_traffic | fields + tag| spath' - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) query_response = transmission.query(query) assert query_response is not None assert query_response['success'] is False assert query_response['code'] == ErrorCode.TRANSMISSION_UNKNOWN.value - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', + autospec=True) def test_status_response(self, mock_status_response): - + """test to check status response""" dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'status_by_sid.json') - mocked_return_value = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as file_obj: + mocked_return_value = file_obj.read() mock_status_response.return_value = get_mock_response(200, mocked_return_value) @@ -137,7 +147,7 @@ def test_status_response(self, mock_status_response): "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 @@ -155,12 +165,14 @@ def test_status_response(self, mock_status_response): assert 'success' in status_response assert status_response['success'] is True - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', + autospec=True) def test_status_response_error(self, mock_status_response): - + """test to check status exception""" dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'status_by_sid_failed.json') - mocked_return_value = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as file_obj: + mocked_return_value = file_obj.read() mock_status_response.return_value = get_mock_response(200, mocked_return_value) @@ -187,12 +199,14 @@ def test_status_response_error(self, mock_status_response): assert 'success' in status_response assert status_response['success'] is True - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', + autospec=True) def test_status_response_running(self, mock_status_response): - + """test to check if status response running""" dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'status_by_sid_running.json') - mocked_return_value = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as file_obj: + mocked_return_value = file_obj.read() mock_status_response.return_value = get_mock_response(200, mocked_return_value) @@ -219,12 +233,14 @@ def test_status_response_running(self, mock_status_response): assert 'success' in status_response assert status_response['success'] is True - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', + autospec=True) def test_status_response_cancelled(self, mock_status_response): - + """test to check if status response cancelled""" dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'status_by_sid_running_cancel.json') - mocked_return_value = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as file_obj: + mocked_return_value = file_obj.read() mock_status_response.return_value = get_mock_response(200, mocked_return_value) @@ -251,9 +267,10 @@ def test_status_response_cancelled(self, mock_status_response): assert 'success' in status_response assert status_response['success'] is True - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', + autospec=True) def test_status_response_exception(self, mock_status_response): - + """test to check status response exception""" # dir_path = os.path.dirname(os.path.realpath(__file__)) # file_path = os.path.join(dir_path, 'api_response', 'status_by_sid.json') # mocked_return_value = open(file_path, 'r').read() @@ -274,19 +291,21 @@ def test_status_response_exception(self, mock_status_response): search_id = "1536832140.4293" - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) status_response = transmission.status(search_id) assert status_response is not None assert status_response['success'] is False - assert ErrorCode.TRANSMISSION_UNKNOWN.value==status_response['code'] + assert ErrorCode.TRANSMISSION_UNKNOWN.value == status_response['code'] - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search_results', + autospec=True) def test_results_response(self, mock_results_response): - + """test to check result response""" dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'result_by_sid.json') - mocked_return_value = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as file_obj: + mocked_return_value = file_obj.read() mock_results_response.return_value = get_mock_response(200, mocked_return_value) @@ -295,17 +314,17 @@ def test_results_response(self, mock_results_response): "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 } - + search_id = "1536832140.4293" offset = 0 length = 1 - - transmission = stix_transmission.StixTransmission('splunk', connection, config) + + transmission = stix_transmission.StixTransmission('splunk', connection, config) results_response = transmission.results(search_id, offset, length) assert 'success' in results_response @@ -316,10 +335,11 @@ def test_results_response(self, mock_results_response): @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search_results', autospec=True) def test_results_response_empty_list(self, mock_results_response): - + """test to check empty result response""" dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'empty_result_by_sid.json') - mocked_return_value = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as file_obj: + mocked_return_value = file_obj.read() mock_results_response.return_value = get_mock_response(200, mocked_return_value) @@ -348,7 +368,7 @@ def test_results_response_empty_list(self, mock_results_response): @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search_results', autospec=True) def test_results_response_exception(self, mock_results_response): - + """test to result response exception""" # dir_path = os.path.dirname(os.path.realpath(__file__)) # file_path = os.path.join(dir_path, 'api_response', 'result_by_sid.json') # mocked_return_value = open(file_path, 'r').read() @@ -370,24 +390,27 @@ def test_results_response_exception(self, mock_results_response): search_id = "1536832140.4293" offset = 0 length = 1 - - transmission = stix_transmission.StixTransmission('splunk', connection, config) + + transmission = stix_transmission.StixTransmission('splunk', connection, config) results_response = transmission.results(search_id, offset, length) assert 'success' in results_response assert results_response['success'] is False assert results_response['code'] == ErrorCode.TRANSMISSION_UNKNOWN.value - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.create_search', autospec=True) - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', autospec=True) - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.create_search', + autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search', + autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.get_search_results', + autospec=True) def test_query_flow(self, mock_results_response, mock_status_response, mock_query_response): - + """test to check query flow""" config = { "auth": { "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 @@ -395,14 +418,17 @@ def test_query_flow(self, mock_results_response, mock_status_response, mock_quer query_mock = '{"sid":"1536832140.4293"}' mock_query_response.return_value = get_mock_response(201, query_mock) - + dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, 'api_response', 'result_by_sid.json') - results_mock = open(file_path, 'r').read() + with open(file_path, 'r') as file_obj: + results_mock = file_obj.read() mock_results_response.return_value = get_mock_response(200, results_mock) - + status_file_path = os.path.join(dir_path, 'api_response', 'status_by_sid.json') - status_mock = open(status_file_path, 'r').read() + with open(status_file_path, 'r', encoding='utf-8') as file_obj: + status_mock = file_obj.read() + mock_status_response.return_value = get_mock_response(200, status_mock) query = 'search eventtype=network_traffic | fields + tag| spath' @@ -435,33 +461,35 @@ def test_query_flow(self, mock_results_response, mock_status_response, mock_quer assert 'data' in results_response assert len(results_response['data']) > 0 - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.delete_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.delete_search', + autospec=True) def test_delete_search(self, mock_results_delete): - + """test to check delete search""" config = { "auth": { "username": "", "password": "" } - } + } connection = { "host": "host", "port": 8080 } - + mocked_return_value = '{"messages":[{"type":"INFO","text":"Search job cancelled."}]}' mock_results_delete.return_value = get_mock_response(200, mocked_return_value) search_id = "1536832140.4293" - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) results_response = transmission.delete(search_id) - + assert results_response is not None assert results_response['success'] is True - @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.delete_search', autospec=True) + @patch('stix_shifter_modules.splunk.stix_transmission.api_client.APIClient.delete_search', + autospec=True) def test_delete_search_exception(self, mock_results_delete): - + """test to check delete search exception""" config = { "auth": { "username": "", @@ -473,12 +501,284 @@ def test_delete_search_exception(self, mock_results_delete): "port": 8080 } - - mocked_return_value = '{"messages":[{"type":"INFO","text":"Unknown sid."}]}' - mock_results_delete.return_value = get_mock_response(201, mocked_return_value) + mocked_return_value = '{"type":"Unknown_sid","messages":"Unknown sid"}' + mock_results_delete.return_value = get_mock_response(404, mocked_return_value) search_id = "1536832140.4293" - transmission = stix_transmission.StixTransmission('splunk', connection, config) + transmission = stix_transmission.StixTransmission('splunk', connection, config) results_response = transmission.delete(search_id) assert results_response is not None assert results_response['success'] is False assert results_response['code'] == ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS.value + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_host_result(self, mock_result): + """Test Invalid host for transmit result""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_result.side_effect = ClientConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + + search_id = "test" + + offset = 0 + length = 1 + results_response = transmission.results(search_id, offset, length) + + assert results_response is not None + assert results_response['success'] is False + assert results_response['error'] is not None + assert "Invalid Host" in results_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_host_status(self, mock_ping): + """Test Invalid host for transmit status""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_ping.side_effect = ClientConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + search_id = "test" + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert status_response['error'] is not None + assert "Invalid Host" in status_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_host_ping(self, mock_ping): + """Test Invalid host for ping connection""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_ping.side_effect = ClientConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['error'] is not None + assert "Invalid Host" in ping_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_quota_invalid_json_exception(self, mock_quota_response): + """test invalid json exception""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mocked_return_value = "invalid json" + mock_quota_response.return_value = get_mock_response(200, mocked_return_value) + transmission = stix_transmission.StixTransmission('splunk', connection, config) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_timeout_error_in_ping(self, mock_ping): + """test connection/timeout error in transmit ping""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_ping.side_effect = TimeoutError("TimeoutError") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['error'] is not None + assert "TimeoutError" in ping_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_timeout_error_in_status(self, mock_ping): + """test connection/timeout error in transmit status""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_ping.side_effect = TimeoutError("TimeoutError") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + search_id = "test" + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert status_response['error'] is not None + assert "TimeoutError" in status_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_timeout_error_in_result(self, mock_result): + """test connection/timeout error in transmit result""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_result.side_effect = TimeoutError("TimeoutError") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + + search_id = "test" + offset = 0 + length = 1 + results_response = transmission.results(search_id, offset, length) + + assert results_response is not None + assert results_response['success'] is False + assert results_response['error'] is not None + assert "TimeoutError" in results_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_timeout_error_in_delete(self, mock_delete): + """test connection/timeout error in delete module in transmit""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_delete.side_effect = TimeoutError("TimeoutError") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + search_id = "test" + delete_response = transmission.delete(search_id) + + assert delete_response is not None + assert delete_response['success'] is False + assert delete_response['error'] is not None + assert "TimeoutError" in delete_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_timeout_error_in_query(self, mock_query): + """test connection/timeout error in transmit query""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_query.side_effect = TimeoutError("TimeoutError") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + + query = "test" + results_response = transmission.query(query) + assert results_response is not None + assert results_response['success'] is False + assert results_response['error'] is not None + assert "TimeoutError" in results_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_host_query(self, mock_query): + """Test Invalid host for transmit query""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_query.side_effect = ClientConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + query = "test" + query_response = transmission.query(query) + assert query_response is not None + assert query_response['success'] is False + assert query_response['error'] is not None + assert "Invalid Host" in query_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_host_delete(self, mock_delete): + """Test Invalid host for delete module in transmit""" + + config = { + "auth": { + "username": "", + "password": "" + } + } + connection = { + "host": "host", + "port": 8080 + } + + mock_delete.side_effect = ClientConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('splunk', connection, config) + search_id = "test" + delete_response = transmission.delete(search_id) + + assert delete_response is not None + assert delete_response['success'] is False + assert delete_response['error'] is not None + assert "Invalid Host" in delete_response['error']