diff --git a/stix_shifter_modules/azure_log_analytics/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/azure_log_analytics/stix_translation/json/stix_2_1/to_stix_map.json index 47f56ab91..3b0ffc533 100644 --- a/stix_shifter_modules/azure_log_analytics/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/azure_log_analytics/stix_translation/json/stix_2_1/to_stix_map.json @@ -47,10 +47,17 @@ "transformer": "ConvertToReal" } ], - "EndTime": { - "key": "x-ibm-finding.end", - "object": "finding" - }, + "EndTime": [ + { + "key": "x-ibm-finding.end", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "last_observed", + "transformer": "TimestampConversion" + } + ], "ExtendedProperties": { "resourceType": { "key": "x-cloud-resource.resource_type", @@ -59,7 +66,8 @@ }, "ProcessingEndTime": { "key": "x-ibm-finding.x_processing_endtime", - "object": "finding" + "object": "finding", + "transformer": "TimestampConversion" }, "ProductComponentName": { "key": "software.x_product_component_name", @@ -88,10 +96,17 @@ "key": "x-cloud-resource.resource_id", "object": "cloud_resource" }, - "StartTime": { - "key": "x-ibm-finding.start", - "object": "finding" - }, + "StartTime": [ + { + "key": "x-ibm-finding.start", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "first_observed", + "transformer": "TimestampConversion" + } + ], "Status": { "key": "x-ibm-finding.x_status", "object": "finding" @@ -118,7 +133,8 @@ }, "TimeGenerated": { "key": "x-ibm-finding.time_observed", - "object": "finding" + "object": "finding", + "transformer": "TimestampConversion" }, "Type": { "key": "x-ibm-finding.finding_type", @@ -165,27 +181,37 @@ }, "ClosedTime": { "key": "x-incident-info.closed_time", - "object": "incident" + "object": "incident", + "transformer": "TimestampConversion" }, "Comments": { "key": "x-incident-info.comments", "object": "incident" }, - "CreatedTime": { - "key": "x-ibm-finding.start", - "object": "finding" - }, + "CreatedTime": [ + { + "key": "x-ibm-finding.start", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "first_observed", + "transformer": "TimestampConversion" + } + ], "Description": { "key": "x-ibm-finding.description", "object": "finding" }, "FirstActivityTime": { "key": "x-incident-info.first_activity", - "object": "incident" + "object": "incident", + "transformer": "TimestampConversion" }, "FirstModifiedTime": { "key": "x-incident-info.first_modified", - "object": "incident" + "object": "incident", + "transformer": "TimestampConversion" }, "IncidentName": { "key": "x-ibm-finding.x_incident_name", @@ -216,12 +242,21 @@ }, "LastActivityTime": { "key": "x-incident-info.last_active", - "object": "incident" - }, - "LastModifiedTime": { - "key": "x-ibm-finding.end", - "object": "finding" + "object": "incident", + "transformer": "TimestampConversion" }, + "LastModifiedTime": [ + { + "key": "x-ibm-finding.end", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "last_observed", + "transformer": "TimestampConversion" + } + + ], "ModifiedBy": { "key": "x-ibm-finding.x_modified_by", "object": "finding" @@ -280,10 +315,17 @@ "key": "x-ibm-finding.alert_id", "object": "finding" }, - "PreviousTime": { - "key": "x-ibm-finding.start", - "object": "finding" - }, + "PreviousTime": [ + { + "key": "x-ibm-finding.start", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "first_observed", + "transformer": "TimestampConversion" + } + ], "EventID": { "key": "x-oca-event.code", "object": "event" @@ -708,7 +750,8 @@ }, "CreationTimeUtc": { "key": "process.created", - "object": "process1" + "object": "process1", + "transformer": "TimestampConversion" }, "ElevationToken": { "key": "process.x_elevation_token", @@ -733,7 +776,8 @@ }, "CreationTimeUtc": { "key": "process.created", - "object": "parent_process" + "object": "parent_process", + "transformer": "TimestampConversion" }, "ElevationToken": { "key": "process.x_elevation_token", @@ -958,11 +1002,13 @@ }, "StartTimeUtc": { "key": "x-host-logon-session.start_time", - "object": "logon_session" + "object": "logon_session", + "transformer": "TimestampConversion" }, "EndTimeUtc": { "key": "x-host-logon-session.end_time", - "object": "logon_session" + "object": "logon_session", + "transformer": "TimestampConversion" } }, "file": { @@ -1054,11 +1100,13 @@ }, "Longitude": { "key": "x-geo-location.longitude", - "object": "location" + "object": "location", + "transformer": "ToFloat" }, "Latitude": { "key": "x-geo-location.latitude", - "object": "location" + "object": "location", + "transformer": "ToFloat" }, "Organization": { "key": "x-geo-location.organization", diff --git a/stix_shifter_modules/azure_log_analytics/stix_translation/json/to_stix_map.json b/stix_shifter_modules/azure_log_analytics/stix_translation/json/to_stix_map.json index f0d1499b9..4a0c63c1d 100644 --- a/stix_shifter_modules/azure_log_analytics/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/azure_log_analytics/stix_translation/json/to_stix_map.json @@ -47,10 +47,17 @@ "transformer": "ConvertToReal" } ], - "EndTime": { - "key": "x-ibm-finding.end", - "object": "finding" - }, + "EndTime": [ + { + "key": "x-ibm-finding.end", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "last_observed", + "transformer": "TimestampConversion" + } + ], "ExtendedProperties": { "resourceType": { "key": "x-cloud-resource.resource_type", @@ -59,7 +66,8 @@ }, "ProcessingEndTime": { "key": "x-ibm-finding.x_processing_endtime", - "object": "finding" + "object": "finding", + "transformer": "TimestampConversion" }, "ProductComponentName": { "key": "software.x_product_component_name", @@ -88,10 +96,17 @@ "key": "x-cloud-resource.resource_id", "object": "cloud_resource" }, - "StartTime": { - "key": "x-ibm-finding.start", - "object": "finding" - }, + "StartTime": [ + { + "key": "x-ibm-finding.start", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "first_observed", + "transformer": "TimestampConversion" + } + ], "Status": { "key": "x-ibm-finding.x_status", "object": "finding" @@ -118,7 +133,8 @@ }, "TimeGenerated": { "key": "x-ibm-finding.time_observed", - "object": "finding" + "object": "finding", + "transformer": "TimestampConversion" }, "Type": { "key": "x-ibm-finding.finding_type", @@ -165,27 +181,37 @@ }, "ClosedTime": { "key": "x-incident-info.closed_time", - "object": "incident" + "object": "incident", + "transformer": "TimestampConversion" }, "Comments": { "key": "x-incident-info.comments", "object": "incident" }, - "CreatedTime": { - "key": "x-ibm-finding.start", - "object": "finding" - }, + "CreatedTime": [ + { + "key": "x-ibm-finding.start", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "first_observed", + "transformer": "TimestampConversion" + } + ], "Description": { "key": "x-ibm-finding.description", "object": "finding" }, "FirstActivityTime": { "key": "x-incident-info.first_activity", - "object": "incident" + "object": "incident", + "transformer": "TimestampConversion" }, "FirstModifiedTime": { "key": "x-incident-info.first_modified", - "object": "incident" + "object": "incident", + "transformer": "TimestampConversion" }, "IncidentName": { "key": "x-ibm-finding.x_incident_name", @@ -216,12 +242,21 @@ }, "LastActivityTime": { "key": "x-incident-info.last_active", - "object": "incident" - }, - "LastModifiedTime": { - "key": "x-ibm-finding.end", - "object": "finding" + "object": "incident", + "transformer": "TimestampConversion" }, + "LastModifiedTime": [ + { + "key": "x-ibm-finding.end", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "last_observed", + "transformer": "TimestampConversion" + } + + ], "ModifiedBy": { "key": "x-ibm-finding.x_modified_by", "object": "finding" @@ -280,10 +315,17 @@ "key": "x-ibm-finding.alert_id", "object": "finding" }, - "PreviousTime": { - "key": "x-ibm-finding.start", - "object": "finding" - }, + "PreviousTime": [ + { + "key": "x-ibm-finding.start", + "object": "finding", + "transformer": "TimestampConversion" + }, + { + "key": "first_observed", + "transformer": "TimestampConversion" + } + ], "EventID": { "key": "x-oca-event.code", "object": "event" @@ -708,7 +750,8 @@ }, "CreationTimeUtc": { "key": "process.created", - "object": "process1" + "object": "process1", + "transformer": "TimestampConversion" }, "ElevationToken": { "key": "process.x_elevation_token", @@ -733,7 +776,8 @@ }, "CreationTimeUtc": { "key": "process.created", - "object": "parent_process" + "object": "parent_process", + "transformer": "TimestampConversion" }, "ElevationToken": { "key": "process.x_elevation_token", @@ -958,11 +1002,13 @@ }, "StartTimeUtc": { "key": "x-host-logon-session.start_time", - "object": "logon_session" + "object": "logon_session", + "transformer": "TimestampConversion" }, "EndTimeUtc": { "key": "x-host-logon-session.end_time", - "object": "logon_session" + "object": "logon_session", + "transformer": "TimestampConversion" } }, "file": { @@ -1053,13 +1099,15 @@ "object": "location" }, "Longitude": { - "key": "x-geo-location.longitude", - "object": "location" - }, + "key": "x-geo-location.longitude", + "object": "location", + "transformer": "ToFloat" + }, "Latitude": { - "key": "x-geo-location.latitude", - "object": "location" - }, + "key": "x-geo-location.latitude", + "object": "location", + "transformer": "ToFloat" + }, "Organization": { "key": "x-geo-location.organization", "object": "location" diff --git a/stix_shifter_modules/azure_log_analytics/stix_translation/transformers.py b/stix_shifter_modules/azure_log_analytics/stix_translation/transformers.py index 86627c951..b7d777cb9 100644 --- a/stix_shifter_modules/azure_log_analytics/stix_translation/transformers.py +++ b/stix_shifter_modules/azure_log_analytics/stix_translation/transformers.py @@ -25,7 +25,7 @@ class RealToNumber(ValueTransformer): @staticmethod def transform(obj): try: - if obj == "None": + if obj in ("None", "nan", "NaN"): obj = 0 else: obj = float(obj)*100 @@ -67,9 +67,14 @@ class TimestampConversion(ValueTransformer): @staticmethod def transform(timestamp): try: + # with milliseconds if re.search(r"\d{4}(-\d{2}){2} \d{2}(:\d{2}){2}.\d{0,6}\+\d{2}:\d{2}", str(timestamp)): converted_date = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f%z") timestamp = datetime.strftime(converted_date, "%Y-%m-%dT%H:%M:%S.%fZ") + # for without milliseconds, setting three zeroes in the millisecond in the converted date + elif re.search(r"\d{4}(-\d{2}){2} \d{2}(:\d{2}){2}\+\d{2}:\d{2}", str(timestamp)): + converted_date = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S%z") + timestamp = datetime.strftime(converted_date, "%Y-%m-%dT%H:%M:%S.%f")[:-3]+'Z' except: LOGGER.error('Cannot convert this timestamp %s', timestamp) return timestamp @@ -86,3 +91,15 @@ def transform(obj): except ValueError: LOGGER.error('Cannot convert input %s to severity score', obj) return obj + + +class ToFloat(ValueTransformer): + """ Transform the int value to float """ + @staticmethod + def transform(obj): + try: + if isinstance(obj, int): + obj = float(obj) + return obj + except ValueError: + LOGGER.error("Cannot convert input {} to float".format(obj)) diff --git a/stix_shifter_modules/azure_log_analytics/tests/stix_translation/test_azure_log_analytics_json_to_stix.py b/stix_shifter_modules/azure_log_analytics/tests/stix_translation/test_azure_log_analytics_json_to_stix.py index 580bcbc82..0cf0f5bee 100644 --- a/stix_shifter_modules/azure_log_analytics/tests/stix_translation/test_azure_log_analytics_json_to_stix.py +++ b/stix_shifter_modules/azure_log_analytics/tests/stix_translation/test_azure_log_analytics_json_to_stix.py @@ -35,7 +35,7 @@ 'ConfidenceScore': '1.0', 'IsIncident': 'False', 'StartTime': '2023-03-22 07:27:09.489644+00:00', - 'EndTime': '2023-03-22 07:27:09.489644+00:00', + 'EndTime': '2023-03-22 07:27:09+00:00', 'ProcessingEndTime': '2023-03-22 07:27:51.489644+00:00', 'RemediationSteps': 'Review with Sample-account the suspicious command process and command line to confirm ' 'that this is legitimate activity that you expect to see on Sample-VM. If not, ' @@ -238,7 +238,33 @@ 'Value': 'Sample-SHA', 'Type': 'filehash', 'SHA256': 'Sample-SHA' - } + }, + 'network-connection': [ + { + '$id': '11', + 'SourceAddress': { + '$id': '10', + 'Address': '11.111.111.111', + 'Location': { + 'CountryCode': 'RU', + 'CountryName': 'Russian Federation', + 'State': 'Bryanskaya Oblast', + 'City': 'Bryansk', + 'Longitude': 34, + 'Latitude': 53.2875, + 'Asn': 55555, + 'Carrier': 'Chang Way Technologies Co. Limited' + }, + 'Asset': 'False', + 'Type': 'ip' + }, + 'DestinationPort': 3333, + 'Protocol': 'Tcp', + 'FriendlyName': '11.111.111.111 -> 3333', + 'Asset': 'False', + 'Type': 'network-connection' + } + ] }, 'SourceSystem': 'Detection', 'WorkspaceSubscriptionId': '00000000-0000-0000-0000-000000000001', @@ -827,3 +853,20 @@ def test_unmapped_attribute_alone(): assert 'objects' in observed_data objects = observed_data['objects'] assert objects == {} + + @staticmethod + def test_x_geo_location_property(): + """ test the x-geo-location object properties """ + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [DATA1], get_module_transformers(MODULE), options) + assert result_bundle['type'] == 'bundle' + result_bundle_objects = result_bundle['objects'] + observed_data = result_bundle_objects[1] + assert 'objects' in observed_data + objects = observed_data['objects'] + x_geo_location = TestLogAnalyticsResultsToStix.get_first_of_type(objects.values(), 'x-geo-location') + assert x_geo_location['longitude'] == 34.0 + assert type(x_geo_location['latitude']) == float + assert observed_data['first_observed'] == '2023-03-22T07:27:09.489644Z' + assert observed_data['last_observed'] == '2023-03-22T07:27:09.000Z' +