Implementing grouped queries #69

Transition all query related API calls to APIv2 #71 Fix the filter property setter issue #https://bugs.python.org/issue14965 Implement and test watchlist remove values
mfesiem · Sep 22, 2020 · 006c374 · 006c374
1 parent df5e69d
commit 006c374
Show file tree

Hide file tree

Showing 10 changed files with 425 additions and 164 deletions.
diff --git a/msiempy/__init__.py b/msiempy/__init__.py
@@ -245,10 +245,13 @@
     DataSource
 )
 from .event import (
+    QueryExecuteManager,
     Event,
     EventManager,
     GroupFilter,
-    FieldFilter
+    FieldFilter,
+    GroupedEventManager,
+    GroupedEvent,
 )
 from .watchlist import (
     Watchlist,

diff --git a/msiempy/__version__.py b/msiempy/__version__.py
@@ -1,4 +1,4 @@
 """
 Project version.
 """
-__version__='0.3.2'
+__version__='0.3.3'
diff --git a/msiempy/alarm.py b/msiempy/alarm.py
@@ -31,12 +31,14 @@ class AlarmManager(FilteredQueryList):
     **Unlike `EventManager`, filters are computed after the data loaded with regex matching.**  
     
     """
-    def __init__(self, *args, status_filter='all', page_size=200, filters=None, event_filters=None, **kwargs):
+    def __init__(self, *args, status_filter='all', page_size=200, event_filters=None, **kwargs):
+
+        #Declaring attributes before calling super() because it would overwrite values 
+        self._alarm_filters = []
+        self._event_filters = []
+
         super().__init__(*args, **kwargs)
 
-        #Declaring attributes
-        self._alarm_filters = list(tuple())
-        self._event_filters = list(tuple())
         self._status_filter = str()
 
         #Setting attributes
@@ -53,16 +55,16 @@ def __init__(self, *args, status_filter='all', page_size=200, filters=None, even
         #uses the parent filter setter
         #TODO : find a soltuion not to use this
         #calling super().filters=filters #https://bugs.python.org/issue14965
-        super(self.__class__, self.__class__).filters.__set__(self, filters)
+        # super(self.__class__, self.__class__).filters.__set__(self, filters)
 
         #Seeting events filters after alarms filters cause it would overwrite it
         self.event_filters=event_filters
 
         #Casting all data to Alarms objects, better way to do it ?
         collections.UserList.__init__(self, [Alarm(adict=item) for item in self.data if isinstance(item, (dict, NitroDict))])
 
-    @property
-    def filters(self):
+
+    def _get_filters(self):
         """
         The alarm related filters
         """
@@ -162,8 +164,8 @@ def clear_filters(self):
         """
         Reset local alarm and event filters.
         """
-        self._alarm_filters = list(tuple())
-        self._event_filters = list(tuple())
+        self._alarm_filters = []
+        self._event_filters = []
 
     def load_data(self, pages=1, **kwargs):
         """

diff --git a/msiempy/core/query.py b/msiempy/core/query.py
@@ -3,6 +3,7 @@
 Base query class with timerange handling.  
 """
 
+from abc import abstractproperty
 import logging
 import abc
 import datetime
@@ -49,8 +50,7 @@ def __init__(self, *arg, time_range=None, start_time=None, end_time=None, filter
         self._start_time=None
         self._end_time=None
 
-        #self.filters=filters filter property setter should be called in the concrete class
-        #TODO find a better solution to integrate the filter propertie
+        self.filters=filters # filter property setter
 
         if start_time != None and end_time != None :
             self.start_time=start_time
@@ -129,7 +129,7 @@ def start_time(self, start_time):
         elif isinstance(start_time, datetime.datetime):
             self._start_time = start_time
         elif start_time==None:
-             self._start_time=None #raise ValueError("Time must be string or datetime object, not None")#self.start_time = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
+             self._start_time=None
         else:
             raise ValueError("Time must be string or datetime object.")
 
@@ -150,24 +150,27 @@ def end_time(self, end_time):
         elif isinstance(end_time, datetime.datetime):
             self._end_time = end_time
         elif end_time==None:
-             self._end_time=None #raise ValueError("Time must be string or datetime object, not None")
+             self._end_time=None
         else:
             raise ValueError("Time must be string or datetime object.")
-
-    @abc.abstractproperty
+    
+    @property
     def filters(self):
         """ 
-        Filter property : Returns a list of filters.
+        Filter property : Returns a list of filters.  
         Can be set with list of tuple(field, [values]), a `msiempy.event.FieldFilter` or `msiempy.event.GroupFilter` in the case of a `msiempy.event.EventManager` query. A single tuple is also accepted.  
-        `None` value will trigger `msiempy.core.query.FilteredQueryList.clear_filters()`.  
-        Raises : `AttributeError` if type not supported.
+        Filters will always be added to the list, use `clear_filters()` to remove all filters from a query.   
+        Raises : `AttributeError` if type not supported.  
         Abstract declaration.
-        TODO find a better solution to integrate the filter property
         """
-        pass
-
+        return self._get_filters()
+    
     @filters.setter
     def filters(self, filters):
+        self._set_filters(filters)
+
+
+    def _set_filters(self, filters):
         if isinstance(filters, list):
             for f in filters :
                 self.add_filter(f)
@@ -180,6 +183,14 @@ def filters(self, filters):
 
         else :
             raise AttributeError("Illegal type for the filter object, it must be a list, a tuple or None.")
+
+    @abc.abstractmethod
+    def _get_filters(self):
+        """
+        Returns the filters in the right format.  
+        Abstract declaration.
+        """
+        pass
 
 
     @abc.abstractmethod
@@ -209,5 +220,4 @@ def qry_load_data(self, *args, **kwargs):
     def load_data(self, *args, **kwargs):
         """Load the data from the SIEM into the list.  
         Abstract declaration."""
-        pass
-
+        pass
diff --git a/msiempy/core/session.py b/msiempy/core/session.py
@@ -27,6 +27,8 @@
                 "os": "Win32"}
                 """)),
 
+    ### DATASOURCES
+
     "get_devtree": ("GRP_GETVIRTUALGROUPIPSLISTDATA",
                     """{"ITEMS": "#{DC1 + DC2}",
                         "DID": "1",
@@ -196,6 +198,7 @@
         "ds_details2": ("dsGetDataSourceDetail",
                         Template("""{"datasourceId": {"value": "$ds_id"}}""")),
 
+        ### ALARMS
 
         "get_alarms_custom_time": (Template("""alarmGetTriggeredAlarms?triggeredTimeRange=$time_range&customStart=$start_time&customEnd=$end_time&status=$status&pageSize=$page_size&pageNumber=$page_number"""),
                     None),
@@ -221,21 +224,17 @@
 
         "delete_alarms_11_2_1": ("""alarmDeleteTriggeredAlarm""", Template("""{"triggeredIds":{"alarmIdList":[$ids]}}""")),
 
-        "get_possible_filters" : ( """qryGetFilterFields""", None ),
-
-        "get_possible_fields" : ( Template("""qryGetSelectFields?type=$type&groupType=$groupType"""), None ),
-
-        "get_esm_time" : ( """essmgtGetESSTime""",None),
-
         "get_alerts_now" : ("""IPS_GETALERTSNOW""", Template("""{"IPSID": "$ds_id"}""")),
 
         "get_flows_now" : ("""IPS_GETFLOWSNOW""", Template("""{"IPSID": "$ds_id"}""")),
 
-        "logout" : ( """userLogout""", None ),
+        ### QUERY MODULE 
 
-        "get_user_locale" : ( """getUserLocale""", None ),
+        "get_possible_filters" : ( """v2/qryGetFilterFields""", None ),
+
+        "get_possible_fields" : ( Template("""v2/qryGetSelectFields?type=$type&groupType=$groupType"""), None ),
 
-        "event_query_custom_time" : ("""qryExecuteDetail?type=EVENT&reverse=false""", Template("""{
+        "event_query_custom_time" : ("""v2/qryExecuteDetail?type=EVENT&reverse=false""", Template("""{
                 "config": {
                     "timeRange": "$time_range",
                     "customStart": "$start_time",
@@ -249,7 +248,7 @@
                     }
                     }""")),
 
-        "event_query" : ("""qryExecuteDetail?type=EVENT&reverse=false""", Template("""{
+        "event_query" : ("""v2/qryExecuteDetail?type=EVENT&reverse=false""", Template("""{
                 "config": {
                     "timeRange":"$time_range",
                     "fields":$fields,
@@ -261,14 +260,34 @@
                     }
                     }""")),
 
-        "query_status" : ("""qryGetStatus""", Template("""{"resultID": $resultID}""")),
+        "query_status" : ("""v2/qryGetStatus""", Template("""{"resultID": $resultID}""")),
 
-        "query_result" : (Template("""qryGetResults?startPos=$startPos&numRows=$numRows&reverse=false"""), Template("""{"resultID": $resultID}""")),
+        "query_result" : (Template("""v2/qryGetResults?startPos=$startPos&numRows=$numRows&reverse=false"""), Template("""{"resultID": $resultID}""")),
 
-        "time_zones" : ("""userGetTimeZones""", None),
+        "grouped_event_query": ("""v2/qryExecuteGrouped?queryType=EVENT""", Template("""{
+                "config": { 
+                    "filters": $filters,
+                    "field": {"name": "$field"},
+                    "timeRange": "$time_range"
+                    }
+        }""")),
+
+        "grouped_event_query_custom_time": ("""v2/qryExecuteGrouped?queryType=EVENT""", Template("""{
+                "config": { 
+                    "filters": $filters,
+                    "field": {"name": "$field"},
+                    "timeRange": "$time_range",
+                    "customStart": "$start_time",
+                    "customEnd": "$end_time",
+                    }
+        }""")),
+
+        "close_query" : ("""v2/qryClose""", Template("""{"resultID": $resultID}""")),
+
+        ### EVENTS OPERATIONS
+
+        "get_alert_data": ("""ipsGetAlertData""", Template("""{"id": {"value":"$id"}}""")),
 
-        "logout" : ("""logout""", None),
-
         "add_note_to_event" : ("""ipsAddAlertNote""", Template("""{
             "id": {"value": "$id"},
             "note": {"note": "$note"}
@@ -277,6 +296,8 @@
         "add_note_to_event_int": ("""IPS_ADDALERTNOTE""", Template("""{"AID": "$id",
                                                             "NOTE": "$note"}""")),
 
+        ### WATCHLISTS OPERATIONS
+
         "get_wl_types": ("""sysGetWatchlistFields""", None),
         "get_watchlists_no_filters" : (Template("""sysGetWatchlists?hidden=$hidden&dynamic=$dynamic&writeOnly=$writeOnly&indexedOnly=$indexedOnly"""), 
             None),
@@ -323,14 +344,27 @@
             "values": $values,
             }""")),
 
+        "remove_watchlist_values": ("""sysRemoveWatchlistValues""", Template("""{
+            "watchlist": $watchlist,
+            "values": $values,
+            }""")),
+
         "get_watchlist_values": ("SYS_GETWATCHLISTDETAILS",
                                         Template("""{"WID": "$id", "LIM": "T"}""")),
 
         "remove_watchlists": ("""sysRemoveWatchlist""", Template("""{"ids": {"watchlistIdList": ["$wl_id_list"]}}""")),
 
-        "get_alert_data": ("""ipsGetAlertData""", Template("""{"id": {"value":"$id"}}""")),
+        ### MISC
+
+        "get_user_locale" : ( """getUserLocale""", None ),
+
+        "time_zones" : ("""userGetTimeZones""", None),
+
+        "logout" : ("""logout""", None),
 
         "get_sys_info"  : ("SYS_GETSYSINFO","""{}"""),
+
+        "get_esm_time" : ( """essmgtGetESSTime""",None),
 
         "build_stamp" : ("essmgtGetBuildStamp",None)
 } 
@@ -623,7 +657,7 @@ def api_request(self, method, data=None, http='post', callback=None, raw=False,
                     log.debug('{} -> Result ({}): {}'.format(
                         str(response),
                         type(result),
-                        str(result)[:200] + '[...]' if len(str(result))>200 else ''
+                        str(result)[:200]
                     ))
 
                     return result
@@ -840,6 +874,7 @@ def _unpack_resp(response) :
             response: requests.Response response object
         Returns a list, a dict or a string
         """
+        log.debug("Unpacking SIEM response: {}".format(str(response.text)[:200]))
         try :
             data = response.json()
             if isinstance(response.json(), dict):

diff --git a/msiempy/core/utils.py b/msiempy/core/utils.py
@@ -236,17 +236,24 @@ def convert_to_time_obj(time_str):
 def parse_query_result(columns, rows):
     """
     For input :
-        columns = ['key1','name','password']
-        rows = [
-            ['67','bob','b08b'],
-            ['68','mike','kaas'],
-            ['69','jean','p992'],
-        ]
+        columns = [{'name': 'Alert.LastTime'}, {'name': 'Rule.msg'}, {'name': 'Alert.DstIP'}, {'name': 'Alert.IPSIDAlertID'}]
+        rows = 
+            [
+                {'values': ['09/22/2020 15:51:14', 'Postfix Disconnect from host', '::', '144116287604260864|547123']}, 
+                {'values': ['09/22/2020 15:51:14', 'Postfix Lost connection from host', '::', '144116287604260864|547122']}
+            } 
+
     Returns :
     [
-        {key1=67, name=bob, password=b08b},
-        {...},
-        {},
+        {
+            "Alert.LastTime":"09/22/2020 15:51:14", 
+            "Rule.msg":"Postfix Disconnect from host",
+            "Alert.DstIP":"::",
+            "Alert.IPSIDAlertID":"144116287604260864|547123"
+        },
+        {
+            ...
+        },
     ]
 
     """