Merge pull request #85 from mfesiem/develop

Develop

.github/workflows/publish-test-docs-only.yml

@@ -17,31 +17,38 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          # Fetches entire history, so we can analyze commits since last tag
+          fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
+          sudo apt-get install graphviz
           python -m pip install --upgrade pip
           python -m pip install -r requirements.txt
           python -m pip install .
           python -m pip show msiempy
 
-      - name: Generate documentation
+      - name: Generate test documentation
         run: |
           python3 -m pdoc msiempy --output-dir docs_tmp --html --force --template-dir ./.pdoc_templates
           
           git clone https://github.com/mfesiem/mfesiem.github.io
           rm -rf ./mfesiem.github.io/docs/test/msiempy
           cp -r ./docs_tmp/msiempy mfesiem.github.io/docs/test/
 
-          # pyreverse -s 1 -f PUB_ONLY -o png -m y msiempy
-          # mv ./classes.png ./mfesiem.github.io/docs/test/msiempy
-          # mv ./packages.png ./mfesiem.github.io/docs/test/msiempy
+          # Ignore any errors on diagram generation
+          set +e
+          pyreverse -s 1 -f PUB_ONLY -o png -m y msiempy
+          mv ./classes.png ./mfesiem.github.io/docs/test/msiempy
+          mv ./packages.png ./mfesiem.github.io/docs/test/msiempy
+          set -e
 
-      - name: Publish documentation
+      - name: Publish documentation to mfesiem.github.io/docs/test/msiempy
         uses: peaceiris/actions-gh-pages@v3
         with:
           deploy_key: ${{ secrets.MFESIEM_GITHUB_IO_PRIVATE_KEY }}
@@ -51,5 +58,6 @@ jobs:
           user_name: 'Github Actions'
           user_email: "[email protected]"
           commit_message: "Generate documentation - develop"
+          enable_jekyll: true
 
 

.github/workflows/publish.yml

@@ -27,6 +27,7 @@ jobs:
 
       - name: Install dependencies
         run: |
+          sudo apt-get install graphviz
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install -r requirements.txt
           python -m pip install .
@@ -55,11 +56,11 @@ jobs:
           rm -rf ./mfesiem.github.io/docs/msiempy
           cp -r ./docs_tmp/msiempy mfesiem.github.io/docs/
 
-          # pyreverse -s 1 -f PUB_ONLY -o png -m y msiempy
-          # mv ./classes.png ./mfesiem.github.io/docs/msiempy
-          # mv ./packages.png ./mfesiem.github.io/docs/msiempy
+          pyreverse -s 1 -f PUB_ONLY -o png -m y msiempy
+          mv ./classes.png ./mfesiem.github.io/docs/msiempy
+          mv ./packages.png ./mfesiem.github.io/docs/msiempy
 
-      - name: Publish documentation
+      - name: Publish documentation to mfesiem.github.io/docs/msiempy
         uses: peaceiris/actions-gh-pages@v3
         with:
           deploy_key: ${{ secrets.MFESIEM_GITHUB_IO_PRIVATE_KEY }}
@@ -69,6 +70,7 @@ jobs:
           user_name: 'Github Actions'
           user_email: "[email protected]"
           commit_message: "Generate documentation - master"
+          enable_jekyll: true
 
       - name: Build msiempy
         run: python3 setup.py --quiet build check sdist bdist_wheel
@@ -85,7 +87,8 @@ jobs:
         run: |
           version="$(python setup.py --version)"
           last_tag="$(git --no-pager tag -l | tail -1)"
-          git --no-pager log ${last_tag}.. --pretty=oneline > ./tmp_tag.txt
+          echo "msiempy ${version}" > ./tmp_tag.txt
+          git --no-pager log ${last_tag}.. --pretty=oneline >> ./tmp_tag.txt
           git config --global user.email "[email protected]"
           git config --global user.name "Github actions"
           git tag -a ${version} -F ./tmp_tag.txt

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 name: test
 on: 
   schedule:
-    - cron:  '0 0 1,15 * *'
+    - cron:  '0 0 1,7,15,23 * *'
 jobs: 
 
   test:
@@ -25,7 +25,6 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements.txt
           python -m pip install .
           python -m pip show msiempy
       

CONTRIBUTING.md

@@ -24,4 +24,18 @@ python3 -m pip install -r requirements.txt
 # Install module
 python3 ./setup.py install
 # Hack and pull request
-```
+```
+
+### Git flow
+- Commits to `master` branch are trigerring: 
+    - Tests + upload coverage
+    - Generate documentation + publish to gh-pages
+    - PyPi realeases and create new tag **if the `__version__` has been bumped**.  
+    - See [publish](https://github.com/mfesiem/msiempy/blob/master/.github/workflows/publish.yml)
+- Commits to `develop` branch are trigerring:
+    - Generate documentation + publish to gh-pages under `test` folder
+    - See [publish-test-docs-only](https://github.com/mfesiem/msiempy/blob/master/.github/workflows/publish-test-docs-only.yml)
+- Tests on Windows and MacOS are scheduled to run once a week. 
+    - See [test](https://github.com/mfesiem/msiempy/blob/master/.github/workflows/test.yml)
+
+See the github actions workflows for more details: 

msiempy/__init__.py

@@ -264,3 +264,4 @@
     GroupedEvent,
 )
 from .watchlist import Watchlist, WatchlistManager
+from .__version__ import __version__ as VERSION

msiempy/__version__.py

@@ -2,7 +2,7 @@
 Project version and meta informations.
 """
 
-__version__ = "0.3.4.dev2"
+__version__ = "0.3.4"
 __title__ = "msiempy"
 __description__ = "msiempy - McAfee SIEM API Python wrapper"
 __author__ = "andywalden, tristanlatr, mathieubeland, and other contributors. "

msiempy/alarm.py

@@ -342,20 +342,23 @@ def _alarm_match(self, alarm):
 
     def _event_match(self, alarm):
         """
-        Internal filter method that is going to return True if the passed alarm match all event related filters.
+        Internal filter method that is going to return True if any triggering event match all passed event filters.
         """
         match = True
-        for event_filter in self._event_filters:
-            match = False
-            try:
-                value = str(alarm["events"][0][event_filter[0]])
-            except KeyError:
-                break
-            for filter_value in event_filter[1]:
-                if regex_match(filter_value.lower(), value.lower()):
-                    match = True
+        for event in alarm["events"]:
+            for event_filter in self._event_filters:
+                match = False
+                try:
+                    value = str(event[event_filter[0]])
+                except KeyError:
                     break
-            if not match:
+                for filter_value in event_filter[1]:
+                    if regex_match(filter_value.lower(), value.lower()):
+                        match = True
+                        break
+                if not match:
+                    break
+            if match:
                 break
         return match
 

msiempy/device.py

@@ -15,7 +15,10 @@
 
 
 class ESM(NitroObject):
-    """Enterprise Security Manager interface"""
+    """
+    Enterprise Security Manager interface.  
+    Object do not contain data, it's a simple interface to data structures / values returned by the SIEM or helper methods.  
+    """
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -66,8 +69,11 @@ def status(self):
         - `backupLastTime`
         - `rulesAndSoftwareCheckEnabled`
         - `rulesAndSoftNextCheck`
-        - `rulesAndSoftLastCheck`
-        Other functions exist to return subsets of this data also.
+        - `rulesAndSoftLastCheck`  
+
+        Other functions exist to return subsets of this data also.  
+
+        .. note:: Uses internal API method `SYS_GETSYSINFO`
         """
         status = self.nitro.request("get_sys_info")
         return self._map_status_int_fields(status)
@@ -104,7 +110,8 @@ def ram(self):
 
     def backup_status(self):
         """
-        Returns: Backup status and timestamps.
+        Returns: Backup status and timestamps.  
+        Use `status()`.  
         Example :
         ```
             {'autoBackupEnabled': True,
@@ -125,14 +132,16 @@ def backup_status(self):
 
     def callhome(self):
         """
-        Returns: `True/False` if there is currently a callhome connection
+        Returns: `True/False` if there is currently a callhome connection.  
+        Use `status()`.  
         """
         if self.status()["callHomeIp"]:
             return True
 
     def rules_status(self):
         """
-        Returns: Rules autocheck status and timestamps.
+        Returns: Rules autocheck status and timestamps.  
+        Use `status()`.  
         Example:
         ```
             { 'rulesAndSoftwareCheckEnabled': True
@@ -161,24 +170,25 @@ def get_alerts(self, ds_id, flows=False):
         - `flows`: (`bool`) Also get flows from the device (default: False)
 
         Returns: `None`
-        # TODO: add test method in tests/auth/test_device.py
+
+        .. note:: Uses internal API methods `IPS_GETALERTSNOW` and `IPS_GETFLOWSNOW`
         """
+        # TODO add test method in `tests/auth/test_device.py`
         self.nitro.request("get_alerts_now", ds_id=ds_id)
         if flows:
             self.nitro.request("get_flows_now", ds_id=ds_id)
 
     @lru_cache(maxsize=None)
     def recs(self):
         """
-        Returns: `list(tuple())`, List of receivers name and id
+        Returns: `list(tuple())`, List of receivers name and id.  
         """
         rec_list = self.nitro.request("get_recs")
         return [(rec["name"], rec["id"]["id"]) for rec in rec_list]
 
     @lru_cache(maxsize=None)
     def _get_timezones(self):
-        """Gets list of timezones from the ESM.
-        Returns: Raw `string` from ESM
+        """Gets `list` of timezones from the ESM.  
         """
         return self.nitro.request("time_zones")
 
@@ -259,6 +269,7 @@ def venmod_to_type_id(self, vendor, model):
 
         Returns: `str` Matching type_id or None if there is no match
         """
+        # TODO Write a test method in `tests/auth/test_device.py`
         for venmod in self._get_ds_types():
             if vendor == venmod[1]:
                 if model == venmod[2]:
@@ -317,8 +328,15 @@ class DevTree(NitroList):
 
     Exemple:
     ```
+    # Quick python code to list all McAfee SIEM Datasources
+    from msiempy.device import DevTree
+    devtree = DevTree()
+    print("All Datasources")
+    print(devtree.get_text(fields=["parent_name", "name", "ds_id"]))
     ```
 
+
+    .. note:: Uses internal API methods such as `GRP_GETVIRTUALGROUPIPSLISTDATA` to assemble `DevTree` object.  
     """
 
     def __init__(self, *args, **kwargs):

msiempy/event.py

@@ -1359,8 +1359,9 @@ def __setitem__(self, key, value):
 
     def get_id(self):
         """
-        Get the event ID.
-        Try to return `e['Alert.IPSIDAlertID']` or e['eventId']` or concatenate `e['ipsId']['id']` and `e['alertId']` depending of the Event dictionnary keys.
+        Get the event ID.  
+
+        Return the full event ID or `None`.  
         """
         the_id = (
             self.data["Alert.IPSIDAlertID"]
@@ -1384,7 +1385,9 @@ def clear_notes(self):
 
     def set_note(self, note, no_date=False):
         """
-        Set the event's note. Desctructive action.
+        Set the event's note. Desctructive action.  
+
+        .. note:: Uses the internal API method `IPS_ADDALERTNOTE`
         """
         the_id = self.get_id()
 

msiempy/watchlist.py

@@ -193,6 +193,8 @@ def load_values(self):
         """
         Load Watchlist values.
         Raises: `KeyError` if watchlist invalid.
+
+        .. note:: Uses the internal API method `SYS_GETWATCHLISTDETAILS`
         """
         wl_details = self.nitro.request("get_watchlist_values", id=self.data["id"])