Merge pull request #25 from splunk-soar-connectors/next

Merging next to main for release 2.6.0

office365_consts.py

@@ -26,18 +26,19 @@
     " displayName eq 'msg folder root' or displayName eq 'outbox' or displayName eq 'recoverable items deletions' or "\
     "displayName eq 'scheduled' or displayName eq 'search folders' or displayName eq 'sent items' or displayName eq 'server failures' or "\
     "displayName eq 'sync issues'"
-MSGOFFICE365_STATE_FILE_CORRUPT_ERROR = "Error occurred while loading the state file due to its unexpected format. Resetting the state file with the default format. \
-Please try again."
+MSGOFFICE365_STATE_FILE_CORRUPT_ERR = "Error occurred while loading the state file. " \
+    "Resetting the state file with the default format. Please test the connectivity."
 MSGOFFICE365_AUTHORIZE_TROUBLESHOOT_MSG = 'If authorization URL fails to communicate with your Phantom instance, check whether you have:  '\
-                                ' 1. Specified the Web Redirect URL of your App -- The Redirect URL should be <POST URL>/result . '\
-                                ' 2. Configured the base URL of your Phantom Instance at Administration -> Company Settings -> Info'
+    ' 1. Specified the Web Redirect URL of your App -- The Redirect URL should be <POST URL>/result . '\
+    ' 2. Configured the base URL of your Phantom Instance at Administration -> Company Settings -> Info'
 MSGOFFICE365_INVALID_PERMISSION_ERR = "Error occurred while saving the newly generated access token "\
     "(in place of the expired token) in the state file."
 MSGOFFICE365_INVALID_PERMISSION_ERR += " Please check the owner, owner group, and the permissions of the state file. The Phantom "
 MSGOFFICE365_INVALID_PERMISSION_ERR += "user should have the correct access rights and ownership for the corresponding state file "\
     "(refer to readme file for more information)."
 MSGOFFICE365_NO_DATA_FOUND = "No data found"
 MSGOFFICE365_DUPLICATE_CONTAINER_FOUND_MSG = "duplicate container found"
+MSGOFFICE365_ERR_EMPTY_RESPONSE = "Status Code {code}. Empty response and no information in the header."
 
 MSGOFFICE365_DEFAULT_REQUEST_TIMEOUT = 30  # in seconds
 MSGOFFICE365_CONTAINER_DESCRIPTION = 'Email ingested using MS Graph API - {last_modified_time}'
@@ -51,3 +52,6 @@
     "Please provide a valid non-zero positive integer value in the {param} parameter"
 )
 MSGOFFICE365_NON_NEG_INT_MSG = "Please provide a valid non-negative integer value in the {param} parameter"
+MSGOFFICE365_ENCRYPTION_ERR = "Error occurred while encrypting the state file"
+MSGOFFICE365_DECRYPTION_ERR = "Error occurred while decrypting the state file"
+MSGOFFICE365_UNEXPECTED_ACCESS_TOKEN_ERR = "Found unexpected value of access token. Please run the test connectivity to generate a new token"

office365_get_email.html

@@ -222,7 +222,7 @@ <h4 class="wf-h4-style">Attachments</h4>
                                     <td>{{ file.name }}</td>
                                     <td>
                                         <a class="no-word-wrap" href="javascript:;" onclick="context_menu(this, [{'contains': ['sha1'],
-                                            'value': '{{ attachment.vaultId }}' }], 0, {{ container.id }}, null, false);">
+                                            'value': '{{ file.vaultId }}' }], 0, {{ container.id }}, null, false);">
                                         {{ file.vaultId }}
                                         &nbsp;
                                         <span class="fa fa-caret-down" style="font-size: smaller;"></span>
@@ -239,13 +239,22 @@ <h4 class="wf-h4-style">Attachments</h4>
                         <table class="phantom-table dataTable">
                             <thead>
                                 <th>Attachment Name</th>
+                                <th>Vault ID</th>
                                 <th>Attachment Type</th>
                                 <th>Item Attachment Type</th>
                                 <th>Content Type</th>
                             </thead>
                             {% for item in email_data.attachment_data.item_attachment %}
                                 <tr>
                                     <td>{{ item.name }}</td>
+                                    <td>
+                                        <a class="no-word-wrap" href="javascript:;" onclick="context_menu(this, [{'contains': ['sha1'],
+                                            'value': '{{ item.vaultId }}' }], 0, {{ container.id }}, null, false);">
+                                        {{ item.vaultId }}
+                                        &nbsp;
+                                        <span class="fa fa-caret-down" style="font-size: smaller;"></span>
+                                        </a>
+                                    </td>
                                     <td>{{ item.attachmentType }}</td>
                                     <td>{{ item.itemType }}</td>
                                     <td>{{ item.contentType }}</td>

office365_view.py

@@ -91,12 +91,12 @@ def display_view(provides, all_app_runs, context):
             results.append(ctx_result)
 
     if provides == "list events":
-       return_page = "office365_list_events.html"
+        return_page = "office365_list_events.html"
 
     if provides == "get email":
-       return_page = "office365_get_email.html"
+        return_page = "office365_get_email.html"
 
     if provides == "run query":
-       return_page = "office365_run_query.html"
+        return_page = "office365_run_query.html"
 
     return return_page

process_email.py

@@ -17,11 +17,9 @@
 import json
 import mimetypes
 import os
-import random
 import re
 import shutil
 import socket
-import string
 import tempfile
 from builtins import str
 from collections import OrderedDict
@@ -35,6 +33,7 @@
 import phantom.utils as ph_utils
 from bs4 import BeautifulSoup, UnicodeDammit
 from django.core.validators import URLValidator
+from phantom.vault import Vault
 from requests.structures import CaseInsensitiveDict
 
 from office365_consts import ERR_MSG_UNAVAILABLE
@@ -98,17 +97,17 @@
 IPV6_REGEX = r'\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|'
 IPV6_REGEX += r'(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))'
 IPV6_REGEX += (r'|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
-r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|')
+               r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|')
 IPV6_REGEX += (r'(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
-r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
+               r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
 IPV6_REGEX += (r'(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
-r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
+               r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
 IPV6_REGEX += (r'(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
-r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
+               r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
 IPV6_REGEX += (r'(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
-r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
+               r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|')
 IPV6_REGEX += (r'(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
-r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*')
+               r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*')
 
 
 uri_regexc = re.compile(URI_REGEX)
@@ -666,23 +665,24 @@ def _handle_attachment(self, part, tmp_dir, file_name, file_path):
         except IOError as ioerr:
             error_msg = _get_error_message_from_exception(ioerr)
             if "File name too long" in error_msg:
-                self.write_with_new_filename(tmp_dir, part_payload, files, file_name, as_byte=False)
+                self.write_with_new_filename(part_payload, files, file_name, as_byte=False)
             else:
                 self._debug_print('Failed to write file: {}'.format(ioerr))
 
-    def write_with_new_filename(self, tmp_dir, data, dict_to_fill, file_name, as_byte=False):
+    def write_with_new_filename(self, data, dict_to_fill, file_name, as_byte=False):
         try:
-            _, file_extension = os.path.splitext(file_name)
-            random_suffix = '_' + ''.join(random.SystemRandom().choice(string.ascii_lowercase) for _ in range(16))
-            new_file_name = "ph_long_file_name_{0}{1}".format(random_suffix, file_extension)
-            full_path = os.path.join(tmp_dir, new_file_name)
+            if hasattr(Vault, 'get_vault_tmp_dir'):
+                fd, full_path = tempfile.mkstemp(dir=Vault.get_vault_tmp_dir())
+            else:
+                fd, full_path = tempfile.mkstemp(dir='/opt/phantom/vault/tmp')
+            os.close(fd)
 
             with open(full_path, 'wb') as f:    # noqa
                 if as_byte:
                     f.write(data.as_bytes())
                 else:
                     f.write(data)
-            dict_to_fill.append({'file_name': new_file_name, 'file_path': full_path})
+            dict_to_fill.append({'file_name': file_name, 'file_path': full_path})
         except Exception as e:
             self._base_connector.debug_print('Exception while writing file: {}'.format(e))
 
@@ -1265,8 +1265,8 @@ def _handle_file(self, curr_file, container_id, run_automation=False):
 
         if vault_id:
             cef_artifact.update({'vaultId': vault_id,
-                'cs6': vault_id,
-                'cs6Label': 'Vault ID'})
+                                 'cs6': vault_id,
+                                 'cs6Label': 'Vault ID'})
 
             # now get the rest of the hashes and add them to the cef artifact
             self._add_vault_hashes_to_dictionary(cef_artifact, vault_id)

readme.html

@@ -30,7 +30,7 @@ <h2>Authentication</h2>
 <br><br>
 Once the app is created, follow the below-mentioned steps:
 <ul>
-    <li>Under <b>Certificates & secrets</b> select <b>New client secret</b>. Note down this key somewhere secure, as it cannot be retrieved after closing the window.</li>
+    <li>Under <b>Certificates & secrets</b> select <b>New client secret</b>. Enter the <b>Description</b> and select the desired duration in <b>Expires</b>. Click on <b>Add</b>. Note down this <b>value</b> somewhere secure, as it cannot be retrieved after closing the window.</li>
     <li>Under <b>Authentication</b>, select <b>Add a platform</b>. In the <b>Add a platform</b> window, select <b>Web</b>. The <b>Redirect URLs</b> should be filled right here. We will get <b>Redirect URLs</b> from the Phantom asset we create below in the section titled <b>Phantom Graph Asset</b>.</li>
     <li>Under <b>API Permissions</b> Click on <b>Add a permission</b>.</li>
     <li>Under the <b>Microsoft API</b> section, select <b>Microsoft Graph</b>.</li>
@@ -40,12 +40,12 @@ <h2>Authentication</h2>
         <li>Mail.ReadWrite (https://graph.microsoft.com/Mail.ReadWrite)</li>
         <li>User.Read.All (https://graph.microsoft.com/User.Read.All)</li>
             <ul><li>For non-admin access, use User.Read (Delegated permission) instead (https://graph.microsoft.com/User.Read)</li></ul>
-        <li>Group.Read.All (https://graph.microsoft.com/Group.Read.All) - It is required only if you want to run the <b>list events</b> action for the group's calendar and for the <b>list groups</b> action.</li>
+        <li>Group.Read.All (https://graph.microsoft.com/Group.Read.All) - It is required only if you want to run the <b>list events</b> action for the group's calendar and for the <b>list groups</b> and the <b>list group members</b> action.</li>
         <li>Calendar.Read (https://graph.microsoft.com/Calendars.Read) - It is required only if you want to run the <b>list events</b> action for the user's calendar.</li>
         <li>MailboxSettings.Read (https://graph.microsoft.com/MailboxSettings.Read) - It is required only if you want to run the <b>oof status</b> action.</li>
     </ul>
 </ul>
-After making these changes, click <b>Add permissions</b>, then select <b>Grant admin consent for Test Phantom</b> at the bottom of the screen.
+After making these changes, click <b>Add permissions</b>, then select <b>Grant admin consent for &lt;your_organization_name_as_on_azure_portal&gt;</b> at the bottom of the screen.
 <h2>Phantom Graph Asset</h2>
 When creating an asset for the <b>MS Graph for Office 365</b> app, place <b>Application ID</b> of the app created during the app registration on the Azure Portal in the <b>Application ID</b> field and place the client secret generated during the app registration process in the <b>Application Secret</b> field. Then, after filling out the <b>Tenant</b> field, click <b>SAVE</b>. Both the Application/Client ID and the Tenant ID can be found in the <b>Overview</b> tab on your app's Azure page.
 <br><br>
@@ -114,8 +114,9 @@ <h3>On-Poll</h3>
       <li>extract_ips - Extracts the IP addresses present in the emails.</li>
       <li>extract_domains - Extract the domain names present in the emails.</li>
       <li>extract_hashes - Extract the hashes present in the emails (MD5).</li>
+      <li>ingest_eml - Fetch the EML file content for the 'item attachment' and ingest it into the vault. This will only ingest the first level 'item attachment' as an EML file. The nested item attachments will not be ingested into the vault. If the extract_attachments flag is set to false, then the application will also skip the EML file ingestion regardless of this flag value.</li>
     </ul>
-
+    <p>If extract_attachments is set to true, only fileAttachment will be ingested. If both ingest_eml and extract_attachments are set to true, then both fileAttachment and itemAttachment will be ingested.</p>
 <h2>Guidelines to provide folder parameter value</h2>
 <p>This is applicable to 'on poll', 'copy email', 'move email', and 'run query' actions.</p>
 <ul>
@@ -153,8 +154,8 @@ <h3>Note</h3>
     <li>An optional parameter <b>Admin Access Required</b> has been added to this app. In most cases, this should remain checked, as admin access is required for email use cases. If the desired integration is to integrate with only one user's calendar, you may consider unchecking this box. If unchecked, it allows a non-admin user to provide access to a specific account. This functionality will ONLY work with the <b>list events</b> functionality. If unchecked, the <b>Access scope</b> <em>must</em> be used. The default scope will work for listing calendar events. Additional information on scope can be found <a href="https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-permissions-and-consent#openid-connect-scopes" target="_blank">here.</a></li>
     <li>As per the Microsoft known issues for <b>Group.Read.All</b> permission (<a href="https://docs.microsoft.com/en-us/graph/known-issues#groups" target="_blank">here</a>), if you want to run the <b>list events</b> for fetching group's calendar events, you have to uncheck an optional parameter <b>Admin Access Required</b> and provide <b>Group.Read.All (https://graph.microsoft.com/Group.Read.All)</b> permission into the scope parameter in the asset configuration parameters. If an asset parameter <b>Admin Access Required</b> checked and configured the app with above mentioned all the application permissions (which includes <b>Group.Read.All</b> application permission), it throws an error like <b>Access is denied</b> while running <b>list events</b> action for fetching group's calendar events. Because of the known issue of <b>Group.Read.All</b> application permission, this permission required admin consent (on behalf of the user permission) to fetch the group's calendar events.</li>
     <li>If the parameter <b>Admin Access Required</b> is unchecked, you have to provide a <b>scope</b> parameter in the asset configuration. All the actions will get executed according to the scopes provided in the <b>scope</b> config parameter. The actions will throw an appropriate error if the scope of the corresponding permission is not provided by the end-user.</li>
-    <li>If both the checkboxes <b>Admin Consent Already Provided</b> and <b>Admin Access Required</b> are kept as checked while running the test connectivity for the first time, then the test connectivity will pass but actions will fail. In order to use the <b>Admin Consent Already Provided</b> checkbox, first, you need to run the test connectivity with only <b>Admin Access Required</b> as checked and provide admin consent, and then while running the test connectivity for the second time the <b>Admin Consent Already Provided</b> should be checked if required. The <b>checkbox Admin Consent Already Provided</b> is used when running the test connectivity if admin consent is already provided.</li>
-    <li>There is an API limitation that will affect run_query action when providing Unicode values in the subject or in the body as parameters and if the result count exceeds 999, the action will fail.</li>
+    <li>There is an API limitation that will affect run_query action when providing Unicode values in the subject or in the body as parameters and if the result count exceeds 999, the action will fail.</li>
+    <li>The sensitive values are stored encrypted in the state file.</li>
 </ul>
 </p>
 

release_notes/2.6.0.md

@@ -0,0 +1,7 @@
+* Added support to ingest 'item attachment' as an EML file for the 'get email' action [PAPP-25609]
+* Feature enhancement for the 'on poll' action [PAPP-25609]
+  * Added new asset configuration parameter to ingest the EML file for item attachments
+  * Added support to process all types of 'item attachment' and create the required artifacts
+* Bug fix for the extraction of attachment having long filename
+* Added the 'list group members' action
+* Introduced encryption for the sensitive values stored in the state file

release_notes/release_notes.html

@@ -1,5 +1,17 @@
-<b>MS Graph for Office 365 Release Notes - Published by Splunk April 29, 2022</b>
+<b>MS Graph for Office 365 Release Notes - Published by Splunk June 28, 2022</b>
 <br><br>
+<b>Version 2.6.0 - Released June 28, 2022</b>
+<ul>
+<li>Added support to ingest 'item attachment' as an EML file for the 'get email' action [PAPP-25609]</li>
+<li>Feature enhancement for the 'on poll' action [PAPP-25609]</li>
+<ul>
+<li>Added new asset configuration parameter to ingest the EML file for item attachments</li>
+<li>Added support to process all types of 'item attachment' and create the required artifacts</li>
+</ul>
+<li>Bug fix for the extraction of attachment having long filename</li>
+<li>Added the 'list group members' action</li>
+<li>Introduced encryption for the sensitive values stored in the state file</li>
+</ul>
 <b>Version 2.5.3 - Released April 29, 2022</b>
 <ul>
 <li>Fixed authentication bug with Admin Consent where actions would still fail even after successful Test Connectivity [PAPP-25448]</li>