Al 2646

assemblyline_ui/api/v4/ingest.py

@@ -21,7 +21,7 @@
     STORAGE, config, FILESTORE, metadata_validator
 from assemblyline_ui.helper.service import ui_to_submission_params
 from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \
-    FETCH_METHODS
+    FETCH_METHODS, update_submission_parameters
 from assemblyline_ui.helper.user import check_async_submission_quota, decrement_submission_ingest_quota, \
     load_user_settings
 
@@ -150,15 +150,17 @@ def ingest_single_file(**kwargs):
       "base64": "<BINARY DATA OF THE FILE TO SCAN... ENCODED AS BASE64 STRING>",
 
       // OPTIONAL VALUES
-      "name": "file.exe",         # Name of the file to scan otherwise the sha256 or base file of the url
+      "name": "file.exe",                   # Name of the file to scan otherwise the sha256 or base file of the url
 
-      "metadata": {               # Submission metadata
-        "key": val,                 # Key/Value pair for metadata parameters
+      "submission_profile": "Static Analysis",    # Name of submission profile to use
+
+      "metadata": {                         # Submission metadata
+        "key": val,                             # Key/Value pair for metadata parameters
       },
 
-      "params": {                 # Submission parameters
-        "key": val,                 # Key/Value pair for params that differ from the user's defaults
-      },                            # Default params can be fetch at /api/v3/user/submission_params/<user>/
+      "params": {                           # Submission parameters
+        "key": val,                             # Key/Value pair for params that differ from the user's defaults
+      },                                        # Default params can be fetch at /api/v3/user/submission_params/<user>/
 
       "generate_alert": False,        # Generate an alert in our alerting system or not
       "notification_queue": None,     # Name of the notification queue
@@ -269,8 +271,11 @@ def ingest_single_file(**kwargs):
             "type": "INGEST"
         })
 
-        # Apply provided params
-        s_params.update(data.get("params", {}))
+        # Update submission parameters as specified by the user
+        try:
+            update_submission_parameters(s_params, data, user)
+        except Exception as e:
+            return make_api_response({}, str(e), 400)
 
         # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings
         default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources

assemblyline_ui/api/v4/submission.py

@@ -312,7 +312,6 @@ def get_full_results(sid, **kwargs):
        },
      "state": "completed",              # State of the submission
      "submission": {                    # Submission Block
-       "profile": true,                     # Should keep stats about execution?
        "description": "",                   # Submission description
        "ttl": 30,                           # Submission days to live
        "ignore_filtering": false,           # Ignore filtering services?
@@ -451,7 +450,6 @@ def get_submission(sid, **kwargs):
        ["FNAME", "sha256"], ...],    # Each file = List of name/sha256
      "errors": [],              # List of error keys (sha256.ServiceName)
      "submission": {            # Submission Block
-       "profile": true,           # Should keep stats about execution?
        "description": "",         # Submission description
        "ttl": 30,                 # Submission days to live
        "ignore_filtering": false, # Ignore filtering services?

assemblyline_ui/api/v4/submit.py

@@ -11,6 +11,7 @@
 from assemblyline.common.dict_utils import flatten
 from assemblyline.common.str_utils import safe_str
 from assemblyline.common.uid import get_random_id
+from assemblyline.odm.models.config import SubmissionProfile
 from assemblyline.odm.messages.submission import Submission
 from assemblyline.odm.models.user import ROLES
 from assemblyline_core.submission_client import SubmissionClient, SubmissionException
@@ -19,7 +20,7 @@
     CLASSIFICATION as Classification, IDENTIFY, metadata_validator
 from assemblyline_ui.helper.service import ui_to_submission_params
 from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \
-    FETCH_METHODS
+    FETCH_METHODS, update_submission_parameters
 from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota, load_user_settings
 
 SUB_API = 'submit'
@@ -251,15 +252,17 @@ def submit(**kwargs):
       "base64": "<BINARY DATA OF THE FILE TO SCAN... ENCODED AS BASE64 STRING>",
 
       // OPTIONAL VALUES
-      "name": "file.exe",         # Name of the file to scan otherwise the sha256 or base file of the url
+      "name": "file.exe",                   # Name of the file to scan otherwise the sha256 or base file of the url
 
-      "metadata": {               # Submission metadata
-        "key": val,                 # Key/Value pair for metadata parameters
+      "submission_profile": "Static Analysis",    # Name of submission profile to use
+
+      "metadata": {                         # Submission metadata
+        "key": val,                             # Key/Value pair for metadata parameters
       },
 
-      "params": {                 # Submission parameters
-        "key": val,                 # Key/Value pair for params that differ from the user's defaults
-      },                            # Default params can be fetch at /api/v3/user/submission_params/<user>/
+      "params": {                           # Submission parameters
+        "key": val,                             # Key/Value pair for params that differ from the user's defaults
+      },                                        # Default params can be fetch at /api/v4/user/submission_params/<user>/
     }
 
     Data Block (Binary):
@@ -335,7 +338,14 @@ def submit(**kwargs):
 
         # Create task object
         s_params = ui_to_submission_params(user_settings)
-        s_params.update(data.get("params", {}))
+
+        # Update submission parameters as specified by the user
+        try:
+            update_submission_parameters(s_params, data, user)
+        except Exception as e:
+            return make_api_response({}, str(e), 400)
+
+
         default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources
         if 'groups' not in s_params:
             s_params['groups'] = [g for g in user['groups'] if g in s_params['classification']]

assemblyline_ui/api/v4/ui.py

@@ -16,7 +16,7 @@
 from assemblyline_ui.config import TEMP_DIR, STORAGE, FILESTORE, config, CLASSIFICATION as Classification, \
     IDENTIFY, metadata_validator
 from assemblyline_ui.helper.service import ui_to_submission_params
-from assemblyline_ui.helper.submission import submission_received
+from assemblyline_ui.helper.submission import submission_received, update_submission_parameters
 from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota
 from assemblyline_core.submission_client import SubmissionClient, SubmissionException
 
@@ -260,6 +260,13 @@ def start_ui_submission(ui_sid, **kwargs):
             # Submit to dispatcher
             try:
                 params = ui_to_submission_params(ui_params)
+
+                # Update submission parameters as specified by the user
+                try:
+                    update_submission_parameters(params, ui_params, user)
+                except Exception as e:
+                    return make_api_response({}, str(e), 400)
+
                 metadata = params.pop("metadata", {})
 
                 # Enforce maximum DTL

assemblyline_ui/api/v4/user.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import List
 from assemblyline.odm.models.config import ExternalLinks
 from flask import request, session as flsk_session
@@ -7,21 +8,21 @@
 from assemblyline.common.security import (check_password_requirements, get_password_hash,
                                           get_password_requirement_message)
 from assemblyline.datastore.exceptions import SearchException
-from assemblyline.odm.models.config import HASH_PATTERN_MAP
+from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES
 from assemblyline.odm.models.user import (ACL_MAP, ROLES, USER_ROLES, USER_TYPE_DEP, USER_TYPES, User, load_roles,
                                           load_roles_form_acls)
 from assemblyline.odm.models.user_favorites import Favorite
 from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint
-from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, CLASSIFICATION_ALIASES, DAILY_QUOTA_TRACKER, LOGGER, \
-    STORAGE, UI_MESSAGING, VERSION, config, AI_AGENT, UI_METADATA_VALIDATION
+from assemblyline_ui.config import AI_AGENT, APPS_LIST, CLASSIFICATION, CLASSIFICATION_ALIASES, \
+    DAILY_QUOTA_TRACKER, LOGGER, STORAGE,SUBMISSION_PROFILES, UI_MESSAGING,UI_METADATA_VALIDATION, VERSION, config
 from assemblyline_ui.helper.search import list_all_fields
 from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params
 from assemblyline_ui.helper.user import (
     get_default_user_quotas, get_dynamic_classification, load_user_settings, save_user_account, save_user_settings,
     API_PRIV_MAP)
 from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException
 
-from .federated_lookup import filtered_tag_names
+from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names
 
 
 SUB_API = 'user'
@@ -91,10 +92,12 @@ def who_am_i(**kwargs):
          "max_dtl": 30,                             # Maximum number of days retrohunt job stay in the system
        },
        "submission": {                            # Submission Configuration
+         "configurable_params": [],                 # Submission parameters that are configurable when using profiles
          "dtl": 10,                                 # Default number of days submission stay in the system
          "max_dtl": 30,                             # Maximum number of days submission stay in the system
          "file_sources": [],                        # List of file sources to perform remote submission into the system
          "metadata": {},                            # Metadata compliance policy to submit to the system
+         "profiles": {},                            # Submission profiles
          "verdicts": {                              # Verdict scoring configuration
             "info": 0,                                # Default minimum score for info
             "suspicious": 300,                        # Default minimum score for suspicious
@@ -193,6 +196,24 @@ def who_am_i(**kwargs):
     [file_sources["sha256"]["sources"].append(x.name) for x in config.submission.sha256_sources
      if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)]
 
+    # Prepare submission profile configurations for UI
+    submission_profiles = {}
+    for name, profile in SUBMISSION_PROFILES.items():
+        if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification):
+            # We want to pass forward the configurations that have been explicitly set as a configuration
+            submission_profiles[name] = profile.params.as_primitives(strip_null=True)
+            submission_profiles[name]["editable_params"] = profile.editable_params
+
+    # Expand service categories if used in submission profiles (assists with the UI locking down service selection)
+    service_categories = list(STORAGE.service.facet('category').keys())
+    for profile in submission_profiles.values():
+        for key, services in profile.get("services", {}).items():
+            expanded_services = list()
+            for srv in services:
+                if srv in service_categories:
+                    expanded_services.extend([i['name'] for i in STORAGE.service.search(f"category:{srv}", as_obj=False, fl="name")['items']])
+            profile['services'][key] = list(set(services).union(set(expanded_services)))
+
     user_data['configuration'] = {
         "auth": {
             "allow_2fa": config.auth.allow_2fa,
@@ -220,6 +241,7 @@ def who_am_i(**kwargs):
             "max_dtl": config.submission.max_dtl,
             "file_sources": file_sources,
             "metadata": UI_METADATA_VALIDATION,
+            "profiles": submission_profiles,
             "verdicts": {
                 "info": config.submission.verdicts.info,
                 "suspicious": config.submission.verdicts.suspicious,
@@ -921,7 +943,6 @@ def set_user_settings(username, **kwargs):
 
     Data Block:
     {
-     "profile": true,                       # Should submissions be profiled
      "classification": "",                  # Default classification for this user sumbissions
      "default_zip_password": "zippy"        # Default password used for protected file downloads
      "description": "",                     # Default description for this user's submissions
@@ -982,7 +1003,6 @@ def get_user_submission_params(username, **kwargs):
 
     Result example:
     {
-     "profile": true,               # Should submissions be profiled
      "classification": "",          # Default classification for this user sumbissions
      "description": "",             # Default description for this user's submissions
      "priority": 1000,              # Default submission priority

assemblyline_ui/config.py

@@ -181,5 +181,7 @@ def get_signup_queue(key):
 ARCHIVE_MANAGER: ArchiveManager = ArchiveManager(
     config=config, datastore=STORAGE, filestore=FILESTORE, identify=IDENTIFY)
 SERVICE_LIST = forge.CachedObject(STORAGE.list_all_services, kwargs=dict(as_obj=False, full=True))
+SUBMISSION_PROFILES = {profile.name: profile for profile in config.submission.profiles}
+
 # End global
 #################################################################

assemblyline_ui/helper/service.py

@@ -1,13 +1,48 @@
 from copy import copy
 from typing import Any, Optional
-from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST
+from assemblyline.common.dict_utils import recursive_update
+from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST, SUBMISSION_PROFILES, STORAGE
 from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, SubmissionParams
 from assemblyline.odm.models.user_settings import UserSettings
 
 # Get the list of fields relating to the models
 USER_SETTINGS_FIELDS = list(UserSettings.fields().keys())
 SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys())
 
+def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED):
+    out = {}
+    for profile in SUBMISSION_PROFILES.values():
+        if CLASSIFICATION.is_accessible(classification, profile.classification):
+            user_default_profile = user_default_values.get(profile.name, {})
+
+            params = copy(profile.params.as_primitives(strip_null=True))
+            out[profile.name] = recursive_update(params, user_default_values.get(profile.name, {}))
+
+            service_spec = []
+            # If there are any editable service parameters that haven't been set, then assign their default values
+            for service, editable_params in profile.editable_params.items():
+                service_obj = STORAGE.get_service_with_delta(service, as_obj=False)
+                if not service_obj:
+                    continue
+                param_object = {'name': service, "params": []}
+                profile_service_spec =  user_default_profile.get('service_spec', {}).get(service, {})
+                for param in service_obj['submission_params']:
+                    if param['name'] not in editable_params:
+                        # Service parameter isn't allowed to be overridden in this profile
+                        continue
+
+                    new_param = copy(param)
+                    if profile_service_spec.get(param['name']):
+                        # Overwrite with user-specific value for profile
+                        new_param['value'] = profile_service_spec[param['name']]
+                    new_param["hide"] = False
+                    param_object["params"].append(new_param)
+                service_spec.append(param_object)
+
+            # Overwrite 'service_spec' value to be compliant with frontend rendering
+            out[profile.name]['service_spec'] = service_spec
+    return out
+
 def get_default_service_spec(srv_list=None, user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED):
     if not srv_list:
         srv_list = SERVICE_LIST

assemblyline_ui/helper/submission.py

@@ -9,14 +9,15 @@
 from typing import List
 from urllib.parse import urlparse
 
+from assemblyline.common.dict_utils import recursive_update
 from assemblyline.common.file import make_uri_file
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.str_utils import safe_str
 from assemblyline.common.iprange import is_ip_reserved
 from assemblyline.odm.models.config import HASH_PATTERN_MAP
 from assemblyline.odm.messages.submission import SubmissionMessage
 from assemblyline.odm.models.user import ROLES
-from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE
+from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES
 
 # Baseline fetch methods
 FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url'])
@@ -30,6 +31,7 @@
     MYIP = '127.0.0.1'
 
 
+
 #############################
 # download functions
 class FileTooBigException(Exception):
@@ -147,7 +149,21 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di
 
     return found, fileinfo
 
-
+def update_submission_parameters(s_params: dict, data: dict, user: dict):
+    s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile'))
+    # Apply provided params (if the user is allowed to)
+    if ROLES.submission_customize in user['roles']:
+        s_params.update(data.get("params", {}))
+    elif s_profile:
+        if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification):
+            # User isn't allowed to use the submission profile specified
+            raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile")
+        # Apply the profile (but allow the user to change some properties)
+        s_params = recursive_update(s_params, data.get("params", {}))
+        s_params = recursive_update(s_params, s_profile.params.as_primitives(strip_null=True))
+    else:
+        # No profile specified, raise an exception back to the user
+        raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}")