From 60b6e1b1b7e769a75da61b76c032ac52ae894d16 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 26 Jul 2024 17:35:17 -0700 Subject: [PATCH 001/115] Add an extra, missing field to the lookup.py model called max_matches that was accidentally dropped. set extra='forbid' for SecurityContentObject --- .../security_content_object_abstract.py | 2 +- contentctl/objects/lookup.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index 430872be..a9700cd5 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -25,7 +25,7 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True) + model_config = ConfigDict(use_enum_values=True,validate_default=True,extra="forbid") # name: str = ... # author: str = Field(...,max_length=255) # date: datetime.date = Field(...) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index d0b88fc8..00823681 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -29,6 +29,7 @@ class Lookup(SecurityContentObject): default_match: Optional[bool] = None match_type: Optional[str] = None min_matches: Optional[int] = None + max_matches: Optional[int] = None case_sensitive_match: Optional[bool] = None @@ -43,6 +44,7 @@ def serialize_model(self): "default_match": "true" if self.default_match is True else "false", "match_type": self.match_type, "min_matches": self.min_matches, + "max_matches": self.max_matches, "case_sensitive_match": "true" if self.case_sensitive_match is True else "false", "collection": self.collection, "fields_list": self.fields_list From fd33140ab656765a544260e926fce3139747be43 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 26 Jul 2024 19:07:39 -0700 Subject: [PATCH 002/115] enable error for extra keys in Pydantic Objects --- contentctl/contentctl.py | 2 +- contentctl/input/director.py | 1 - .../detection_abstract.py | 1 - contentctl/objects/alert_action.py | 3 ++- contentctl/objects/atomic.py | 7 +++--- contentctl/objects/base_test.py | 3 ++- contentctl/objects/baseline_tags.py | 3 ++- contentctl/objects/config.py | 2 +- contentctl/objects/data_source.py | 5 ++-- contentctl/objects/deployment.py | 5 ++-- contentctl/objects/deployment_email.py | 3 ++- contentctl/objects/deployment_notable.py | 3 ++- contentctl/objects/deployment_phantom.py | 3 ++- contentctl/objects/deployment_rba.py | 3 ++- contentctl/objects/deployment_scheduling.py | 3 ++- contentctl/objects/deployment_slack.py | 3 ++- contentctl/objects/detection_tags.py | 6 ++--- contentctl/objects/event_source.py | 11 -------- contentctl/objects/investigation_tags.py | 3 ++- contentctl/objects/mitre_attack_enrichment.py | 2 +- contentctl/objects/observable.py | 3 ++- contentctl/objects/playbook_tags.py | 6 ++++- contentctl/objects/ssa_detection.py | 25 +++++-------------- contentctl/objects/ssa_detection_tags.py | 3 ++- contentctl/objects/unit_test_attack_data.py | 3 ++- contentctl/objects/unit_test_baseline.py | 3 ++- contentctl/objects/unit_test_old.py | 3 ++- contentctl/output/data_source_writer.py | 1 - 28 files changed, 55 insertions(+), 64 deletions(-) delete mode 100644 contentctl/objects/event_source.py diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index eac735d1..3a9a208b 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -153,7 +153,7 @@ def main(): else: #The file exists, so load it up! - config_obj = YmlReader().load_file(configFile) + config_obj = YmlReader().load_file(configFile,add_fields=False) t = test.model_validate(config_obj) except Exception as e: print(f"Error validating 'contentctl.yml':\n{str(e)}") diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 0740abe3..256e8bc5 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -22,7 +22,6 @@ from contentctl.objects.atomic import AtomicTest from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.data_source import DataSource -from contentctl.objects.event_source import EventSource from contentctl.enrichments.attack_enrichment import AttackEnrichment from contentctl.enrichments.cve_enrichment import CveEnrichment diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 6f35d9cb..c0c4a688 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -22,7 +22,6 @@ from contentctl.objects.unit_test import UnitTest from contentctl.objects.test_group import TestGroup from contentctl.objects.integration_test import IntegrationTest -from contentctl.objects.event_source import EventSource from contentctl.objects.data_source import DataSource #from contentctl.objects.playbook import Playbook diff --git a/contentctl/objects/alert_action.py b/contentctl/objects/alert_action.py index f2f745d4..d2855292 100644 --- a/contentctl/objects/alert_action.py +++ b/contentctl/objects/alert_action.py @@ -1,5 +1,5 @@ from __future__ import annotations -from pydantic import BaseModel, model_serializer +from pydantic import BaseModel, model_serializer, ConfigDict from typing import Optional from contentctl.objects.deployment_email import DeploymentEmail @@ -9,6 +9,7 @@ from contentctl.objects.deployment_phantom import DeploymentPhantom class AlertAction(BaseModel): + model_config = ConfigDict(extra="forbid") email: Optional[DeploymentEmail] = None notable: Optional[DeploymentNotable] = None rba: Optional[DeploymentRBA] = DeploymentRBA() diff --git a/contentctl/objects/atomic.py b/contentctl/objects/atomic.py index e0abc30e..3fcaa757 100644 --- a/contentctl/objects/atomic.py +++ b/contentctl/objects/atomic.py @@ -41,6 +41,7 @@ class InputArgumentType(StrEnum): Url = "Url" class AtomicExecutor(BaseModel): + model_config = ConfigDict(extra="forbid") name: str elevation_required: Optional[bool] = False #Appears to be optional command: Optional[str] = None @@ -48,7 +49,7 @@ class AtomicExecutor(BaseModel): cleanup_command: Optional[str] = None @model_validator(mode='after') - def ensure_mutually_exclusive_fields(self)->AtomicExecutor: + def ensure_mutually_exclusive_fields(self): if self.command is not None and self.steps is not None: raise ValueError("command and steps cannot both be defined in the executor section. Exactly one must be defined.") elif self.command is None and self.steps is None: @@ -88,7 +89,7 @@ class AtomicTest(BaseModel): dependency_executor_name: Optional[DependencyExecutorType] = None @staticmethod - def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4)->Self: + def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4)->AtomicTest: return AtomicTest(name="Placeholder Atomic Test (enrichment disabled)", auto_generated_guid=auto_generated_guid, description="This is a placeholder AtomicTest. Because enrichments were not enabled, it has not been validated against the real Atomic Red Team Repo.", @@ -97,7 +98,7 @@ def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4)->Self: command="Placeholder command (enrichment disabled)")) @staticmethod - def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4)->Self: + def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4)->AtomicTest: return AtomicTest(name="Missing Atomic", auto_generated_guid=auto_generated_guid, description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile..", diff --git a/contentctl/objects/base_test.py b/contentctl/objects/base_test.py index c5ae1e02..853215ff 100644 --- a/contentctl/objects/base_test.py +++ b/contentctl/objects/base_test.py @@ -2,7 +2,7 @@ from typing import Union from abc import ABC, abstractmethod -from pydantic import BaseModel +from pydantic import BaseModel,ConfigDict from contentctl.objects.base_test_result import BaseTestResult @@ -20,6 +20,7 @@ def __str__(self) -> str: # TODO (cmcginley): enforce distinct test names w/in detections class BaseTest(BaseModel, ABC): + model_config = ConfigDict(extra="forbid") """ A test case for a detection """ diff --git a/contentctl/objects/baseline_tags.py b/contentctl/objects/baseline_tags.py index 4817ce4b..b2382c13 100644 --- a/contentctl/objects/baseline_tags.py +++ b/contentctl/objects/baseline_tags.py @@ -1,5 +1,5 @@ from __future__ import annotations -from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer +from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer, ConfigDict from typing import List, Any, Union from contentctl.objects.story import Story @@ -12,6 +12,7 @@ class BaselineTags(BaseModel): + model_config = ConfigDict(extra="forbid") analytic_story: List[Story] = Field(...) detections: List[Union[Detection,str]] = Field(...) product: List[SecurityContentProductName] = Field(...,min_length=1) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 4b13c568..215a8786 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -28,7 +28,7 @@ SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/{uid}/release/{version}/download" class App_Base(BaseModel,ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True, extra='forbid') uid: Optional[int] = Field(default=None) title: str = Field(description="Human-readable name used by the app. This can have special characters.") appid: Optional[Annotated[str, Field(pattern="^[a-zA-Z0-9_-]+$")]]= Field(default=None,description="Internal name used by your app. " diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index 7e31a9a4..2e488eeb 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -1,14 +1,13 @@ from __future__ import annotations from typing import Optional, Any -from pydantic import Field, FilePath, model_serializer +from pydantic import Field, model_serializer from contentctl.objects.security_content_object import SecurityContentObject -from contentctl.objects.event_source import EventSource class DataSource(SecurityContentObject): source: str = Field(...) sourcetype: str = Field(...) separator: Optional[str] = None - configuration: Optional[str] = None + configuration: Optional[str] = None supported_TA: Optional[list] = None fields: Optional[list] = None field_mappings: Optional[list] = None diff --git a/contentctl/objects/deployment.py b/contentctl/objects/deployment.py index f2b2f391..cf721dab 100644 --- a/contentctl/objects/deployment.py +++ b/contentctl/objects/deployment.py @@ -1,6 +1,6 @@ from __future__ import annotations -from pydantic import Field, computed_field, model_validator,ValidationInfo, model_serializer -from typing import Optional,Any +from pydantic import Field, computed_field,ValidationInfo, model_serializer +from typing import Any from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.deployment_scheduling import DeploymentScheduling @@ -57,7 +57,6 @@ def serialize_model(self): "tags": self.tags } - #Combine fields from this model with fields from parent model.update(super_fields) diff --git a/contentctl/objects/deployment_email.py b/contentctl/objects/deployment_email.py index a607502c..1d1269fe 100644 --- a/contentctl/objects/deployment_email.py +++ b/contentctl/objects/deployment_email.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentEmail(BaseModel): + model_config = ConfigDict(extra="forbid") message: str subject: str to: str \ No newline at end of file diff --git a/contentctl/objects/deployment_notable.py b/contentctl/objects/deployment_notable.py index b6e2c463..7f064b43 100644 --- a/contentctl/objects/deployment_notable.py +++ b/contentctl/objects/deployment_notable.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from typing import List class DeploymentNotable(BaseModel): + model_config = ConfigDict(extra="forbid") rule_description: str rule_title: str nes_fields: List[str] \ No newline at end of file diff --git a/contentctl/objects/deployment_phantom.py b/contentctl/objects/deployment_phantom.py index 11df2feb..1d4a9975 100644 --- a/contentctl/objects/deployment_phantom.py +++ b/contentctl/objects/deployment_phantom.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentPhantom(BaseModel): + model_config = ConfigDict(extra="forbid") cam_workers : str label : str phantom_server : str diff --git a/contentctl/objects/deployment_rba.py b/contentctl/objects/deployment_rba.py index b3412b3f..58917c70 100644 --- a/contentctl/objects/deployment_rba.py +++ b/contentctl/objects/deployment_rba.py @@ -1,6 +1,7 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentRBA(BaseModel): + model_config = ConfigDict(extra="forbid") enabled: bool = False \ No newline at end of file diff --git a/contentctl/objects/deployment_scheduling.py b/contentctl/objects/deployment_scheduling.py index 6c5a75a8..b21673d8 100644 --- a/contentctl/objects/deployment_scheduling.py +++ b/contentctl/objects/deployment_scheduling.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentScheduling(BaseModel): + model_config = ConfigDict(extra="forbid") cron_schedule: str earliest_time: str latest_time: str diff --git a/contentctl/objects/deployment_slack.py b/contentctl/objects/deployment_slack.py index 294836e2..03cf5ebb 100644 --- a/contentctl/objects/deployment_slack.py +++ b/contentctl/objects/deployment_slack.py @@ -1,7 +1,8 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentSlack(BaseModel): + model_config = ConfigDict(extra="forbid") channel: str message: str \ No newline at end of file diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index 87667c29..e775cb37 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -18,11 +18,11 @@ class DetectionTags(BaseModel): # detection spec - model_config = ConfigDict(use_enum_values=True,validate_default=False) + model_config = ConfigDict(use_enum_values=True,validate_default=False, extra='forbid') analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) - - + group: list[str] = [] + context: list[str] = [] confidence: NonNegativeInt = Field(...,le=100) impact: NonNegativeInt = Field(...,le=100) @computed_field diff --git a/contentctl/objects/event_source.py b/contentctl/objects/event_source.py deleted file mode 100644 index 0ed61979..00000000 --- a/contentctl/objects/event_source.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations -from typing import Union, Optional, List -from pydantic import BaseModel, Field - -from contentctl.objects.security_content_object import SecurityContentObject - -class EventSource(SecurityContentObject): - fields: Optional[list[str]] = None - field_mappings: Optional[list[dict]] = None - convert_to_log_source: Optional[list[dict]] = None - example_log: Optional[str] = None diff --git a/contentctl/objects/investigation_tags.py b/contentctl/objects/investigation_tags.py index 6db99eff..17d4b2d5 100644 --- a/contentctl/objects/investigation_tags.py +++ b/contentctl/objects/investigation_tags.py @@ -1,10 +1,11 @@ from __future__ import annotations from typing import List -from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer +from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer,ConfigDict from contentctl.objects.story import Story from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain class InvestigationTags(BaseModel): + model_config = ConfigDict(extra="forbid") analytic_story: List[Story] = Field([],min_length=1) product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1) required_fields: List[str] = Field(min_length=1) diff --git a/contentctl/objects/mitre_attack_enrichment.py b/contentctl/objects/mitre_attack_enrichment.py index bf00e18a..2fb61761 100644 --- a/contentctl/objects/mitre_attack_enrichment.py +++ b/contentctl/objects/mitre_attack_enrichment.py @@ -22,7 +22,7 @@ class MitreTactics(StrEnum): class MitreAttackEnrichment(BaseModel): - ConfigDict(use_enum_values=True) + ConfigDict(use_enum_values=True,extra='forbid') mitre_attack_id: Annotated[str, Field(pattern="^T\d{4}(.\d{3})?$")] = Field(...) mitre_attack_technique: str = Field(...) mitre_attack_tactics: List[MitreTactics] = Field(...) diff --git a/contentctl/objects/observable.py b/contentctl/objects/observable.py index 3a6134fe..1ca93b3e 100644 --- a/contentctl/objects/observable.py +++ b/contentctl/objects/observable.py @@ -1,11 +1,12 @@ from __future__ import annotations -from pydantic import BaseModel, validator +from pydantic import BaseModel, validator,ConfigDict from contentctl.objects.constants import * class Observable(BaseModel): + model_config = ConfigDict(extra="forbid") name: str type: str role: list[str] diff --git a/contentctl/objects/playbook_tags.py b/contentctl/objects/playbook_tags.py index fd4a21e6..10d90ac1 100644 --- a/contentctl/objects/playbook_tags.py +++ b/contentctl/objects/playbook_tags.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING, Optional, List -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field,ConfigDict import enum from contentctl.objects.detection import Detection @@ -36,6 +36,7 @@ class DefendTechnique(str,enum.Enum): D3_SRA = "D3-SRA" D3_RUAA = "D3-RUAA" class PlaybookTag(BaseModel): + model_config = ConfigDict(extra="forbid") analytic_story: Optional[list] = None detections: Optional[list] = None platform_tags: list[str] = Field(...,min_length=0) @@ -46,5 +47,8 @@ class PlaybookTag(BaseModel): use_cases: list[PlaybookUseCase] = Field([],min_length=0) defend_technique_id: Optional[List[DefendTechnique]] = None + labels:list[str] = [] + playbook_outputs:list[str] = [] + detection_objects: list[Detection] = [] \ No newline at end of file diff --git a/contentctl/objects/ssa_detection.py b/contentctl/objects/ssa_detection.py index 036f0b77..2a501aa7 100644 --- a/contentctl/objects/ssa_detection.py +++ b/contentctl/objects/ssa_detection.py @@ -1,30 +1,19 @@ from __future__ import annotations -import uuid -import string -import requests -import time -from pydantic import BaseModel, validator, root_validator -from dataclasses import dataclass -from datetime import datetime + +from pydantic import BaseModel,ConfigDict from typing import Union -import re -from contentctl.objects.abstract_security_content_objects.detection_abstract import Detection_Abstract + + from contentctl.objects.enums import AnalyticsType -from contentctl.objects.enums import DataModel from contentctl.objects.enums import DetectionStatus -from contentctl.objects.deployment import Deployment from contentctl.objects.ssa_detection_tags import SSADetectionTags from contentctl.objects.unit_test_ssa import UnitTestSSA from contentctl.objects.unit_test_old import UnitTestOld -from contentctl.objects.macro import Macro -from contentctl.objects.lookup import Lookup -from contentctl.objects.baseline import Baseline -from contentctl.objects.playbook import Playbook -from contentctl.helper.link_validator import LinkValidator -from contentctl.objects.enums import SecurityContentType + class SSADetection(BaseModel): + model_config = ConfigDict(extra="forbid",use_enum_values=True) # detection spec name: str id: str @@ -59,8 +48,6 @@ class SSADetection(BaseModel): # raise ValueError('name is longer then 67 chars: ' + v) # return v - class Config: - use_enum_values = True ''' @validator("name") diff --git a/contentctl/objects/ssa_detection_tags.py b/contentctl/objects/ssa_detection_tags.py index 62fef564..ccf7c073 100644 --- a/contentctl/objects/ssa_detection_tags.py +++ b/contentctl/objects/ssa_detection_tags.py @@ -1,13 +1,14 @@ from __future__ import annotations import re from typing import List -from pydantic import BaseModel, validator, ValidationError, model_validator, Field +from pydantic import BaseModel, validator, ValidationError, model_validator, Field,ConfigDict from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment from contentctl.objects.constants import * from contentctl.objects.enums import SecurityContentProductName class SSADetectionTags(BaseModel): + model_config = ConfigDict(extra="forbid") # detection spec #name: str analytic_story: list diff --git a/contentctl/objects/unit_test_attack_data.py b/contentctl/objects/unit_test_attack_data.py index 7a4d5d8a..c5dbc58d 100644 --- a/contentctl/objects/unit_test_attack_data.py +++ b/contentctl/objects/unit_test_attack_data.py @@ -1,9 +1,10 @@ from __future__ import annotations -from pydantic import BaseModel, HttpUrl, FilePath, Field +from pydantic import BaseModel, HttpUrl, FilePath, Field,ConfigDict from typing import Union, Optional class UnitTestAttackData(BaseModel): + model_config = ConfigDict(extra="forbid") data: Union[HttpUrl, FilePath] = Field(...) # TODO - should source and sourcetype should be mapped to a list # of supported source and sourcetypes in a given environment? diff --git a/contentctl/objects/unit_test_baseline.py b/contentctl/objects/unit_test_baseline.py index 9ba49336..66a60594 100644 --- a/contentctl/objects/unit_test_baseline.py +++ b/contentctl/objects/unit_test_baseline.py @@ -1,9 +1,10 @@ -from pydantic import BaseModel +from pydantic import BaseModel,ConfigDict from typing import Union class UnitTestBaseline(BaseModel): + model_config = ConfigDict(extra="forbid") name: str file: str pass_condition: str diff --git a/contentctl/objects/unit_test_old.py b/contentctl/objects/unit_test_old.py index 3858e01a..346e837d 100644 --- a/contentctl/objects/unit_test_old.py +++ b/contentctl/objects/unit_test_old.py @@ -1,10 +1,11 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from contentctl.objects.unit_test_ssa import UnitTestSSA class UnitTestOld(BaseModel): + model_config = ConfigDict(extra="forbid") name: str tests: list[UnitTestSSA] \ No newline at end of file diff --git a/contentctl/output/data_source_writer.py b/contentctl/output/data_source_writer.py index ba505905..0b573168 100644 --- a/contentctl/output/data_source_writer.py +++ b/contentctl/output/data_source_writer.py @@ -1,6 +1,5 @@ import csv from contentctl.objects.data_source import DataSource -from contentctl.objects.event_source import EventSource from typing import List import pathlib From e4f7dccab040f04aa852a6b271cac2b36363161a Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 26 Jul 2024 19:10:45 -0700 Subject: [PATCH 003/115] update template to remove risk_score since it is a comptued_field and should not be in the yml --- .../templates/detections/endpoint/anomalous_usage_of_7zip.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml index 77465781..3c3d85d3 100644 --- a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +++ b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml @@ -73,11 +73,10 @@ tags: - Processes.parent_process - Processes.process_id - Processes.parent_process_id - risk_score: 64 security_domain: endpoint tests: - name: True Positive Test attack_data: - data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1560.001/archive_utility/windows-sysmon.log source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational - sourcetype: xmlwineventlog \ No newline at end of file + sourcetype: xmlwineventlog From 830d201b54d87dfcdeeba71d4162a60e3e4b5296 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 15 Aug 2024 14:07:09 -0700 Subject: [PATCH 004/115] initial lookup updates --- contentctl/objects/lookup.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 9cc36007..1e28562d 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -15,6 +15,7 @@ LOOKUPS_TO_IGNORE.add("cim_corporate_web_domain_lookup") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("alexa_lookup_by_str") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("interesting_ports_lookup") #Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add("asset_lookup_by_str") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("admon_groups_def") #Shipped with the SA-admon addon #Special case for the Detection "Exploit Public Facing Application via Apache Commons Text" @@ -139,8 +140,23 @@ def ensure_mutually_exclusive_fields(self)->Lookup: @staticmethod def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: - lookups_to_get = set(re.findall(r'[^output]lookup (?:update=true)?(?:append=t)?\s*([^\s]*)', text_field)) + inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s]+)', text_field)) + #outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) + # Don't match inputlookup or outputlookup. Allow local=true or update=true or local=t or update=t + lookups_to_get = set(re.findall(r'(?:(? 0: + for l in my_lookups: + print(f"\n\nABCDEF {l.name}\n\n") + + return my_lookups \ No newline at end of file From e96fbd46bbde90ee865278239e4cd6a8dc076377 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 19 Aug 2024 14:20:53 -0700 Subject: [PATCH 005/115] continuing to make lookup improvements. --- contentctl/objects/lookup.py | 62 ++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 1e28562d..3a1cf8ee 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -1,8 +1,10 @@ from __future__ import annotations -from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer -from typing import TYPE_CHECKING, Optional, Any, Union +from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field +from typing import TYPE_CHECKING, Optional, Any, Union, Literal import re import csv +from enum import StrEnum +import abc if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto from contentctl.objects.config import validate @@ -17,6 +19,7 @@ LOOKUPS_TO_IGNORE.add("interesting_ports_lookup") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("asset_lookup_by_str") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("admon_groups_def") #Shipped with the SA-admon addon +LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Enterprise Security #Special case for the Detection "Exploit Public Facing Application via Apache Commons Text" LOOKUPS_TO_IGNORE.add("=") @@ -24,7 +27,7 @@ # TODO (#220): Split Lookup into 2 classes -class Lookup(SecurityContentObject): +class Lookup(SecurityContentObject, abc.ABC): collection: Optional[str] = None fields_list: Optional[str] = None @@ -141,22 +144,55 @@ def ensure_mutually_exclusive_fields(self)->Lookup: @staticmethod def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s]+)', text_field)) - #outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) + outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) # Don't match inputlookup or outputlookup. Allow local=true or update=true or local=t or update=t lookups_to_get = set(re.findall(r'(?:(? 0: - for l in my_lookups: - print(f"\n\nABCDEF {l.name}\n\n") return my_lookups - \ No newline at end of file + +class Lookup_external_type(StrEnum): + PYTHON = "python" + EXECUTABLE = "executable" + KVSTORE = "kvstore" + GEO = "geo" + GEO_HEX = "geo_hex" + +class ExternalLookup(Lookup, abc.ABC): + fields_list: list[str] = Field(...,min_length=1) + external_type: Lookup_external_type = Field(...) + + +class PythonLookup(ExternalLookup): + external_type = Lookup_external_type.PYTHON + +class ExecutableLookup(ExternalLookup): + external_type = Lookup_external_type.EXECUTABLE + +class KVStoreLookup(ExternalLookup): + external_type = Lookup_external_type.KVSTORE + +class GeoLookup(ExternalLookup): + external_type = Lookup_external_type.GEO + +class GeoHexLookup(ExternalLookup): + external_type = Lookup_external_type.GEO_HEX + + + + +class CSVLookup(Lookup): + pass \ No newline at end of file From 32ed03fd2e52f3ca3ff414816e3b7db326bd58fc Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 19 Aug 2024 14:21:18 -0700 Subject: [PATCH 006/115] more lookup changes --- contentctl/objects/lookup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 3a1cf8ee..5deae8e8 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -28,8 +28,8 @@ # TODO (#220): Split Lookup into 2 classes class Lookup(SecurityContentObject, abc.ABC): - - collection: Optional[str] = None + #collection will always be the name of the lookup + fields_list: Optional[str] = None filename: Optional[FilePath] = None default_match: Optional[bool] = None @@ -175,7 +175,6 @@ class ExternalLookup(Lookup, abc.ABC): fields_list: list[str] = Field(...,min_length=1) external_type: Lookup_external_type = Field(...) - class PythonLookup(ExternalLookup): external_type = Lookup_external_type.PYTHON @@ -195,4 +194,4 @@ class GeoHexLookup(ExternalLookup): class CSVLookup(Lookup): - pass \ No newline at end of file + filename: FilePath = Field(...) From 2a5663d973d7f826ad4777bdd0fa7c6aa02f1dec Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 22 Aug 2024 11:20:54 -0500 Subject: [PATCH 007/115] First crack at default config --- .vscode/settings.json | 10 +++++- pyproject.toml | 79 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8a62413d..74d85aa8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,7 +4,15 @@ "python.testing.cwd": "${workspaceFolder}", "python.languageServer": "Pylance", "python.analysis.typeCheckingMode": "strict", - "editor.defaultFormatter": "ms-python.black-formatter" + "[python]": { + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + "source.organizeImports": "explicit" + }, + "editor.defaultFormatter": "charliermarsh.ruff", + }, + "ruff.nativeServer": "on" } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e43c8bdf..6176e4d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,3 +33,82 @@ setuptools = ">=69.5.1,<74.0.0" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 88 +indent-width = 4 + +# Assume Python 3.8 +target-version = "py38" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" \ No newline at end of file From 3f7a585bea187ac06f5be04ef662eb4df1bb3fbe Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 22 Aug 2024 12:41:55 -0500 Subject: [PATCH 008/115] Adding suggested extension config --- .vscode/extensions.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..8d1a435f --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "charliermarsh.ruff" + ] +} \ No newline at end of file From 66e743ec77cca4ca314572aaaf168acda3ccea3c Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 27 Aug 2024 08:44:17 -0500 Subject: [PATCH 009/115] initial sketch --- .../detection_abstract.py | 2 + contentctl/objects/enums.py | 1 + contentctl/objects/rba.py | 49 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 contentctl/objects/rba.py diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index b6cac6be..702b4125 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -30,6 +30,7 @@ from contentctl.objects.test_group import TestGroup from contentctl.objects.integration_test import IntegrationTest from contentctl.objects.data_source import DataSource +from contentctl.objects.rba import rba # from contentctl.objects.playbook import Playbook from contentctl.objects.enums import ProvidingTechnology @@ -49,6 +50,7 @@ class Detection_Abstract(SecurityContentObject): search: str = Field(...) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) + rba: rba = Field(...) enabled_by_default: bool = False file_path: FilePath = Field(...) diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py index fa294302..395cb524 100644 --- a/contentctl/objects/enums.py +++ b/contentctl/objects/enums.py @@ -418,3 +418,4 @@ class RiskSeverity(str,enum.Enum): LOW = "low" MEDIUM = "medium" HIGH = "high" + diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py new file mode 100644 index 00000000..ceb56fbf --- /dev/null +++ b/contentctl/objects/rba.py @@ -0,0 +1,49 @@ +import enum +from pydantic import BaseModel +from abc import ABC +from typing import Union, List + + + +class RiskObjectType(str,enum.Enum): + SYSTEM = "system" + USER = "user" + OTHER = "other" + +class ThreatObjectType(str,enum.Enum): + CERTIFICATE_COMMON_NAME = "certificate_common_name" + CERTIFICATE_ORGANIZATION = "certificate_organization" + CERTIFICATE_SERIAL = "certificate_serial" + CERTIFICATE_UNIT = "certificate_unit" + COMMAND = "command" + DOMAIN = "domain" + EMAIL_ADDRESS = "email_address" + EMAIL_SUBJECT = "email_subject" + FILE_HASH = "file_hash" + FILE_NAME = "file_name" + HTTP_USER_AGENT = "http_user_agent" + IP_ADDRESS = "ip_address" + PROCESS = "process" + PROCESS_NAME = "process_name" + PARENT_PROCESS = "parent_process" + PARENT_PROCESS_NAME = "parent_process_name" + PROCESS_HASH = "process_hash" + REGISTRY_PATH = "registry_path" + REGISTRY_VALUE_NAME = "registry_value_name" + REGISTRY_VALUE_TEXT = "regstiry_value_text" + SERVICE = "service" + URL = "url" + +class risk_object(BaseModel, ABC): + field: str + type: RiskObjectType + score: int + +class threat_object(BaseModel, ABC): + field: str + type: ThreatObjectType + +class rba(BaseModel, ABC): + message: str + risk_objects: List[risk_object] + threat_object: Union[List[threat_object], None] \ No newline at end of file From 0a88668f3812d616ddc1ee8b854d3d4bd77420f1 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 30 Aug 2024 14:56:36 -0500 Subject: [PATCH 010/115] Bumping target_version --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6176e4d1..618a9ca8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ line-length = 88 indent-width = 4 # Assume Python 3.8 -target-version = "py38" +target-version = "py311" [tool.ruff.lint] # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. @@ -111,4 +111,4 @@ docstring-code-format = false # # This only has an effect when the `docstring-code-format` setting is # enabled. -docstring-code-line-length = "dynamic" \ No newline at end of file +docstring-code-line-length = "dynamic" From dd5b52dbd7db260a73f03586a64c4a3f9f7b3967 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 5 Sep 2024 09:28:25 -0500 Subject: [PATCH 011/115] save point --- .../detection_abstract.py | 4 ++-- contentctl/objects/rba.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 12fa03fa..8a5bb55e 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -35,7 +35,7 @@ from contentctl.objects.integration_test import IntegrationTest from contentctl.objects.data_source import DataSource -from contentctl.objects.rba import rba +from contentctl.objects.rba import rba_object from contentctl.objects.base_test_result import TestResultStatus @@ -63,7 +63,7 @@ class Detection_Abstract(SecurityContentObject): search: str = Field(...) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) - rba: rba = Field(...) + rba: rba_object = Field(...) enabled_by_default: bool = False file_path: FilePath = Field(...) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index ceb56fbf..89493616 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -1,16 +1,16 @@ -import enum +from enum import Enum from pydantic import BaseModel from abc import ABC -from typing import Union, List +from typing import Set -class RiskObjectType(str,enum.Enum): +class RiskObjectType(str, Enum): SYSTEM = "system" USER = "user" OTHER = "other" -class ThreatObjectType(str,enum.Enum): +class ThreatObjectType(str, Enum): CERTIFICATE_COMMON_NAME = "certificate_common_name" CERTIFICATE_ORGANIZATION = "certificate_organization" CERTIFICATE_SERIAL = "certificate_serial" @@ -34,16 +34,16 @@ class ThreatObjectType(str,enum.Enum): SERVICE = "service" URL = "url" -class risk_object(BaseModel, ABC): +class risk_object(BaseModel): field: str type: RiskObjectType score: int -class threat_object(BaseModel, ABC): +class threat_object(BaseModel): field: str type: ThreatObjectType -class rba(BaseModel, ABC): +class rba_object(BaseModel, ABC): message: str - risk_objects: List[risk_object] - threat_object: Union[List[threat_object], None] \ No newline at end of file + risk_objects: Set[risk_object] + threat_objects: Set[threat_object] = set() \ No newline at end of file From 2568f7110aeab2554c5605c1c1168f4577874020 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 10 Sep 2024 14:12:04 -0500 Subject: [PATCH 012/115] Updated pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 56df65ee..57e8d3bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,6 @@ exclude = [ line-length = 88 indent-width = 4 -# Assume Python 3.8 target-version = "py311" [tool.ruff.lint] From 0b4158b3f81e799a576e33943aeee34673282665 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Sep 2024 09:18:47 -0500 Subject: [PATCH 013/115] Github CI --- .github/workflows/ruff.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 00000000..6d84a781 --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,23 @@ +name: lint & format +on: + push: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install ruff + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run lint + run: ruff check --output-format=github contentctl/ + - name: Run Formatter + run: ruff format --check contentctl/ \ No newline at end of file From f99655d4657a6649c043359c2df32e93bebd278a Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Sep 2024 09:24:45 -0500 Subject: [PATCH 014/115] Add precommit hook --- .pre-commit-config.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4c0bcb35 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 # Use the ref you want to point at + hooks: + - id: check-json + - id: check-symlinks + - id: check-yaml + - id: detect-aws-credentials + - id: detect-private-key + - id: forbid-submodules + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.4 + hooks: + - id: ruff + args: [ --fix ] + - id: ruff-format \ No newline at end of file From c1bdfbcebde244f58a0b0cd6285d6ffd4613d20a Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Sep 2024 09:25:10 -0500 Subject: [PATCH 015/115] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2e4fcc96..156005c0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ poetry.lock # usual mac files .DS_Store */.DS_Store +.ruff_cache # custom dist/* From 7ed5e02a2872f00b4d5d8dd66976852dec3f46ce Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Sep 2024 09:46:09 -0500 Subject: [PATCH 016/115] Updating ruff workflow --- .github/workflows/ruff.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 6d84a781..4843361a 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -1,6 +1,5 @@ name: lint & format on: - push: pull_request: types: [opened, reopened, synchronize] From 9aa1607fff43ffc3c2487f153f33a1b16c50d217 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Sep 2024 10:49:43 -0500 Subject: [PATCH 017/115] Adding ruff as dev dependency --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3c9e4f6d..001d38a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ gitpython = "^3.1.43" setuptools = ">=69.5.1,<75.0.0" [tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] +ruff = "^0.6.4" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From 0c8d8e581f6291c5eb6f8de78e39944674373d5d Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 7 Oct 2024 15:06:08 -0500 Subject: [PATCH 018/115] Added python 3.13 to End to End testing --- .github/workflows/testEndToEnd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testEndToEnd.yml b/.github/workflows/testEndToEnd.yml index 29e0958e..54e80065 100644 --- a/.github/workflows/testEndToEnd.yml +++ b/.github/workflows/testEndToEnd.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ["3.11", "3.12"] + python_version: ["3.11", "3.12", "3.13"] operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"] #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"] From 1520f3b0cd631eedcb6187cb6f45a35da00d92fc Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 8 Oct 2024 11:51:42 -0500 Subject: [PATCH 019/115] Tweak run conditions and matrix --- .github/workflows/testEndToEnd.yml | 5 ++--- .github/workflows/test_against_escu.yml | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/testEndToEnd.yml b/.github/workflows/testEndToEnd.yml index 54e80065..3dc027a9 100644 --- a/.github/workflows/testEndToEnd.yml +++ b/.github/workflows/testEndToEnd.yml @@ -1,8 +1,7 @@ name: testEndToEnd on: - push: pull_request: - types: [opened, reopened] + types: [opened, reopened, sychronize] schedule: - cron: "44 4 * * *" @@ -12,7 +11,7 @@ jobs: fail-fast: false matrix: python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"] + operating_system: ["ubuntu-20.04", "ubuntu-24.04", "macos-15", "macos-14", "windows-2022"] #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"] diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index b527a6ee..63f8ea52 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -6,9 +6,8 @@ # note yet been fixed in security_content, we may see this workflow fail. name: test_against_escu on: - push: pull_request: - types: [opened, reopened] + types: [opened, reopened, sychronize] schedule: - cron: "44 4 * * *" @@ -17,9 +16,9 @@ jobs: strategy: fail-fast: false matrix: - python_version: ["3.11", "3.12"] + python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14"] + operating_system: ["ubuntu-20.04", "ubuntu-24.04", "macos-15", "macos-14"] # Do not test against ESCU until known character encoding issue is resolved # operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"] From 3ba8a09f0903ea8445839a61ee5774387d2bb9a0 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 8 Oct 2024 11:53:48 -0500 Subject: [PATCH 020/115] Typo --- .github/workflows/testEndToEnd.yml | 2 +- .github/workflows/test_against_escu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testEndToEnd.yml b/.github/workflows/testEndToEnd.yml index 3dc027a9..444ee96a 100644 --- a/.github/workflows/testEndToEnd.yml +++ b/.github/workflows/testEndToEnd.yml @@ -1,7 +1,7 @@ name: testEndToEnd on: pull_request: - types: [opened, reopened, sychronize] + types: [opened, reopened, synchronize] schedule: - cron: "44 4 * * *" diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index 63f8ea52..9758b6c0 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -7,7 +7,7 @@ name: test_against_escu on: pull_request: - types: [opened, reopened, sychronize] + types: [opened, reopened, synchronize] schedule: - cron: "44 4 * * *" From 030ae920c49f2ea42249c4e088c87f7b1fffb3ef Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 29 Oct 2024 09:16:53 -0500 Subject: [PATCH 021/115] bumped version --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c0bcb35..d8891b2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,8 +9,8 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.7.1 hooks: - id: ruff args: [ --fix ] - - id: ruff-format \ No newline at end of file + - id: ruff-format diff --git a/pyproject.toml b/pyproject.toml index ccd805c2..1a1dcb62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.6.4" +ruff = "^0.7.1" [build-system] requires = ["poetry-core>=1.0.0"] From b294765b1711a384e9d5438d5c6cc359df07f06c Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 14:17:02 -0600 Subject: [PATCH 022/115] Implement hashing --- contentctl/objects/rba.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 89493616..64ce7ecb 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -39,11 +39,17 @@ class risk_object(BaseModel): type: RiskObjectType score: int + def __hash__(self): + return hash((self.field, self.type, self.score)) + class threat_object(BaseModel): field: str type: ThreatObjectType + def __hash__(self): + return hash((self.field, self.type)) + class rba_object(BaseModel, ABC): message: str - risk_objects: Set[risk_object] - threat_objects: Set[threat_object] = set() \ No newline at end of file + risk_objects: Set[risk_object] + threat_objects: Set[threat_object] \ No newline at end of file From 7f7724c711ddc0ac8bccdbacd741df76273ca933 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 15:16:50 -0600 Subject: [PATCH 023/115] Updated default detection --- .../endpoint/anomalous_usage_of_7zip.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml index a101fd7d..df4cab65 100644 --- a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +++ b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml @@ -38,6 +38,22 @@ drilldown_searches: search: '| from datamodel Risk.All_Risk | search normalized_risk_object IN ($user$, $dest$) starthoursago=168 endhoursago=1 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`' earliest_offset: $info_min_time$ latest_offset: $info_max_time$ +rba: + message: An instance of $parent_process_name$ spawning $process_name$ was identified + on endpoint $dest$ by user $user$. This behavior is indicative of suspicious loading + of 7zip. + risk_objects: + - field: user + type: user + score: 56 + - field: dest + type: system + score: 60 + threat_objects: + - field: parent_process_name + type: parent_process_name + - field: process_name + type: process_name tags: analytic_story: - Cobalt Strike From 3be2c3ae3b8ca57c064e37feef6a6c68eafc85ed Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 15:24:29 -0600 Subject: [PATCH 024/115] Remove tags.message and tags.observable --- .../endpoint/anomalous_usage_of_7zip.yml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml index df4cab65..09bf1088 100644 --- a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +++ b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml @@ -60,29 +60,9 @@ tags: asset_type: Endpoint confidence: 80 impact: 80 - message: An instance of $parent_process_name$ spawning $process_name$ was identified - on endpoint $dest$ by user $user$. This behavior is indicative of suspicious loading - of 7zip. mitre_attack_id: - T1560.001 - T1560 - observable: - - name: user - type: User - role: - - Victim - - name: dest - type: Hostname - role: - - Victim - - name: parent_process_name - type: Process - role: - - Attacker - - name: process_name - type: Process - role: - - Attacker product: - Splunk Enterprise - Splunk Enterprise Security From 9c138f1042d0dd997e0636ddac719a25c38d81e5 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 15:26:42 -0600 Subject: [PATCH 025/115] remove code for tags.message --- contentctl/objects/detection_tags.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index b1d489f4..f672b721 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -72,7 +72,6 @@ def severity(self)->RiskSeverity: # TODO (#249): Add pydantic validator to ensure observables are unique within a detection observable: List[Observable] = [] - message: str = Field(...) product: list[SecurityContentProductName] = Field(..., min_length=1) required_fields: list[str] = Field(min_length=1) throttling: Optional[Throttling] = None @@ -157,7 +156,6 @@ def serialize_model(self): "kill_chain_phases": self.kill_chain_phases, "nist": self.nist, "observable": self.observable, - "message": self.message, "risk_score": self.risk_score, "security_domain": self.security_domain, "risk_severity": self.severity, From 11a1ca90a8dc16ed98db10fdc91801e77de5ff7c Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 16:05:14 -0600 Subject: [PATCH 026/115] reworking validations --- .../detection_abstract.py | 149 +++++++++++++----- 1 file changed, 110 insertions(+), 39 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 23c6dedb..e1282511 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -70,7 +70,7 @@ class Detection_Abstract(SecurityContentObject): search: str = Field(...) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) - rba: rba_object = Field(...) + rba: Optional[rba_object] = Field(...) explanation: None | str = Field( default=None, exclude=True, #Don't serialize this value when dumping the object @@ -761,50 +761,83 @@ def ensureThrottlingFieldsExist(self): @model_validator(mode="after") - def ensureProperObservablesExist(self): + def ensureProperRBAConfig(self): """ - If a detections is PRODUCTION and either TTP or ANOMALY, then it MUST have an Observable with the VICTIM role. - + If a detection has an RBA deployment and is PRODUCTION, then it must have an RBA config, with at least one risk object + Returns: - self: Returns itself if the valdiation passes + self: Returns itself if the validation passes """ + # NOTE: we ignore the type error around self.status because we are using Pydantic's # use_enum_values configuration # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name - if self.status not in [DetectionStatus.production.value]: # type: ignore - # Only perform this validation on production detections + if self.status not in [DetectionStatus.production.value]: # type: ignore return self + + if self.deployment.alert_action.rba is None: + # confirm we don't have an RBA config + if self.rba is None: + return self + else: + raise ValueError( + "Detection does not have a matching RBA deployment config, the RBA portion should be omitted." + ) + else: + if len(self.rba.risk_objects) > 0: # type: ignore + return self + else: + raise ValueError( + "Detection expects an RBA config with at least one risk object." + ) - if self.type not in [AnalyticsType.TTP.value, AnalyticsType.Anomaly.value]: - # Only perform this validation on TTP and Anomaly detections - return self - - # Detection is required to have a victim - roles: list[str] = [] - for observable in self.tags.observable: - roles.extend(observable.role) - - if roles.count("Victim") == 0: - raise ValueError( - "Error, there must be AT LEAST 1 Observable with the role 'Victim' declared in " - "Detection.tags.observables. However, none were found." - ) - - # Exactly one victim was found - return self + # TODO - Remove old observable code + # @model_validator(mode="after") + # def ensureProperObservablesExist(self): + # """ + # If a detections is PRODUCTION and either TTP or ANOMALY, then it MUST have an Observable with the VICTIM role. + + # Returns: + # self: Returns itself if the valdiation passes + # """ + # # NOTE: we ignore the type error around self.status because we are using Pydantic's + # # use_enum_values configuration + # # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name + # if self.status not in [DetectionStatus.production.value]: # type: ignore + # # Only perform this validation on production detections + # return self + + # if self.type not in [AnalyticsType.TTP.value, AnalyticsType.Anomaly.value]: + # # Only perform this validation on TTP and Anomaly detections + # return self + + # # Detection is required to have a victim + # roles: list[str] = [] + # for observable in self.tags.observable: + # roles.extend(observable.role) + + # if roles.count("Victim") == 0: + # raise ValueError( + # "Error, there must be AT LEAST 1 Observable with the role 'Victim' declared in " + # "Detection.tags.observables. However, none were found." + # ) + + # # Exactly one victim was found + # return self @model_validator(mode="after") - def search_observables_exist_validate(self): - observable_fields = [ob.name.lower() for ob in self.tags.observable] + def search_rba_fields_exist_validate(self): + risk_fields = [ob.field.lower() for ob in self.rba.risk_objects] + threat_fields = [ob.field.lower() for ob in self.rba.threat_objects] + rba_fields = risk_fields + threat_fields - # All $field$ fields from the message must appear in the search field_match_regex = r"\$([^\s.]*)\$" missing_fields: set[str] - if self.tags.message: - matches = re.findall(field_match_regex, self.tags.message.lower()) + if self.rba.message: + matches = re.findall(field_match_regex, self.rba.message.lower()) message_fields = [match.replace("$", "").lower() for match in matches] - missing_fields = set([field for field in observable_fields if field not in self.search.lower()]) + missing_fields = set([field for field in rba_fields if field not in self.search.lower()]) else: message_fields = [] missing_fields = set() @@ -812,10 +845,9 @@ def search_observables_exist_validate(self): error_messages: list[str] = [] if len(missing_fields) > 0: error_messages.append( - "The following fields are declared as observables, but do not exist in the " + "The following fields are declared in the rba config, but do not exist in the " f"search: {missing_fields}" ) - missing_fields = set([field for field in message_fields if field not in self.search.lower()]) if len(missing_fields) > 0: error_messages.append( @@ -823,19 +855,58 @@ def search_observables_exist_validate(self): f"the search: {missing_fields}" ) - # NOTE: we ignore the type error around self.status because we are using Pydantic's - # use_enum_values configuration - # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name - if len(error_messages) > 0 and self.status == DetectionStatus.production.value: # type: ignore + if len(error_messages) > 0 and self.status == DetectionStatus.production.value: #type: ignore msg = ( - "Use of fields in observables/messages that do not appear in search:\n\t- " + "Use of fields in rba/messages that do not appear in search:\n\t- " "\n\t- ".join(error_messages) ) raise ValueError(msg) - - # Found everything return self + # TODO: Remove old observable code + # @model_validator(mode="after") + # def search_observables_exist_validate(self): + # observable_fields = [ob.name.lower() for ob in self.tags.observable] + + # # All $field$ fields from the message must appear in the search + # field_match_regex = r"\$([^\s.]*)\$" + + # missing_fields: set[str] + # if self.tags.message: + # matches = re.findall(field_match_regex, self.tags.message.lower()) + # message_fields = [match.replace("$", "").lower() for match in matches] + # missing_fields = set([field for field in observable_fields if field not in self.search.lower()]) + # else: + # message_fields = [] + # missing_fields = set() + + # error_messages: list[str] = [] + # if len(missing_fields) > 0: + # error_messages.append( + # "The following fields are declared as observables, but do not exist in the " + # f"search: {missing_fields}" + # ) + + # missing_fields = set([field for field in message_fields if field not in self.search.lower()]) + # if len(missing_fields) > 0: + # error_messages.append( + # "The following fields are used as fields in the message, but do not exist in " + # f"the search: {missing_fields}" + # ) + + # # NOTE: we ignore the type error around self.status because we are using Pydantic's + # # use_enum_values configuration + # # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name + # if len(error_messages) > 0 and self.status == DetectionStatus.production.value: # type: ignore + # msg = ( + # "Use of fields in observables/messages that do not appear in search:\n\t- " + # "\n\t- ".join(error_messages) + # ) + # raise ValueError(msg) + + # # Found everything + # return self + @field_validator("tests", mode="before") def ensure_yml_test_is_unittest(cls, v:list[dict]): """The typing for the tests field allows it to be one of From 3882b9bcb7fd1ab11f785520392920eee7f487b6 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 16:15:22 -0600 Subject: [PATCH 027/115] new rba location --- contentctl/output/templates/savedsearches_detections.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/output/templates/savedsearches_detections.j2 b/contentctl/output/templates/savedsearches_detections.j2 index 396bb2c6..2a719133 100644 --- a/contentctl/output/templates/savedsearches_detections.j2 +++ b/contentctl/output/templates/savedsearches_detections.j2 @@ -44,7 +44,7 @@ action.escu.providing_technologies = null action.escu.analytic_story = {{ objectListToNameList(detection.tags.analytic_story) | tojson }} {% if detection.deployment.alert_action.rba.enabled%} action.risk = 1 -action.risk.param._risk_message = {{ detection.tags.message | escapeNewlines() }} +action.risk.param._risk_message = {{ detection.rba.message | escapeNewlines() }} action.risk.param._risk = {{ detection.risk | tojson }} action.risk.param._risk_score = 0 action.risk.param.verbose = 0 From d5848228b540fd53cf4ed82a6ce9eea588a4df97 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 16:36:03 -0600 Subject: [PATCH 028/115] Refactor risk() --- .../detection_abstract.py | 127 ++++++++++-------- 1 file changed, 74 insertions(+), 53 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index e1282511..d02c321c 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -366,66 +366,87 @@ def nes_fields(self) -> Optional[str]: def providing_technologies(self) -> List[ProvidingTechnology]: return ProvidingTechnology.getProvidingTechFromSearch(self.search) - # TODO (#247): Refactor the risk property of detection_abstract + @computed_field @property def risk(self) -> list[dict[str, Any]]: risk_objects: list[dict[str, str | int]] = [] - # TODO (#246): "User Name" type should map to a "user" risk object and not "other" - risk_object_user_types = {'user', 'username', 'email address'} - risk_object_system_types = {'device', 'endpoint', 'hostname', 'ip address'} - process_threat_object_types = {'process name', 'process'} - file_threat_object_types = {'file name', 'file', 'file hash'} - url_threat_object_types = {'url string', 'url'} - ip_threat_object_types = {'ip address'} - - for entity in self.tags.observable: + + for entity in self.rba.risk_objects: risk_object: dict[str, str | int] = dict() - if 'Victim' in entity.role and entity.type.lower() in risk_object_user_types: - risk_object['risk_object_type'] = 'user' - risk_object['risk_object_field'] = entity.name - risk_object['risk_score'] = self.tags.risk_score - risk_objects.append(risk_object) - - elif 'Victim' in entity.role and entity.type.lower() in risk_object_system_types: - risk_object['risk_object_type'] = 'system' - risk_object['risk_object_field'] = entity.name - risk_object['risk_score'] = self.tags.risk_score - risk_objects.append(risk_object) - - elif 'Attacker' in entity.role and entity.type.lower() in process_threat_object_types: - risk_object['threat_object_field'] = entity.name - risk_object['threat_object_type'] = "process" - risk_objects.append(risk_object) - - elif 'Attacker' in entity.role and entity.type.lower() in file_threat_object_types: - risk_object['threat_object_field'] = entity.name - risk_object['threat_object_type'] = "file_name" - risk_objects.append(risk_object) - - elif 'Attacker' in entity.role and entity.type.lower() in ip_threat_object_types: - risk_object['threat_object_field'] = entity.name - risk_object['threat_object_type'] = "ip_address" - risk_objects.append(risk_object) - - elif 'Attacker' in entity.role and entity.type.lower() in url_threat_object_types: - risk_object['threat_object_field'] = entity.name - risk_object['threat_object_type'] = "url" - risk_objects.append(risk_object) - - elif 'Attacker' in entity.role: - risk_object['threat_object_field'] = entity.name - risk_object['threat_object_type'] = entity.type.lower() - risk_objects.append(risk_object) + risk_object['risk_object_type'] = entity.type + risk_object['risk_object_field'] = entity.field + risk_object['risk_score'] = entity.score + risk_objects.append(risk_object) + + for entity in self.rba.threat_objects: + threat_object: dict[str, str] = dict() + threat_object['threat_object_field'] = entity.field + threat_object['threat_object_type'] = entity.type + risk_objects.append(threat_object) + return risk_objects - else: - risk_object['risk_object_type'] = 'other' - risk_object['risk_object_field'] = entity.name - risk_object['risk_score'] = self.tags.risk_score - risk_objects.append(risk_object) - continue - return risk_objects + # TODO Remove observable code + # @computed_field + # @property + # def risk(self) -> list[dict[str, Any]]: + # risk_objects: list[dict[str, str | int]] = [] + # # TODO (#246): "User Name" type should map to a "user" risk object and not "other" + # risk_object_user_types = {'user', 'username', 'email address'} + # risk_object_system_types = {'device', 'endpoint', 'hostname', 'ip address'} + # process_threat_object_types = {'process name', 'process'} + # file_threat_object_types = {'file name', 'file', 'file hash'} + # url_threat_object_types = {'url string', 'url'} + # ip_threat_object_types = {'ip address'} + + # for entity in self.tags.observable: + # risk_object: dict[str, str | int] = dict() + # if 'Victim' in entity.role and entity.type.lower() in risk_object_user_types: + # risk_object['risk_object_type'] = 'user' + # risk_object['risk_object_field'] = entity.name + # risk_object['risk_score'] = self.tags.risk_score + # risk_objects.append(risk_object) + + # elif 'Victim' in entity.role and entity.type.lower() in risk_object_system_types: + # risk_object['risk_object_type'] = 'system' + # risk_object['risk_object_field'] = entity.name + # risk_object['risk_score'] = self.tags.risk_score + # risk_objects.append(risk_object) + + # elif 'Attacker' in entity.role and entity.type.lower() in process_threat_object_types: + # risk_object['threat_object_field'] = entity.name + # risk_object['threat_object_type'] = "process" + # risk_objects.append(risk_object) + + # elif 'Attacker' in entity.role and entity.type.lower() in file_threat_object_types: + # risk_object['threat_object_field'] = entity.name + # risk_object['threat_object_type'] = "file_name" + # risk_objects.append(risk_object) + + # elif 'Attacker' in entity.role and entity.type.lower() in ip_threat_object_types: + # risk_object['threat_object_field'] = entity.name + # risk_object['threat_object_type'] = "ip_address" + # risk_objects.append(risk_object) + + # elif 'Attacker' in entity.role and entity.type.lower() in url_threat_object_types: + # risk_object['threat_object_field'] = entity.name + # risk_object['threat_object_type'] = "url" + # risk_objects.append(risk_object) + + # elif 'Attacker' in entity.role: + # risk_object['threat_object_field'] = entity.name + # risk_object['threat_object_type'] = entity.type.lower() + # risk_objects.append(risk_object) + + # else: + # risk_object['risk_object_type'] = 'other' + # risk_object['risk_object_field'] = entity.name + # risk_object['risk_score'] = self.tags.risk_score + # risk_objects.append(risk_object) + # continue + + # return risk_objects @computed_field @property From 3cde4a6d5cd06db1b17449d66571243cbddb9f44 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 8 Nov 2024 16:45:18 -0600 Subject: [PATCH 029/115] slight tweak --- .../abstract_security_content_objects/detection_abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index d02c321c..239c78c2 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -796,7 +796,7 @@ def ensureProperRBAConfig(self): if self.status not in [DetectionStatus.production.value]: # type: ignore return self - if self.deployment.alert_action.rba is None: + if self.deployment.alert_action.rba.enabled is False: # confirm we don't have an RBA config if self.rba is None: return self From f4739cc996779423a745ddad6e620264af12a88e Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 10:53:24 -0600 Subject: [PATCH 030/115] Better guard against None --- .../detection_abstract.py | 7 +++++-- contentctl/objects/detection_tags.py | 1 - contentctl/output/yml_output.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 239c78c2..1ef4059b 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -70,7 +70,7 @@ class Detection_Abstract(SecurityContentObject): search: str = Field(...) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) - rba: Optional[rba_object] = Field(...) + rba: Optional[rba_object] = Field(default=None) explanation: None | str = Field( default=None, exclude=True, #Don't serialize this value when dumping the object @@ -796,7 +796,7 @@ def ensureProperRBAConfig(self): if self.status not in [DetectionStatus.production.value]: # type: ignore return self - if self.deployment.alert_action.rba.enabled is False: + if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: # confirm we don't have an RBA config if self.rba is None: return self @@ -848,6 +848,9 @@ def ensureProperRBAConfig(self): @model_validator(mode="after") def search_rba_fields_exist_validate(self): + if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: + return self + risk_fields = [ob.field.lower() for ob in self.rba.risk_objects] threat_fields = [ob.field.lower() for ob in self.rba.threat_objects] rba_fields = risk_fields + threat_fields diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index f672b721..e8e3010d 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -155,7 +155,6 @@ def serialize_model(self): "cis20": self.cis20, "kill_chain_phases": self.kill_chain_phases, "nist": self.nist, - "observable": self.observable, "risk_score": self.risk_score, "security_domain": self.security_domain, "risk_severity": self.severity, diff --git a/contentctl/output/yml_output.py b/contentctl/output/yml_output.py index 93eae5dc..196c5300 100644 --- a/contentctl/output/yml_output.py +++ b/contentctl/output/yml_output.py @@ -29,6 +29,7 @@ def writeDetections(self, objects: list, output_path : str) -> None: "how_to_implement": True, "known_false_positives": True, "references": True, + "rba": True "tags": { "analytic_story": True, @@ -41,7 +42,6 @@ def writeDetections(self, objects: list, output_path : str) -> None: "message": True, "mitre_attack_id": True, "kill_chain_phases:": True, - "observable": True, "product": True, "required_fields": True, "risk_score": True, From d6b848eb97ae1bf451aed10681b4997b0ac35c4d Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 11:24:14 -0600 Subject: [PATCH 031/115] Another None case --- .../abstract_security_content_objects/detection_abstract.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 1ef4059b..4c1d669a 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -805,6 +805,11 @@ def ensureProperRBAConfig(self): "Detection does not have a matching RBA deployment config, the RBA portion should be omitted." ) else: + print(self.name) + if self.rba is None: + raise ValueError( + "Detection is expected to have an RBA object based on its deployment config" + ) if len(self.rba.risk_objects) > 0: # type: ignore return self else: From 9cda91ec082f3e83af3d72fe81497f685aed89e7 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 11:35:54 -0600 Subject: [PATCH 032/115] remove print --- .../detection_abstract.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 4c1d669a..86202baa 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -805,17 +805,17 @@ def ensureProperRBAConfig(self): "Detection does not have a matching RBA deployment config, the RBA portion should be omitted." ) else: - print(self.name) if self.rba is None: raise ValueError( "Detection is expected to have an RBA object based on its deployment config" ) - if len(self.rba.risk_objects) > 0: # type: ignore - return self else: - raise ValueError( - "Detection expects an RBA config with at least one risk object." - ) + if len(self.rba.risk_objects) > 0: # type: ignore + return self + else: + raise ValueError( + "Detection expects an RBA config with at least one risk object." + ) # TODO - Remove old observable code # @model_validator(mode="after") From 8e5676c72188dadb6cd21e856ad278d2834802dc Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 11:41:49 -0600 Subject: [PATCH 033/115] Another None guard --- .../abstract_security_content_objects/detection_abstract.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 86202baa..c958fa89 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -853,9 +853,15 @@ def ensureProperRBAConfig(self): @model_validator(mode="after") def search_rba_fields_exist_validate(self): + # Return immediately if RBA isn't required if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: return self + # Raise error if RBA isn't present + if self.rba is None: + raise ValueError( + "RBA is required for this detection based on its deployment config" + ) risk_fields = [ob.field.lower() for ob in self.rba.risk_objects] threat_fields = [ob.field.lower() for ob in self.rba.threat_objects] rba_fields = risk_fields + threat_fields From 6f77c477efbdfa865e7a14fc0b8127a2716eba71 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 11:50:17 -0600 Subject: [PATCH 034/115] Just production --- .../abstract_security_content_objects/detection_abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index c958fa89..1c0dc94b 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -854,7 +854,7 @@ def ensureProperRBAConfig(self): @model_validator(mode="after") def search_rba_fields_exist_validate(self): # Return immediately if RBA isn't required - if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: + if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None) or self.status != DetectionStatus.production.value: #type: ignore return self # Raise error if RBA isn't present From ef7784da653f213f05f7b4f83d304955b78914d1 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 12 Nov 2024 14:57:05 -0800 Subject: [PATCH 035/115] Move Baseline datamodel from YML field to computed_field --- contentctl/objects/baseline.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index 5dc59d8f..e5fb243d 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import Annotated, Optional, List,Any -from pydantic import field_validator, ValidationInfo, Field, model_serializer +from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field from contentctl.objects.deployment import Deployment from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.enums import DataModel @@ -15,7 +15,6 @@ class Baseline(SecurityContentObject): name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) type: Annotated[str,Field(pattern="^Baseline$")] = Field(...) - datamodel: Optional[List[DataModel]] = None search: str = Field(..., min_length=4) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) @@ -34,6 +33,10 @@ def get_conf_stanza_name(self, app:CustomApp)->str: def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment: return Deployment.getDeployment(v,info) + @computed_field + @property + def datamodel(self) -> List[DataModel]: + return [dm for dm in DataModel if dm.value in self.search] @model_serializer def serialize_model(self): From a27f7903a054301218deca8246f7eaeb4ef705cf Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 12 Nov 2024 15:04:09 -0800 Subject: [PATCH 036/115] make datamodel a computed field for investigation --- contentctl/objects/baseline.py | 2 +- contentctl/objects/investigation.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index e5fb243d..a41acbb4 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Annotated, Optional, List,Any +from typing import Annotated, List,Any from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field from contentctl.objects.deployment import Deployment from contentctl.objects.security_content_object import SecurityContentObject diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 293e3331..6e058783 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -16,13 +16,10 @@ class Investigation(SecurityContentObject): model_config = ConfigDict(use_enum_values=True,validate_default=False) type: str = Field(...,pattern="^Investigation$") - datamodel: list[DataModel] = Field(...) name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) search: str = Field(...) how_to_implement: str = Field(...) known_false_positives: str = Field(...) - - tags: InvestigationTags # enrichment @@ -38,6 +35,11 @@ def inputs(self)->List[str]: return inputs + @computed_field + @property + def datamodel(self) -> List[DataModel]: + return [dm for dm in DataModel if dm.value in self.search] + @computed_field @property def lowercase_name(self)->str: From 1a4ea933910e1a8f51958183c24f00cf4c5a17bb Mon Sep 17 00:00:00 2001 From: ljstella Date: Wed, 13 Nov 2024 09:38:08 -0600 Subject: [PATCH 037/115] Validate all, not just production --- .../abstract_security_content_objects/detection_abstract.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index a8578aad..78e7951f 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -794,8 +794,7 @@ def ensureProperRBAConfig(self): # NOTE: we ignore the type error around self.status because we are using Pydantic's # use_enum_values configuration # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name - if self.status not in [DetectionStatus.production.value]: # type: ignore - return self + if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: # confirm we don't have an RBA config @@ -855,7 +854,7 @@ def ensureProperRBAConfig(self): @model_validator(mode="after") def search_rba_fields_exist_validate(self): # Return immediately if RBA isn't required - if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None) or self.status != DetectionStatus.production.value: #type: ignore + if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None): #type: ignore return self # Raise error if RBA isn't present From e2565f4eb4ebfa451f145a4a8062c5d966eadd78 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 14 Nov 2024 09:24:46 -0600 Subject: [PATCH 038/115] Remove comment --- .../abstract_security_content_objects/detection_abstract.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 78e7951f..8d0810b4 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -791,10 +791,6 @@ def ensureProperRBAConfig(self): self: Returns itself if the validation passes """ - # NOTE: we ignore the type error around self.status because we are using Pydantic's - # use_enum_values configuration - # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name - if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: # confirm we don't have an RBA config From 12c88812b89da1790f15750c5d6159003cd3ea2f Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 14 Nov 2024 09:25:11 -0600 Subject: [PATCH 039/115] Temporary tweak for testing companion branch --- .github/workflows/test_against_escu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index b527a6ee..f3b1dd94 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -36,6 +36,7 @@ jobs: with: path: security_content repository: splunk/security_content + ref: rba_migration #Install the given version of Python we will test against - name: Install Required Python Version From afa864bb8884297b261be5f3566efefd2d8d874a Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 15 Nov 2024 09:56:11 -0600 Subject: [PATCH 040/115] tweak to required --- .../abstract_security_content_objects/detection_abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 8d0810b4..dc37aed2 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -850,7 +850,7 @@ def ensureProperRBAConfig(self): @model_validator(mode="after") def search_rba_fields_exist_validate(self): # Return immediately if RBA isn't required - if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None): #type: ignore + if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None) and self.rba is None: #type: ignore return self # Raise error if RBA isn't present From e7fd4660a32ad8b21b0b21155b7e79df20905e3e Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 15 Nov 2024 11:13:44 -0600 Subject: [PATCH 041/115] threat object type typo --- contentctl/objects/rba.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 64ce7ecb..3ad29cc4 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -30,7 +30,7 @@ class ThreatObjectType(str, Enum): PROCESS_HASH = "process_hash" REGISTRY_PATH = "registry_path" REGISTRY_VALUE_NAME = "registry_value_name" - REGISTRY_VALUE_TEXT = "regstiry_value_text" + REGISTRY_VALUE_TEXT = "registry_value_text" SERVICE = "service" URL = "url" From 2fe24e68ba1afd911e75c377a1973f9cd0d9f60f Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 15 Nov 2024 14:56:48 -0600 Subject: [PATCH 042/115] more threat object types --- contentctl/objects/rba.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 3ad29cc4..58c5b5aa 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -21,6 +21,7 @@ class ThreatObjectType(str, Enum): EMAIL_SUBJECT = "email_subject" FILE_HASH = "file_hash" FILE_NAME = "file_name" + FILE_PATH = "file_path" HTTP_USER_AGENT = "http_user_agent" IP_ADDRESS = "ip_address" PROCESS = "process" @@ -32,6 +33,7 @@ class ThreatObjectType(str, Enum): REGISTRY_VALUE_NAME = "registry_value_name" REGISTRY_VALUE_TEXT = "registry_value_text" SERVICE = "service" + SYSTEM = "system" URL = "url" class risk_object(BaseModel): From 3435f4ccb37306c1929cd30af58931320a7d158f Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 15 Nov 2024 15:04:03 -0600 Subject: [PATCH 043/115] one more threat object type --- contentctl/objects/rba.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 58c5b5aa..05a32a41 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -34,6 +34,7 @@ class ThreatObjectType(str, Enum): REGISTRY_VALUE_TEXT = "registry_value_text" SERVICE = "service" SYSTEM = "system" + TLS_HASH = "tls_hash" URL = "url" class risk_object(BaseModel): From d1c78f2c3edaebd7df8603b596c94ebd97e4c223 Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 18 Nov 2024 08:35:18 -0600 Subject: [PATCH 044/115] bump tyro version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 35936c6f..1eab2eda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ semantic-version = "^2.10.0" bottle = ">=0.12.25,<0.14.0" tqdm = "^4.66.5" pygit2 = "^1.15.1" -tyro = "^0.8.3" +tyro = "^0.9.1" gitpython = "^3.1.43" setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] From 9a546c8528da3071c3216f2fed3698dbc1f5b625 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 18 Nov 2024 11:44:35 -0800 Subject: [PATCH 045/115] Removal of required_fields from pydantic object definitions. --- contentctl/actions/new_content.py | 1 - contentctl/helper/utils.py | 14 -------------- contentctl/objects/baseline_tags.py | 2 -- contentctl/objects/detection_tags.py | 1 - contentctl/objects/investigation_tags.py | 2 -- contentctl/output/templates/doc_detections.j2 | 5 ----- contentctl/output/yml_output.py | 1 - .../endpoint/anomalous_usage_of_7zip.yml | 11 ----------- 8 files changed, 37 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 0a54cf11..2e451704 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -49,7 +49,6 @@ def buildDetection(self)->dict[str,Any]: answers['tags']['mitre_attack_id'] = [x.strip() for x in answers['mitre_attack_ids'].split(',')] answers['tags']['observable'] = [{'name': 'UPDATE', 'type': 'UPDATE', 'role': ['UPDATE']}] answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] - answers['tags']['required_fields'] = ['UPDATE'] answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100' answers['tags']['security_domain'] = answers['security_domain'] del answers["security_domain"] diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 261ecb64..e0649f2d 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -247,20 +247,6 @@ def validate_git_pull_request(repo_path: str, pr_number: int) -> str: return hash - # @staticmethod - # def check_required_fields( - # thisField: str, definedFields: dict, requiredFields: list[str] - # ): - # missing_fields = [ - # field for field in requiredFields if field not in definedFields - # ] - # if len(missing_fields) > 0: - # raise ( - # ValueError( - # f"Could not validate - please resolve other errors resulting in missing fields {missing_fields}" - # ) - # ) - @staticmethod def verify_file_exists( file_path: str, verbose_print=False, timeout_seconds: int = 10 diff --git a/contentctl/objects/baseline_tags.py b/contentctl/objects/baseline_tags.py index 397edaf3..db5f8048 100644 --- a/contentctl/objects/baseline_tags.py +++ b/contentctl/objects/baseline_tags.py @@ -18,7 +18,6 @@ class BaselineTags(BaseModel): # TODO (#223): can we remove str from the possible types here? detections: List[Union[Detection,str]] = Field(...) product: List[SecurityContentProductName] = Field(...,min_length=1) - required_fields: List[str] = Field(...,min_length=1) security_domain: SecurityDomain = Field(...) @@ -34,7 +33,6 @@ def serialize_model(self): "analytic_story": [story.name for story in self.analytic_story], "detections": [detection.name for detection in self.detections if isinstance(detection,Detection)], "product": self.product, - "required_fields":self.required_fields, "security_domain":self.security_domain, "deployments": None } diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index 7c6b6028..69c6bafe 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -74,7 +74,6 @@ def severity(self)->RiskSeverity: observable: List[Observable] = [] message: str = Field(...) product: list[SecurityContentProductName] = Field(..., min_length=1) - required_fields: list[str] = Field(min_length=1) throttling: Optional[Throttling] = None security_domain: SecurityDomain = Field(...) cve: List[CVE_TYPE] = [] diff --git a/contentctl/objects/investigation_tags.py b/contentctl/objects/investigation_tags.py index 17d4b2d5..c4b812e6 100644 --- a/contentctl/objects/investigation_tags.py +++ b/contentctl/objects/investigation_tags.py @@ -8,7 +8,6 @@ class InvestigationTags(BaseModel): model_config = ConfigDict(extra="forbid") analytic_story: List[Story] = Field([],min_length=1) product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1) - required_fields: List[str] = Field(min_length=1) security_domain: SecurityDomain = Field(...) @@ -24,7 +23,6 @@ def serialize_model(self): model= { "analytic_story": [story.name for story in self.analytic_story], "product": self.product, - "required_fields": self.required_fields, "security_domain": self.security_domain, } diff --git a/contentctl/output/templates/doc_detections.j2 b/contentctl/output/templates/doc_detections.j2 index 5430b0ed..60f0282f 100644 --- a/contentctl/output/templates/doc_detections.j2 +++ b/contentctl/output/templates/doc_detections.j2 @@ -162,11 +162,6 @@ The SPL above uses the following Lookups: {% endfor %} {% endif -%} -#### Required field -{% for field in object.tags.required_fields -%} -* {{ field }} -{% endfor %} - #### How To Implement {{ object.how_to_implement}} diff --git a/contentctl/output/yml_output.py b/contentctl/output/yml_output.py index 93eae5dc..b4da5412 100644 --- a/contentctl/output/yml_output.py +++ b/contentctl/output/yml_output.py @@ -43,7 +43,6 @@ def writeDetections(self, objects: list, output_path : str) -> None: "kill_chain_phases:": True, "observable": True, "product": True, - "required_fields": True, "risk_score": True, "security_domain": True }, diff --git a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml index 22db7857..fbf847e1 100644 --- a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +++ b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml @@ -71,17 +71,6 @@ tags: - Splunk Enterprise - Splunk Enterprise Security - Splunk Cloud - required_fields: - - _time - - Processes.process_name - - Processes.process - - Processes.dest - - Processes.user - - Processes.parent_process_name - - Processes.process_name - - Processes.parent_process - - Processes.process_id - - Processes.parent_process_id security_domain: endpoint tests: - name: True Positive Test From 73ef9e38d176d9a2db3e4efe13b70921c84ddeeb Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 18 Nov 2024 11:49:44 -0800 Subject: [PATCH 046/115] Remove context from detection_tags --- contentctl/objects/detection_tags.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index 69c6bafe..185bc190 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -42,7 +42,6 @@ class DetectionTags(BaseModel): analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) group: list[str] = [] - context: list[str] = [] confidence: NonNegativeInt = Field(...,le=100) impact: NonNegativeInt = Field(...,le=100) @computed_field From 0bf6a5f5c526dd266ffc2d9752dc265dc2e6858e Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 18 Nov 2024 14:36:49 -0600 Subject: [PATCH 047/115] Update pyproject.toml for python 3.13 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bdbe8d51..e27c9d06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ readme = "README.md" contentctl = 'contentctl.contentctl:main' [tool.poetry.dependencies] -python = "^3.11,<3.13" +python = "^3.11,<3.14" pydantic = "^2.8.2" PyYAML = "^6.0.2" requests = "~2.32.3" From 39ce0ef905eb173b258841da54d7691537695027 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 19 Nov 2024 15:33:51 -0800 Subject: [PATCH 048/115] Removed risk_Score from contentctl new template. Added drilldowns, if appropriate, and made the link to attack_data set invalid, so that if it is not updated it fails validation. This prevents an incorrect attack_data from failing silently. --- contentctl/actions/new_content.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 2e451704..1f53ca10 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -13,6 +13,20 @@ from contentctl.output.yml_writer import YmlWriter class NewContent: + DEFAULT_DRILLDOWN_DEF = [ + { + "name": 'View the detection results for - "$first_observable_name_here$" and "$second_observable_name_here$"', + "search": '%original_detection_search% | search first_observable_type_here = "$first_observable_name_here$" second_observable_type_here = $second_observable_name_here$', + "earliest_offset": '$info_min_time$', + "latest_offset": '$info_max_time$' + }, + { + "name": 'View risk events for the last 7 days for - "$first_observable_name_here$" and "$second_observable_name_here$"', + "search": '| from datamodel Risk.All_Risk | search normalized_risk_object IN ("$first_observable_name_here$", "$second_observable_name_here$") starthoursago=168 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`', + "earliest_offset": '$info_min_time$', + "latest_offset": '$info_max_time$' + } + ] def buildDetection(self)->dict[str,Any]: questions = NewContentQuestions.get_questions_detection() @@ -40,6 +54,8 @@ def buildDetection(self)->dict[str,Any]: answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT' answers['known_false_positives'] = 'UPDATE_KNOWN_FALSE_POSITIVES' answers['references'] = ['REFERENCE'] + if answers['type'] in ["TTP", "Correlation", "Anomaly", "TTP"]: + answers['drilldown_searches'] = NewContent.DEFAULT_DRILLDOWN_DEF answers['tags'] = dict() answers['tags']['analytic_story'] = ['UPDATE_STORY_NAME'] answers['tags']['asset_type'] = 'UPDATE asset_type' @@ -49,7 +65,6 @@ def buildDetection(self)->dict[str,Any]: answers['tags']['mitre_attack_id'] = [x.strip() for x in answers['mitre_attack_ids'].split(',')] answers['tags']['observable'] = [{'name': 'UPDATE', 'type': 'UPDATE', 'role': ['UPDATE']}] answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] - answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100' answers['tags']['security_domain'] = answers['security_domain'] del answers["security_domain"] answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE'] @@ -60,7 +75,7 @@ def buildDetection(self)->dict[str,Any]: 'name': "True Positive Test", 'attack_data': [ { - 'data': "https://github.com/splunk/contentctl/wiki", + 'data': "Go to https://github.com/splunk/contentctl/wiki for information about the format of this field", "sourcetype": "UPDATE SOURCETYPE", "source": "UPDATE SOURCE" } From c647a9f41cf34cf82373080e91a23c75cc536b8d Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 19 Nov 2024 15:38:16 -0800 Subject: [PATCH 049/115] make a change to github workflow, temporarily, to test against relevant updated content --- .github/workflows/test_against_escu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index b527a6ee..d4f427ff 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@v4 with: path: security_content - repository: splunk/security_content + ref: strict_yml_fields #quick test against updated content #Install the given version of Python we will test against - name: Install Required Python Version From 0a910ce6747169e57bf73dc13feac70009e6ed19 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 19 Nov 2024 16:24:29 -0800 Subject: [PATCH 050/115] clean up new content template --- contentctl/actions/new_content.py | 147 +++++++++++++++++------------- 1 file changed, 82 insertions(+), 65 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 1f53ca10..3cf5579d 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -1,5 +1,3 @@ - - from dataclasses import dataclass import questionary from typing import Any @@ -28,64 +26,87 @@ class NewContent: } ] - def buildDetection(self)->dict[str,Any]: + def buildDetection(self) -> tuple[dict[str, Any], str]: questions = NewContentQuestions.get_questions_detection() - answers: dict[str,str] = questionary.prompt( - questions, - kbi_msg="User did not answer all of the prompt questions. Exiting...") + answers: dict[str, str] = questionary.prompt( + questions, + kbi_msg="User did not answer all of the prompt questions. Exiting...", + ) if not answers: raise ValueError("User didn't answer one or more questions!") - answers.update(answers) - answers['name'] = answers['detection_name'] - del answers['detection_name'] - answers['id'] = str(uuid.uuid4()) - answers['version'] = 1 - answers['date'] = datetime.today().strftime('%Y-%m-%d') - answers['author'] = answers['detection_author'] - del answers['detection_author'] - answers['data_source'] = answers['data_source'] - answers['type'] = answers['detection_type'] - del answers['detection_type'] - answers['status'] = "production" #start everything as production since that's what we INTEND the content to become - answers['description'] = 'UPDATE_DESCRIPTION' - file_name = answers['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() - answers['search'] = answers['detection_search'] + ' | `' + file_name + '_filter`' - del answers['detection_search'] - answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT' - answers['known_false_positives'] = 'UPDATE_KNOWN_FALSE_POSITIVES' - answers['references'] = ['REFERENCE'] - if answers['type'] in ["TTP", "Correlation", "Anomaly", "TTP"]: - answers['drilldown_searches'] = NewContent.DEFAULT_DRILLDOWN_DEF - answers['tags'] = dict() - answers['tags']['analytic_story'] = ['UPDATE_STORY_NAME'] - answers['tags']['asset_type'] = 'UPDATE asset_type' - answers['tags']['confidence'] = 'UPDATE value between 1-100' - answers['tags']['impact'] = 'UPDATE value between 1-100' - answers['tags']['message'] = 'UPDATE message' - answers['tags']['mitre_attack_id'] = [x.strip() for x in answers['mitre_attack_ids'].split(',')] - answers['tags']['observable'] = [{'name': 'UPDATE', 'type': 'UPDATE', 'role': ['UPDATE']}] - answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] - answers['tags']['security_domain'] = answers['security_domain'] - del answers["security_domain"] - answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE'] - - #generate the tests section - answers['tests'] = [ - { - 'name': "True Positive Test", - 'attack_data': [ - { - 'data': "Go to https://github.com/splunk/contentctl/wiki for information about the format of this field", - "sourcetype": "UPDATE SOURCETYPE", - "source": "UPDATE SOURCE" - } - ] - } - ] - del answers["mitre_attack_ids"] - return answers - def buildStory(self)->dict[str,Any]: + data_source_field = ( + answers["data_source"] if len(answers["data_source"]) > 0 else ["UPDATE"] + ) + file_name = ( + answers["detection_name"] + .replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") + .lower() + ) + + #Minimum lenght for a mitre tactic is 5 characters: T1000 + if len(answers["mitre_attack_ids"]) >= 5: + mitre_attack_ids = [x.strip() for x in answers["mitre_attack_ids"].split(",")] + else: + #string was too short, so just put a placeholder + mitre_attack_ids = ["UPDATE"] + + output_file_answers: dict[str, Any] = { + "name": answers["detection_name"], + "id": str(uuid.uuid4()), + "version": 1, + "date": datetime.today().strftime("%Y-%m-%d"), + "author": answers["detection_author"], + "status": "production", # start everything as production since that's what we INTEND the content to become + "type": answers["detection_type"], + "description": "UPDATE_DESCRIPTION", + "data_source": data_source_field, + "search": f"{answers['detection_search']} | `{file_name}_filter`'", + "how_to_implement": "UPDATE_HOW_TO_IMPLEMENT", + "known_false_positives": "UPDATE_KNOWN_FALSE_POSITIVES", + "references": ["REFERENCE"], + "drilldown_searches": NewContent.DEFAULT_DRILLDOWN_DEF, + "tags": { + "analytic_story": ["UPDATE_STORY_NAME"], + "asset_type": "UPDATE asset_type", + "confidence": "UPDATE value between 1-100", + "impact": "UPDATE value between 1-100", + "message": "UPDATE message", + "mitre_attack_id": mitre_attack_ids, + "observable": [ + {"name": "UPDATE", "type": "UPDATE", "role": ["UPDATE"]} + ], + "product": [ + "Splunk Enterprise", + "Splunk Enterprise Security", + "Splunk Cloud", + ], + "security_domain": answers["security_domain"], + "cve": ["UPDATE WITH CVE(S) IF APPLICABLE"], + }, + "tests": [ + { + "name": "True Positive Test", + "attack_data": [ + { + "data": "Go to https://github.com/splunk/contentctl/wiki for information about the format of this field", + "sourcetype": "UPDATE SOURCETYPE", + "source": "UPDATE SOURCE", + } + ], + } + ], + } + + if answers["detection_type"] not in ["TTP", "Correlation", "Anomaly", "TTP"]: + del output_file_answers["drilldown_searches"] + + return output_file_answers, answers['detection_kind'] + + def buildStory(self) -> dict[str, Any]: questions = NewContentQuestions.get_questions_story() answers = questionary.prompt( questions, @@ -110,12 +131,11 @@ def buildStory(self)->dict[str,Any]: del answers['usecase'] answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE'] return answers - def execute(self, input_dto: new) -> None: if input_dto.type == NewContentType.detection: - content_dict = self.buildDetection() - subdirectory = pathlib.Path('detections') / content_dict.pop('detection_kind') + content_dict, detection_kind = self.buildDetection() + subdirectory = pathlib.Path('detections') / detection_kind elif input_dto.type == NewContentType.story: content_dict = self.buildStory() subdirectory = pathlib.Path('stories') @@ -125,23 +145,20 @@ def execute(self, input_dto: new) -> None: full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name')) YmlWriter.writeYmlFile(str(full_output_path), content_dict) - - def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewContentType) -> None: if type == NewContentType.detection: file_path = os.path.join(self.output_path, 'detections', subdirectory_name, self.convertNameToFileName(object['name'], object['tags']['product'])) output_folder = pathlib.Path(self.output_path)/'detections'/subdirectory_name - #make sure the output folder exists for this detection + # make sure the output folder exists for this detection output_folder.mkdir(exist_ok=True) YmlWriter.writeDetection(file_path, object) print("Successfully created detection " + file_path) - + elif type == NewContentType.story: file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product'])) YmlWriter.writeStory(file_path, object) print("Successfully created story " + file_path) - + else: raise(Exception(f"Object Must be Story or Detection, but is not: {object}")) - From 9856df50820bebbf8f4e718bfcfad113aaec2a12 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 19 Nov 2024 16:27:46 -0800 Subject: [PATCH 051/115] fix out security_content repo reference --- .github/workflows/test_against_escu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index d4f427ff..7e2b8fce 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -35,6 +35,7 @@ jobs: uses: actions/checkout@v4 with: path: security_content + repository: splunk/security_content ref: strict_yml_fields #quick test against updated content #Install the given version of Python we will test against From aa99df8007fb7b021afb804aa61cbde9b9c68864 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 19 Nov 2024 16:37:17 -0800 Subject: [PATCH 052/115] remove duplicate in list --- contentctl/actions/new_content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 3cf5579d..92145958 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -101,7 +101,7 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: ], } - if answers["detection_type"] not in ["TTP", "Correlation", "Anomaly", "TTP"]: + if answers["detection_type"] not in ["TTP", "Anomaly", "Correlation"]: del output_file_answers["drilldown_searches"] return output_file_answers, answers['detection_kind'] From 4bb9d415a71a80d071c4e6e8ba0f75e8d87140d9 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 20 Nov 2024 10:28:55 -0800 Subject: [PATCH 053/115] Revert test to security_content develop branch. Bump version of contentctl to v4.5.0 in prep for release. --- .github/workflows/test_against_escu.yml | 1 - pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index 7e2b8fce..b527a6ee 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -36,7 +36,6 @@ jobs: with: path: security_content repository: splunk/security_content - ref: strict_yml_fields #quick test against updated content #Install the given version of Python we will test against - name: Install Required Python Version diff --git a/pyproject.toml b/pyproject.toml index ed1eebd9..6e9f51d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contentctl" -version = "4.4.6" +version = "4.5.0" description = "Splunk Content Control Tool" authors = ["STRT "] From f97597b515c07015f2e27d47a8907c1b2664fa96 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 20 Nov 2024 11:32:13 -0800 Subject: [PATCH 054/115] Update new_content generation to give a repeatable value when a field has not been updated. Provide more context for enum fields as to what can be set. Finally, throw an error during YML read if an un-UPDATED field still exists in any of the YMLs. --- contentctl/actions/new_content.py | 36 ++++++++++++----------- contentctl/input/new_content_questions.py | 2 +- contentctl/input/yml_reader.py | 17 +++++++---- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 92145958..fb33c507 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -9,7 +9,8 @@ import pathlib from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract from contentctl.output.yml_writer import YmlWriter - +from contentctl.objects.enums import AssetType +from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, SES_OBSERVABLE_ROLE_MAPPING class NewContent: DEFAULT_DRILLDOWN_DEF = [ { @@ -25,6 +26,7 @@ class NewContent: "latest_offset": '$info_max_time$' } ] + UPDATE_PREFIX = "_UPDATE_" def buildDetection(self) -> tuple[dict[str, Any], str]: questions = NewContentQuestions.get_questions_detection() @@ -36,7 +38,7 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: raise ValueError("User didn't answer one or more questions!") data_source_field = ( - answers["data_source"] if len(answers["data_source"]) > 0 else ["UPDATE"] + answers["data_source"] if len(answers["data_source"]) > 0 else [f"{NewContent.UPDATE_PREFIX} zero or more data_sources"] ) file_name = ( answers["detection_name"] @@ -52,7 +54,7 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: mitre_attack_ids = [x.strip() for x in answers["mitre_attack_ids"].split(",")] else: #string was too short, so just put a placeholder - mitre_attack_ids = ["UPDATE"] + mitre_attack_ids = [f"{NewContent.UPDATE_PREFIX} zero or more mitre_attack_ids"] output_file_answers: dict[str, Any] = { "name": answers["detection_name"], @@ -62,22 +64,22 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: "author": answers["detection_author"], "status": "production", # start everything as production since that's what we INTEND the content to become "type": answers["detection_type"], - "description": "UPDATE_DESCRIPTION", + "description": f"{NewContent.UPDATE_PREFIX} by providing a description of your search", "data_source": data_source_field, "search": f"{answers['detection_search']} | `{file_name}_filter`'", - "how_to_implement": "UPDATE_HOW_TO_IMPLEMENT", - "known_false_positives": "UPDATE_KNOWN_FALSE_POSITIVES", - "references": ["REFERENCE"], + "how_to_implement": f"{NewContent.UPDATE_PREFIX} how to implement your search", + "known_false_positives": f"{NewContent.UPDATE_PREFIX} known false positives for your search", + "references": [f"{NewContent.UPDATE_PREFIX} zero or more http references to provide more information about your search"], "drilldown_searches": NewContent.DEFAULT_DRILLDOWN_DEF, "tags": { - "analytic_story": ["UPDATE_STORY_NAME"], - "asset_type": "UPDATE asset_type", - "confidence": "UPDATE value between 1-100", - "impact": "UPDATE value between 1-100", - "message": "UPDATE message", + "analytic_story": [f"{NewContent.UPDATE_PREFIX} by providing zero or more analytic stories"], + "asset_type": f"{NewContent.UPDATE_PREFIX} by providing and asset type from {list(AssetType._value2member_map_)}", + "confidence": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100", + "impact": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100", + "message": f"{NewContent.UPDATE_PREFIX} by providing a risk message. Fields in your search results can be referenced using $fieldName$", "mitre_attack_id": mitre_attack_ids, "observable": [ - {"name": "UPDATE", "type": "UPDATE", "role": ["UPDATE"]} + {"name": f"{NewContent.UPDATE_PREFIX} the field name of the observable. This is a field that exists in your search results.", "type": f"{NewContent.UPDATE_PREFIX} the type of your observable from the list {list(SES_OBSERVABLE_TYPE_MAPPING.keys())}.", "role": [f"{NewContent.UPDATE_PREFIX} the role from the list {list(SES_OBSERVABLE_ROLE_MAPPING.keys())}"]} ], "product": [ "Splunk Enterprise", @@ -85,16 +87,16 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: "Splunk Cloud", ], "security_domain": answers["security_domain"], - "cve": ["UPDATE WITH CVE(S) IF APPLICABLE"], + "cve": [f"{NewContent.UPDATE_PREFIX} with CVE(s) if applicable"], }, "tests": [ { "name": "True Positive Test", "attack_data": [ { - "data": "Go to https://github.com/splunk/contentctl/wiki for information about the format of this field", - "sourcetype": "UPDATE SOURCETYPE", - "source": "UPDATE SOURCE", + "data": f"{NewContent.UPDATE_PREFIX} the data file to replay. Go to https://github.com/splunk/contentctl/wiki for information about the format of this field", + "sourcetype": f"{NewContent.UPDATE_PREFIX} the sourcetype of your data file.", + "source": f"{NewContent.UPDATE_PREFIX} the source of your datafile", } ], } diff --git a/contentctl/input/new_content_questions.py b/contentctl/input/new_content_questions.py index 02b20f46..98595776 100644 --- a/contentctl/input/new_content_questions.py +++ b/contentctl/input/new_content_questions.py @@ -57,7 +57,7 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: "type": "text", "message": "enter search (spl)", "name": "detection_search", - "default": "| UPDATE_SPL", + "default": "| _UPDATE_ SPL", }, { "type": "text", diff --git a/contentctl/input/yml_reader.py b/contentctl/input/yml_reader.py index 11bea479..8df243fd 100644 --- a/contentctl/input/yml_reader.py +++ b/contentctl/input/yml_reader.py @@ -1,15 +1,12 @@ from typing import Dict, Any - import yaml - - import sys import pathlib class YmlReader(): @staticmethod - def load_file(file_path: pathlib.Path, add_fields=True, STRICT_YML_CHECKING=False) -> Dict[str,Any]: + def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING:bool=False) -> Dict[str,Any]: try: file_handler = open(file_path, 'r', encoding="utf-8") @@ -27,8 +24,16 @@ def load_file(file_path: pathlib.Path, add_fields=True, STRICT_YML_CHECKING=Fals print(f"Error loading YML file {file_path}: {str(e)}") sys.exit(1) try: - #yml_obj = list(yaml.safe_load_all(file_handler))[0] - yml_obj = yaml.load(file_handler, Loader=yaml.CSafeLoader) + #Ideally we should use + # from contentctl.actions.new_content import NewContent + # and use NewContent.UPDATE_PREFIX, + # but there is a circular dependency right now which makes that difficult. + # We have instead hardcoded UPDATE_PREFIX + UPDATE_PREFIX = "_UPDATE_" + data = file_handler.read() + if UPDATE_PREFIX in data: + raise Exception(f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required.") + yml_obj = yaml.load(data, Loader=yaml.CSafeLoader) except yaml.YAMLError as exc: print(exc) sys.exit(1) From 3fea2f60c886e54bffd30d8f58408303beb3fc9f Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 20 Nov 2024 12:51:13 -0800 Subject: [PATCH 055/115] update drilldowns --- contentctl/actions/new_content.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index fb33c507..136528b8 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -12,21 +12,23 @@ from contentctl.objects.enums import AssetType from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, SES_OBSERVABLE_ROLE_MAPPING class NewContent: + UPDATE_PREFIX = "_UPDATE_" + DEFAULT_DRILLDOWN_DEF = [ { - "name": 'View the detection results for - "$first_observable_name_here$" and "$second_observable_name_here$"', - "search": '%original_detection_search% | search first_observable_type_here = "$first_observable_name_here$" second_observable_type_here = $second_observable_name_here$', + "name": f'View the detection results for - "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" and "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"', + "search": f'%original_detection_search% | search "${UPDATE_PREFIX}FIRST_RISK_OBJECT = "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" second_observable_type_here = "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"', "earliest_offset": '$info_min_time$', "latest_offset": '$info_max_time$' }, { - "name": 'View risk events for the last 7 days for - "$first_observable_name_here$" and "$second_observable_name_here$"', - "search": '| from datamodel Risk.All_Risk | search normalized_risk_object IN ("$first_observable_name_here$", "$second_observable_name_here$") starthoursago=168 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`', + "name": f'View risk events for the last 7 days for - "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" and "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"', + "search": f'| from datamodel Risk.All_Risk | search normalized_risk_object IN ("${UPDATE_PREFIX}FIRST_RISK_OBJECT$", "${UPDATE_PREFIX}SECOND_RISK_OBJECT$") starthoursago=168 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`', "earliest_offset": '$info_min_time$', "latest_offset": '$info_max_time$' } ] - UPDATE_PREFIX = "_UPDATE_" + def buildDetection(self) -> tuple[dict[str, Any], str]: questions = NewContentQuestions.get_questions_detection() From db1996973cdff1ba959595b1f4a9cc1f6d88381d Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 20 Nov 2024 17:01:07 -0800 Subject: [PATCH 056/115] change _UPDATE_ string to __UPDATE__ Remove extra pair of quotes from new detection template --- contentctl/actions/new_content.py | 4 ++-- contentctl/input/new_content_questions.py | 2 +- contentctl/input/yml_reader.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 136528b8..0d3ffc4a 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -12,7 +12,7 @@ from contentctl.objects.enums import AssetType from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, SES_OBSERVABLE_ROLE_MAPPING class NewContent: - UPDATE_PREFIX = "_UPDATE_" + UPDATE_PREFIX = "__UPDATE__" DEFAULT_DRILLDOWN_DEF = [ { @@ -68,7 +68,7 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: "type": answers["detection_type"], "description": f"{NewContent.UPDATE_PREFIX} by providing a description of your search", "data_source": data_source_field, - "search": f"{answers['detection_search']} | `{file_name}_filter`'", + "search": f"{answers['detection_search']} | `{file_name}_filter`", "how_to_implement": f"{NewContent.UPDATE_PREFIX} how to implement your search", "known_false_positives": f"{NewContent.UPDATE_PREFIX} known false positives for your search", "references": [f"{NewContent.UPDATE_PREFIX} zero or more http references to provide more information about your search"], diff --git a/contentctl/input/new_content_questions.py b/contentctl/input/new_content_questions.py index 98595776..dbc47cdd 100644 --- a/contentctl/input/new_content_questions.py +++ b/contentctl/input/new_content_questions.py @@ -57,7 +57,7 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: "type": "text", "message": "enter search (spl)", "name": "detection_search", - "default": "| _UPDATE_ SPL", + "default": "| __UPDATE__ SPL", }, { "type": "text", diff --git a/contentctl/input/yml_reader.py b/contentctl/input/yml_reader.py index 8df243fd..49dfb812 100644 --- a/contentctl/input/yml_reader.py +++ b/contentctl/input/yml_reader.py @@ -29,7 +29,7 @@ def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING # and use NewContent.UPDATE_PREFIX, # but there is a circular dependency right now which makes that difficult. # We have instead hardcoded UPDATE_PREFIX - UPDATE_PREFIX = "_UPDATE_" + UPDATE_PREFIX = "__UPDATE__" data = file_handler.read() if UPDATE_PREFIX in data: raise Exception(f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required.") From b382d440ad038ba5db48d84e113d2581cec1abb3 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 20 Nov 2024 17:21:19 -0800 Subject: [PATCH 057/115] pin pydantic to older minor version to resolve bug in our code --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6e9f51d4..4d6172e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ contentctl = 'contentctl.contentctl:main' [tool.poetry.dependencies] python = "^3.11,<3.13" -pydantic = "^2.8.2" +pydantic = "~2.9.2" PyYAML = "^6.0.2" requests = "~2.32.3" pycvesearch = "^1.2" From 140089f842495ece46ae028ec169e99024c3b2b5 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 22 Nov 2024 08:22:58 -0600 Subject: [PATCH 058/115] Oopsied the merge --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 35936c6f..fcb016b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ contentctl = 'contentctl.contentctl:main' [tool.poetry.dependencies] python = "^3.11" -pydantic = "^2.8.2" +pydantic = "~2.9.2" PyYAML = "^6.0.2" requests = "~2.32.3" pycvesearch = "^1.2" From 042a53a27e16f6f57e04708a057f14b2530f05ec Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 22 Nov 2024 08:24:27 -0600 Subject: [PATCH 059/115] Wrong branch for 3.13 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ae581e3..ec8ec960 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ readme = "README.md" contentctl = 'contentctl.contentctl:main' [tool.poetry.dependencies] -python = "^3.11,<3.14" +python = "^3.11,<3.13" pydantic = "~2.9.2" PyYAML = "^6.0.2" requests = "~2.32.3" From 559195229ebfa73bdbe9ebc6f062a2c00801ad0d Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 22 Nov 2024 11:24:06 -0600 Subject: [PATCH 060/115] Ruff version bump --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8891b2a..a8c73774 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.8.0 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index 745bd4ab..00f568d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.7.1" +ruff = "^0.8.0" [build-system] requires = ["poetry-core>=1.0.0"] From 5191ff821ba202ee50a50aefeb488478605ac659 Mon Sep 17 00:00:00 2001 From: Lou Stella Date: Sat, 23 Nov 2024 13:33:11 -0600 Subject: [PATCH 061/115] bump tyro version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82c9a47a..6350a4ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ semantic-version = "^2.10.0" bottle = ">=0.12.25,<0.14.0" tqdm = "^4.66.5" pygit2 = "^1.15.1" -tyro = "^0.9.1" +tyro = "^0.9.2" gitpython = "^3.1.43" setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] From e671f2b3a9a30b377bb2f387badb9b0aea8be980 Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 25 Nov 2024 12:59:44 -0600 Subject: [PATCH 062/115] Create new rba object via new content workflow --- contentctl/actions/new_content.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 0a54cf11..283b3f4c 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -40,6 +40,24 @@ def buildDetection(self)->dict[str,Any]: answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT' answers['known_false_positives'] = 'UPDATE_KNOWN_FALSE_POSITIVES' answers['references'] = ['REFERENCE'] + if answers['type'] == "TTP" or answers['type'] == "Anomaly": + answers['rba'] = { + 'message': "UPDATE Risk Message", + 'risk_objects': [ + { + 'field': 'UPDATE RISK OBJECT FIELD NAME', + 'type': 'UPDATE SYSTEM, USER, OR OTHER', + 'score': 10 + } + ], + 'threat_objects': [ + { + 'field': 'UPDATE THREAT OBJECT FIELD', + 'type': 'UPDATE THREAT OBJECT TYPE' + } + ] + } + answers['tags'] = dict() answers['tags']['analytic_story'] = ['UPDATE_STORY_NAME'] answers['tags']['asset_type'] = 'UPDATE asset_type' @@ -47,10 +65,8 @@ def buildDetection(self)->dict[str,Any]: answers['tags']['impact'] = 'UPDATE value between 1-100' answers['tags']['message'] = 'UPDATE message' answers['tags']['mitre_attack_id'] = [x.strip() for x in answers['mitre_attack_ids'].split(',')] - answers['tags']['observable'] = [{'name': 'UPDATE', 'type': 'UPDATE', 'role': ['UPDATE']}] answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] answers['tags']['required_fields'] = ['UPDATE'] - answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100' answers['tags']['security_domain'] = answers['security_domain'] del answers["security_domain"] answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE'] From 1107ae121c7cb41470c31397bb32cf0e52da79cd Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 25 Nov 2024 13:01:04 -0600 Subject: [PATCH 063/115] Reordering output --- contentctl/actions/new_content.py | 3 ++- contentctl/input/new_content_questions.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 283b3f4c..a56e5e83 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -29,10 +29,11 @@ def buildDetection(self)->dict[str,Any]: answers['date'] = datetime.today().strftime('%Y-%m-%d') answers['author'] = answers['detection_author'] del answers['detection_author'] - answers['data_source'] = answers['data_source'] answers['type'] = answers['detection_type'] del answers['detection_type'] answers['status'] = "production" #start everything as production since that's what we INTEND the content to become + answers['data_source'] = answers['data_sources'] + del answers['data_sources'] answers['description'] = 'UPDATE_DESCRIPTION' file_name = answers['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() answers['search'] = answers['detection_search'] + ' | `' + file_name + '_filter`' diff --git a/contentctl/input/new_content_questions.py b/contentctl/input/new_content_questions.py index 02b20f46..29071ee2 100644 --- a/contentctl/input/new_content_questions.py +++ b/contentctl/input/new_content_questions.py @@ -48,7 +48,7 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: { 'type': 'checkbox', 'message': 'Your data source', - 'name': 'data_source', + 'name': 'data_sources', #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory 'choices': sorted(DataSource._value2member_map_ ) From e5c150db5e39b10e5ed0a52372411832b6808814 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 11:15:30 -0800 Subject: [PATCH 064/115] Placeholder for contentctl 5 prs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ec8ec960..db99e2a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contentctl" -version = "4.4.7" +version = "5.0.0" description = "Splunk Content Control Tool" authors = ["STRT "] From 8f24494baa1a142980429492eb1e92b467fbac34 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 11:26:37 -0800 Subject: [PATCH 065/115] convert plain enums, or enums with multiple inheritance, to StrEnum or IntEnum --- contentctl/actions/build.py | 2 +- .../DetectionTestingManager.py | 1 - contentctl/objects/enums.py | 79 +++++-------------- 3 files changed, 22 insertions(+), 60 deletions(-) diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index feb0351b..43daf360 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from contentctl.objects.enums import SecurityContentProduct, SecurityContentType +from contentctl.objects.enums import SecurityContentType from contentctl.input.director import Director, DirectorOutputDto from contentctl.output.conf_output import ConfOutput from contentctl.output.conf_writer import ConfWriter diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 5ad5e117..13058e2f 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -5,7 +5,6 @@ from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer from urllib.parse import urlparse from copy import deepcopy -from contentctl.objects.enums import DetectionTestingTargetInfrastructure import signal import datetime # from queue import Queue diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py index 333ef358..8070d4a4 100644 --- a/contentctl/objects/enums.py +++ b/contentctl/objects/enums.py @@ -1,15 +1,15 @@ from __future__ import annotations from typing import List -import enum +from enum import StrEnum, IntEnum -class AnalyticsType(str, enum.Enum): +class AnalyticsType(StrEnum): TTP = "TTP" Anomaly = "Anomaly" Hunting = "Hunting" Correlation = "Correlation" -class DeploymentType(str, enum.Enum): +class DeploymentType(StrEnum): TTP = "TTP" Anomaly = "Anomaly" Hunting = "Hunting" @@ -18,7 +18,7 @@ class DeploymentType(str, enum.Enum): Embedded = "Embedded" -class DataModel(str,enum.Enum): +class DataModel(StrEnum): ENDPOINT = "Endpoint" NETWORK_TRAFFIC = "Network_Traffic" AUTHENTICATION = "Authentication" @@ -40,11 +40,11 @@ class DataModel(str,enum.Enum): SPLUNK_AUDIT = "Splunk_Audit" -class PlaybookType(str, enum.Enum): +class PlaybookType(StrEnum): INVESTIGATION = "Investigation" RESPONSE = "Response" -class SecurityContentType(enum.Enum): +class SecurityContentType(IntEnum): detections = 1 baselines = 2 stories = 3 @@ -68,20 +68,15 @@ class SecurityContentType(enum.Enum): # json_objects = "json_objects" -class SecurityContentProduct(enum.Enum): - SPLUNK_APP = 1 - API = 3 - CUSTOM = 4 - -class SecurityContentProductName(str, enum.Enum): +class SecurityContentProductName(StrEnum): SPLUNK_ENTERPRISE = "Splunk Enterprise" SPLUNK_ENTERPRISE_SECURITY = "Splunk Enterprise Security" SPLUNK_CLOUD = "Splunk Cloud" SPLUNK_SECURITY_ANALYTICS_FOR_AWS = "Splunk Security Analytics for AWS" SPLUNK_BEHAVIORAL_ANALYTICS = "Splunk Behavioral Analytics" -class SecurityContentInvestigationProductName(str, enum.Enum): +class SecurityContentInvestigationProductName(StrEnum): SPLUNK_ENTERPRISE = "Splunk Enterprise" SPLUNK_ENTERPRISE_SECURITY = "Splunk Enterprise Security" SPLUNK_CLOUD = "Splunk Cloud" @@ -90,33 +85,20 @@ class SecurityContentInvestigationProductName(str, enum.Enum): SPLUNK_PHANTOM = "Splunk Phantom" -class DetectionStatus(enum.Enum): - production = "production" - deprecated = "deprecated" - experimental = "experimental" - validation = "validation" - - -class DetectionStatusSSA(enum.Enum): +class DetectionStatus(StrEnum): production = "production" deprecated = "deprecated" experimental = "experimental" validation = "validation" -class LogLevel(enum.Enum): +class LogLevel(StrEnum): NONE = "NONE" ERROR = "ERROR" INFO = "INFO" -class AlertActions(enum.Enum): - notable = "notable" - rba = "rba" - email = "email" - - -class StoryCategory(str, enum.Enum): +class StoryCategory(StrEnum): ABUSE = "Abuse" ADVERSARY_TACTICS = "Adversary Tactics" BEST_PRACTICES = "Best Practices" @@ -139,37 +121,18 @@ class StoryCategory(str, enum.Enum): UNAUTHORIZED_SOFTWARE = "Unauthorized Software" -class PostTestBehavior(str, enum.Enum): +class PostTestBehavior(StrEnum): always_pause = "always_pause" pause_on_failure = "pause_on_failure" never_pause = "never_pause" -class DetectionTestingMode(str, enum.Enum): +class DetectionTestingMode(StrEnum): selected = "selected" all = "all" changes = "changes" -class DetectionTestingTargetInfrastructure(str, enum.Enum): - container = "container" - server = "server" - - -class InstanceState(str, enum.Enum): - starting = "starting" - running = "running" - error = "error" - stopping = "stopping" - stopped = "stopped" - - -class SigmaConverterTarget(enum.Enum): - CIM = 1 - RAW = 2 - OCSF = 3 - ALL = 4 - # It's unclear why we use a mix of constants and enums. The following list was taken from: # contentctl/contentctl/helper/constants.py. # We convect it to an enum here @@ -183,7 +146,7 @@ class SigmaConverterTarget(enum.Enum): # "Command And Control": 6, # "Actions on Objectives": 7 # } -class KillChainPhase(str, enum.Enum): +class KillChainPhase(StrEnum): UNKNOWN ="Unknown" RECONNAISSANCE = "Reconnaissance" WEAPONIZATION = "Weaponization" @@ -194,7 +157,7 @@ class KillChainPhase(str, enum.Enum): ACTIONS_ON_OBJECTIVES = "Actions on Objectives" -class DataSource(str,enum.Enum): +class DataSource(StrEnum): OSQUERY_ES_PROCESS_EVENTS = "OSQuery ES Process Events" POWERSHELL_4104 = "Powershell 4104" SYSMON_EVENT_ID_1 = "Sysmon EventID 1" @@ -234,7 +197,7 @@ class DataSource(str,enum.Enum): WINDOWS_SECURITY_5145 = "Windows Security 5145" WINDOWS_SYSTEM_7045 = "Windows System 7045" -class ProvidingTechnology(str, enum.Enum): +class ProvidingTechnology(StrEnum): AMAZON_SECURITY_LAKE = "Amazon Security Lake" AMAZON_WEB_SERVICES_CLOUDTRAIL = "Amazon Web Services - Cloudtrail" AZURE_AD = "Azure AD" @@ -302,7 +265,7 @@ def getProvidingTechFromSearch(search_string:str)->List[ProvidingTechnology]: return sorted(list(matched_technologies)) -class Cis18Value(str,enum.Enum): +class Cis18Value(StrEnum): CIS_0 = "CIS 0" CIS_1 = "CIS 1" CIS_2 = "CIS 2" @@ -323,7 +286,7 @@ class Cis18Value(str,enum.Enum): CIS_17 = "CIS 17" CIS_18 = "CIS 18" -class SecurityDomain(str, enum.Enum): +class SecurityDomain(StrEnum): ENDPOINT = "endpoint" NETWORK = "network" THREAT = "threat" @@ -331,7 +294,7 @@ class SecurityDomain(str, enum.Enum): ACCESS = "access" AUDIT = "audit" -class AssetType(str, enum.Enum): +class AssetType(StrEnum): AWS_ACCOUNT = "AWS Account" AWS_EKS_KUBERNETES_CLUSTER = "AWS EKS Kubernetes cluster" AWS_FEDERATED_ACCOUNT = "AWS Federated Account" @@ -382,7 +345,7 @@ class AssetType(str, enum.Enum): WEB_APPLICATION = "Web Application" WINDOWS = "Windows" -class NistCategory(str, enum.Enum): +class NistCategory(StrEnum): ID_AM = "ID.AM" ID_BE = "ID.BE" ID_GV = "ID.GV" @@ -406,7 +369,7 @@ class NistCategory(str, enum.Enum): RC_IM = "RC.IM" RC_CO = "RC.CO" -class RiskSeverity(str,enum.Enum): +class RiskSeverity(StrEnum): # Levels taken from the following documentation link # https://docs.splunk.com/Documentation/ES/7.3.2/User/RiskScoring # 20 - info (0-20 for us) From 09992707a3ee7e22b6b59a682d39438eb427d356 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 11:31:16 -0800 Subject: [PATCH 066/115] Remove all usage of use_enum_values. However, this has not received any testing yet. --- .../detection_abstract.py | 4 +- .../security_content_object_abstract.py | 3 +- contentctl/objects/config.py | 44 +++++++------------ contentctl/objects/detection_tags.py | 3 +- contentctl/objects/investigation.py | 3 +- contentctl/objects/mitre_attack_enrichment.py | 2 - contentctl/objects/story_tags.py | 3 +- 7 files changed, 20 insertions(+), 42 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index dc0350d5..070fbc0f 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -55,9 +55,7 @@ } -# TODO (#266): disable the use_enum_values configuration class Detection_Abstract(SecurityContentObject): - model_config = ConfigDict(use_enum_values=True) name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) #contentType: SecurityContentType = SecurityContentType.detections type: AnalyticsType = Field(...) @@ -210,7 +208,7 @@ def adjust_tests_and_groups(self) -> None: # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name # Skip tests for non-production detections - if self.status != DetectionStatus.production.value: # type: ignore + if self.status != DetectionStatus.production.value: self.skip_all_tests(f"TEST SKIPPED: Detection is non-production ({self.status})") # Skip tests for detecton types like Correlation which are not supported via contentctl diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index f93602f1..a69fac65 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -31,9 +31,8 @@ NO_FILE_NAME = "NO_FILE_NAME" -# TODO (#266): disable the use_enum_values configuration class SecurityContentObject_Abstract(BaseModel, abc.ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True) + model_config = ConfigDict(validate_default=True) name: str = Field(...,max_length=99) author: str = Field(...,max_length=255) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 659d1113..b1b6aed5 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -33,9 +33,8 @@ SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/{uid}/release/{version}/download" -# TODO (#266): disable the use_enum_values configuration class App_Base(BaseModel,ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) uid: Optional[int] = Field(default=None) title: str = Field(description="Human-readable name used by the app. This can have special characters.") appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. " @@ -59,9 +58,8 @@ def ensureAppPathExists(self, config:test, stage_file:bool=False): config.getLocalAppDir().mkdir(parents=True) -# TODO (#266): disable the use_enum_values configuration class TestApp(App_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) hardcoded_path: Optional[Union[FilePath,HttpUrl]] = Field(default=None, description="This may be a relative or absolute link to a file OR an HTTP URL linking to your app.") @@ -99,9 +97,8 @@ def getApp(self, config:test,stage_file:bool=False)->str: return str(destination) -# TODO (#266): disable the use_enum_values configuration class CustomApp(App_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) # Fields required for app.conf based on # https://docs.splunk.com/Documentation/Splunk/9.0.4/Admin/Appconf uid: int = Field(ge=2, lt=100000, default_factory=lambda:random.randint(20000,100000)) @@ -159,9 +156,8 @@ def getApp(self, config:test, stage_file=True)->str: verbose_print=True) return str(destination) -# TODO (#266): disable the use_enum_values configuration class Config_Base(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) path: DirectoryPath = Field(default=DirectoryPath("."), description="The root of your app.") app:CustomApp = Field(default_factory=CustomApp) @@ -175,7 +171,7 @@ def serialize_path(path: DirectoryPath)->str: return str(path) class init(Config_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) bare: bool = Field(default=False, description="contentctl normally provides some some example content " "(macros, stories, data_sources, and/or analytic stories). This option disables " "initialization with that additional contnet. Note that even if --bare is used, it " @@ -184,9 +180,8 @@ class init(Config_Base): "the deployment/ directory (since it is not yet easily customizable).") -# TODO (#266): disable the use_enum_values configuration class validate(Config_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) enrichments: bool = Field(default=False, description="Enable MITRE, APP, and CVE Enrichments. "\ "This is useful when outputting a release build "\ "and validating these values, but should otherwise "\ @@ -241,9 +236,8 @@ def getReportingPath(self)->pathlib.Path: return self.path/"reporting/" -# TODO (#266): disable the use_enum_values configuration class build(validate): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) build_path: DirectoryPath = Field(default=DirectoryPath("dist/"), title="Target path for all build outputs") @field_serializer('build_path',when_used='always') @@ -395,17 +389,15 @@ class new(Config_Base): type: NewContentType = Field(default=NewContentType.detection, description="Specify the type of content you would like to create.") -# TODO (#266): disable the use_enum_values configuration class deploy_acs(inspect): - model_config = ConfigDict(use_enum_values=True,validate_default=False, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=False, arbitrary_types_allowed=True) #ignore linter error splunk_cloud_jwt_token: str = Field(exclude=True, description="Splunk JWT used for performing ACS operations on a Splunk Cloud Instance") splunk_cloud_stack: str = Field(description="The name of your Splunk Cloud Stack") -# TODO (#266): disable the use_enum_values configuration class Infrastructure(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) splunk_app_username:str = Field(default="admin", description="Username for logging in to your Splunk Server") splunk_app_password:str = Field(exclude=True, default="password", description="Password for logging in to your Splunk Server.") instance_address:str = Field(..., description="Address of your splunk server.") @@ -415,15 +407,13 @@ class Infrastructure(BaseModel): instance_name: str = Field(...) -# TODO (#266): disable the use_enum_values configuration class Container(Infrastructure): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) instance_address:str = Field(default="localhost", description="Address of your splunk server.") -# TODO (#266): disable the use_enum_values configuration class ContainerSettings(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) leave_running: bool = Field(default=True, description="Leave container running after it is first " "set up to speed up subsequent test runs.") num_containers: PositiveInt = Field(default=1, description="Number of containers to start in parallel. " @@ -447,15 +437,13 @@ class All(BaseModel): pass -# TODO (#266): disable the use_enum_values configuration class Changes(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) target_branch:str = Field(...,description="The target branch to diff against. Note that this includes uncommitted changes in the working directory as well.") -# TODO (#266): disable the use_enum_values configuration class Selected(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) files:List[FilePath] = Field(...,description="List of detection files to test, separated by spaces.") @field_serializer('files',when_used='always') @@ -826,9 +814,8 @@ def getModeName(self)->str: return DetectionTestingMode.selected.value -# TODO (#266): disable the use_enum_values configuration class test(test_common): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) container_settings:ContainerSettings = ContainerSettings() test_instances: List[Container] = Field([], exclude = True, validate_default=True) splunk_api_username: Optional[str] = Field(default=None, exclude = True,description="Splunk API username used for running appinspect or installating apps from Splunkbase") @@ -893,9 +880,8 @@ def getAppFilePath(self): TEST_ARGS_ENV = "CONTENTCTL_TEST_INFRASTRUCTURES" -# TODO (#266): disable the use_enum_values configuration class test_servers(test_common): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) test_instances:List[Infrastructure] = Field([],description="Test against one or more preconfigured servers.", validate_default=True) server_info:Optional[str] = Field(None, validate_default=True, description='String of pre-configured servers to use for testing. The list MUST be in the format:\n' 'address,username,web_ui_port,hec_port,api_port;address_2,username_2,web_ui_port_2,hec_port_2,api_port_2' diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index b1d489f4..d64cbb46 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -35,10 +35,9 @@ from contentctl.objects.atomic import AtomicEnrichment, AtomicTest from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE, CVE_TYPE -# TODO (#266): disable the use_enum_values configuration class DetectionTags(BaseModel): # detection spec - model_config = ConfigDict(use_enum_values=True, validate_default=False) + model_config = ConfigDict(validate_default=False) analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 293e3331..d1ecf52b 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -12,9 +12,8 @@ ) from contentctl.objects.config import CustomApp -# TODO (#266): disable the use_enum_values configuration class Investigation(SecurityContentObject): - model_config = ConfigDict(use_enum_values=True,validate_default=False) + model_config = ConfigDict(validate_default=False) type: str = Field(...,pattern="^Investigation$") datamodel: list[DataModel] = Field(...) name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) diff --git a/contentctl/objects/mitre_attack_enrichment.py b/contentctl/objects/mitre_attack_enrichment.py index 401774e9..888754c8 100644 --- a/contentctl/objects/mitre_attack_enrichment.py +++ b/contentctl/objects/mitre_attack_enrichment.py @@ -83,9 +83,7 @@ def standardize_contributors(cls, contributors:list[str] | None) -> list[str]: return [] return contributors -# TODO (#266): disable the use_enum_values configuration class MitreAttackEnrichment(BaseModel): - ConfigDict(use_enum_values=True) mitre_attack_id: MITRE_ATTACK_ID_TYPE = Field(...) mitre_attack_technique: str = Field(...) mitre_attack_tactics: List[MitreTactics] = Field(...) diff --git a/contentctl/objects/story_tags.py b/contentctl/objects/story_tags.py index 42eb2f37..e1bd45dc 100644 --- a/contentctl/objects/story_tags.py +++ b/contentctl/objects/story_tags.py @@ -18,9 +18,8 @@ class StoryUseCase(str,Enum): OTHER = "Other" -# TODO (#266): disable the use_enum_values configuration class StoryTags(BaseModel): - model_config = ConfigDict(extra='forbid', use_enum_values=True) + model_config = ConfigDict(extra='forbid') category: List[StoryCategory] = Field(...,min_length=1) product: List[SecurityContentProductName] = Field(...,min_length=1) usecase: StoryUseCase = Field(...) From 31f46a22f5c93f9ecad87176f8a6b8c2b4d15933 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 14:09:11 -0800 Subject: [PATCH 067/115] Remove use of .value on enums in code now that they have been more strictly defined as IntEnum or StrEnum. This has not yet been tested. --- .../DetectionTestingInfrastructure.py | 50 +++++++++---------- .../actions/detection_testing/progress_bar.py | 12 ++--- .../views/DetectionTestingView.py | 8 +-- contentctl/actions/test.py | 9 ++-- .../detection_abstract.py | 40 +++++++-------- .../security_content_object_abstract.py | 8 +-- contentctl/objects/base_test.py | 4 +- contentctl/objects/base_test_result.py | 6 +-- contentctl/objects/config.py | 26 ++++------ contentctl/objects/constants.py | 2 +- contentctl/objects/correlation_search.py | 50 +++++++++---------- contentctl/objects/detection_tags.py | 2 +- contentctl/output/svg_output.py | 2 +- .../templates/analyticstories_detections.j2 | 2 +- .../templates/savedsearches_detections.j2 | 4 +- enums_test.py | 46 +++++++++++++++++ 16 files changed, 155 insertions(+), 116 deletions(-) create mode 100644 enums_test.py diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 8e816025..7804b29e 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -442,7 +442,7 @@ def test_detection(self, detection: Detection) -> None: self.format_pbar_string( TestReportingType.GROUP, test_group.name, - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=time.time(), set_pbar=False, ) @@ -483,7 +483,7 @@ def test_detection(self, detection: Detection) -> None: self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.DONE_GROUP.value, + TestingStates.DONE_GROUP, start_time=setup_results.start_time, set_pbar=False, ) @@ -504,7 +504,7 @@ def setup_test_group(self, test_group: TestGroup) -> SetupTestGroupResults: self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.BEGINNING_GROUP.value, + TestingStates.BEGINNING_GROUP, start_time=setup_start_time ) # https://github.com/WoLpH/python-progressbar/issues/164 @@ -544,7 +544,7 @@ def cleanup_test_group( self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.DELETING.value, + TestingStates.DELETING, start_time=test_group_start_time, ) @@ -632,7 +632,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=test_start_time, set_pbar=False, ) @@ -664,7 +664,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.ERROR.value, + FinalTestingStates.ERROR, start_time=test_start_time, set_pbar=False, ) @@ -724,7 +724,7 @@ def execute_unit_test( res = "ERROR" link = detection.search else: - res = test.result.status.value.upper() # type: ignore + res = test.result.status.upper() # type: ignore link = test.result.get_summary_dict()["sid_link"] self.format_pbar_string( @@ -755,7 +755,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.PASS.value, + FinalTestingStates.PASS, start_time=test_start_time, set_pbar=False, ) @@ -766,7 +766,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=test_start_time, set_pbar=False, ) @@ -777,7 +777,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.FAIL.value, + FinalTestingStates.FAIL, start_time=test_start_time, set_pbar=False, ) @@ -788,7 +788,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.ERROR.value, + FinalTestingStates.ERROR, start_time=test_start_time, set_pbar=False, ) @@ -821,7 +821,7 @@ def execute_integration_test( test_start_time = time.time() # First, check to see if the test should be skipped (Hunting or Correlation) - if detection.type in [AnalyticsType.Hunting.value, AnalyticsType.Correlation.value]: + if detection.type in [AnalyticsType.Hunting, AnalyticsType.Correlation]: test.skip( f"TEST SKIPPED: detection is type {detection.type} and cannot be integration " "tested at this time" @@ -843,11 +843,11 @@ def execute_integration_test( # Determine the reporting state (we should only encounter SKIP/FAIL/ERROR) state: str if test.result.status == TestResultStatus.SKIP: - state = FinalTestingStates.SKIP.value + state = FinalTestingStates.SKIP elif test.result.status == TestResultStatus.FAIL: - state = FinalTestingStates.FAIL.value + state = FinalTestingStates.FAIL elif test.result.status == TestResultStatus.ERROR: - state = FinalTestingStates.ERROR.value + state = FinalTestingStates.ERROR else: raise ValueError( f"Status for (integration) '{detection.name}:{test.name}' was preemptively set" @@ -891,7 +891,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.FAIL.value, + FinalTestingStates.FAIL, start_time=test_start_time, set_pbar=False, ) @@ -935,7 +935,7 @@ def execute_integration_test( if test.result is None: res = "ERROR" else: - res = test.result.status.value.upper() # type: ignore + res = test.result.status.upper() # type: ignore # Get the link to the saved search in this specific instance link = f"https://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}" @@ -968,7 +968,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.PASS.value, + FinalTestingStates.PASS, start_time=test_start_time, set_pbar=False, ) @@ -979,7 +979,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=test_start_time, set_pbar=False, ) @@ -990,7 +990,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.FAIL.value, + FinalTestingStates.FAIL, start_time=test_start_time, set_pbar=False, ) @@ -1001,7 +1001,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.ERROR.value, + FinalTestingStates.ERROR, start_time=test_start_time, set_pbar=False, ) @@ -1077,7 +1077,7 @@ def retry_search_until_timeout( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - TestingStates.PROCESSING.value, + TestingStates.PROCESSING, start_time=start_time ) @@ -1086,7 +1086,7 @@ def retry_search_until_timeout( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - TestingStates.SEARCHING.value, + TestingStates.SEARCHING, start_time=start_time, ) @@ -1289,7 +1289,7 @@ def replay_attack_data_file( self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.DOWNLOADING.value, + TestingStates.DOWNLOADING, start_time=test_group_start_time ) @@ -1307,7 +1307,7 @@ def replay_attack_data_file( self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.REPLAYING.value, + TestingStates.REPLAYING, start_time=test_group_start_time ) diff --git a/contentctl/actions/detection_testing/progress_bar.py b/contentctl/actions/detection_testing/progress_bar.py index 45e30e06..5b5abd1a 100644 --- a/contentctl/actions/detection_testing/progress_bar.py +++ b/contentctl/actions/detection_testing/progress_bar.py @@ -1,10 +1,10 @@ import time -from enum import Enum +from enum import StrEnum from tqdm import tqdm import datetime -class TestReportingType(str, Enum): +class TestReportingType(StrEnum): """ 5-char identifiers for the type of testing being reported on """ @@ -21,7 +21,7 @@ class TestReportingType(str, Enum): INTEGRATION = "INTEG" -class TestingStates(str, Enum): +class TestingStates(StrEnum): """ Defined testing states """ @@ -40,10 +40,10 @@ class TestingStates(str, Enum): # the longest length of any state -LONGEST_STATE = max(len(w.value) for w in TestingStates) +LONGEST_STATE = max(len(w) for w in TestingStates) -class FinalTestingStates(str, Enum): +class FinalTestingStates(StrEnum): """ The possible final states for a test (for pbar reporting) """ @@ -82,7 +82,7 @@ def format_pbar_string( :returns: a formatted string for use w/ pbar """ # Extract and ljust our various fields - field_one = test_reporting_type.value + field_one = test_reporting_type field_two = test_name.ljust(MAX_TEST_NAME_LENGTH) field_three = state.ljust(LONGEST_STATE) field_four = datetime.timedelta(seconds=round(time.time() - start_time)) diff --git a/contentctl/actions/detection_testing/views/DetectionTestingView.py b/contentctl/actions/detection_testing/views/DetectionTestingView.py index 8ff6e583..98cc7122 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingView.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingView.py @@ -110,11 +110,11 @@ def getSummaryObject( total_skipped += 1 # Aggregate production status metrics - if detection.status == DetectionStatus.production.value: # type: ignore + if detection.status == DetectionStatus.production: total_production += 1 - elif detection.status == DetectionStatus.experimental.value: # type: ignore + elif detection.status == DetectionStatus.experimental: total_experimental += 1 - elif detection.status == DetectionStatus.deprecated.value: # type: ignore + elif detection.status == DetectionStatus.deprecated: total_deprecated += 1 # Check if the detection is manual_test @@ -178,7 +178,7 @@ def getSummaryObject( # Construct and return the larger results dict result_dict = { "summary": { - "mode": self.config.getModeName(), + "mode": self.config.mode.mode_name, "enable_integration_testing": self.config.enable_integration_testing, "success": overall_success, "total_detections": total_detections, diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index 716ecd71..b3437cef 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import List -from contentctl.objects.config import test_common +from contentctl.objects.config import test_common, Selected, Changes from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType from contentctl.objects.detection import Detection @@ -78,10 +78,9 @@ def execute(self, input_dto: TestInputDto) -> bool: input_dto=manager_input_dto, output_dto=output_dto ) - mode = input_dto.config.getModeName() if len(input_dto.detections) == 0: print( - f"With Detection Testing Mode '{mode}', there were [0] detections found to test." + f"With Detection Testing Mode '{input_dto.config.mode.mode_name}', there were [0] detections found to test." "\nAs such, we will quit immediately." ) # Directly call stop so that the summary.yml will be generated. Of course it will not @@ -89,8 +88,8 @@ def execute(self, input_dto: TestInputDto) -> bool: # detections were tested. file.stop() else: - print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections") - if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]: + print(f"MODE: [{input_dto.config.mode.mode_name}] - Test [{len(input_dto.detections)}] detections") + if isinstance(input_dto.config.mode, Selected) or isinstance(input_dto.config.mode, Changes): files_string = '\n- '.join( [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections] ) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 070fbc0f..1f00d141 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Union, Optional, List, Any, Annotated import re import pathlib -from enum import Enum +from enum import StrEnum from pydantic import ( field_validator, @@ -51,7 +51,7 @@ # Those AnalyticsTypes that we do not test via contentctl SKIPPED_ANALYTICS_TYPES: set[str] = { - AnalyticsType.Correlation.value + AnalyticsType.Correlation } @@ -99,7 +99,7 @@ def get_conf_stanza_name(self, app:CustomApp)->str: def get_action_dot_correlationsearch_dot_label(self, app:CustomApp, max_stanza_length:int=ES_MAX_STANZA_LENGTH)->str: stanza_name = self.get_conf_stanza_name(app) stanza_name_after_saving_in_es = ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format( - security_domain_value = self.tags.security_domain.value, + security_domain_value = self.tags.security_domain, search_name = stanza_name ) @@ -208,7 +208,7 @@ def adjust_tests_and_groups(self) -> None: # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name # Skip tests for non-production detections - if self.status != DetectionStatus.production.value: + if self.status != DetectionStatus.production: self.skip_all_tests(f"TEST SKIPPED: Detection is non-production ({self.status})") # Skip tests for detecton types like Correlation which are not supported via contentctl @@ -261,7 +261,7 @@ def test_status(self) -> TestResultStatus | None: @computed_field @property def datamodel(self) -> List[DataModel]: - return [dm for dm in DataModel if dm.value in self.search] + return [dm for dm in DataModel if dm in self.search] @@ -306,13 +306,13 @@ def annotations(self) -> dict[str, Union[List[str], int, str]]: def mappings(self) -> dict[str, List[str]]: mappings: dict[str, Any] = {} if len(self.tags.cis20) > 0: - mappings["cis20"] = [tag.value for tag in self.tags.cis20] + mappings["cis20"] = [tag for tag in self.tags.cis20] if len(self.tags.kill_chain_phases) > 0: - mappings['kill_chain_phases'] = [phase.value for phase in self.tags.kill_chain_phases] + mappings['kill_chain_phases'] = [phase for phase in self.tags.kill_chain_phases] if len(self.tags.mitre_attack_id) > 0: mappings['mitre_attack'] = self.tags.mitre_attack_id if len(self.tags.nist) > 0: - mappings['nist'] = [category.value for category in self.tags.nist] + mappings['nist'] = [category for category in self.tags.nist] # No need to sort the dict! It has been constructed in-order. # However, if this logic is changed, then consider reordering or @@ -433,7 +433,7 @@ def metadata(self) -> dict[str, str|float]: # break the `inspect` action. return { 'detection_id': str(self.id), - 'deprecated': '1' if self.status == DetectionStatus.deprecated.value else '0', # type: ignore + 'deprecated': '1' if self.status == DetectionStatus.deprecated else '0', # type: ignore 'detection_version': str(self.version), 'publish_time': datetime.datetime(self.date.year,self.date.month,self.date.day,0,0,0,0,tzinfo=datetime.timezone.utc).timestamp() } @@ -568,7 +568,7 @@ def model_post_init(self, __context: Any) -> None: # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting - if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + if self.type == AnalyticsType.Hunting or self.status != DetectionStatus.production: #No additional check need to happen on the potential drilldowns. pass else: @@ -711,14 +711,14 @@ def only_enabled_if_production_status(cls, v: Any, info: ValidationInfo) -> bool if status != DetectionStatus.production: errors.append( f"status is '{status.name}'. Detections that are enabled by default MUST be " - f"'{DetectionStatus.production.value}'" + f"'{DetectionStatus.production}'" ) if searchType not in [AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]: errors.append( - f"type is '{searchType.value}'. Detections that are enabled by default MUST be one" + f"type is '{searchType}'. Detections that are enabled by default MUST be one" " of the following types: " - f"{[AnalyticsType.Anomaly.value, AnalyticsType.Correlation.value, AnalyticsType.TTP.value]}") + f"{[AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]}") if len(errors) > 0: error_message = "\n - ".join(errors) raise ValueError(f"Detection is 'enabled_by_default: true' however \n - {error_message}") @@ -727,7 +727,7 @@ def only_enabled_if_production_status(cls, v: Any, info: ValidationInfo) -> bool @model_validator(mode="after") def addTags_nist(self): - if self.type == AnalyticsType.TTP.value: + if self.type == AnalyticsType.TTP: self.tags.nist = [NistCategory.DE_CM] else: self.tags.nist = [NistCategory.DE_AE] @@ -765,11 +765,11 @@ def ensureProperObservablesExist(self): # NOTE: we ignore the type error around self.status because we are using Pydantic's # use_enum_values configuration # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name - if self.status not in [DetectionStatus.production.value]: # type: ignore + if self.status not in [DetectionStatus.production]: # type: ignore # Only perform this validation on production detections return self - if self.type not in [AnalyticsType.TTP.value, AnalyticsType.Anomaly.value]: + if self.type not in [AnalyticsType.TTP, AnalyticsType.Anomaly]: # Only perform this validation on TTP and Anomaly detections return self @@ -820,7 +820,7 @@ def search_observables_exist_validate(self): # NOTE: we ignore the type error around self.status because we are using Pydantic's # use_enum_values configuration # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name - if len(error_messages) > 0 and self.status == DetectionStatus.production.value: # type: ignore + if len(error_messages) > 0 and self.status == DetectionStatus.production: # type: ignore msg = ( "Use of fields in observables/messages that do not appear in search:\n\t- " "\n\t- ".join(error_messages) @@ -876,7 +876,7 @@ def tests_validate( info: ValidationInfo ) -> list[UnitTest | IntegrationTest | ManualTest]: # Only production analytics require tests - if info.data.get("status", "") != DetectionStatus.production.value: + if info.data.get("status", "") != DetectionStatus.production: return v # All types EXCEPT Correlation MUST have test(s). Any other type, including newly defined @@ -989,7 +989,7 @@ def get_summary( value = getattr(self, field) # Enums and Path objects cannot be serialized directly, so we convert it to a string - if isinstance(value, Enum) or isinstance(value, pathlib.Path): + if isinstance(value, StrEnum) or isinstance(value, pathlib.Path): value = str(value) # Alias any fields as needed @@ -1011,7 +1011,7 @@ def get_summary( # Initialize the dict as a mapping of strings to str/bool result: dict[str, Union[str, bool]] = { "name": test.name, - "test_type": test.test_type.value + "test_type": test.test_type } # If result is not None, get a summary of the test result w/ the requested fields diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index a69fac65..b0fccab8 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -161,10 +161,10 @@ def getDeploymentFromType(typeField: Union[str, None], info: ValidationInfo) -> raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context") type_to_deployment_name_map = { - AnalyticsType.TTP.value: "ESCU Default Configuration TTP", - AnalyticsType.Hunting.value: "ESCU Default Configuration Hunting", - AnalyticsType.Correlation.value: "ESCU Default Configuration Correlation", - AnalyticsType.Anomaly.value: "ESCU Default Configuration Anomaly", + AnalyticsType.TTP: "ESCU Default Configuration TTP", + AnalyticsType.Hunting: "ESCU Default Configuration Hunting", + AnalyticsType.Correlation: "ESCU Default Configuration Correlation", + AnalyticsType.Anomaly: "ESCU Default Configuration Anomaly", "Baseline": "ESCU Default Configuration Baseline" } converted_type_field = type_to_deployment_name_map[typeField] diff --git a/contentctl/objects/base_test.py b/contentctl/objects/base_test.py index 20e681cf..0bbb2289 100644 --- a/contentctl/objects/base_test.py +++ b/contentctl/objects/base_test.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import Union from abc import ABC, abstractmethod @@ -7,7 +7,7 @@ from contentctl.objects.base_test_result import BaseTestResult -class TestType(str, Enum): +class TestType(StrEnum): """ Types of tests """ diff --git a/contentctl/objects/base_test_result.py b/contentctl/objects/base_test_result.py index d29f93cb..6f9ce11a 100644 --- a/contentctl/objects/base_test_result.py +++ b/contentctl/objects/base_test_result.py @@ -1,5 +1,5 @@ from typing import Union, Any -from enum import Enum +from enum import StrEnum from pydantic import ConfigDict, BaseModel from splunklib.data import Record # type: ignore @@ -10,7 +10,7 @@ # TODO (#267): Align test reporting more closely w/ status enums (as it relates to "untested") # TODO (PEX-432): add status "UNSET" so that we can make sure the result is always of this enum # type; remove mypy ignores associated w/ these typing issues once we do -class TestResultStatus(str, Enum): +class TestResultStatus(StrEnum): """Enum for test status (e.g. pass/fail)""" # Test failed (detection did NOT fire appropriately) FAIL = "fail" @@ -113,7 +113,7 @@ def get_summary_dict( # Exceptions and enums cannot be serialized, so convert to str if isinstance(getattr(self, field), Exception): summary_dict[field] = str(getattr(self, field)) - elif isinstance(getattr(self, field), Enum): + elif isinstance(getattr(self, field), StrEnum): summary_dict[field] = str(getattr(self, field)) else: summary_dict[field] = getattr(self, field) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index b1b6aed5..361ddab7 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -434,16 +434,19 @@ def getContainers(self)->List[Container]: class All(BaseModel): #Doesn't need any extra logic + mode_name = "All" pass class Changes(BaseModel): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) + mode_name: str = "Changes" target_branch:str = Field(...,description="The target branch to diff against. Note that this includes uncommitted changes in the working directory as well.") class Selected(BaseModel): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) + mode_name = "Selected" files:List[FilePath] = Field(...,description="List of detection files to test, separated by spaces.") @field_serializer('files',when_used='always') @@ -672,12 +675,12 @@ def serialize_path(paths: List[FilePath])->List[str]: class test_common(build): mode:Union[Changes, Selected, All] = Field(All(), union_mode='left_to_right') post_test_behavior: PostTestBehavior = Field(default=PostTestBehavior.pause_on_failure, description="Controls what to do when a test completes.\n\n" - f"'{PostTestBehavior.always_pause.value}' - the state of " + f"'{PostTestBehavior.always_pause}' - the state of " "the test will always pause after a test, allowing the user to log into the " "server and experiment with the search and data before it is removed.\n\n" - f"'{PostTestBehavior.pause_on_failure.value}' - pause execution ONLY when a test fails. The user may press ENTER in the terminal " + f"'{PostTestBehavior.pause_on_failure}' - pause execution ONLY when a test fails. The user may press ENTER in the terminal " "running the test to move on to the next test.\n\n" - f"'{PostTestBehavior.never_pause.value}' - never stop testing, even if a test fails.\n\n" + f"'{PostTestBehavior.never_pause}' - never stop testing, even if a test fails.\n\n" "***SPECIAL NOTE FOR CI/CD*** 'never_pause' MUST be used for a test to " "run in an unattended manner or in a CI/CD system - otherwise a single failed test " "will result in the testing never finishing as the tool waits for input.") @@ -694,7 +697,7 @@ class test_common(build): " interactive command line workflow that can display progress bars and status information frequently. " "Unfortunately it is incompatible with, or may cause poorly formatted logs, in many CI/CD systems or other unattended environments. " "If you are running contentctl in CI/CD, then please set this argument to True. Note that if you are running in a CI/CD context, " - f"you also MUST set post_test_behavior to {PostTestBehavior.never_pause.value}. Otherwiser, a failed detection will cause" + f"you also MUST set post_test_behavior to {PostTestBehavior.never_pause}. Otherwiser, a failed detection will cause" "the CI/CD running to pause indefinitely.") apps: List[TestApp] = Field(default=DEFAULT_APPS, exclude=False, description="List of apps to install in test environment") @@ -703,7 +706,7 @@ class test_common(build): def dumpCICDPlanAndQuit(self, githash: str, detections:List[Detection]): output_file = self.path / "test_plan.yml" self.mode = Selected(files=sorted([detection.file_path for detection in detections], key=lambda path: str(path))) - self.post_test_behavior = PostTestBehavior.never_pause.value + self.post_test_behavior = PostTestBehavior.never_pause #required so that CI/CD does not get too much output or hang self.disable_tqdm = True @@ -770,12 +773,12 @@ def ensureCommonInformationModel(self)->Self: def suppressTQDM(self)->Self: if self.disable_tqdm: tqdm.tqdm.__init__ = partialmethod(tqdm.tqdm.__init__, disable=True) - if self.post_test_behavior != PostTestBehavior.never_pause.value: + if self.post_test_behavior != PostTestBehavior.never_pause: raise ValueError(f"You have disabled tqdm, presumably because you are " f"running in CI/CD or another unattended context.\n" f"However, post_test_behavior is set to [{self.post_test_behavior}].\n" f"If that is the case, then you MUST set post_test_behavior " - f"to [{PostTestBehavior.never_pause.value}].\n" + f"to [{PostTestBehavior.never_pause}].\n" "Otherwise, if a detection fails in CI/CD, your CI/CD runner will hang forever.") return self @@ -805,15 +808,6 @@ def checkPlanOnlyUse(self)->Self: return self - def getModeName(self)->str: - if isinstance(self.mode, All): - return DetectionTestingMode.all.value - elif isinstance(self.mode, Changes): - return DetectionTestingMode.changes.value - else: - return DetectionTestingMode.selected.value - - class test(test_common): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) container_settings:ContainerSettings = ContainerSettings() diff --git a/contentctl/objects/constants.py b/contentctl/objects/constants.py index c295ec86..8a32f28d 100644 --- a/contentctl/objects/constants.py +++ b/contentctl/objects/constants.py @@ -149,7 +149,7 @@ # errors, if its name is longer than 99 characters. # When an saved search is cloned in Enterprise Security User Interface, # it is wrapped in the following: -# {Detection.tags.security_domain.value} - {SEARCH_STANZA_NAME} - Rule +# {Detection.tags.security_domain} - {SEARCH_STANZA_NAME} - Rule # Similarly, when we generate the search stanza name in contentctl, it # is app.label - detection.name - Rule # However, in product the search name is: diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index c64eed6b..2a8d1e9c 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -2,7 +2,7 @@ import time import json from typing import Any -from enum import Enum +from enum import StrEnum, IntEnum from functools import cached_property from pydantic import ConfigDict, BaseModel, computed_field, Field, PrivateAttr @@ -76,7 +76,7 @@ def get_logger() -> logging.Logger: return logger -class SavedSearchKeys(str, Enum): +class SavedSearchKeys(StrEnum): """ Various keys into the SavedSearch content """ @@ -89,7 +89,7 @@ class SavedSearchKeys(str, Enum): DISBALED_KEY = "disabled" -class Indexes(str, Enum): +class Indexes(StrEnum): """ Indexes we search against """ @@ -98,7 +98,7 @@ class Indexes(str, Enum): NOTABLE_INDEX = "notable" -class TimeoutConfig(int, Enum): +class TimeoutConfig(IntEnum): """ Configuration values for the exponential backoff timer """ @@ -115,7 +115,7 @@ class TimeoutConfig(int, Enum): # TODO (#226): evaluate sane defaults for timeframe for integration testing (e.g. 5y is good # now, but maybe not always...); maybe set latest/earliest to None? -class ScheduleConfig(str, Enum): +class ScheduleConfig(StrEnum): """ Configuraton values for the saved search schedule """ @@ -310,7 +310,7 @@ def earliest_time(self) -> str: The earliest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY.value] + return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY] else: raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") @@ -320,7 +320,7 @@ def latest_time(self) -> str: The latest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY.value] + return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY] else: raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") @@ -330,7 +330,7 @@ def cron_schedule(self) -> str: The cron schedule configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY.value] + return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY] else: raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") @@ -340,7 +340,7 @@ def enabled(self) -> bool: Whether the saved search is enabled """ if self.saved_search is not None: - if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY.value]): + if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY]): return False else: return True @@ -368,7 +368,7 @@ def _get_risk_analysis_action(content: dict[str, Any]) -> RiskAnalysisAction | N :param content: a dict of strings to values :returns: a RiskAnalysisAction, or None if none exists """ - if int(content[SavedSearchKeys.RISK_ACTION_KEY.value]): + if int(content[SavedSearchKeys.RISK_ACTION_KEY]): try: return RiskAnalysisAction.parse_from_dict(content) except ValueError as e: @@ -383,7 +383,7 @@ def _get_notable_action(content: dict[str, Any]) -> NotableAction | None: :returns: a NotableAction, or None if none exists """ # grab notable details if present - if int(content[SavedSearchKeys.NOTABLE_ACTION_KEY.value]): + if int(content[SavedSearchKeys.NOTABLE_ACTION_KEY]): return NotableAction.parse_from_dict(content) return None @@ -463,9 +463,9 @@ def disable(self, refresh: bool = True) -> None: def update_timeframe( self, - earliest_time: str = ScheduleConfig.EARLIEST_TIME.value, - latest_time: str = ScheduleConfig.LATEST_TIME.value, - cron_schedule: str = ScheduleConfig.CRON_SCHEDULE.value, + earliest_time: str = ScheduleConfig.EARLIEST_TIME, + latest_time: str = ScheduleConfig.LATEST_TIME, + cron_schedule: str = ScheduleConfig.CRON_SCHEDULE, refresh: bool = True ) -> None: """Updates the correlation search timeframe to work with test data @@ -481,9 +481,9 @@ def update_timeframe( """ # update the SavedSearch accordingly data = { - SavedSearchKeys.EARLIEST_TIME_KEY.value: earliest_time, - SavedSearchKeys.LATEST_TIME_KEY.value: latest_time, - SavedSearchKeys.CRON_SCHEDULE_KEY.value: cron_schedule + SavedSearchKeys.EARLIEST_TIME_KEY: earliest_time, + SavedSearchKeys.LATEST_TIME_KEY: latest_time, + SavedSearchKeys.CRON_SCHEDULE_KEY: cron_schedule } self.logger.info(data) self.logger.info(f"Updating timeframe for '{self.name}': {data}") @@ -554,7 +554,7 @@ def get_risk_events(self, force_update: bool = False) -> list[RiskEvent]: for result in result_iterator: # sanity check that this result from the iterator is a risk event and not some # other metadata - if result["index"] == Indexes.RISK_INDEX.value: + if result["index"] == Indexes.RISK_INDEX: try: parsed_raw = json.loads(result["_raw"]) event = RiskEvent.parse_obj(parsed_raw) @@ -619,7 +619,7 @@ def get_notable_events(self, force_update: bool = False) -> list[NotableEvent]: for result in result_iterator: # sanity check that this result from the iterator is a notable event and not some # other metadata - if result["index"] == Indexes.NOTABLE_INDEX.value: + if result["index"] == Indexes.NOTABLE_INDEX: try: parsed_raw = json.loads(result["_raw"]) event = NotableEvent.parse_obj(parsed_raw) @@ -746,7 +746,7 @@ def validate_notable_events(self) -> None: # NOTE: it would be more ideal to switch this to a system which gets the handle of the saved search job and polls # it for completion, but that seems more tricky - def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: bool = False) -> IntegrationTestResult: + def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = False) -> IntegrationTestResult: """Execute the integration test Executes an integration test for this CorrelationSearch. First, ensures no matching risk/notables already exist @@ -760,10 +760,10 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: boo """ # max_sleep must be greater than the base value we must wait for the scheduled searchjob to run (jobs run every # 60s) - if max_sleep < TimeoutConfig.BASE_SLEEP.value: + if max_sleep < TimeoutConfig.BASE_SLEEP: raise ClientError( f"max_sleep value of {max_sleep} is less than the base sleep required " - f"({TimeoutConfig.BASE_SLEEP.value})" + f"({TimeoutConfig.BASE_SLEEP})" ) # initialize result as None @@ -774,7 +774,7 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: boo num_tries = 0 # set the initial base sleep time - time_to_sleep = TimeoutConfig.BASE_SLEEP.value + time_to_sleep = TimeoutConfig.BASE_SLEEP try: # first make sure the indexes are currently empty and the detection is starting from a disabled state @@ -999,9 +999,9 @@ def cleanup(self, delete_test_index=False) -> None: if delete_test_index: self.indexes_to_purge.add(self.test_index) # type: ignore if self._risk_events is not None: - self.indexes_to_purge.add(Indexes.RISK_INDEX.value) + self.indexes_to_purge.add(Indexes.RISK_INDEX) if self._notable_events is not None: - self.indexes_to_purge.add(Indexes.NOTABLE_INDEX.value) + self.indexes_to_purge.add(Indexes.NOTABLE_INDEX) # delete the indexes for index in self.indexes_to_purge: diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index d64cbb46..7b1e9331 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -151,7 +151,7 @@ def serialize_model(self): # Since this field has no parent, there is no need to call super() serialization function return { "analytic_story": [story.name for story in self.analytic_story], - "asset_type": self.asset_type.value, + "asset_type": self.asset_type, "cis20": self.cis20, "kill_chain_phases": self.kill_chain_phases, "nist": self.nist, diff --git a/contentctl/output/svg_output.py b/contentctl/output/svg_output.py index d454ccb2..2d0c9d56 100644 --- a/contentctl/output/svg_output.py +++ b/contentctl/output/svg_output.py @@ -35,7 +35,7 @@ def writeObjects(self, detections: List[Detection], output_path: pathlib.Path, t total_dict:dict[str,Any] = self.get_badge_dict("Detections", detections, detections) - production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production.value]) + production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production]) #deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated]) #experimental_dict = self.get_badge_dict("Experimental", detections, [detection for detection in detections if detection.status == DetectionStatus.experimental]) diff --git a/contentctl/output/templates/analyticstories_detections.j2 b/contentctl/output/templates/analyticstories_detections.j2 index e97f82a8..d24a1217 100644 --- a/contentctl/output/templates/analyticstories_detections.j2 +++ b/contentctl/output/templates/analyticstories_detections.j2 @@ -5,7 +5,7 @@ {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} [savedsearch://{{ detection.get_conf_stanza_name(app) }}] type = detection -asset_type = {{ detection.tags.asset_type.value }} +asset_type = {{ detection.tags.asset_type }} confidence = medium explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }} {% if detection.how_to_implement is defined %} diff --git a/contentctl/output/templates/savedsearches_detections.j2 b/contentctl/output/templates/savedsearches_detections.j2 index 396bb2c6..e3e7154a 100644 --- a/contentctl/output/templates/savedsearches_detections.j2 +++ b/contentctl/output/templates/savedsearches_detections.j2 @@ -70,8 +70,8 @@ action.notable.param.nes_fields = {{ detection.nes_fields }} {% endif %} action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}} action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%} -action.notable.param.security_domain = {{ detection.tags.security_domain.value }} -action.notable.param.severity = {{ detection.tags.severity.value }} +action.notable.param.security_domain = {{ detection.tags.security_domain }} +action.notable.param.severity = {{ detection.tags.severity }} {% endif %} {% if detection.deployment.alert_action.email %} action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }} diff --git a/enums_test.py b/enums_test.py new file mode 100644 index 00000000..fea6a545 --- /dev/null +++ b/enums_test.py @@ -0,0 +1,46 @@ +from pydantic import BaseModel, ConfigDict +from enum import StrEnum, IntEnum + +class SomeString(StrEnum): + one = "one" + two = "TWO" + +class SomeInt(IntEnum): + one = 1 + two = 2 + +class WithUseEnum(BaseModel): + ConfigDict(use_enum_values=True) + strval: SomeString + intval: SomeInt + + +class WithOutUseEnum(BaseModel): + strval: SomeString + intval: SomeInt + +withObj = WithUseEnum.model_validate({"strval": "one", "intval": "2"}) +withoutObj = WithOutUseEnum.model_validate({"strval": "one", "intval": "2"}) + + +print("With tests") +print(withObj.strval) +print(withObj.strval.upper()) +print(withObj.strval.value) +print(withObj.intval) +print(withObj.intval.value) +print(withObj.strval == SomeString.one) +print(withObj.strval == "ONE") +print(withObj.intval == SomeInt.two) +print(withObj.intval == 2) + + +print("Without tests") +print(withoutObj.strval) +print(withoutObj.strval.value) +print(withoutObj.intval) +print(withoutObj.intval.value) +print(withoutObj.strval == SomeString.one) +print(withoutObj.strval == "ONE") +print(withoutObj.intval == SomeInt.two) +print(withoutObj.intval == 2) \ No newline at end of file From ed958cc00e7479ab4dfaeb89efa7c6fd7a6bd283 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 14:11:31 -0800 Subject: [PATCH 068/115] fix missing typing of mode_name --- contentctl/objects/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 361ddab7..93e33de9 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -434,7 +434,7 @@ def getContainers(self)->List[Container]: class All(BaseModel): #Doesn't need any extra logic - mode_name = "All" + mode_name:str = "All" pass @@ -446,7 +446,7 @@ class Changes(BaseModel): class Selected(BaseModel): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) - mode_name = "Selected" + mode_name:str = "Selected" files:List[FilePath] = Field(...,description="List of detection files to test, separated by spaces.") @field_serializer('files',when_used='always') From 0237ccdb486bbbd632314ee85b396096956d18b1 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 5 Dec 2024 16:31:00 -0800 Subject: [PATCH 069/115] remove files that are no longer used anymore. Add logic to serialize StrEnum and IntEnum when writing YMLs from objects that contain them. --- contentctl/actions/new_content.py | 1 - contentctl/output/detection_writer.py | 28 --------- contentctl/output/new_content_yml_output.py | 56 ----------------- contentctl/output/yml_output.py | 66 --------------------- contentctl/output/yml_writer.py | 15 +++++ enums_test.py | 46 -------------- 6 files changed, 15 insertions(+), 197 deletions(-) delete mode 100644 contentctl/output/detection_writer.py delete mode 100644 contentctl/output/new_content_yml_output.py delete mode 100644 contentctl/output/yml_output.py delete mode 100644 enums_test.py diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 0a54cf11..74b4dbce 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -4,7 +4,6 @@ import questionary from typing import Any from contentctl.input.new_content_questions import NewContentQuestions -from contentctl.output.new_content_yml_output import NewContentYmlOutput from contentctl.objects.config import new, NewContentType import uuid from datetime import datetime diff --git a/contentctl/output/detection_writer.py b/contentctl/output/detection_writer.py deleted file mode 100644 index 2f439ca9..00000000 --- a/contentctl/output/detection_writer.py +++ /dev/null @@ -1,28 +0,0 @@ - -import yaml - - -class DetectionWriter: - - @staticmethod - def writeYmlFile(file_path : str, obj : dict) -> None: - - new_obj = dict() - new_obj["name"] = obj["name"] - new_obj["id"] = obj["id"] - new_obj["version"] = obj["version"] - new_obj["date"] = obj["date"] - new_obj["author"] = obj["author"] - new_obj["type"] = obj["type"] - new_obj["status"] = obj["status"] - new_obj["description"] = obj["description"] - new_obj["data_source"] = obj["data_source"] - new_obj["search"] = obj["search"] - new_obj["how_to_implement"] = obj["how_to_implement"] - new_obj["known_false_positives"] = obj["known_false_positives"] - new_obj["references"] = obj["references"] - new_obj["tags"] = obj["tags"] - new_obj["tests"] = obj["tests"] - - with open(file_path, 'w') as outfile: - yaml.safe_dump(new_obj, outfile, default_flow_style=False, sort_keys=False) \ No newline at end of file diff --git a/contentctl/output/new_content_yml_output.py b/contentctl/output/new_content_yml_output.py deleted file mode 100644 index 38730b37..00000000 --- a/contentctl/output/new_content_yml_output.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import pathlib -from contentctl.objects.enums import SecurityContentType -from contentctl.output.yml_writer import YmlWriter -import pathlib -from contentctl.objects.config import NewContentType -class NewContentYmlOutput(): - output_path: pathlib.Path - - def __init__(self, output_path:pathlib.Path): - self.output_path = output_path - - - def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewContentType) -> None: - if type == NewContentType.detection: - - file_path = os.path.join(self.output_path, 'detections', subdirectory_name, self.convertNameToFileName(object['name'], object['tags']['product'])) - output_folder = pathlib.Path(self.output_path)/'detections'/subdirectory_name - #make sure the output folder exists for this detection - output_folder.mkdir(exist_ok=True) - - YmlWriter.writeYmlFile(file_path, object) - print("Successfully created detection " + file_path) - - elif type == NewContentType.story: - file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product'])) - YmlWriter.writeYmlFile(file_path, object) - print("Successfully created story " + file_path) - - else: - raise(Exception(f"Object Must be Story or Detection, but is not: {object}")) - - - - def convertNameToFileName(self, name: str, product: list): - file_name = name \ - .replace(' ', '_') \ - .replace('-','_') \ - .replace('.','_') \ - .replace('/','_') \ - .lower() - - file_name = file_name + '.yml' - return file_name - - - def convertNameToTestFileName(self, name: str, product: list): - file_name = name \ - .replace(' ', '_') \ - .replace('-','_') \ - .replace('.','_') \ - .replace('/','_') \ - .lower() - - file_name = file_name + '.test.yml' - return file_name \ No newline at end of file diff --git a/contentctl/output/yml_output.py b/contentctl/output/yml_output.py deleted file mode 100644 index 93eae5dc..00000000 --- a/contentctl/output/yml_output.py +++ /dev/null @@ -1,66 +0,0 @@ -import os - -from contentctl.output.detection_writer import DetectionWriter -from contentctl.objects.detection import Detection - - -class YmlOutput(): - - - def writeDetections(self, objects: list, output_path : str) -> None: - for obj in objects: - file_path = obj.file_path - obj.id = str(obj.id) - - DetectionWriter.writeYmlFile(os.path.join(output_path, file_path), obj.dict( - exclude_none=True, - include = - { - "name": True, - "id": True, - "version": True, - "date": True, - "author": True, - "type": True, - "status": True, - "description": True, - "data_source": True, - "search": True, - "how_to_implement": True, - "known_false_positives": True, - "references": True, - "tags": - { - "analytic_story": True, - "asset_type": True, - "atomic_guid": True, - "confidence": True, - "impact": True, - "drilldown_search": True, - "mappings": True, - "message": True, - "mitre_attack_id": True, - "kill_chain_phases:": True, - "observable": True, - "product": True, - "required_fields": True, - "risk_score": True, - "security_domain": True - }, - "tests": - { - '__all__': - { - "name": True, - "attack_data": { - '__all__': - { - "data": True, - "source": True, - "sourcetype": True - } - } - } - } - } - )) \ No newline at end of file diff --git a/contentctl/output/yml_writer.py b/contentctl/output/yml_writer.py index 7d71762b..2e408c83 100644 --- a/contentctl/output/yml_writer.py +++ b/contentctl/output/yml_writer.py @@ -1,6 +1,21 @@ import yaml from typing import Any +from enum import StrEnum, IntEnum + +# Set the following so that we can write StrEnum and IntEnum +# to files. Otherwise, we will get the following errors when trying +# to write to files: +# yaml.representer.RepresenterError: ('cannot represent an object',..... +yaml.SafeDumper.add_multi_representer( + StrEnum, + yaml.representer.SafeRepresenter.represent_str +) + +yaml.SafeDumper.add_multi_representer( + IntEnum, + yaml.representer.SafeRepresenter.represent_int +) class YmlWriter: diff --git a/enums_test.py b/enums_test.py deleted file mode 100644 index fea6a545..00000000 --- a/enums_test.py +++ /dev/null @@ -1,46 +0,0 @@ -from pydantic import BaseModel, ConfigDict -from enum import StrEnum, IntEnum - -class SomeString(StrEnum): - one = "one" - two = "TWO" - -class SomeInt(IntEnum): - one = 1 - two = 2 - -class WithUseEnum(BaseModel): - ConfigDict(use_enum_values=True) - strval: SomeString - intval: SomeInt - - -class WithOutUseEnum(BaseModel): - strval: SomeString - intval: SomeInt - -withObj = WithUseEnum.model_validate({"strval": "one", "intval": "2"}) -withoutObj = WithOutUseEnum.model_validate({"strval": "one", "intval": "2"}) - - -print("With tests") -print(withObj.strval) -print(withObj.strval.upper()) -print(withObj.strval.value) -print(withObj.intval) -print(withObj.intval.value) -print(withObj.strval == SomeString.one) -print(withObj.strval == "ONE") -print(withObj.intval == SomeInt.two) -print(withObj.intval == 2) - - -print("Without tests") -print(withoutObj.strval) -print(withoutObj.strval.value) -print(withoutObj.intval) -print(withoutObj.intval.value) -print(withoutObj.strval == SomeString.one) -print(withoutObj.strval == "ONE") -print(withoutObj.intval == SomeInt.two) -print(withoutObj.intval == 2) \ No newline at end of file From 6f394cc0c37ff1398765f17ce69f6dae28938a6c Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 5 Dec 2024 16:35:07 -0800 Subject: [PATCH 070/115] Remove dead code from new_content.py --- contentctl/actions/new_content.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 74b4dbce..0315c47f 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -110,23 +110,3 @@ def execute(self, input_dto: new) -> None: full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name')) YmlWriter.writeYmlFile(str(full_output_path), content_dict) - - - def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewContentType) -> None: - if type == NewContentType.detection: - file_path = os.path.join(self.output_path, 'detections', subdirectory_name, self.convertNameToFileName(object['name'], object['tags']['product'])) - output_folder = pathlib.Path(self.output_path)/'detections'/subdirectory_name - #make sure the output folder exists for this detection - output_folder.mkdir(exist_ok=True) - - YmlWriter.writeDetection(file_path, object) - print("Successfully created detection " + file_path) - - elif type == NewContentType.story: - file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product'])) - YmlWriter.writeStory(file_path, object) - print("Successfully created story " + file_path) - - else: - raise(Exception(f"Object Must be Story or Detection, but is not: {object}")) - From 61d99ede5ce9a93c8286fcc23521a2c1bd645c96 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 10 Dec 2024 16:36:46 -0800 Subject: [PATCH 071/115] remove the 'forbid' from a few classes that restated it. but this is not necessary if they inherit from SecurityContentObject --- contentctl/objects/data_source.py | 1 - contentctl/objects/deployment.py | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index ed8a8f86..2ed9c80c 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -9,7 +9,6 @@ class TA(BaseModel): url: HttpUrl | None = None version: str class DataSource(SecurityContentObject): - model_config = ConfigDict(extra="forbid") source: str = Field(...) sourcetype: str = Field(...) separator: Optional[str] = None diff --git a/contentctl/objects/deployment.py b/contentctl/objects/deployment.py index 8fd264b6..6e2cc6d2 100644 --- a/contentctl/objects/deployment.py +++ b/contentctl/objects/deployment.py @@ -10,15 +10,7 @@ from contentctl.objects.enums import DeploymentType -class Deployment(SecurityContentObject): - model_config = ConfigDict(extra="forbid") - #id: str = None - #date: str = None - #author: str = None - #description: str = None - #contentType: SecurityContentType = SecurityContentType.deployments - - +class Deployment(SecurityContentObject): scheduling: DeploymentScheduling = Field(...) alert_action: AlertAction = AlertAction() type: DeploymentType = Field(...) From 5b865523850341d0028c681a39a774bba7b463a7 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 10 Dec 2024 16:42:17 -0800 Subject: [PATCH 072/115] Clean up two more use of .value on StrEnums --- contentctl/objects/baseline.py | 2 +- contentctl/objects/investigation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index a41acbb4..f59c7dce 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -36,7 +36,7 @@ def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment: @computed_field @property def datamodel(self) -> List[DataModel]: - return [dm for dm in DataModel if dm.value in self.search] + return [dm for dm in DataModel if dm in self.search] @model_serializer def serialize_model(self): diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 433105ce..0d35a9db 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -37,7 +37,7 @@ def inputs(self)->List[str]: @computed_field @property def datamodel(self) -> List[DataModel]: - return [dm for dm in DataModel if dm.value in self.search] + return [dm for dm in DataModel if dm in self.search] @computed_field @property From ed1c8b0c257b397a7cade76d3372cffb09c7345b Mon Sep 17 00:00:00 2001 From: ljstella Date: Wed, 11 Dec 2024 09:06:36 -0600 Subject: [PATCH 073/115] Version bump --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8c73774..85e96877 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.0 + rev: v0.8.2 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index 00f568d6..05292a06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.8.0" +ruff = "^0.8.2" [build-system] requires = ["poetry-core>=1.0.0"] From c0d440c0817f4a23b9e1da21aea3dffbe3e5257c Mon Sep 17 00:00:00 2001 From: ljstella Date: Wed, 11 Dec 2024 09:38:42 -0600 Subject: [PATCH 074/115] other precommit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85e96877..4ac592eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 # Use the ref you want to point at + rev: v5.0.0 # Use the ref you want to point at hooks: - id: check-json - id: check-symlinks From ea437a8e7707ecd6b9eec24d7cae5ef58e6dd1ff Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Dec 2024 12:27:38 -0600 Subject: [PATCH 075/115] Add GH Actions to Dependabot --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 650a5e95..b1bdacd5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,8 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 6 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" \ No newline at end of file From 5cbf82a23a372bc5634e24c71dea8dcc86d982f8 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Dec 2024 12:28:19 -0600 Subject: [PATCH 076/115] Reduce matrix for simplicity --- .github/workflows/testEndToEnd.yml | 2 +- .github/workflows/test_against_escu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testEndToEnd.yml b/.github/workflows/testEndToEnd.yml index 444ee96a..6b1a4b20 100644 --- a/.github/workflows/testEndToEnd.yml +++ b/.github/workflows/testEndToEnd.yml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-20.04", "ubuntu-24.04", "macos-15", "macos-14", "windows-2022"] + operating_system: ["ubuntu-24.04", "macos-15", "windows-2022"] #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"] diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index 9758b6c0..ef07d445 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -18,7 +18,7 @@ jobs: matrix: python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-20.04", "ubuntu-24.04", "macos-15", "macos-14"] + operating_system: ["ubuntu-24.04", "macos-15"] # Do not test against ESCU until known character encoding issue is resolved # operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"] From f88bca614c3317f1061686b0608c19c0968d7c78 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 11:26:37 -0800 Subject: [PATCH 077/115] convert plain enums, or enums with multiple inheritance, to StrEnum or IntEnum --- contentctl/actions/build.py | 2 +- .../DetectionTestingManager.py | 1 - contentctl/objects/enums.py | 79 +++++-------------- 3 files changed, 22 insertions(+), 60 deletions(-) diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index feb0351b..43daf360 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from contentctl.objects.enums import SecurityContentProduct, SecurityContentType +from contentctl.objects.enums import SecurityContentType from contentctl.input.director import Director, DirectorOutputDto from contentctl.output.conf_output import ConfOutput from contentctl.output.conf_writer import ConfWriter diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 5ad5e117..13058e2f 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -5,7 +5,6 @@ from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer from urllib.parse import urlparse from copy import deepcopy -from contentctl.objects.enums import DetectionTestingTargetInfrastructure import signal import datetime # from queue import Queue diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py index 333ef358..8070d4a4 100644 --- a/contentctl/objects/enums.py +++ b/contentctl/objects/enums.py @@ -1,15 +1,15 @@ from __future__ import annotations from typing import List -import enum +from enum import StrEnum, IntEnum -class AnalyticsType(str, enum.Enum): +class AnalyticsType(StrEnum): TTP = "TTP" Anomaly = "Anomaly" Hunting = "Hunting" Correlation = "Correlation" -class DeploymentType(str, enum.Enum): +class DeploymentType(StrEnum): TTP = "TTP" Anomaly = "Anomaly" Hunting = "Hunting" @@ -18,7 +18,7 @@ class DeploymentType(str, enum.Enum): Embedded = "Embedded" -class DataModel(str,enum.Enum): +class DataModel(StrEnum): ENDPOINT = "Endpoint" NETWORK_TRAFFIC = "Network_Traffic" AUTHENTICATION = "Authentication" @@ -40,11 +40,11 @@ class DataModel(str,enum.Enum): SPLUNK_AUDIT = "Splunk_Audit" -class PlaybookType(str, enum.Enum): +class PlaybookType(StrEnum): INVESTIGATION = "Investigation" RESPONSE = "Response" -class SecurityContentType(enum.Enum): +class SecurityContentType(IntEnum): detections = 1 baselines = 2 stories = 3 @@ -68,20 +68,15 @@ class SecurityContentType(enum.Enum): # json_objects = "json_objects" -class SecurityContentProduct(enum.Enum): - SPLUNK_APP = 1 - API = 3 - CUSTOM = 4 - -class SecurityContentProductName(str, enum.Enum): +class SecurityContentProductName(StrEnum): SPLUNK_ENTERPRISE = "Splunk Enterprise" SPLUNK_ENTERPRISE_SECURITY = "Splunk Enterprise Security" SPLUNK_CLOUD = "Splunk Cloud" SPLUNK_SECURITY_ANALYTICS_FOR_AWS = "Splunk Security Analytics for AWS" SPLUNK_BEHAVIORAL_ANALYTICS = "Splunk Behavioral Analytics" -class SecurityContentInvestigationProductName(str, enum.Enum): +class SecurityContentInvestigationProductName(StrEnum): SPLUNK_ENTERPRISE = "Splunk Enterprise" SPLUNK_ENTERPRISE_SECURITY = "Splunk Enterprise Security" SPLUNK_CLOUD = "Splunk Cloud" @@ -90,33 +85,20 @@ class SecurityContentInvestigationProductName(str, enum.Enum): SPLUNK_PHANTOM = "Splunk Phantom" -class DetectionStatus(enum.Enum): - production = "production" - deprecated = "deprecated" - experimental = "experimental" - validation = "validation" - - -class DetectionStatusSSA(enum.Enum): +class DetectionStatus(StrEnum): production = "production" deprecated = "deprecated" experimental = "experimental" validation = "validation" -class LogLevel(enum.Enum): +class LogLevel(StrEnum): NONE = "NONE" ERROR = "ERROR" INFO = "INFO" -class AlertActions(enum.Enum): - notable = "notable" - rba = "rba" - email = "email" - - -class StoryCategory(str, enum.Enum): +class StoryCategory(StrEnum): ABUSE = "Abuse" ADVERSARY_TACTICS = "Adversary Tactics" BEST_PRACTICES = "Best Practices" @@ -139,37 +121,18 @@ class StoryCategory(str, enum.Enum): UNAUTHORIZED_SOFTWARE = "Unauthorized Software" -class PostTestBehavior(str, enum.Enum): +class PostTestBehavior(StrEnum): always_pause = "always_pause" pause_on_failure = "pause_on_failure" never_pause = "never_pause" -class DetectionTestingMode(str, enum.Enum): +class DetectionTestingMode(StrEnum): selected = "selected" all = "all" changes = "changes" -class DetectionTestingTargetInfrastructure(str, enum.Enum): - container = "container" - server = "server" - - -class InstanceState(str, enum.Enum): - starting = "starting" - running = "running" - error = "error" - stopping = "stopping" - stopped = "stopped" - - -class SigmaConverterTarget(enum.Enum): - CIM = 1 - RAW = 2 - OCSF = 3 - ALL = 4 - # It's unclear why we use a mix of constants and enums. The following list was taken from: # contentctl/contentctl/helper/constants.py. # We convect it to an enum here @@ -183,7 +146,7 @@ class SigmaConverterTarget(enum.Enum): # "Command And Control": 6, # "Actions on Objectives": 7 # } -class KillChainPhase(str, enum.Enum): +class KillChainPhase(StrEnum): UNKNOWN ="Unknown" RECONNAISSANCE = "Reconnaissance" WEAPONIZATION = "Weaponization" @@ -194,7 +157,7 @@ class KillChainPhase(str, enum.Enum): ACTIONS_ON_OBJECTIVES = "Actions on Objectives" -class DataSource(str,enum.Enum): +class DataSource(StrEnum): OSQUERY_ES_PROCESS_EVENTS = "OSQuery ES Process Events" POWERSHELL_4104 = "Powershell 4104" SYSMON_EVENT_ID_1 = "Sysmon EventID 1" @@ -234,7 +197,7 @@ class DataSource(str,enum.Enum): WINDOWS_SECURITY_5145 = "Windows Security 5145" WINDOWS_SYSTEM_7045 = "Windows System 7045" -class ProvidingTechnology(str, enum.Enum): +class ProvidingTechnology(StrEnum): AMAZON_SECURITY_LAKE = "Amazon Security Lake" AMAZON_WEB_SERVICES_CLOUDTRAIL = "Amazon Web Services - Cloudtrail" AZURE_AD = "Azure AD" @@ -302,7 +265,7 @@ def getProvidingTechFromSearch(search_string:str)->List[ProvidingTechnology]: return sorted(list(matched_technologies)) -class Cis18Value(str,enum.Enum): +class Cis18Value(StrEnum): CIS_0 = "CIS 0" CIS_1 = "CIS 1" CIS_2 = "CIS 2" @@ -323,7 +286,7 @@ class Cis18Value(str,enum.Enum): CIS_17 = "CIS 17" CIS_18 = "CIS 18" -class SecurityDomain(str, enum.Enum): +class SecurityDomain(StrEnum): ENDPOINT = "endpoint" NETWORK = "network" THREAT = "threat" @@ -331,7 +294,7 @@ class SecurityDomain(str, enum.Enum): ACCESS = "access" AUDIT = "audit" -class AssetType(str, enum.Enum): +class AssetType(StrEnum): AWS_ACCOUNT = "AWS Account" AWS_EKS_KUBERNETES_CLUSTER = "AWS EKS Kubernetes cluster" AWS_FEDERATED_ACCOUNT = "AWS Federated Account" @@ -382,7 +345,7 @@ class AssetType(str, enum.Enum): WEB_APPLICATION = "Web Application" WINDOWS = "Windows" -class NistCategory(str, enum.Enum): +class NistCategory(StrEnum): ID_AM = "ID.AM" ID_BE = "ID.BE" ID_GV = "ID.GV" @@ -406,7 +369,7 @@ class NistCategory(str, enum.Enum): RC_IM = "RC.IM" RC_CO = "RC.CO" -class RiskSeverity(str,enum.Enum): +class RiskSeverity(StrEnum): # Levels taken from the following documentation link # https://docs.splunk.com/Documentation/ES/7.3.2/User/RiskScoring # 20 - info (0-20 for us) From 5b9cb9581a58c5c190b41a1d70e45835a786dbc0 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 11:31:16 -0800 Subject: [PATCH 078/115] Remove all usage of use_enum_values. However, this has not received any testing yet. --- .../detection_abstract.py | 4 +- .../security_content_object_abstract.py | 4 +- contentctl/objects/config.py | 44 +++++++------------ contentctl/objects/detection_tags.py | 3 +- contentctl/objects/investigation.py | 3 +- contentctl/objects/mitre_attack_enrichment.py | 2 - contentctl/objects/story_tags.py | 3 +- 7 files changed, 21 insertions(+), 42 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index dc37aed2..3b7517eb 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -58,9 +58,7 @@ } -# TODO (#266): disable the use_enum_values configuration class Detection_Abstract(SecurityContentObject): - model_config = ConfigDict(use_enum_values=True) name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) #contentType: SecurityContentType = SecurityContentType.detections type: AnalyticsType = Field(...) @@ -215,7 +213,7 @@ def adjust_tests_and_groups(self) -> None: # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name # Skip tests for non-production detections - if self.status != DetectionStatus.production.value: # type: ignore + if self.status != DetectionStatus.production.value: self.skip_all_tests(f"TEST SKIPPED: Detection is non-production ({self.status})") # Skip tests for detecton types like Correlation which are not supported via contentctl diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index af1b3674..a69fac65 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -31,9 +31,9 @@ NO_FILE_NAME = "NO_FILE_NAME" -# TODO (#266): disable the use_enum_values configuration class SecurityContentObject_Abstract(BaseModel, abc.ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True,extra="forbid") + model_config = ConfigDict(validate_default=True) + name: str = Field(...,max_length=99) author: str = Field(...,max_length=255) date: datetime.date = Field(...) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index c41b93ea..b1b6aed5 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -33,9 +33,8 @@ SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/{uid}/release/{version}/download" -# TODO (#266): disable the use_enum_values configuration class App_Base(BaseModel,ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True, extra='forbid') + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) uid: Optional[int] = Field(default=None) title: str = Field(description="Human-readable name used by the app. This can have special characters.") appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. " @@ -59,9 +58,8 @@ def ensureAppPathExists(self, config:test, stage_file:bool=False): config.getLocalAppDir().mkdir(parents=True) -# TODO (#266): disable the use_enum_values configuration class TestApp(App_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) hardcoded_path: Optional[Union[FilePath,HttpUrl]] = Field(default=None, description="This may be a relative or absolute link to a file OR an HTTP URL linking to your app.") @@ -99,9 +97,8 @@ def getApp(self, config:test,stage_file:bool=False)->str: return str(destination) -# TODO (#266): disable the use_enum_values configuration class CustomApp(App_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) # Fields required for app.conf based on # https://docs.splunk.com/Documentation/Splunk/9.0.4/Admin/Appconf uid: int = Field(ge=2, lt=100000, default_factory=lambda:random.randint(20000,100000)) @@ -159,9 +156,8 @@ def getApp(self, config:test, stage_file=True)->str: verbose_print=True) return str(destination) -# TODO (#266): disable the use_enum_values configuration class Config_Base(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) path: DirectoryPath = Field(default=DirectoryPath("."), description="The root of your app.") app:CustomApp = Field(default_factory=CustomApp) @@ -175,7 +171,7 @@ def serialize_path(path: DirectoryPath)->str: return str(path) class init(Config_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) bare: bool = Field(default=False, description="contentctl normally provides some some example content " "(macros, stories, data_sources, and/or analytic stories). This option disables " "initialization with that additional contnet. Note that even if --bare is used, it " @@ -184,9 +180,8 @@ class init(Config_Base): "the deployment/ directory (since it is not yet easily customizable).") -# TODO (#266): disable the use_enum_values configuration class validate(Config_Base): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) enrichments: bool = Field(default=False, description="Enable MITRE, APP, and CVE Enrichments. "\ "This is useful when outputting a release build "\ "and validating these values, but should otherwise "\ @@ -241,9 +236,8 @@ def getReportingPath(self)->pathlib.Path: return self.path/"reporting/" -# TODO (#266): disable the use_enum_values configuration class build(validate): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) build_path: DirectoryPath = Field(default=DirectoryPath("dist/"), title="Target path for all build outputs") @field_serializer('build_path',when_used='always') @@ -395,17 +389,15 @@ class new(Config_Base): type: NewContentType = Field(default=NewContentType.detection, description="Specify the type of content you would like to create.") -# TODO (#266): disable the use_enum_values configuration class deploy_acs(inspect): - model_config = ConfigDict(use_enum_values=True,validate_default=False, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=False, arbitrary_types_allowed=True) #ignore linter error splunk_cloud_jwt_token: str = Field(exclude=True, description="Splunk JWT used for performing ACS operations on a Splunk Cloud Instance") splunk_cloud_stack: str = Field(description="The name of your Splunk Cloud Stack") -# TODO (#266): disable the use_enum_values configuration class Infrastructure(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) splunk_app_username:str = Field(default="admin", description="Username for logging in to your Splunk Server") splunk_app_password:str = Field(exclude=True, default="password", description="Password for logging in to your Splunk Server.") instance_address:str = Field(..., description="Address of your splunk server.") @@ -415,15 +407,13 @@ class Infrastructure(BaseModel): instance_name: str = Field(...) -# TODO (#266): disable the use_enum_values configuration class Container(Infrastructure): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) instance_address:str = Field(default="localhost", description="Address of your splunk server.") -# TODO (#266): disable the use_enum_values configuration class ContainerSettings(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) leave_running: bool = Field(default=True, description="Leave container running after it is first " "set up to speed up subsequent test runs.") num_containers: PositiveInt = Field(default=1, description="Number of containers to start in parallel. " @@ -447,15 +437,13 @@ class All(BaseModel): pass -# TODO (#266): disable the use_enum_values configuration class Changes(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) target_branch:str = Field(...,description="The target branch to diff against. Note that this includes uncommitted changes in the working directory as well.") -# TODO (#266): disable the use_enum_values configuration class Selected(BaseModel): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) files:List[FilePath] = Field(...,description="List of detection files to test, separated by spaces.") @field_serializer('files',when_used='always') @@ -826,9 +814,8 @@ def getModeName(self)->str: return DetectionTestingMode.selected.value -# TODO (#266): disable the use_enum_values configuration class test(test_common): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) container_settings:ContainerSettings = ContainerSettings() test_instances: List[Container] = Field([], exclude = True, validate_default=True) splunk_api_username: Optional[str] = Field(default=None, exclude = True,description="Splunk API username used for running appinspect or installating apps from Splunkbase") @@ -893,9 +880,8 @@ def getAppFilePath(self): TEST_ARGS_ENV = "CONTENTCTL_TEST_INFRASTRUCTURES" -# TODO (#266): disable the use_enum_values configuration class test_servers(test_common): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) test_instances:List[Infrastructure] = Field([],description="Test against one or more preconfigured servers.", validate_default=True) server_info:Optional[str] = Field(None, validate_default=True, description='String of pre-configured servers to use for testing. The list MUST be in the format:\n' 'address,username,web_ui_port,hec_port,api_port;address_2,username_2,web_ui_port_2,hec_port_2,api_port_2' diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index e6574cab..d8207225 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -35,10 +35,9 @@ from contentctl.objects.atomic import AtomicEnrichment, AtomicTest from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE, CVE_TYPE -# TODO (#266): disable the use_enum_values configuration class DetectionTags(BaseModel): # detection spec - model_config = ConfigDict(use_enum_values=True,validate_default=False, extra='forbid') + model_config = ConfigDict(validate_default=False) analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) group: list[str] = [] diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 6e058783..433105ce 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -12,9 +12,8 @@ ) from contentctl.objects.config import CustomApp -# TODO (#266): disable the use_enum_values configuration class Investigation(SecurityContentObject): - model_config = ConfigDict(use_enum_values=True,validate_default=False) + model_config = ConfigDict(validate_default=False) type: str = Field(...,pattern="^Investigation$") name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) search: str = Field(...) diff --git a/contentctl/objects/mitre_attack_enrichment.py b/contentctl/objects/mitre_attack_enrichment.py index 85df2c4b..888754c8 100644 --- a/contentctl/objects/mitre_attack_enrichment.py +++ b/contentctl/objects/mitre_attack_enrichment.py @@ -83,9 +83,7 @@ def standardize_contributors(cls, contributors:list[str] | None) -> list[str]: return [] return contributors -# TODO (#266): disable the use_enum_values configuration class MitreAttackEnrichment(BaseModel): - ConfigDict(use_enum_values=True,extra='forbid') mitre_attack_id: MITRE_ATTACK_ID_TYPE = Field(...) mitre_attack_technique: str = Field(...) mitre_attack_tactics: List[MitreTactics] = Field(...) diff --git a/contentctl/objects/story_tags.py b/contentctl/objects/story_tags.py index 42eb2f37..e1bd45dc 100644 --- a/contentctl/objects/story_tags.py +++ b/contentctl/objects/story_tags.py @@ -18,9 +18,8 @@ class StoryUseCase(str,Enum): OTHER = "Other" -# TODO (#266): disable the use_enum_values configuration class StoryTags(BaseModel): - model_config = ConfigDict(extra='forbid', use_enum_values=True) + model_config = ConfigDict(extra='forbid') category: List[StoryCategory] = Field(...,min_length=1) product: List[SecurityContentProductName] = Field(...,min_length=1) usecase: StoryUseCase = Field(...) From 827a8f44348c002a0bacabd6f2ca5c5578810c66 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 14:09:11 -0800 Subject: [PATCH 079/115] Remove use of .value on enums in code now that they have been more strictly defined as IntEnum or StrEnum. This has not yet been tested. --- .../DetectionTestingInfrastructure.py | 50 +++++++++---------- .../actions/detection_testing/progress_bar.py | 12 ++--- .../views/DetectionTestingView.py | 8 +-- contentctl/actions/test.py | 9 ++-- .../detection_abstract.py | 36 ++++++------- .../security_content_object_abstract.py | 8 +-- contentctl/objects/base_test.py | 4 +- contentctl/objects/base_test_result.py | 6 +-- contentctl/objects/config.py | 26 ++++------ contentctl/objects/constants.py | 2 +- contentctl/objects/correlation_search.py | 50 +++++++++---------- contentctl/objects/detection_tags.py | 2 +- contentctl/output/svg_output.py | 2 +- .../templates/analyticstories_detections.j2 | 2 +- .../templates/savedsearches_detections.j2 | 4 +- enums_test.py | 46 +++++++++++++++++ 16 files changed, 153 insertions(+), 114 deletions(-) create mode 100644 enums_test.py diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 8e816025..7804b29e 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -442,7 +442,7 @@ def test_detection(self, detection: Detection) -> None: self.format_pbar_string( TestReportingType.GROUP, test_group.name, - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=time.time(), set_pbar=False, ) @@ -483,7 +483,7 @@ def test_detection(self, detection: Detection) -> None: self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.DONE_GROUP.value, + TestingStates.DONE_GROUP, start_time=setup_results.start_time, set_pbar=False, ) @@ -504,7 +504,7 @@ def setup_test_group(self, test_group: TestGroup) -> SetupTestGroupResults: self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.BEGINNING_GROUP.value, + TestingStates.BEGINNING_GROUP, start_time=setup_start_time ) # https://github.com/WoLpH/python-progressbar/issues/164 @@ -544,7 +544,7 @@ def cleanup_test_group( self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.DELETING.value, + TestingStates.DELETING, start_time=test_group_start_time, ) @@ -632,7 +632,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=test_start_time, set_pbar=False, ) @@ -664,7 +664,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.ERROR.value, + FinalTestingStates.ERROR, start_time=test_start_time, set_pbar=False, ) @@ -724,7 +724,7 @@ def execute_unit_test( res = "ERROR" link = detection.search else: - res = test.result.status.value.upper() # type: ignore + res = test.result.status.upper() # type: ignore link = test.result.get_summary_dict()["sid_link"] self.format_pbar_string( @@ -755,7 +755,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.PASS.value, + FinalTestingStates.PASS, start_time=test_start_time, set_pbar=False, ) @@ -766,7 +766,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=test_start_time, set_pbar=False, ) @@ -777,7 +777,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.FAIL.value, + FinalTestingStates.FAIL, start_time=test_start_time, set_pbar=False, ) @@ -788,7 +788,7 @@ def execute_unit_test( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - FinalTestingStates.ERROR.value, + FinalTestingStates.ERROR, start_time=test_start_time, set_pbar=False, ) @@ -821,7 +821,7 @@ def execute_integration_test( test_start_time = time.time() # First, check to see if the test should be skipped (Hunting or Correlation) - if detection.type in [AnalyticsType.Hunting.value, AnalyticsType.Correlation.value]: + if detection.type in [AnalyticsType.Hunting, AnalyticsType.Correlation]: test.skip( f"TEST SKIPPED: detection is type {detection.type} and cannot be integration " "tested at this time" @@ -843,11 +843,11 @@ def execute_integration_test( # Determine the reporting state (we should only encounter SKIP/FAIL/ERROR) state: str if test.result.status == TestResultStatus.SKIP: - state = FinalTestingStates.SKIP.value + state = FinalTestingStates.SKIP elif test.result.status == TestResultStatus.FAIL: - state = FinalTestingStates.FAIL.value + state = FinalTestingStates.FAIL elif test.result.status == TestResultStatus.ERROR: - state = FinalTestingStates.ERROR.value + state = FinalTestingStates.ERROR else: raise ValueError( f"Status for (integration) '{detection.name}:{test.name}' was preemptively set" @@ -891,7 +891,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.FAIL.value, + FinalTestingStates.FAIL, start_time=test_start_time, set_pbar=False, ) @@ -935,7 +935,7 @@ def execute_integration_test( if test.result is None: res = "ERROR" else: - res = test.result.status.value.upper() # type: ignore + res = test.result.status.upper() # type: ignore # Get the link to the saved search in this specific instance link = f"https://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}" @@ -968,7 +968,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.PASS.value, + FinalTestingStates.PASS, start_time=test_start_time, set_pbar=False, ) @@ -979,7 +979,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.SKIP.value, + FinalTestingStates.SKIP, start_time=test_start_time, set_pbar=False, ) @@ -990,7 +990,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.FAIL.value, + FinalTestingStates.FAIL, start_time=test_start_time, set_pbar=False, ) @@ -1001,7 +1001,7 @@ def execute_integration_test( self.format_pbar_string( TestReportingType.INTEGRATION, f"{detection.name}:{test.name}", - FinalTestingStates.ERROR.value, + FinalTestingStates.ERROR, start_time=test_start_time, set_pbar=False, ) @@ -1077,7 +1077,7 @@ def retry_search_until_timeout( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - TestingStates.PROCESSING.value, + TestingStates.PROCESSING, start_time=start_time ) @@ -1086,7 +1086,7 @@ def retry_search_until_timeout( self.format_pbar_string( TestReportingType.UNIT, f"{detection.name}:{test.name}", - TestingStates.SEARCHING.value, + TestingStates.SEARCHING, start_time=start_time, ) @@ -1289,7 +1289,7 @@ def replay_attack_data_file( self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.DOWNLOADING.value, + TestingStates.DOWNLOADING, start_time=test_group_start_time ) @@ -1307,7 +1307,7 @@ def replay_attack_data_file( self.format_pbar_string( TestReportingType.GROUP, test_group.name, - TestingStates.REPLAYING.value, + TestingStates.REPLAYING, start_time=test_group_start_time ) diff --git a/contentctl/actions/detection_testing/progress_bar.py b/contentctl/actions/detection_testing/progress_bar.py index 45e30e06..5b5abd1a 100644 --- a/contentctl/actions/detection_testing/progress_bar.py +++ b/contentctl/actions/detection_testing/progress_bar.py @@ -1,10 +1,10 @@ import time -from enum import Enum +from enum import StrEnum from tqdm import tqdm import datetime -class TestReportingType(str, Enum): +class TestReportingType(StrEnum): """ 5-char identifiers for the type of testing being reported on """ @@ -21,7 +21,7 @@ class TestReportingType(str, Enum): INTEGRATION = "INTEG" -class TestingStates(str, Enum): +class TestingStates(StrEnum): """ Defined testing states """ @@ -40,10 +40,10 @@ class TestingStates(str, Enum): # the longest length of any state -LONGEST_STATE = max(len(w.value) for w in TestingStates) +LONGEST_STATE = max(len(w) for w in TestingStates) -class FinalTestingStates(str, Enum): +class FinalTestingStates(StrEnum): """ The possible final states for a test (for pbar reporting) """ @@ -82,7 +82,7 @@ def format_pbar_string( :returns: a formatted string for use w/ pbar """ # Extract and ljust our various fields - field_one = test_reporting_type.value + field_one = test_reporting_type field_two = test_name.ljust(MAX_TEST_NAME_LENGTH) field_three = state.ljust(LONGEST_STATE) field_four = datetime.timedelta(seconds=round(time.time() - start_time)) diff --git a/contentctl/actions/detection_testing/views/DetectionTestingView.py b/contentctl/actions/detection_testing/views/DetectionTestingView.py index 8ff6e583..98cc7122 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingView.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingView.py @@ -110,11 +110,11 @@ def getSummaryObject( total_skipped += 1 # Aggregate production status metrics - if detection.status == DetectionStatus.production.value: # type: ignore + if detection.status == DetectionStatus.production: total_production += 1 - elif detection.status == DetectionStatus.experimental.value: # type: ignore + elif detection.status == DetectionStatus.experimental: total_experimental += 1 - elif detection.status == DetectionStatus.deprecated.value: # type: ignore + elif detection.status == DetectionStatus.deprecated: total_deprecated += 1 # Check if the detection is manual_test @@ -178,7 +178,7 @@ def getSummaryObject( # Construct and return the larger results dict result_dict = { "summary": { - "mode": self.config.getModeName(), + "mode": self.config.mode.mode_name, "enable_integration_testing": self.config.enable_integration_testing, "success": overall_success, "total_detections": total_detections, diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index 716ecd71..b3437cef 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import List -from contentctl.objects.config import test_common +from contentctl.objects.config import test_common, Selected, Changes from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType from contentctl.objects.detection import Detection @@ -78,10 +78,9 @@ def execute(self, input_dto: TestInputDto) -> bool: input_dto=manager_input_dto, output_dto=output_dto ) - mode = input_dto.config.getModeName() if len(input_dto.detections) == 0: print( - f"With Detection Testing Mode '{mode}', there were [0] detections found to test." + f"With Detection Testing Mode '{input_dto.config.mode.mode_name}', there were [0] detections found to test." "\nAs such, we will quit immediately." ) # Directly call stop so that the summary.yml will be generated. Of course it will not @@ -89,8 +88,8 @@ def execute(self, input_dto: TestInputDto) -> bool: # detections were tested. file.stop() else: - print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections") - if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]: + print(f"MODE: [{input_dto.config.mode.mode_name}] - Test [{len(input_dto.detections)}] detections") + if isinstance(input_dto.config.mode, Selected) or isinstance(input_dto.config.mode, Changes): files_string = '\n- '.join( [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections] ) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 3b7517eb..ff5b94f0 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Union, Optional, List, Any, Annotated import re import pathlib -from enum import Enum +from enum import StrEnum from pydantic import ( field_validator, @@ -54,7 +54,7 @@ # Those AnalyticsTypes that we do not test via contentctl SKIPPED_ANALYTICS_TYPES: set[str] = { - AnalyticsType.Correlation.value + AnalyticsType.Correlation } @@ -104,7 +104,7 @@ def get_conf_stanza_name(self, app:CustomApp)->str: def get_action_dot_correlationsearch_dot_label(self, app:CustomApp, max_stanza_length:int=ES_MAX_STANZA_LENGTH)->str: stanza_name = self.get_conf_stanza_name(app) stanza_name_after_saving_in_es = ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format( - security_domain_value = self.tags.security_domain.value, + security_domain_value = self.tags.security_domain, search_name = stanza_name ) @@ -213,7 +213,7 @@ def adjust_tests_and_groups(self) -> None: # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name # Skip tests for non-production detections - if self.status != DetectionStatus.production.value: + if self.status != DetectionStatus.production: self.skip_all_tests(f"TEST SKIPPED: Detection is non-production ({self.status})") # Skip tests for detecton types like Correlation which are not supported via contentctl @@ -266,7 +266,7 @@ def test_status(self) -> TestResultStatus | None: @computed_field @property def datamodel(self) -> List[DataModel]: - return [dm for dm in DataModel if dm.value in self.search] + return [dm for dm in DataModel if dm in self.search] @@ -311,13 +311,13 @@ def annotations(self) -> dict[str, Union[List[str], int, str]]: def mappings(self) -> dict[str, List[str]]: mappings: dict[str, Any] = {} if len(self.tags.cis20) > 0: - mappings["cis20"] = [tag.value for tag in self.tags.cis20] + mappings["cis20"] = [tag for tag in self.tags.cis20] if len(self.tags.kill_chain_phases) > 0: - mappings['kill_chain_phases'] = [phase.value for phase in self.tags.kill_chain_phases] + mappings['kill_chain_phases'] = [phase for phase in self.tags.kill_chain_phases] if len(self.tags.mitre_attack_id) > 0: mappings['mitre_attack'] = self.tags.mitre_attack_id if len(self.tags.nist) > 0: - mappings['nist'] = [category.value for category in self.tags.nist] + mappings['nist'] = [category for category in self.tags.nist] # No need to sort the dict! It has been constructed in-order. # However, if this logic is changed, then consider reordering or @@ -459,7 +459,7 @@ def metadata(self) -> dict[str, str|float]: # break the `inspect` action. return { 'detection_id': str(self.id), - 'deprecated': '1' if self.status == DetectionStatus.deprecated.value else '0', # type: ignore + 'deprecated': '1' if self.status == DetectionStatus.deprecated else '0', # type: ignore 'detection_version': str(self.version), 'publish_time': datetime.datetime(self.date.year,self.date.month,self.date.day,0,0,0,0,tzinfo=datetime.timezone.utc).timestamp() } @@ -594,7 +594,7 @@ def model_post_init(self, __context: Any) -> None: # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting - if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + if self.type == AnalyticsType.Hunting or self.status != DetectionStatus.production: #No additional check need to happen on the potential drilldowns. pass else: @@ -737,14 +737,14 @@ def only_enabled_if_production_status(cls, v: Any, info: ValidationInfo) -> bool if status != DetectionStatus.production: errors.append( f"status is '{status.name}'. Detections that are enabled by default MUST be " - f"'{DetectionStatus.production.value}'" + f"'{DetectionStatus.production}'" ) if searchType not in [AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]: errors.append( - f"type is '{searchType.value}'. Detections that are enabled by default MUST be one" + f"type is '{searchType}'. Detections that are enabled by default MUST be one" " of the following types: " - f"{[AnalyticsType.Anomaly.value, AnalyticsType.Correlation.value, AnalyticsType.TTP.value]}") + f"{[AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]}") if len(errors) > 0: error_message = "\n - ".join(errors) raise ValueError(f"Detection is 'enabled_by_default: true' however \n - {error_message}") @@ -753,7 +753,7 @@ def only_enabled_if_production_status(cls, v: Any, info: ValidationInfo) -> bool @model_validator(mode="after") def addTags_nist(self): - if self.type == AnalyticsType.TTP.value: + if self.type == AnalyticsType.TTP: self.tags.nist = [NistCategory.DE_CM] else: self.tags.nist = [NistCategory.DE_AE] @@ -884,7 +884,7 @@ def search_rba_fields_exist_validate(self): f"the search: {missing_fields}" ) - if len(error_messages) > 0 and self.status == DetectionStatus.production.value: #type: ignore + if len(error_messages) > 0 and self.status == DetectionStatus.production: msg = ( "Use of fields in rba/messages that do not appear in search:\n\t- " "\n\t- ".join(error_messages) @@ -982,7 +982,7 @@ def tests_validate( info: ValidationInfo ) -> list[UnitTest | IntegrationTest | ManualTest]: # Only production analytics require tests - if info.data.get("status", "") != DetectionStatus.production.value: + if info.data.get("status", "") != DetectionStatus.production: return v # All types EXCEPT Correlation MUST have test(s). Any other type, including newly defined @@ -1095,7 +1095,7 @@ def get_summary( value = getattr(self, field) # Enums and Path objects cannot be serialized directly, so we convert it to a string - if isinstance(value, Enum) or isinstance(value, pathlib.Path): + if isinstance(value, StrEnum) or isinstance(value, pathlib.Path): value = str(value) # Alias any fields as needed @@ -1117,7 +1117,7 @@ def get_summary( # Initialize the dict as a mapping of strings to str/bool result: dict[str, Union[str, bool]] = { "name": test.name, - "test_type": test.test_type.value + "test_type": test.test_type } # If result is not None, get a summary of the test result w/ the requested fields diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index a69fac65..b0fccab8 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -161,10 +161,10 @@ def getDeploymentFromType(typeField: Union[str, None], info: ValidationInfo) -> raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context") type_to_deployment_name_map = { - AnalyticsType.TTP.value: "ESCU Default Configuration TTP", - AnalyticsType.Hunting.value: "ESCU Default Configuration Hunting", - AnalyticsType.Correlation.value: "ESCU Default Configuration Correlation", - AnalyticsType.Anomaly.value: "ESCU Default Configuration Anomaly", + AnalyticsType.TTP: "ESCU Default Configuration TTP", + AnalyticsType.Hunting: "ESCU Default Configuration Hunting", + AnalyticsType.Correlation: "ESCU Default Configuration Correlation", + AnalyticsType.Anomaly: "ESCU Default Configuration Anomaly", "Baseline": "ESCU Default Configuration Baseline" } converted_type_field = type_to_deployment_name_map[typeField] diff --git a/contentctl/objects/base_test.py b/contentctl/objects/base_test.py index 8a377dfc..a47ed574 100644 --- a/contentctl/objects/base_test.py +++ b/contentctl/objects/base_test.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import Union from abc import ABC, abstractmethod @@ -7,7 +7,7 @@ from contentctl.objects.base_test_result import BaseTestResult -class TestType(str, Enum): +class TestType(StrEnum): """ Types of tests """ diff --git a/contentctl/objects/base_test_result.py b/contentctl/objects/base_test_result.py index d29f93cb..6f9ce11a 100644 --- a/contentctl/objects/base_test_result.py +++ b/contentctl/objects/base_test_result.py @@ -1,5 +1,5 @@ from typing import Union, Any -from enum import Enum +from enum import StrEnum from pydantic import ConfigDict, BaseModel from splunklib.data import Record # type: ignore @@ -10,7 +10,7 @@ # TODO (#267): Align test reporting more closely w/ status enums (as it relates to "untested") # TODO (PEX-432): add status "UNSET" so that we can make sure the result is always of this enum # type; remove mypy ignores associated w/ these typing issues once we do -class TestResultStatus(str, Enum): +class TestResultStatus(StrEnum): """Enum for test status (e.g. pass/fail)""" # Test failed (detection did NOT fire appropriately) FAIL = "fail" @@ -113,7 +113,7 @@ def get_summary_dict( # Exceptions and enums cannot be serialized, so convert to str if isinstance(getattr(self, field), Exception): summary_dict[field] = str(getattr(self, field)) - elif isinstance(getattr(self, field), Enum): + elif isinstance(getattr(self, field), StrEnum): summary_dict[field] = str(getattr(self, field)) else: summary_dict[field] = getattr(self, field) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index b1b6aed5..361ddab7 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -434,16 +434,19 @@ def getContainers(self)->List[Container]: class All(BaseModel): #Doesn't need any extra logic + mode_name = "All" pass class Changes(BaseModel): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) + mode_name: str = "Changes" target_branch:str = Field(...,description="The target branch to diff against. Note that this includes uncommitted changes in the working directory as well.") class Selected(BaseModel): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) + mode_name = "Selected" files:List[FilePath] = Field(...,description="List of detection files to test, separated by spaces.") @field_serializer('files',when_used='always') @@ -672,12 +675,12 @@ def serialize_path(paths: List[FilePath])->List[str]: class test_common(build): mode:Union[Changes, Selected, All] = Field(All(), union_mode='left_to_right') post_test_behavior: PostTestBehavior = Field(default=PostTestBehavior.pause_on_failure, description="Controls what to do when a test completes.\n\n" - f"'{PostTestBehavior.always_pause.value}' - the state of " + f"'{PostTestBehavior.always_pause}' - the state of " "the test will always pause after a test, allowing the user to log into the " "server and experiment with the search and data before it is removed.\n\n" - f"'{PostTestBehavior.pause_on_failure.value}' - pause execution ONLY when a test fails. The user may press ENTER in the terminal " + f"'{PostTestBehavior.pause_on_failure}' - pause execution ONLY when a test fails. The user may press ENTER in the terminal " "running the test to move on to the next test.\n\n" - f"'{PostTestBehavior.never_pause.value}' - never stop testing, even if a test fails.\n\n" + f"'{PostTestBehavior.never_pause}' - never stop testing, even if a test fails.\n\n" "***SPECIAL NOTE FOR CI/CD*** 'never_pause' MUST be used for a test to " "run in an unattended manner or in a CI/CD system - otherwise a single failed test " "will result in the testing never finishing as the tool waits for input.") @@ -694,7 +697,7 @@ class test_common(build): " interactive command line workflow that can display progress bars and status information frequently. " "Unfortunately it is incompatible with, or may cause poorly formatted logs, in many CI/CD systems or other unattended environments. " "If you are running contentctl in CI/CD, then please set this argument to True. Note that if you are running in a CI/CD context, " - f"you also MUST set post_test_behavior to {PostTestBehavior.never_pause.value}. Otherwiser, a failed detection will cause" + f"you also MUST set post_test_behavior to {PostTestBehavior.never_pause}. Otherwiser, a failed detection will cause" "the CI/CD running to pause indefinitely.") apps: List[TestApp] = Field(default=DEFAULT_APPS, exclude=False, description="List of apps to install in test environment") @@ -703,7 +706,7 @@ class test_common(build): def dumpCICDPlanAndQuit(self, githash: str, detections:List[Detection]): output_file = self.path / "test_plan.yml" self.mode = Selected(files=sorted([detection.file_path for detection in detections], key=lambda path: str(path))) - self.post_test_behavior = PostTestBehavior.never_pause.value + self.post_test_behavior = PostTestBehavior.never_pause #required so that CI/CD does not get too much output or hang self.disable_tqdm = True @@ -770,12 +773,12 @@ def ensureCommonInformationModel(self)->Self: def suppressTQDM(self)->Self: if self.disable_tqdm: tqdm.tqdm.__init__ = partialmethod(tqdm.tqdm.__init__, disable=True) - if self.post_test_behavior != PostTestBehavior.never_pause.value: + if self.post_test_behavior != PostTestBehavior.never_pause: raise ValueError(f"You have disabled tqdm, presumably because you are " f"running in CI/CD or another unattended context.\n" f"However, post_test_behavior is set to [{self.post_test_behavior}].\n" f"If that is the case, then you MUST set post_test_behavior " - f"to [{PostTestBehavior.never_pause.value}].\n" + f"to [{PostTestBehavior.never_pause}].\n" "Otherwise, if a detection fails in CI/CD, your CI/CD runner will hang forever.") return self @@ -805,15 +808,6 @@ def checkPlanOnlyUse(self)->Self: return self - def getModeName(self)->str: - if isinstance(self.mode, All): - return DetectionTestingMode.all.value - elif isinstance(self.mode, Changes): - return DetectionTestingMode.changes.value - else: - return DetectionTestingMode.selected.value - - class test(test_common): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) container_settings:ContainerSettings = ContainerSettings() diff --git a/contentctl/objects/constants.py b/contentctl/objects/constants.py index c295ec86..8a32f28d 100644 --- a/contentctl/objects/constants.py +++ b/contentctl/objects/constants.py @@ -149,7 +149,7 @@ # errors, if its name is longer than 99 characters. # When an saved search is cloned in Enterprise Security User Interface, # it is wrapped in the following: -# {Detection.tags.security_domain.value} - {SEARCH_STANZA_NAME} - Rule +# {Detection.tags.security_domain} - {SEARCH_STANZA_NAME} - Rule # Similarly, when we generate the search stanza name in contentctl, it # is app.label - detection.name - Rule # However, in product the search name is: diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index c64eed6b..2a8d1e9c 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -2,7 +2,7 @@ import time import json from typing import Any -from enum import Enum +from enum import StrEnum, IntEnum from functools import cached_property from pydantic import ConfigDict, BaseModel, computed_field, Field, PrivateAttr @@ -76,7 +76,7 @@ def get_logger() -> logging.Logger: return logger -class SavedSearchKeys(str, Enum): +class SavedSearchKeys(StrEnum): """ Various keys into the SavedSearch content """ @@ -89,7 +89,7 @@ class SavedSearchKeys(str, Enum): DISBALED_KEY = "disabled" -class Indexes(str, Enum): +class Indexes(StrEnum): """ Indexes we search against """ @@ -98,7 +98,7 @@ class Indexes(str, Enum): NOTABLE_INDEX = "notable" -class TimeoutConfig(int, Enum): +class TimeoutConfig(IntEnum): """ Configuration values for the exponential backoff timer """ @@ -115,7 +115,7 @@ class TimeoutConfig(int, Enum): # TODO (#226): evaluate sane defaults for timeframe for integration testing (e.g. 5y is good # now, but maybe not always...); maybe set latest/earliest to None? -class ScheduleConfig(str, Enum): +class ScheduleConfig(StrEnum): """ Configuraton values for the saved search schedule """ @@ -310,7 +310,7 @@ def earliest_time(self) -> str: The earliest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY.value] + return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY] else: raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") @@ -320,7 +320,7 @@ def latest_time(self) -> str: The latest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY.value] + return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY] else: raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") @@ -330,7 +330,7 @@ def cron_schedule(self) -> str: The cron schedule configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY.value] + return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY] else: raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") @@ -340,7 +340,7 @@ def enabled(self) -> bool: Whether the saved search is enabled """ if self.saved_search is not None: - if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY.value]): + if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY]): return False else: return True @@ -368,7 +368,7 @@ def _get_risk_analysis_action(content: dict[str, Any]) -> RiskAnalysisAction | N :param content: a dict of strings to values :returns: a RiskAnalysisAction, or None if none exists """ - if int(content[SavedSearchKeys.RISK_ACTION_KEY.value]): + if int(content[SavedSearchKeys.RISK_ACTION_KEY]): try: return RiskAnalysisAction.parse_from_dict(content) except ValueError as e: @@ -383,7 +383,7 @@ def _get_notable_action(content: dict[str, Any]) -> NotableAction | None: :returns: a NotableAction, or None if none exists """ # grab notable details if present - if int(content[SavedSearchKeys.NOTABLE_ACTION_KEY.value]): + if int(content[SavedSearchKeys.NOTABLE_ACTION_KEY]): return NotableAction.parse_from_dict(content) return None @@ -463,9 +463,9 @@ def disable(self, refresh: bool = True) -> None: def update_timeframe( self, - earliest_time: str = ScheduleConfig.EARLIEST_TIME.value, - latest_time: str = ScheduleConfig.LATEST_TIME.value, - cron_schedule: str = ScheduleConfig.CRON_SCHEDULE.value, + earliest_time: str = ScheduleConfig.EARLIEST_TIME, + latest_time: str = ScheduleConfig.LATEST_TIME, + cron_schedule: str = ScheduleConfig.CRON_SCHEDULE, refresh: bool = True ) -> None: """Updates the correlation search timeframe to work with test data @@ -481,9 +481,9 @@ def update_timeframe( """ # update the SavedSearch accordingly data = { - SavedSearchKeys.EARLIEST_TIME_KEY.value: earliest_time, - SavedSearchKeys.LATEST_TIME_KEY.value: latest_time, - SavedSearchKeys.CRON_SCHEDULE_KEY.value: cron_schedule + SavedSearchKeys.EARLIEST_TIME_KEY: earliest_time, + SavedSearchKeys.LATEST_TIME_KEY: latest_time, + SavedSearchKeys.CRON_SCHEDULE_KEY: cron_schedule } self.logger.info(data) self.logger.info(f"Updating timeframe for '{self.name}': {data}") @@ -554,7 +554,7 @@ def get_risk_events(self, force_update: bool = False) -> list[RiskEvent]: for result in result_iterator: # sanity check that this result from the iterator is a risk event and not some # other metadata - if result["index"] == Indexes.RISK_INDEX.value: + if result["index"] == Indexes.RISK_INDEX: try: parsed_raw = json.loads(result["_raw"]) event = RiskEvent.parse_obj(parsed_raw) @@ -619,7 +619,7 @@ def get_notable_events(self, force_update: bool = False) -> list[NotableEvent]: for result in result_iterator: # sanity check that this result from the iterator is a notable event and not some # other metadata - if result["index"] == Indexes.NOTABLE_INDEX.value: + if result["index"] == Indexes.NOTABLE_INDEX: try: parsed_raw = json.loads(result["_raw"]) event = NotableEvent.parse_obj(parsed_raw) @@ -746,7 +746,7 @@ def validate_notable_events(self) -> None: # NOTE: it would be more ideal to switch this to a system which gets the handle of the saved search job and polls # it for completion, but that seems more tricky - def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: bool = False) -> IntegrationTestResult: + def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = False) -> IntegrationTestResult: """Execute the integration test Executes an integration test for this CorrelationSearch. First, ensures no matching risk/notables already exist @@ -760,10 +760,10 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: boo """ # max_sleep must be greater than the base value we must wait for the scheduled searchjob to run (jobs run every # 60s) - if max_sleep < TimeoutConfig.BASE_SLEEP.value: + if max_sleep < TimeoutConfig.BASE_SLEEP: raise ClientError( f"max_sleep value of {max_sleep} is less than the base sleep required " - f"({TimeoutConfig.BASE_SLEEP.value})" + f"({TimeoutConfig.BASE_SLEEP})" ) # initialize result as None @@ -774,7 +774,7 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: boo num_tries = 0 # set the initial base sleep time - time_to_sleep = TimeoutConfig.BASE_SLEEP.value + time_to_sleep = TimeoutConfig.BASE_SLEEP try: # first make sure the indexes are currently empty and the detection is starting from a disabled state @@ -999,9 +999,9 @@ def cleanup(self, delete_test_index=False) -> None: if delete_test_index: self.indexes_to_purge.add(self.test_index) # type: ignore if self._risk_events is not None: - self.indexes_to_purge.add(Indexes.RISK_INDEX.value) + self.indexes_to_purge.add(Indexes.RISK_INDEX) if self._notable_events is not None: - self.indexes_to_purge.add(Indexes.NOTABLE_INDEX.value) + self.indexes_to_purge.add(Indexes.NOTABLE_INDEX) # delete the indexes for index in self.indexes_to_purge: diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index d8207225..ce073186 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -148,7 +148,7 @@ def serialize_model(self): # Since this field has no parent, there is no need to call super() serialization function return { "analytic_story": [story.name for story in self.analytic_story], - "asset_type": self.asset_type.value, + "asset_type": self.asset_type, "cis20": self.cis20, "kill_chain_phases": self.kill_chain_phases, "nist": self.nist, diff --git a/contentctl/output/svg_output.py b/contentctl/output/svg_output.py index d454ccb2..2d0c9d56 100644 --- a/contentctl/output/svg_output.py +++ b/contentctl/output/svg_output.py @@ -35,7 +35,7 @@ def writeObjects(self, detections: List[Detection], output_path: pathlib.Path, t total_dict:dict[str,Any] = self.get_badge_dict("Detections", detections, detections) - production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production.value]) + production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production]) #deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated]) #experimental_dict = self.get_badge_dict("Experimental", detections, [detection for detection in detections if detection.status == DetectionStatus.experimental]) diff --git a/contentctl/output/templates/analyticstories_detections.j2 b/contentctl/output/templates/analyticstories_detections.j2 index e97f82a8..d24a1217 100644 --- a/contentctl/output/templates/analyticstories_detections.j2 +++ b/contentctl/output/templates/analyticstories_detections.j2 @@ -5,7 +5,7 @@ {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} [savedsearch://{{ detection.get_conf_stanza_name(app) }}] type = detection -asset_type = {{ detection.tags.asset_type.value }} +asset_type = {{ detection.tags.asset_type }} confidence = medium explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }} {% if detection.how_to_implement is defined %} diff --git a/contentctl/output/templates/savedsearches_detections.j2 b/contentctl/output/templates/savedsearches_detections.j2 index 2a719133..55e99e60 100644 --- a/contentctl/output/templates/savedsearches_detections.j2 +++ b/contentctl/output/templates/savedsearches_detections.j2 @@ -70,8 +70,8 @@ action.notable.param.nes_fields = {{ detection.nes_fields }} {% endif %} action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}} action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%} -action.notable.param.security_domain = {{ detection.tags.security_domain.value }} -action.notable.param.severity = {{ detection.tags.severity.value }} +action.notable.param.security_domain = {{ detection.tags.security_domain }} +action.notable.param.severity = {{ detection.tags.severity }} {% endif %} {% if detection.deployment.alert_action.email %} action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }} diff --git a/enums_test.py b/enums_test.py new file mode 100644 index 00000000..fea6a545 --- /dev/null +++ b/enums_test.py @@ -0,0 +1,46 @@ +from pydantic import BaseModel, ConfigDict +from enum import StrEnum, IntEnum + +class SomeString(StrEnum): + one = "one" + two = "TWO" + +class SomeInt(IntEnum): + one = 1 + two = 2 + +class WithUseEnum(BaseModel): + ConfigDict(use_enum_values=True) + strval: SomeString + intval: SomeInt + + +class WithOutUseEnum(BaseModel): + strval: SomeString + intval: SomeInt + +withObj = WithUseEnum.model_validate({"strval": "one", "intval": "2"}) +withoutObj = WithOutUseEnum.model_validate({"strval": "one", "intval": "2"}) + + +print("With tests") +print(withObj.strval) +print(withObj.strval.upper()) +print(withObj.strval.value) +print(withObj.intval) +print(withObj.intval.value) +print(withObj.strval == SomeString.one) +print(withObj.strval == "ONE") +print(withObj.intval == SomeInt.two) +print(withObj.intval == 2) + + +print("Without tests") +print(withoutObj.strval) +print(withoutObj.strval.value) +print(withoutObj.intval) +print(withoutObj.intval.value) +print(withoutObj.strval == SomeString.one) +print(withoutObj.strval == "ONE") +print(withoutObj.intval == SomeInt.two) +print(withoutObj.intval == 2) \ No newline at end of file From eeaeb4d052cd36671d6d71ba94fbf28099846c46 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 4 Dec 2024 14:11:31 -0800 Subject: [PATCH 080/115] fix missing typing of mode_name --- contentctl/objects/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 361ddab7..93e33de9 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -434,7 +434,7 @@ def getContainers(self)->List[Container]: class All(BaseModel): #Doesn't need any extra logic - mode_name = "All" + mode_name:str = "All" pass @@ -446,7 +446,7 @@ class Changes(BaseModel): class Selected(BaseModel): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) - mode_name = "Selected" + mode_name:str = "Selected" files:List[FilePath] = Field(...,description="List of detection files to test, separated by spaces.") @field_serializer('files',when_used='always') From b794d15495dbacbe638e25bea6928ff83358d159 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 5 Dec 2024 16:31:00 -0800 Subject: [PATCH 081/115] remove files that are no longer used anymore. Add logic to serialize StrEnum and IntEnum when writing YMLs from objects that contain them. --- contentctl/actions/new_content.py | 1 - contentctl/output/detection_writer.py | 28 --------- contentctl/output/new_content_yml_output.py | 56 ------------------ contentctl/output/yml_output.py | 65 --------------------- contentctl/output/yml_writer.py | 15 +++++ enums_test.py | 46 --------------- 6 files changed, 15 insertions(+), 196 deletions(-) delete mode 100644 contentctl/output/detection_writer.py delete mode 100644 contentctl/output/new_content_yml_output.py delete mode 100644 contentctl/output/yml_output.py delete mode 100644 enums_test.py diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 0d3ffc4a..205ef787 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -2,7 +2,6 @@ import questionary from typing import Any from contentctl.input.new_content_questions import NewContentQuestions -from contentctl.output.new_content_yml_output import NewContentYmlOutput from contentctl.objects.config import new, NewContentType import uuid from datetime import datetime diff --git a/contentctl/output/detection_writer.py b/contentctl/output/detection_writer.py deleted file mode 100644 index 2f439ca9..00000000 --- a/contentctl/output/detection_writer.py +++ /dev/null @@ -1,28 +0,0 @@ - -import yaml - - -class DetectionWriter: - - @staticmethod - def writeYmlFile(file_path : str, obj : dict) -> None: - - new_obj = dict() - new_obj["name"] = obj["name"] - new_obj["id"] = obj["id"] - new_obj["version"] = obj["version"] - new_obj["date"] = obj["date"] - new_obj["author"] = obj["author"] - new_obj["type"] = obj["type"] - new_obj["status"] = obj["status"] - new_obj["description"] = obj["description"] - new_obj["data_source"] = obj["data_source"] - new_obj["search"] = obj["search"] - new_obj["how_to_implement"] = obj["how_to_implement"] - new_obj["known_false_positives"] = obj["known_false_positives"] - new_obj["references"] = obj["references"] - new_obj["tags"] = obj["tags"] - new_obj["tests"] = obj["tests"] - - with open(file_path, 'w') as outfile: - yaml.safe_dump(new_obj, outfile, default_flow_style=False, sort_keys=False) \ No newline at end of file diff --git a/contentctl/output/new_content_yml_output.py b/contentctl/output/new_content_yml_output.py deleted file mode 100644 index 38730b37..00000000 --- a/contentctl/output/new_content_yml_output.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import pathlib -from contentctl.objects.enums import SecurityContentType -from contentctl.output.yml_writer import YmlWriter -import pathlib -from contentctl.objects.config import NewContentType -class NewContentYmlOutput(): - output_path: pathlib.Path - - def __init__(self, output_path:pathlib.Path): - self.output_path = output_path - - - def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewContentType) -> None: - if type == NewContentType.detection: - - file_path = os.path.join(self.output_path, 'detections', subdirectory_name, self.convertNameToFileName(object['name'], object['tags']['product'])) - output_folder = pathlib.Path(self.output_path)/'detections'/subdirectory_name - #make sure the output folder exists for this detection - output_folder.mkdir(exist_ok=True) - - YmlWriter.writeYmlFile(file_path, object) - print("Successfully created detection " + file_path) - - elif type == NewContentType.story: - file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product'])) - YmlWriter.writeYmlFile(file_path, object) - print("Successfully created story " + file_path) - - else: - raise(Exception(f"Object Must be Story or Detection, but is not: {object}")) - - - - def convertNameToFileName(self, name: str, product: list): - file_name = name \ - .replace(' ', '_') \ - .replace('-','_') \ - .replace('.','_') \ - .replace('/','_') \ - .lower() - - file_name = file_name + '.yml' - return file_name - - - def convertNameToTestFileName(self, name: str, product: list): - file_name = name \ - .replace(' ', '_') \ - .replace('-','_') \ - .replace('.','_') \ - .replace('/','_') \ - .lower() - - file_name = file_name + '.test.yml' - return file_name \ No newline at end of file diff --git a/contentctl/output/yml_output.py b/contentctl/output/yml_output.py deleted file mode 100644 index aca446bf..00000000 --- a/contentctl/output/yml_output.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -from contentctl.output.detection_writer import DetectionWriter -from contentctl.objects.detection import Detection - - -class YmlOutput(): - - - def writeDetections(self, objects: list, output_path : str) -> None: - for obj in objects: - file_path = obj.file_path - obj.id = str(obj.id) - - DetectionWriter.writeYmlFile(os.path.join(output_path, file_path), obj.dict( - exclude_none=True, - include = - { - "name": True, - "id": True, - "version": True, - "date": True, - "author": True, - "type": True, - "status": True, - "description": True, - "data_source": True, - "search": True, - "how_to_implement": True, - "known_false_positives": True, - "references": True, - "rba": True - "tags": - { - "analytic_story": True, - "asset_type": True, - "atomic_guid": True, - "confidence": True, - "impact": True, - "drilldown_search": True, - "mappings": True, - "message": True, - "mitre_attack_id": True, - "kill_chain_phases:": True, - "product": True, - "risk_score": True, - "security_domain": True - }, - "tests": - { - '__all__': - { - "name": True, - "attack_data": { - '__all__': - { - "data": True, - "source": True, - "sourcetype": True - } - } - } - } - } - )) \ No newline at end of file diff --git a/contentctl/output/yml_writer.py b/contentctl/output/yml_writer.py index 7d71762b..2e408c83 100644 --- a/contentctl/output/yml_writer.py +++ b/contentctl/output/yml_writer.py @@ -1,6 +1,21 @@ import yaml from typing import Any +from enum import StrEnum, IntEnum + +# Set the following so that we can write StrEnum and IntEnum +# to files. Otherwise, we will get the following errors when trying +# to write to files: +# yaml.representer.RepresenterError: ('cannot represent an object',..... +yaml.SafeDumper.add_multi_representer( + StrEnum, + yaml.representer.SafeRepresenter.represent_str +) + +yaml.SafeDumper.add_multi_representer( + IntEnum, + yaml.representer.SafeRepresenter.represent_int +) class YmlWriter: diff --git a/enums_test.py b/enums_test.py deleted file mode 100644 index fea6a545..00000000 --- a/enums_test.py +++ /dev/null @@ -1,46 +0,0 @@ -from pydantic import BaseModel, ConfigDict -from enum import StrEnum, IntEnum - -class SomeString(StrEnum): - one = "one" - two = "TWO" - -class SomeInt(IntEnum): - one = 1 - two = 2 - -class WithUseEnum(BaseModel): - ConfigDict(use_enum_values=True) - strval: SomeString - intval: SomeInt - - -class WithOutUseEnum(BaseModel): - strval: SomeString - intval: SomeInt - -withObj = WithUseEnum.model_validate({"strval": "one", "intval": "2"}) -withoutObj = WithOutUseEnum.model_validate({"strval": "one", "intval": "2"}) - - -print("With tests") -print(withObj.strval) -print(withObj.strval.upper()) -print(withObj.strval.value) -print(withObj.intval) -print(withObj.intval.value) -print(withObj.strval == SomeString.one) -print(withObj.strval == "ONE") -print(withObj.intval == SomeInt.two) -print(withObj.intval == 2) - - -print("Without tests") -print(withoutObj.strval) -print(withoutObj.strval.value) -print(withoutObj.intval) -print(withoutObj.intval.value) -print(withoutObj.strval == SomeString.one) -print(withoutObj.strval == "ONE") -print(withoutObj.intval == SomeInt.two) -print(withoutObj.intval == 2) \ No newline at end of file From 334062cf71453269137559fd03aa10ed0640eda8 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 5 Dec 2024 16:35:07 -0800 Subject: [PATCH 082/115] Remove dead code from new_content.py --- contentctl/actions/new_content.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 205ef787..3d5fa5b6 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -148,20 +148,3 @@ def execute(self, input_dto: new) -> None: full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name')) YmlWriter.writeYmlFile(str(full_output_path), content_dict) - def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewContentType) -> None: - if type == NewContentType.detection: - file_path = os.path.join(self.output_path, 'detections', subdirectory_name, self.convertNameToFileName(object['name'], object['tags']['product'])) - output_folder = pathlib.Path(self.output_path)/'detections'/subdirectory_name - # make sure the output folder exists for this detection - output_folder.mkdir(exist_ok=True) - - YmlWriter.writeDetection(file_path, object) - print("Successfully created detection " + file_path) - - elif type == NewContentType.story: - file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product'])) - YmlWriter.writeStory(file_path, object) - print("Successfully created story " + file_path) - - else: - raise(Exception(f"Object Must be Story or Detection, but is not: {object}")) From 4bc5e68a34ba3c9fa2de227e28cfa06d2de61446 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 10 Dec 2024 16:36:46 -0800 Subject: [PATCH 083/115] remove the 'forbid' from a few classes that restated it. but this is not necessary if they inherit from SecurityContentObject --- contentctl/objects/data_source.py | 1 - contentctl/objects/deployment.py | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index ed8a8f86..2ed9c80c 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -9,7 +9,6 @@ class TA(BaseModel): url: HttpUrl | None = None version: str class DataSource(SecurityContentObject): - model_config = ConfigDict(extra="forbid") source: str = Field(...) sourcetype: str = Field(...) separator: Optional[str] = None diff --git a/contentctl/objects/deployment.py b/contentctl/objects/deployment.py index 8fd264b6..6e2cc6d2 100644 --- a/contentctl/objects/deployment.py +++ b/contentctl/objects/deployment.py @@ -10,15 +10,7 @@ from contentctl.objects.enums import DeploymentType -class Deployment(SecurityContentObject): - model_config = ConfigDict(extra="forbid") - #id: str = None - #date: str = None - #author: str = None - #description: str = None - #contentType: SecurityContentType = SecurityContentType.deployments - - +class Deployment(SecurityContentObject): scheduling: DeploymentScheduling = Field(...) alert_action: AlertAction = AlertAction() type: DeploymentType = Field(...) From 84715bf720413c393884f04c8bd2c0341b367a73 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 10 Dec 2024 16:42:17 -0800 Subject: [PATCH 084/115] Clean up two more use of .value on StrEnums --- contentctl/objects/baseline.py | 2 +- contentctl/objects/investigation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index a41acbb4..f59c7dce 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -36,7 +36,7 @@ def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment: @computed_field @property def datamodel(self) -> List[DataModel]: - return [dm for dm in DataModel if dm.value in self.search] + return [dm for dm in DataModel if dm in self.search] @model_serializer def serialize_model(self): diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 433105ce..0d35a9db 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -37,7 +37,7 @@ def inputs(self)->List[str]: @computed_field @property def datamodel(self) -> List[DataModel]: - return [dm for dm in DataModel if dm.value in self.search] + return [dm for dm in DataModel if dm in self.search] @computed_field @property From 8cc34517d0abcce901b79669d06a78c77fcfce96 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Dec 2024 12:27:38 -0600 Subject: [PATCH 085/115] Add GH Actions to Dependabot --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 650a5e95..b1bdacd5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,8 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 6 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" \ No newline at end of file From b4848be669ae9b1aee8e3ed200197631a64f6482 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 12 Dec 2024 12:28:19 -0600 Subject: [PATCH 086/115] Reduce matrix for simplicity --- .github/workflows/testEndToEnd.yml | 2 +- .github/workflows/test_against_escu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testEndToEnd.yml b/.github/workflows/testEndToEnd.yml index 444ee96a..6b1a4b20 100644 --- a/.github/workflows/testEndToEnd.yml +++ b/.github/workflows/testEndToEnd.yml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-20.04", "ubuntu-24.04", "macos-15", "macos-14", "windows-2022"] + operating_system: ["ubuntu-24.04", "macos-15", "windows-2022"] #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"] diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index 424eebcc..f29e6a6f 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -18,7 +18,7 @@ jobs: matrix: python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-20.04", "ubuntu-24.04", "macos-15", "macos-14"] + operating_system: ["ubuntu-24.04", "macos-15"] # Do not test against ESCU until known character encoding issue is resolved # operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"] From d71674f1ddec5603def74725768f59517709fb68 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 19 Dec 2024 08:23:26 -0600 Subject: [PATCH 087/115] version bump --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4ac592eb..61eb7b55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.2 + rev: v0.8.4 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index a3fd626a..7c4df1bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.8.2" +ruff = "^0.8.4" [build-system] requires = ["poetry-core>=1.0.0"] From cc51953d3ea3bc3a6d8b9f78546f8cb41894969e Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 23 Dec 2024 11:12:54 -0800 Subject: [PATCH 088/115] More cleanup with different classes --- contentctl/objects/lookup.py | 226 ++++++++++++++++++++--------------- 1 file changed, 130 insertions(+), 96 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 5deae8e8..fd0c8f84 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -1,10 +1,13 @@ from __future__ import annotations -from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field -from typing import TYPE_CHECKING, Optional, Any, Union, Literal + +from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt, computed_field +from enum import StrEnum, auto +from typing import TYPE_CHECKING, Optional, Any, Union, Literal, Annotated, Self import re import csv -from enum import StrEnum import abc +from functools import cached_property +import pathlib if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto from contentctl.objects.config import validate @@ -26,18 +29,25 @@ LOOKUPS_TO_IGNORE.add("other_lookups") -# TODO (#220): Split Lookup into 2 classes -class Lookup(SecurityContentObject, abc.ABC): - #collection will always be the name of the lookup +class Lookup_Type(StrEnum): + csv = auto() + kvstore = auto() + mlmodel = auto() + + - fields_list: Optional[str] = None - filename: Optional[FilePath] = None +# TODO (#220): Split Lookup into 2 classes +class Lookup(SecurityContentObject, abc.ABC): default_match: Optional[bool] = None - match_type: Optional[str] = None - min_matches: Optional[int] = None - case_sensitive_match: Optional[bool] = None + # Per the documentation for transforms.conf, EXACT should not be specified in this list, + # so we include only WILDCARD and CIDR + match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field(default=[]) + min_matches: None | NonNegativeInt = Field(default=None) + max_matches: None | Annotated[NonNegativeInt, Field(ge=1, le=1000)] = Field(default=None) + case_sensitive_match: None | bool = Field(default=None) + @model_serializer def serialize_model(self): #Call parent serializer @@ -45,13 +55,11 @@ def serialize_model(self): #All fields custom to this model model= { - "filename": self.filename.name if self.filename is not None else None, + "default_match": "true" if self.default_match is True else "false", "match_type": self.match_type, "min_matches": self.min_matches, "case_sensitive_match": "true" if self.case_sensitive_match is True else "false", - "collection": self.collection, - "fields_list": self.fields_list } #return the model @@ -69,31 +77,102 @@ def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any: return data - def model_post_init(self, ctx:dict[str,Any]): - if not self.filename: - return - import pathlib - filenamePath = pathlib.Path(self.filename) + + - if filenamePath.suffix not in [".csv", ".mlmodel"]: - raise ValueError(f"All Lookup files must be CSV files and end in .csv. The following file does not: '{filenamePath}'") + @staticmethod + def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: + inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s]+)', text_field)) + outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) + # Don't match inputlookup or outputlookup. Allow local=true or update=true or local=t or update=t + lookups_to_get = set(re.findall(r'(?:(?Self: + if not self.filename.exists(): + raise ValueError(f"Expected lookup filename {self.filename} does not exist") + return self + + @computed_field + @cached_property + def filename(self)->FilePath: + if self.file_path is None: + raise ValueError("Cannot get the filename of the lookup CSV because the YML file_path attribute is None") + + csv_file = self.file_path.parent / f"{self.file_path.stem}.csv" + return csv_file + + @computed_field + @cached_property + def app_filename(self)->FilePath: + ''' + We may consider two options: + 1. Always apply the datetime stamp to the end of the file. This makes the code easier + 2. Only apply the datetime stamp if it is version > 1. This makes the code a small fraction + more complicated, but preserves longstanding CSV that have not been modified in a long time + ''' + return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.csv") + + + + @model_serializer + def serialize_model(self): + #Call parent serializer + super_fields = super().serialize_model() + + #All fields custom to this model + model= { + "filename": self.filename.name + } + + #return the model + model.update(super_fields) + return model + + @model_validator(mode="after") + def ensure_correct_csv_structure(self)->Self: + + + if self.filename.suffix != ".csv": + raise ValueError(f"All Lookup files must be CSV files and end in .csv. The following file does not: '{self.filename}'") + + # https://docs.python.org/3/library/csv.html#csv.DictReader # Column Names (fieldnames) determine by the number of columns in the first row. # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception. csv_errors:list[str] = [] - with open(filenamePath, "r") as csv_fp: + with open(self.filename, "r") as csv_fp: RESTKEY = "extra_fields_in_a_row" csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY) if csv_dict.fieldnames is None: - raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t" + raise ValueError(f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t" "Unable to read fieldnames from CSV. Is the CSV empty?\n" " Please try opening the file with a CSV Editor to ensure that it is correct.") # Remember that row 1 has the headers and we do not iterate over it in the loop below @@ -110,88 +189,43 @@ def model_post_init(self, ctx:dict[str,Any]): f"but instead had [{column_index}].") if len(csv_errors) > 0: err_string = '\n\t'.join(csv_errors) - raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t{err_string}\n" + raise ValueError(f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t{err_string}\n" f" Please try opening the file with a CSV Editor to ensure that it is correct.") - return - - - @field_validator('match_type') - @classmethod - def match_type_valid(cls, v: Union[str,None], info: ValidationInfo): - if not v: - #Match type can be None and that's okay - return v + return self - if not (v.startswith("WILDCARD(") or v.endswith(")")) : - raise ValueError(f"All match_types must take the format 'WILDCARD(field_name)'. The following file does not: '{v}'") - return v - #Ensure that exactly one of location or filename are defined - @model_validator(mode='after') - def ensure_mutually_exclusive_fields(self)->Lookup: - if self.filename is not None and self.collection is not None: - raise ValueError("filename and collection cannot be defined in the lookup file. Exactly one must be defined.") - elif self.filename is None and self.collection is None: - raise ValueError("Neither filename nor collection were defined in the lookup file. Exactly one must " - "be defined.") +class KVStoreLookup(Lookup): + lookup_type: Literal[Lookup_Type.kvstore] + collection: str = Field(description="Name of the KVStore Collection. Note that collection MUST equal the name.") + fields: list[str] = Field(description="The names of the fields/headings for the KVStore.", min_length=1) + @model_validator(mode="after") + def validate_collection(self)->Self: + if self.collection != self.name: + raise ValueError("Collection MUST be the same as Name of the lookup, but they do not match") return self - - - @staticmethod - def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: - inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s]+)', text_field)) - outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) - # Don't match inputlookup or outputlookup. Allow local=true or update=true or local=t or update=t - lookups_to_get = set(re.findall(r'(?:(? Date: Mon, 23 Dec 2024 12:33:20 -0800 Subject: [PATCH 089/115] initial working cleanup of lookups code --- contentctl/input/director.py | 5 +++-- contentctl/objects/lookup.py | 36 +++++++++++++++--------------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/contentctl/input/director.py b/contentctl/input/director.py index e18dc596..8585f001 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -14,7 +14,7 @@ from contentctl.objects.playbook import Playbook from contentctl.objects.deployment import Deployment from contentctl.objects.macro import Macro -from contentctl.objects.lookup import Lookup +from contentctl.objects.lookup import LookupAdapter, Lookup from contentctl.objects.atomic import AtomicEnrichment from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.data_source import DataSource @@ -157,7 +157,8 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None: modelDict = YmlReader.load_file(file) if contentType == SecurityContentType.lookups: - lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto}) + lookup = LookupAdapter.validate_python(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto}) + #lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto}) self.output_dto.addContentToDictMappings(lookup) elif contentType == SecurityContentType.macros: diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index fddacef7..b07b3bb2 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -1,6 +1,6 @@ from __future__ import annotations -from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt, computed_field +from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt, computed_field, TypeAdapter from enum import StrEnum, auto from typing import TYPE_CHECKING, Optional, Any, Union, Literal, Annotated, Self import re @@ -45,6 +45,7 @@ class Lookup(SecurityContentObject, abc.ABC): min_matches: None | NonNegativeInt = Field(default=None) max_matches: None | Annotated[NonNegativeInt, Field(ge=1, le=1000)] = Field(default=None) case_sensitive_match: None | bool = Field(default=None) + @@ -108,11 +109,10 @@ def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[ - - -class CSVLookup(Lookup): - lookup_type: Literal[Lookup_Type.csv] - +class FileBackedLookup(Lookup, abc.ABC): + # For purposes of the disciminated union, the child classes which + # inherit from this class must declare the typing of lookup_type + # themselves, hence it is not defined in the Lookup class @model_validator(mode="after") def ensure_lookup_file_exists(self)->Self: @@ -124,9 +124,9 @@ def ensure_lookup_file_exists(self)->Self: @cached_property def filename(self)->FilePath: if self.file_path is None: - raise ValueError("Cannot get the filename of the lookup CSV because the YML file_path attribute is None") + raise ValueError(f"Cannot get the filename of the lookup {self.lookup_type} because the YML file_path attribute is None") #type: ignore - csv_file = self.file_path.parent / f"{self.file_path.stem}.csv" + csv_file = self.file_path.parent / f"{self.file_path.stem}.{self.lookup_type}" #type: ignore return csv_file @computed_field @@ -138,10 +138,11 @@ def app_filename(self)->FilePath: 2. Only apply the datetime stamp if it is version > 1. This makes the code a small fraction more complicated, but preserves longstanding CSV that have not been modified in a long time ''' - return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.csv") - - + return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore +class CSVLookup(FileBackedLookup): + lookup_type:Literal[Lookup_Type.csv] + @model_serializer def serialize_model(self): #Call parent serializer @@ -158,13 +159,6 @@ def serialize_model(self): @model_validator(mode="after") def ensure_correct_csv_structure(self)->Self: - - - if self.filename.suffix != ".csv": - raise ValueError(f"All Lookup files must be CSV files and end in .csv. The following file does not: '{self.filename}'") - - - # https://docs.python.org/3/library/csv.html#csv.DictReader # Column Names (fieldnames) determine by the number of columns in the first row. # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception @@ -200,7 +194,7 @@ def ensure_correct_csv_structure(self)->Self: class KVStoreLookup(Lookup): lookup_type: Literal[Lookup_Type.kvstore] - collection: str = Field(description="Name of the KVStore Collection. Note that collection MUST equal the name.") + collection: str = Field(description="Name of the KVStore Collection. Note that collection MUST equal the name. This is a duplicate field, so it will be removed eventually.") fields: list[str] = Field(description="The names of the fields/headings for the KVStore.", min_length=1) @@ -225,9 +219,9 @@ def serialize_model(self): model.update(super_fields) return model -class MlModel(Lookup): +class MlModel(FileBackedLookup): lookup_type: Literal[Lookup_Type.mlmodel] - +LookupAdapter = TypeAdapter(Annotated[CSVLookup | KVStoreLookup | MlModel, Field(discriminator="lookup_type")]) From e7eb947f69bb7fd9d663da84ffdf18df6c769aaa Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 23 Dec 2024 14:52:00 -0800 Subject: [PATCH 090/115] include inputlookup and outputlookup --- contentctl/objects/lookup.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index b07b3bb2..d1394438 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -85,26 +85,19 @@ def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any: @staticmethod def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: + # Comprehensively match all kinds of lookups, including inputlookup and outputlookup inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s]+)', text_field)) outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) - # Don't match inputlookup or outputlookup. Allow local=true or update=true or local=t or update=t - lookups_to_get = set(re.findall(r'(?:(? Date: Thu, 2 Jan 2025 13:58:02 -0600 Subject: [PATCH 091/115] Version bump --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61eb7b55..994a349c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.8.5 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index 7c4df1bc..b8331444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.8.4" +ruff = "^0.8.5" [build-system] requires = ["poetry-core>=1.0.0"] From deefd575584cd9d067c4d2024bc022ea5a2557ea Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 3 Jan 2025 15:09:22 -0800 Subject: [PATCH 092/115] more cleanup on lookup object. remove confidence, impact, and risk_score from the tags field in prep for integration with the RBA changes PR --- contentctl/objects/detection_tags.py | 9 --------- contentctl/objects/lookup.py | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index a5901d2f..7e4388c9 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -41,12 +41,6 @@ class DetectionTags(BaseModel): analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) group: list[str] = [] - confidence: NonNegativeInt = Field(...,le=100) - impact: NonNegativeInt = Field(...,le=100) - @computed_field - @property - def risk_score(self) -> int: - return round((self.confidence * self.impact)/100) @computed_field @property @@ -80,9 +74,6 @@ def severity(self)->RiskSeverity: # enrichment mitre_attack_enrichments: List[MitreAttackEnrichment] = Field([], validate_default=True) - confidence_id: Optional[PositiveInt] = Field(None, ge=1, le=3) - impact_id: Optional[PositiveInt] = Field(None, ge=1, le=5) - evidence_str: Optional[str] = None @computed_field @property diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index d1394438..245b926a 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -131,7 +131,10 @@ def app_filename(self)->FilePath: 2. Only apply the datetime stamp if it is version > 1. This makes the code a small fraction more complicated, but preserves longstanding CSV that have not been modified in a long time ''' - return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore + if self.version > 1: + return pathlib.Path(f"{self.filename.stem}.{self.lookup_type}") #type: ignore + else: + return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore class CSVLookup(FileBackedLookup): lookup_type:Literal[Lookup_Type.csv] @@ -187,15 +190,14 @@ def ensure_correct_csv_structure(self)->Self: class KVStoreLookup(Lookup): lookup_type: Literal[Lookup_Type.kvstore] - collection: str = Field(description="Name of the KVStore Collection. Note that collection MUST equal the name. This is a duplicate field, so it will be removed eventually.") fields: list[str] = Field(description="The names of the fields/headings for the KVStore.", min_length=1) - - @model_validator(mode="after") - def validate_collection(self)->Self: - if self.collection != self.name: - raise ValueError("Collection MUST be the same as Name of the lookup, but they do not match") - return self + @field_validator("fields", mode='after') + @classmethod + def ensure_key(cls, values: list[str]): + if values[0] != "_key": + raise ValueError(f"fields MUST begin with '_key', not '{values[0]}'") + return values @model_serializer def serialize_model(self): From 6f60e75d88e432f7a20c859ddb36bbfc31a97811 Mon Sep 17 00:00:00 2001 From: Lou Stella Date: Sun, 5 Jan 2025 16:12:24 -0600 Subject: [PATCH 093/115] version update --- .pre-commit-config.yaml | 2 +- pyproject.toml | 54 ++++++++++++++++++++--------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 994a349c..77eb04cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.5 + rev: v0.8.6 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index b8331444..d59fb265 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.8.5" +ruff = "^0.8.6" [build-system] requires = ["poetry-core>=1.0.0"] @@ -42,32 +42,32 @@ build-backend = "poetry.core.masonry.api" [tool.ruff] # Exclude a variety of commonly ignored directories. exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".ipynb_checkpoints", - ".mypy_cache", - ".nox", - ".pants.d", - ".pyenv", - ".pytest_cache", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - ".vscode", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "site-packages", - "venv", + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", ] # Same as Black. From 24b003c366f425f3bd4af83097a8aac55ff77301 Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 6 Jan 2025 15:14:54 -0600 Subject: [PATCH 094/115] Update CI to temporarily test against #3269 on security_content --- .github/workflows/test_against_escu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index f29e6a6f..b3f43434 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -35,7 +35,7 @@ jobs: with: path: security_content repository: splunk/security_content - ref: rba_migration + ref: strict_yml_from_rba #Install the given version of Python we will test against - name: Install Required Python Version From 41fab0f5f6a71f6d6d907b04592a691c942ce6a5 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 6 Jan 2025 13:14:58 -0800 Subject: [PATCH 095/115] Fix regex to step matching before hitting | character which may occur WITHOUT whitespace being present. Ensure that any CSV or MLMODEL file written to the app ends in the appropriate datetime stamp. --- contentctl/objects/lookup.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 245b926a..5fcb18db 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -86,9 +86,9 @@ def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any: @staticmethod def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: # Comprehensively match all kinds of lookups, including inputlookup and outputlookup - inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s]+)', text_field)) - outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s]+)',text_field)) - lookupsToGet = set(re.findall(r'(?:(?FilePath: 2. Only apply the datetime stamp if it is version > 1. This makes the code a small fraction more complicated, but preserves longstanding CSV that have not been modified in a long time ''' - if self.version > 1: - return pathlib.Path(f"{self.filename.stem}.{self.lookup_type}") #type: ignore - else: - return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore + return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore class CSVLookup(FileBackedLookup): lookup_type:Literal[Lookup_Type.csv] From f04d92c091665fece1dc2584b43f5cc74dc6cd46 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 6 Jan 2025 17:07:22 -0800 Subject: [PATCH 096/115] Progress and cleanup for outputs with better structure and type annotations in prep for new lookup type --- contentctl/actions/build.py | 36 +-- contentctl/objects/lookup.py | 2 +- contentctl/output/api_json_output.py | 449 ++++++++++++++------------- contentctl/output/conf_output.py | 83 +++-- contentctl/output/conf_writer.py | 10 +- contentctl/output/json_writer.py | 6 +- 6 files changed, 310 insertions(+), 276 deletions(-) diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index 43daf360..d8aef19e 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -14,7 +14,7 @@ import pathlib import json import datetime -from typing import Union + from contentctl.objects.config import build @@ -44,13 +44,13 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto: name="data_sources")) updated_conf_files.update(conf_output.writeHeaders()) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections)) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.stories, SecurityContentType.stories)) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.baselines, SecurityContentType.baselines)) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.investigations, SecurityContentType.investigations)) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups)) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros)) - updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards)) + updated_conf_files.update(conf_output.writeDetections(input_dto.director_output_dto.detections)) + updated_conf_files.update(conf_output.writeStories(input_dto.director_output_dto.stories)) + updated_conf_files.update(conf_output.writeBaselines(input_dto.director_output_dto.baselines)) + updated_conf_files.update(conf_output.writeInvestigations(input_dto.director_output_dto.investigations)) + updated_conf_files.update(conf_output.writeLookups(input_dto.director_output_dto.lookups)) + updated_conf_files.update(conf_output.writeMacros(input_dto.director_output_dto.macros)) + updated_conf_files.update(conf_output.writeDashboards(input_dto.director_output_dto.dashboards)) updated_conf_files.update(conf_output.writeMiscellaneousAppFiles()) @@ -67,17 +67,15 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto: if input_dto.config.build_api: shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True) input_dto.config.getAPIPath().mkdir(parents=True) - api_json_output = ApiJsonOutput() - for output_objects, output_type in [(input_dto.director_output_dto.detections, SecurityContentType.detections), - (input_dto.director_output_dto.stories, SecurityContentType.stories), - (input_dto.director_output_dto.baselines, SecurityContentType.baselines), - (input_dto.director_output_dto.investigations, SecurityContentType.investigations), - (input_dto.director_output_dto.lookups, SecurityContentType.lookups), - (input_dto.director_output_dto.macros, SecurityContentType.macros), - (input_dto.director_output_dto.deployments, SecurityContentType.deployments)]: - api_json_output.writeObjects(output_objects, input_dto.config.getAPIPath(), input_dto.config.app.label, output_type ) - - + api_json_output = ApiJsonOutput(input_dto.config.getAPIPath(), input_dto.config.app.label) + api_json_output.writeDetections(input_dto.director_output_dto.detections) + api_json_output.writeStories(input_dto.director_output_dto.stories) + api_json_output.writeBaselines(input_dto.director_output_dto.baselines) + api_json_output.writeInvestigations(input_dto.director_output_dto.investigations) + api_json_output.writeLookups(input_dto.director_output_dto.lookups) + api_json_output.writeMacros(input_dto.director_output_dto.macros) + api_json_output.writeDeployments(input_dto.director_output_dto.deployments) + #create version file for sse api version_file = input_dto.config.getAPIPath()/"version.json" diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 5fcb18db..cfa11a0d 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -143,7 +143,7 @@ def serialize_model(self): #All fields custom to this model model= { - "filename": self.filename.name + "filename": self.app_filename.name } #return the model diff --git a/contentctl/output/api_json_output.py b/contentctl/output/api_json_output.py index d81b8162..a62e3d3c 100644 --- a/contentctl/output/api_json_output.py +++ b/contentctl/output/api_json_output.py @@ -1,3 +1,15 @@ +from __future__ import annotations +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from contentctl.objects.detection import Detection + from contentctl.objects.lookup import Lookup, FileBackedLookup + from contentctl.objects.macro import Macro + from contentctl.objects.dashboard import Dashboard + from contentctl.objects.story import Story + from contentctl.objects.baseline import Baseline + from contentctl.objects.investigation import Investigation + from contentctl.objects.deployment import Deployment + import os import json import pathlib @@ -11,236 +23,243 @@ class ApiJsonOutput: + output_path: pathlib.Path + app_label: str + + def __init__(self, output_path:pathlib.Path, app_label: str): + self.output_path = output_path + self.app_label = app_label - def writeObjects( + def writeDetections( self, - objects: list[SecurityContentObject_Abstract], - output_path: pathlib.Path, - app_label:str = "ESCU", - contentType: SecurityContentType = None + objects: list[Detection], ) -> None: - """#Serialize all objects - try: - for obj in objects: - - serialized_objects.append(obj.model_dump()) - except Exception as e: - raise Exception(f"Error serializing object with name '{obj.name}' and type '{type(obj).__name__}': '{str(e)}'") - """ - - if contentType == SecurityContentType.detections: - detections = [ - detection.model_dump( - include=set( - [ - "name", - "author", - "date", - "version", - "id", - "description", - "tags", - "search", - "how_to_implement", - "known_false_positives", - "references", - "datamodel", - "macros", - "lookups", - "source", - "nes_fields", - ] - ) + detections = [ + detection.model_dump( + include=set( + [ + "name", + "author", + "date", + "version", + "id", + "description", + "tags", + "search", + "how_to_implement", + "known_false_positives", + "references", + "datamodel", + "macros", + "lookups", + "source", + "nes_fields", + ] ) - for detection in objects - ] - #Only a subset of macro fields are required: - # for detection in detections: - # new_macros = [] - # for macro in detection.get("macros",[]): - # new_macro_fields = {} - # new_macro_fields["name"] = macro.get("name") - # new_macro_fields["definition"] = macro.get("definition") - # new_macro_fields["description"] = macro.get("description") - # if len(macro.get("arguments", [])) > 0: - # new_macro_fields["arguments"] = macro.get("arguments") - # new_macros.append(new_macro_fields) - # detection["macros"] = new_macros - # del() - - - JsonWriter.writeJsonObject( - os.path.join(output_path, "detections.json"), "detections", detections - ) - - elif contentType == SecurityContentType.macros: - macros = [ - macro.model_dump(include=set(["definition", "description", "name"])) - for macro in objects - ] - for macro in macros: - for k in ["author", "date","version","id","references"]: - if k in macro: - del(macro[k]) - JsonWriter.writeJsonObject( - os.path.join(output_path, "macros.json"), "macros", macros ) - - elif contentType == SecurityContentType.stories: - stories = [ - story.model_dump( - include=set( - [ - "name", - "author", - "date", - "version", - "id", - "description", - "narrative", - "references", - "tags", - "detections_names", - "investigation_names", - "baseline_names", - "detections", - ] - ) - ) - for story in objects - ] - # Only get certain fields from detections - for story in stories: - # Only use a small subset of fields from the detection - story["detections"] = [ - { - "name": detection["name"], - "source": detection["source"], - "type": detection["type"], - "tags": detection["tags"].get("mitre_attack_enrichments", []), - } - for detection in story["detections"] - ] - story["detection_names"] = [f"{app_label} - {name} - Rule" for name in story["detection_names"]] + for detection in objects + ] + #Only a subset of macro fields are required: + # for detection in detections: + # new_macros = [] + # for macro in detection.get("macros",[]): + # new_macro_fields = {} + # new_macro_fields["name"] = macro.get("name") + # new_macro_fields["definition"] = macro.get("definition") + # new_macro_fields["description"] = macro.get("description") + # if len(macro.get("arguments", [])) > 0: + # new_macro_fields["arguments"] = macro.get("arguments") + # new_macros.append(new_macro_fields) + # detection["macros"] = new_macros + # del() - - JsonWriter.writeJsonObject( - os.path.join(output_path, "stories.json"), "stories", stories + + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "detections.json"), "detections", detections + ) + + def writeMacros( + self, + objects: list[Macro], + ) -> None: + macros = [ + macro.model_dump(include=set(["definition", "description", "name"])) + for macro in objects + ] + for macro in macros: + for k in ["author", "date","version","id","references"]: + if k in macro: + del(macro[k]) + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "macros.json"), "macros", macros + ) + + def writeStories( + self, + objects: list[Story], + ) -> None: + stories = [ + story.model_dump( + include=set( + [ + "name", + "author", + "date", + "version", + "id", + "description", + "narrative", + "references", + "tags", + "detections_names", + "investigation_names", + "baseline_names", + "detections", + ] + ) ) + for story in objects + ] + # Only get certain fields from detections + for story in stories: + # Only use a small subset of fields from the detection + story["detections"] = [ + { + "name": detection["name"], + "source": detection["source"], + "type": detection["type"], + "tags": detection["tags"].get("mitre_attack_enrichments", []), + } + for detection in story["detections"] + ] + story["detection_names"] = [f"{self.app_label} - {name} - Rule" for name in story["detection_names"]] + - elif contentType == SecurityContentType.baselines: - try: - baselines = [ - baseline.model_dump( - include=set( - [ - "name", - "author", - "date", - "version", - "id", - "description", - "type", - "datamodel", - "search", - "how_to_implement", - "known_false_positives", - "references", - "tags", - ] - ) - ) - for baseline in objects - ] - except Exception as e: - print(e) - print('wait') + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "stories.json"), "stories", stories + ) - JsonWriter.writeJsonObject( - os.path.join(output_path, "baselines.json"), "baselines", baselines + def writeBaselines( + self, + objects: list[Baseline], + ) -> None: + baselines = [ + baseline.model_dump( + include=set( + [ + "name", + "author", + "date", + "version", + "id", + "description", + "type", + "datamodel", + "search", + "how_to_implement", + "known_false_positives", + "references", + "tags", + ] ) + ) + for baseline in objects + ] + + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "baselines.json"), "baselines", baselines + ) - elif contentType == SecurityContentType.investigations: - investigations = [ - investigation.model_dump( - include=set( - [ - "name", - "author", - "date", - "version", - "id", - "description", - "type", - "datamodel", - "search", - "how_to_implemnet", - "known_false_positives", - "references", - "inputs", - "tags", - "lowercase_name", - ] - ) + def writeInvestigations( + self, + objects: list[Investigation], + ) -> None: + investigations = [ + investigation.model_dump( + include=set( + [ + "name", + "author", + "date", + "version", + "id", + "description", + "type", + "datamodel", + "search", + "how_to_implemnet", + "known_false_positives", + "references", + "inputs", + "tags", + "lowercase_name", + ] ) - for investigation in objects - ] - JsonWriter.writeJsonObject( - os.path.join(output_path, "response_tasks.json"), - "response_tasks", - investigations, ) + for investigation in objects + ] + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "response_tasks.json"), + "response_tasks", + investigations, + ) - elif contentType == SecurityContentType.lookups: - lookups = [ - lookup.model_dump( - include=set( - [ - "name", - "description", - "collection", - "fields_list", - "filename", - "default_match", - "match_type", - "min_matches", - "case_sensitive_match", - ] - ) + def writeLookups( + self, + objects: list[Lookup], + ) -> None: + lookups = [ + lookup.model_dump( + include=set( + [ + "name", + "description", + "collection", + "fields_list", + "filename", + "default_match", + "match_type", + "min_matches", + "case_sensitive_match", + ] ) - for lookup in objects - ] - for lookup in lookups: - for k in ["author","date","version","id","references"]: - if k in lookup: - del(lookup[k]) - JsonWriter.writeJsonObject( - os.path.join(output_path, "lookups.json"), "lookups", lookups ) + for lookup in objects + ] + for lookup in lookups: + for k in ["author","date","version","id","references"]: + if k in lookup: + del(lookup[k]) + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "lookups.json"), "lookups", lookups + ) - elif contentType == SecurityContentType.deployments: - deployments = [ - deployment.model_dump( - include=set( - [ - "name", - "author", - "date", - "version", - "id", - "description", - "scheduling", - "rba", - "tags" - ] - ) + def writeDeployments( + self, + objects: list[Deployment], + ) -> None: + deployments = [ + deployment.model_dump( + include=set( + [ + "name", + "author", + "date", + "version", + "id", + "description", + "scheduling", + "rba", + "tags" + ] ) - for deployment in objects - ] - #references are not to be included, but have been deleted in the - #model_serialization logic - JsonWriter.writeJsonObject( - os.path.join(output_path, "deployments.json"), - "deployments", - deployments, - ) \ No newline at end of file + ) + for deployment in objects + ] + #references are not to be included, but have been deleted in the + #model_serialization logic + JsonWriter.writeJsonObject( + os.path.join(self.output_path, "deployments.json"), + "deployments", + deployments, + ) \ No newline at end of file diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index e53aeba0..604f67c8 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -1,3 +1,15 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Callable +if TYPE_CHECKING: + from contentctl.objects.security_content_object import SecurityContentObject + from contentctl.objects.detection import Detection + from contentctl.objects.lookup import Lookup, FileBackedLookup + from contentctl.objects.macro import Macro + from contentctl.objects.dashboard import Dashboard + from contentctl.objects.story import Story + from contentctl.objects.baseline import Baseline + from contentctl.objects.investigation import Investigation + from dataclasses import dataclass import os import glob @@ -80,25 +92,33 @@ def writeMiscellaneousAppFiles(self)->set[pathlib.Path]: return written_files - def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[pathlib.Path]: + def writeDetections(self, objects:list[Detection]) -> set[pathlib.Path]: written_files:set[pathlib.Path] = set() - if type == SecurityContentType.detections: - for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_detections.j2'), - ('default/analyticstories.conf', 'analyticstories_detections.j2')]: - written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, self.config, objects)) - - elif type == SecurityContentType.stories: + for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_detections.j2'), + ('default/analyticstories.conf', 'analyticstories_detections.j2')]: + written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), + template_name, self.config, objects)) + return written_files + + + def writeStories(self, objects:list[Story]) -> set[pathlib.Path]: + written_files:set[pathlib.Path] = set() written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/analyticstories.conf'), 'analyticstories_stories.j2', self.config, objects)) + return written_files + - elif type == SecurityContentType.baselines: + def writeBaselines(self, objects:list[Baseline]) -> set[pathlib.Path]: + written_files:set[pathlib.Path] = set() written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/savedsearches.conf'), 'savedsearches_baselines.j2', self.config, objects)) + return written_files + - elif type == SecurityContentType.investigations: + def writeInvestigations(self, objects:list[Investigation]) -> set[pathlib.Path]: + written_files:set[pathlib.Path] = set() for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_investigations.j2'), ('default/analyticstories.conf', 'analyticstories_investigations.j2')]: ConfWriter.writeConfFile(pathlib.Path(output_app_path), @@ -106,7 +126,7 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[p self.config, objects) - workbench_panels = [] + workbench_panels:list[Investigation] = [] for investigation in objects: if investigation.inputs: response_file_name_xml = investigation.lowercase_name + "___response_task.xml" @@ -128,8 +148,11 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[p template_name, self.config, workbench_panels)) + return written_files + - elif type == SecurityContentType.lookups: + def writeLookups(self, objects:list[Lookup]) -> set[pathlib.Path]: + written_files:set[pathlib.Path] = set() for output_app_path, template_name in [ ('default/collections.conf', 'collections.j2'), ('default/transforms.conf', 'transforms.j2')]: written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), @@ -137,9 +160,7 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[p self.config, objects)) - - #we want to copy all *.mlmodel files as well, not just csvs - files = list(glob.iglob(str(self.config.path/ 'lookups/*.csv'))) + list(glob.iglob(str(self.config.path / 'lookups/*.mlmodel'))) + #Get the path to the lookups folder lookup_folder = self.config.getPackageDirectoryPath()/"lookups" # Make the new folder for the lookups @@ -147,26 +168,24 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[p lookup_folder.mkdir(exist_ok=True) #Copy each lookup into the folder - for lookup_name in files: - lookup_path = pathlib.Path(lookup_name) - if lookup_path.is_file(): - shutil.copy(lookup_path, lookup_folder/lookup_path.name) - else: - raise(Exception(f"Error copying lookup/mlmodel file. Path {lookup_path} does not exist or is not a file.")) - - elif type == SecurityContentType.macros: + for lookup in objects: + if isinstance(lookup, FileBackedLookup): + shutil.copy(lookup.filename, lookup_folder/lookup.app_filename.name) + return written_files + + + def writeMacros(self, objects:list[Macro]) -> set[pathlib.Path]: + written_files:set[pathlib.Path] = set() written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/macros.conf'), 'macros.j2', self.config, objects)) - - elif type == SecurityContentType.dashboards: - written_files.update(ConfWriter.writeDashboardFiles(self.config, objects)) - - - return written_files - - + return written_files + + def writeDashboards(self, objects:list[Dashboard]) -> set[pathlib.Path]: + written_files:set[pathlib.Path] = set() + written_files.update(ConfWriter.writeDashboardFiles(self.config, objects)) + return written_files def packageAppTar(self) -> None: @@ -202,7 +221,7 @@ def packageAppSlim(self) -> None: - def packageApp(self, method=packageAppTar)->None: + def packageApp(self, method: Callable[[ConfOutput],None]=packageAppTar)->None: return method(self) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 410ce4f6..3791b045 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Sequence import datetime import re import os @@ -9,7 +9,7 @@ import pathlib from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.dashboard import Dashboard -from contentctl.objects.config import build +from contentctl.objects.config import build, CustomApp import xml.etree.ElementTree as ET # This list is not exhaustive of all default conf files, but should be @@ -174,7 +174,7 @@ def writeAppConf(config: build) -> pathlib.Path: return output_path @staticmethod - def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path: + def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list[CustomApp]) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) @@ -206,7 +206,7 @@ def writeFileHeader(app_output_path:pathlib.Path, config: build) -> str: @staticmethod - def writeXmlFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> None: + def writeXmlFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list[str]) -> None: j2_env = ConfWriter.getJ2Environment() @@ -271,7 +271,7 @@ def getJ2Environment()->Environment: return j2_env @staticmethod - def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path: + def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : Sequence[SecurityContentObject] | list[CustomApp]) -> pathlib.Path: output_path = config.getPackageDirectoryPath()/app_output_path j2_env = ConfWriter.getJ2Environment() diff --git a/contentctl/output/json_writer.py b/contentctl/output/json_writer.py index fe3696d9..7a32c009 100644 --- a/contentctl/output/json_writer.py +++ b/contentctl/output/json_writer.py @@ -1,11 +1,9 @@ import json -from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract -from typing import List -from io import TextIOWrapper +from typing import Any class JsonWriter(): @staticmethod - def writeJsonObject(file_path : str, object_name: str, objs: List[dict],readable_output=False) -> None: + def writeJsonObject(file_path : str, object_name: str, objs: list[dict[str,Any]],readable_output:bool=False) -> None: try: with open(file_path, 'w') as outfile: if readable_output: From 825beaf7444e45bde837b682f81ee96ae0dab128 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 8 Jan 2025 12:30:08 -0800 Subject: [PATCH 097/115] Able to build without any errors, but that does not mean it is correct yet. Outputs must be diffed against prior versions to make sure there are no unintended changes. --- contentctl/actions/build.py | 32 +++++++++++---- contentctl/actions/validate.py | 3 +- contentctl/input/director.py | 5 +-- .../detection_abstract.py | 22 +++++----- contentctl/objects/detection_tags.py | 20 --------- contentctl/objects/lookup.py | 17 +++++++- contentctl/objects/macro.py | 5 +-- contentctl/objects/rba.py | 41 ++++++++++++++++--- contentctl/output/api_json_output.py | 2 +- contentctl/output/conf_output.py | 16 +------- contentctl/output/conf_writer.py | 8 +++- contentctl/output/templates/collections.j2 | 2 +- .../templates/savedsearches_detections.j2 | 7 +++- contentctl/output/templates/transforms.j2 | 4 +- 14 files changed, 111 insertions(+), 73 deletions(-) diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index d8aef19e..5e3acdb3 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -10,11 +10,11 @@ from contentctl.output.conf_writer import ConfWriter from contentctl.output.api_json_output import ApiJsonOutput from contentctl.output.data_source_writer import DataSourceWriter -from contentctl.objects.lookup import Lookup +from contentctl.objects.lookup import CSVLookup, Lookup_Type import pathlib import json import datetime - +import uuid from contentctl.objects.config import build @@ -34,27 +34,41 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto: updated_conf_files:set[pathlib.Path] = set() conf_output = ConfOutput(input_dto.config) + + # Construct a path to a YML that does not actually exist. + # We mock this "fake" path since the YML does not exist. + # This ensures the checking for the existence of the CSV is correct + data_sources_fake_yml_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.yml" + # Construct a special lookup whose CSV is created at runtime and - # written directly into the output folder. It is created with model_construct, - # not model_validate, because the CSV does not exist yet. + # written directly into the lookups folder. We will delete this after a build, + # assuming that it is successful. data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv" - DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path) - input_dto.director_output_dto.addContentToDictMappings(Lookup.model_construct(description= "A lookup file that will contain the data source objects for detections.", - filename=data_sources_lookup_csv_path, - name="data_sources")) + + + DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path) + input_dto.director_output_dto.addContentToDictMappings(CSVLookup.model_construct(name="data_sources", + id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"), + version=1, + author=input_dto.config.app.author_name, + date = datetime.date.today(), + description= "A lookup file that will contain the data source objects for detections.", + lookup_type=Lookup_Type.csv, + file_path=data_sources_fake_yml_path)) updated_conf_files.update(conf_output.writeHeaders()) + updated_conf_files.update(conf_output.writeLookups(input_dto.director_output_dto.lookups)) updated_conf_files.update(conf_output.writeDetections(input_dto.director_output_dto.detections)) updated_conf_files.update(conf_output.writeStories(input_dto.director_output_dto.stories)) updated_conf_files.update(conf_output.writeBaselines(input_dto.director_output_dto.baselines)) updated_conf_files.update(conf_output.writeInvestigations(input_dto.director_output_dto.investigations)) - updated_conf_files.update(conf_output.writeLookups(input_dto.director_output_dto.lookups)) updated_conf_files.update(conf_output.writeMacros(input_dto.director_output_dto.macros)) updated_conf_files.update(conf_output.writeDashboards(input_dto.director_output_dto.dashboards)) updated_conf_files.update(conf_output.writeMiscellaneousAppFiles()) + #Ensure that the conf file we just generated/update is syntactically valid for conf_file in updated_conf_files: ConfWriter.validateConfFile(conf_file) diff --git a/contentctl/actions/validate.py b/contentctl/actions/validate.py index 9d394d07..c2756f75 100644 --- a/contentctl/actions/validate.py +++ b/contentctl/actions/validate.py @@ -6,6 +6,7 @@ from contentctl.enrichments.attack_enrichment import AttackEnrichment from contentctl.enrichments.cve_enrichment import CveEnrichment from contentctl.objects.atomic import AtomicEnrichment +from contentctl.objects.lookup import FileBackedLookup from contentctl.helper.utils import Utils from contentctl.objects.data_source import DataSource from contentctl.helper.splunk_app import SplunkApp @@ -64,7 +65,7 @@ def ensure_no_orphaned_files_in_lookups(self, repo_path:pathlib.Path, director_o lookupsDirectory = repo_path/"lookups" # Get all of the files referneced by Lookups - usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if lookup.filename is not None] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None] + usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if isinstance(lookup, FileBackedLookup)] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None] # Get all of the mlmodel and csv files in the lookups directory csvAndMlmodelFiles = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"]) diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 8585f001..8462d61e 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -58,13 +58,12 @@ def addContentToDictMappings(self, content: SecurityContentObject): f" - {content.file_path}\n" f" - {self.name_to_content_map[content_name].file_path}" ) - + if content.id in self.uuid_to_content_map: raise ValueError( f"Duplicate id '{content.id}' with paths:\n" f" - {content.file_path}\n" - f" - {self.uuid_to_content_map[content.id].file_path}" - ) + f" - {self.uuid_to_content_map[content.id].file_path}") if isinstance(content, Lookup): self.lookups.append(content) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 81297df9..8dcf01ae 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -16,7 +16,7 @@ ) from contentctl.objects.macro import Macro -from contentctl.objects.lookup import Lookup +from contentctl.objects.lookup import Lookup, FileBackedLookup, KVStoreLookup if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto from contentctl.objects.baseline import Baseline @@ -285,10 +285,8 @@ def annotations(self) -> dict[str, Union[List[str], int, str]]: annotations_dict: dict[str, str | list[str] | int] = {} annotations_dict["analytic_story"] = [story.name for story in self.tags.analytic_story] - annotations_dict["confidence"] = self.tags.confidence if len(self.tags.cve or []) > 0: annotations_dict["cve"] = self.tags.cve - annotations_dict["impact"] = self.tags.impact annotations_dict["type"] = self.type annotations_dict["type_list"] = [self.type] # annotations_dict["version"] = self.version @@ -480,6 +478,11 @@ def serialize_model(self): "source": self.source, "nes_fields": self.nes_fields, } + if self.rba is not None: + model["risk_severity"] = self.rba.severity + model['tags']['risk_score'] = self.rba.risk_score + else: + model['tags']['risk_score'] = 0 # Only a subset of macro fields are required: all_macros: list[dict[str, str | list[str]]] = [] @@ -497,17 +500,17 @@ def serialize_model(self): all_lookups: list[dict[str, str | int | None]] = [] for lookup in self.lookups: - if lookup.collection is not None: + if isinstance(lookup, KVStoreLookup): all_lookups.append( { "name": lookup.name, "description": lookup.description, "collection": lookup.collection, "case_sensitive_match": None, - "fields_list": lookup.fields_list + "fields_list": lookup.fields_to_fields_list_conf_format } ) - elif lookup.filename is not None: + elif isinstance(lookup, FileBackedLookup): all_lookups.append( { "name": lookup.name, @@ -515,9 +518,8 @@ def serialize_model(self): "filename": lookup.filename.name, "default_match": "true" if lookup.default_match else "false", "case_sensitive_match": "true" if lookup.case_sensitive_match else "false", - "match_type": lookup.match_type, - "min_matches": lookup.min_matches, - "fields_list": lookup.fields_list + "match_type": lookup.match_type_to_conf_format, + "min_matches": lookup.min_matches } ) model['lookups'] = all_lookups # type: ignore @@ -790,7 +792,7 @@ def ensureProperRBAConfig(self): """ - if self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None: + if self.deployment.alert_action.rba is None or self.deployment.alert_action.rba.enabled is False: # confirm we don't have an RBA config if self.rba is None: return self diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index b4184927..3f72d38d 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -27,7 +27,6 @@ Cis18Value, AssetType, SecurityDomain, - RiskSeverity, KillChainPhase, NistCategory, SecurityContentProductName @@ -43,23 +42,6 @@ class DetectionTags(BaseModel): asset_type: AssetType = Field(...) group: list[str] = [] - @computed_field - @property - def severity(self)->RiskSeverity: - if 0 <= self.risk_score <= 20: - return RiskSeverity.INFORMATIONAL - elif 20 < self.risk_score <= 40: - return RiskSeverity.LOW - elif 40 < self.risk_score <= 60: - return RiskSeverity.MEDIUM - elif 60 < self.risk_score <= 80: - return RiskSeverity.HIGH - elif 80 < self.risk_score <= 100: - return RiskSeverity.CRITICAL - else: - raise Exception(f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}") - - mitre_attack_id: List[MITRE_ATTACK_ID_TYPE] = [] nist: list[NistCategory] = [] @@ -144,9 +126,7 @@ def serialize_model(self): "cis20": self.cis20, "kill_chain_phases": self.kill_chain_phases, "nist": self.nist, - "risk_score": self.risk_score, "security_domain": self.security_domain, - "risk_severity": self.severity, "mitre_attack_id": self.mitre_attack_id, "mitre_attack_enrichments": self.mitre_attack_enrichments } diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index cfa11a0d..9c1cabda 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -80,7 +80,10 @@ def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any: return data - + @computed_field + @cached_property + def match_type_to_conf_format(self)->str: + return ', '.join(self.match_type) @staticmethod @@ -196,6 +199,16 @@ def ensure_key(cls, values: list[str]): raise ValueError(f"fields MUST begin with '_key', not '{values[0]}'") return values + @computed_field + @cached_property + def collection(self)->str: + return self.name + + @computed_field + @cached_property + def fields_to_fields_list_conf_format(self)->str: + return ', '.join(self.fields) + @model_serializer def serialize_model(self): #Call parent serializer @@ -204,7 +217,7 @@ def serialize_model(self): #All fields custom to this model model= { "collection": self.collection, - "fields_list": ", ".join(self.fields) + "fields_list": self.fields_to_fields_list_conf_format } #return the model diff --git a/contentctl/objects/macro.py b/contentctl/objects/macro.py index ba5faa8f..8c25dff7 100644 --- a/contentctl/objects/macro.py +++ b/contentctl/objects/macro.py @@ -48,7 +48,6 @@ def serialize_model(self): return model @staticmethod - def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[str]=MACROS_TO_IGNORE)->list[Macro]: #Remove any comments, allowing there to be macros (which have a single backtick) inside those comments #If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here```` @@ -59,10 +58,10 @@ def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[st "This may have occurred when a macro was commented out.\n" "Please ammend your search to remove the substring '````'") - # replace all the macros with a space + # Replace all the comments with a space. This prevents a comment from looking like a macro to the parser below text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field) - + # Find all the macros, which start and end with a '`' character macros_to_get = re.findall(r'`([^\s]+)`', text_field) #If macros take arguments, stop at the first argument. We just want the name of the macro macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get]) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 05a32a41..efa9fbc1 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -1,9 +1,11 @@ from enum import Enum -from pydantic import BaseModel +from pydantic import BaseModel, computed_field, Field from abc import ABC -from typing import Set +from typing import Set, Annotated +from contentctl.objects.enums import RiskSeverity +RiskScoreValue_Type = Annotated[int, Field(ge=1, le=100)] class RiskObjectType(str, Enum): SYSTEM = "system" @@ -40,7 +42,7 @@ class ThreatObjectType(str, Enum): class risk_object(BaseModel): field: str type: RiskObjectType - score: int + score: RiskScoreValue_Type def __hash__(self): return hash((self.field, self.type, self.score)) @@ -54,5 +56,34 @@ def __hash__(self): class rba_object(BaseModel, ABC): message: str - risk_objects: Set[risk_object] - threat_objects: Set[threat_object] \ No newline at end of file + risk_objects: Annotated[Set[risk_object], Field(min_length=1)] + threat_objects: Set[threat_object] + + + + @computed_field + @property + def risk_score(self)->RiskScoreValue_Type: + # First get the maximum score associated with + # a risk object. If there are no objects, then + # we should throw an exception. + if len(self.risk_objects) == 0: + raise Exception("There must be at least one Risk Object present to get Severity.") + return max([risk_object.score for risk_object in self.risk_objects]) + + @computed_field + @property + def severity(self)->RiskSeverity: + if 0 <= self.risk_score <= 20: + return RiskSeverity.INFORMATIONAL + elif 20 < self.risk_score <= 40: + return RiskSeverity.LOW + elif 40 < self.risk_score <= 60: + return RiskSeverity.MEDIUM + elif 60 < self.risk_score <= 80: + return RiskSeverity.HIGH + elif 80 < self.risk_score <= 100: + return RiskSeverity.CRITICAL + else: + raise Exception(f"Error getting severity - risk_score must be between 0-100, but was actually {max_score}") + diff --git a/contentctl/output/api_json_output.py b/contentctl/output/api_json_output.py index a62e3d3c..02af11d9 100644 --- a/contentctl/output/api_json_output.py +++ b/contentctl/output/api_json_output.py @@ -215,7 +215,7 @@ def writeLookups( "name", "description", "collection", - "fields_list", + "fields_to_fields_list_conf_format", "filename", "default_match", "match_type", diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index 604f67c8..c5a67673 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -1,34 +1,22 @@ from __future__ import annotations from typing import TYPE_CHECKING, Callable if TYPE_CHECKING: - from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.detection import Detection - from contentctl.objects.lookup import Lookup, FileBackedLookup + from contentctl.objects.lookup import Lookup from contentctl.objects.macro import Macro from contentctl.objects.dashboard import Dashboard from contentctl.objects.story import Story from contentctl.objects.baseline import Baseline from contentctl.objects.investigation import Investigation -from dataclasses import dataclass -import os -import glob +from contentctl.objects.lookup import FileBackedLookup import shutil -import sys import tarfile -from typing import Union -from pathlib import Path import pathlib -import time import timeit import datetime -import shutil -import json from contentctl.output.conf_writer import ConfWriter -from contentctl.objects.enums import SecurityContentType from contentctl.objects.config import build -from requests import Session, post, get -from requests.auth import HTTPBasicAuth class ConfOutput: config: build diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 3791b045..1b6ab841 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -276,7 +276,13 @@ def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: bui j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - output = template.render(objects=objects, app=config.app) + try: + for obj in objects: + output = template.render(objects=[obj], app=config.app) + except Exception as e: + print(e) + import code + code.interact(local=locals()) output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, 'a') as f: diff --git a/contentctl/output/templates/collections.j2 b/contentctl/output/templates/collections.j2 index 06e49140..cdf0eda7 100644 --- a/contentctl/output/templates/collections.j2 +++ b/contentctl/output/templates/collections.j2 @@ -1,6 +1,6 @@ {% for lookup in objects %} -{% if lookup.collection is defined and lookup.collection != None %} +{% if lookup.collection is defined %} [{{ lookup.name }}] enforceTypes = false replicate = false diff --git a/contentctl/output/templates/savedsearches_detections.j2 b/contentctl/output/templates/savedsearches_detections.j2 index 55e99e60..0a5c634f 100644 --- a/contentctl/output/templates/savedsearches_detections.j2 +++ b/contentctl/output/templates/savedsearches_detections.j2 @@ -71,7 +71,12 @@ action.notable.param.nes_fields = {{ detection.nes_fields }} action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}} action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%} action.notable.param.security_domain = {{ detection.tags.security_domain }} -action.notable.param.severity = {{ detection.tags.severity }} +{% if detection.rba %} +action.notable.param.severity = {{ detection.rba.severity }} +{% else %} +{# Correlations do not have detection.rba defined, but should get a default severity #} +action.notable.param.severity = high +{% endif %} {% endif %} {% if detection.deployment.alert_action.email %} action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }} diff --git a/contentctl/output/templates/transforms.j2 b/contentctl/output/templates/transforms.j2 index 2fd029ec..58011189 100644 --- a/contentctl/output/templates/transforms.j2 +++ b/contentctl/output/templates/transforms.j2 @@ -25,8 +25,8 @@ max_matches = {{ lookup.max_matches }} {% if lookup.min_matches is defined and lookup.min_matches != None %} min_matches = {{ lookup.min_matches }} {% endif %} -{% if lookup.fields_list is defined and lookup.fields_list != None %} -fields_list = {{ lookup.fields_list }} +{% if lookup.fields_to_fields_list_conf_format is defined %} +fields_list = {{ lookup.fields_to_fields_list_conf_format }} {% endif %} {% if lookup.filter is defined and lookup.filter != None %} filter = {{ lookup.filter }} From a31d484c5760360c9f7494e921c99375885aab47 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 8 Jan 2025 13:48:08 -0800 Subject: [PATCH 098/115] improve api output serialization for better readability --- contentctl/objects/lookup.py | 2 +- contentctl/output/api_json_output.py | 5 ++--- contentctl/output/json_writer.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 9c1cabda..cba13ab2 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -59,7 +59,7 @@ def serialize_model(self): model= { "default_match": "true" if self.default_match is True else "false", - "match_type": self.match_type, + "match_type": self.match_type_to_conf_format, "min_matches": self.min_matches, "max_matches": self.max_matches, "case_sensitive_match": "true" if self.case_sensitive_match is True else "false", diff --git a/contentctl/output/api_json_output.py b/contentctl/output/api_json_output.py index 02af11d9..20006466 100644 --- a/contentctl/output/api_json_output.py +++ b/contentctl/output/api_json_output.py @@ -2,9 +2,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from contentctl.objects.detection import Detection - from contentctl.objects.lookup import Lookup, FileBackedLookup + from contentctl.objects.lookup import Lookup from contentctl.objects.macro import Macro - from contentctl.objects.dashboard import Dashboard from contentctl.objects.story import Story from contentctl.objects.baseline import Baseline from contentctl.objects.investigation import Investigation @@ -215,7 +214,7 @@ def writeLookups( "name", "description", "collection", - "fields_to_fields_list_conf_format", + "fields_list", "filename", "default_match", "match_type", diff --git a/contentctl/output/json_writer.py b/contentctl/output/json_writer.py index 7a32c009..ee272255 100644 --- a/contentctl/output/json_writer.py +++ b/contentctl/output/json_writer.py @@ -3,7 +3,7 @@ class JsonWriter(): @staticmethod - def writeJsonObject(file_path : str, object_name: str, objs: list[dict[str,Any]],readable_output:bool=False) -> None: + def writeJsonObject(file_path : str, object_name: str, objs: list[dict[str,Any]],readable_output:bool=True) -> None: try: with open(file_path, 'w') as outfile: if readable_output: From 0f7017252b4365aebde0c1f251b199e54a71768f Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 9 Jan 2025 12:03:45 -0800 Subject: [PATCH 099/115] Clean up bad imports. Give more explicit error when something fails to be written correctly to a conf file. Otherwise, it is hard to find out which specific object caused the failure. --- contentctl/output/api_json_output.py | 5 ----- contentctl/output/conf_writer.py | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/contentctl/output/api_json_output.py b/contentctl/output/api_json_output.py index 20006466..87760373 100644 --- a/contentctl/output/api_json_output.py +++ b/contentctl/output/api_json_output.py @@ -10,14 +10,9 @@ from contentctl.objects.deployment import Deployment import os -import json import pathlib from contentctl.output.json_writer import JsonWriter -from contentctl.objects.enums import SecurityContentType -from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import ( - SecurityContentObject_Abstract, -) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 1b6ab841..31b776b2 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -276,17 +276,20 @@ def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: bui j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - try: - for obj in objects: - output = template.render(objects=[obj], app=config.app) - except Exception as e: - print(e) - import code - code.interact(local=locals()) + outputs: list[str] = [] + for obj in objects: + try: + outputs.append(template.render(objects=[obj], app=config.app)) + except Exception as e: + raise Exception(f"Failed writing the following object to file:\n" + f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n" + f"Type {type(obj)}: \n" + f"Output File: {app_output_path}\n" + f"Error: {str(e)}\n") output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + output = ''.join(outputs).encode('utf-8', 'ignore').decode('utf-8') f.write(output) return output_path From 8795e97f3402bae8b30de0e2205cefcf98ff29dc Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 9 Jan 2025 15:26:42 -0600 Subject: [PATCH 100/115] bump to 0.9.0 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77eb04cf..e49aaf16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.0 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index d59fb265..264b7278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.8.6" +ruff = "^0.9.0" [build-system] requires = ["poetry-core>=1.0.0"] From 285acf14f9acaf2047487fe918ebfe6890525b83 Mon Sep 17 00:00:00 2001 From: ljstella Date: Fri, 10 Jan 2025 14:07:08 -0600 Subject: [PATCH 101/115] New threat object type --- contentctl/objects/rba.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 05a32a41..1f9a6c31 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -33,6 +33,7 @@ class ThreatObjectType(str, Enum): REGISTRY_VALUE_NAME = "registry_value_name" REGISTRY_VALUE_TEXT = "registry_value_text" SERVICE = "service" + SIGNATURE = "signature" SYSTEM = "system" TLS_HASH = "tls_hash" URL = "url" @@ -55,4 +56,4 @@ def __hash__(self): class rba_object(BaseModel, ABC): message: str risk_objects: Set[risk_object] - threat_objects: Set[threat_object] \ No newline at end of file + threat_objects: Set[threat_object] From 633f0d5dc73f7040a069e77c150f7265255d3752 Mon Sep 17 00:00:00 2001 From: Lou Stella Date: Sun, 12 Jan 2025 14:29:02 -0600 Subject: [PATCH 102/115] version bump to 0.9.1 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e49aaf16..164a8deb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.0 + rev: v0.9.1 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index 264b7278..c5d0a4b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.9.0" +ruff = "^0.9.1" [build-system] requires = ["poetry-core>=1.0.0"] From 78aa05e58108f479f211c5ef84735c2384fc773f Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 13 Jan 2025 15:20:20 -0800 Subject: [PATCH 103/115] Fix access of variable that does not exist. --- contentctl/objects/rba.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 22adf74e..96294d9a 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -86,5 +86,5 @@ def severity(self)->RiskSeverity: elif 80 < self.risk_score <= 100: return RiskSeverity.CRITICAL else: - raise Exception(f"Error getting severity - risk_score must be between 0-100, but was actually {max_score}") + raise Exception(f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}") From 9b158cef28513797ad99d3b1cc1e338715a29911 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Wed, 15 Jan 2025 16:32:50 -0800 Subject: [PATCH 104/115] initial commit; migrated integration testing to RBA structures; littered code w/ comments for cleanup before merge --- .../DetectionTestingInfrastructure.py | 8 +- .../detection_abstract.py | 1 + contentctl/objects/constants.py | 3 + contentctl/objects/correlation_search.py | 137 +++++------ contentctl/objects/detection_tags.py | 9 +- contentctl/objects/drilldown.py | 1 + contentctl/objects/observable.py | 1 + contentctl/objects/rba.py | 1 + contentctl/objects/risk_event.py | 223 +++++++----------- 9 files changed, 175 insertions(+), 209 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 7804b29e..42cad6c0 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -1094,6 +1094,7 @@ def retry_search_until_timeout( job = self.get_conn().search(query=search, **kwargs) results = JSONResultsReader(job.results(output_mode="json")) + # TODO (cmcginley): @ljstella you're removing this ultimately, right? # Consolidate a set of the distinct observable field names observable_fields_set = set([o.name for o in detection.tags.observable]) # keeping this around for later risk_object_fields_set = set([o.name for o in detection.tags.observable if "Victim" in o.role ]) # just the "Risk Objects" @@ -1121,7 +1122,10 @@ def retry_search_until_timeout( missing_risk_objects = risk_object_fields_set - results_fields_set if len(missing_risk_objects) > 0: # Report a failure in such cases - e = Exception(f"The observable field(s) {missing_risk_objects} are missing in the detection results") + e = Exception( + f"The risk object field(s) {missing_risk_objects} are missing in the " + "detection results" + ) test.result.set_job_content( job.content, self.infrastructure, @@ -1137,6 +1141,8 @@ def retry_search_until_timeout( # on a field. In this case, the field will appear but will not contain any values current_empty_fields: set[str] = set() + # TODO (cmcginley): @ljstella is this something we're keeping for testing as + # well? for field in observable_fields_set: if result.get(field, 'null') == 'null': if field in risk_object_fields_set: diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 81297df9..7d2b0386 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -279,6 +279,7 @@ def source(self) -> str: deployment: Deployment = Field({}) + # TODO (cmcginley): @ljstella removing the refs to confidence and impact? @computed_field @property def annotations(self) -> dict[str, Union[List[str], int, str]]: diff --git a/contentctl/objects/constants.py b/contentctl/objects/constants.py index 8a32f28d..f0f0d8f3 100644 --- a/contentctl/objects/constants.py +++ b/contentctl/objects/constants.py @@ -79,6 +79,7 @@ "Actions on Objectives": 7 } +# TODO (cmcginley): @ljstella should this be removed? also referenced in new_content.py SES_OBSERVABLE_ROLE_MAPPING = { "Other": -1, "Unknown": 0, @@ -93,6 +94,7 @@ "Observer": 9 } +# TODO (cmcginley): @ljstella should this be removed? also referenced in new_content.py SES_OBSERVABLE_TYPE_MAPPING = { "Unknown": 0, "Hostname": 1, @@ -135,6 +137,7 @@ "Impact": "TA0040" } +# TODO (cmcginley): is this just for the transition testing? RBA_OBSERVABLE_ROLE_MAPPING = { "Attacker": 0, "Victim": 1 diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 2a8d1e9c..b0cffe4e 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -29,7 +29,6 @@ from contentctl.objects.detection import Detection from contentctl.objects.risk_event import RiskEvent from contentctl.objects.notable_event import NotableEvent -from contentctl.objects.observable import Observable # Suppress logging by default; enable for local testing @@ -145,24 +144,24 @@ def __init__(self, response_reader: ResponseReader) -> None: def __iter__(self) -> "ResultIterator": return self - def __next__(self) -> dict: + def __next__(self) -> dict[Any, Any]: # Use a reader for JSON format so we can iterate over our results for result in self.results_reader: # log messages, or raise if error if isinstance(result, Message): # convert level string to level int - level_name = result.type.strip().upper() + level_name = result.type.strip().upper() # type: ignore level: int = logging.getLevelName(level_name) # log message at appropriate level and raise if needed - message = f"SPLUNK: {result.message}" + message = f"SPLUNK: {result.message}" # type: ignore self.logger.log(level, message) if level == logging.ERROR: raise ServerError(message) # if dict, just return elif isinstance(result, dict): - return result + return result # type: ignore # raise for any unexpected types else: @@ -310,9 +309,11 @@ def earliest_time(self) -> str: The earliest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY] + return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY] # type: ignore else: - raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") + raise ClientError( + "Something unexpected went wrong in initialization; saved_search was not populated" + ) @property def latest_time(self) -> str: @@ -320,9 +321,11 @@ def latest_time(self) -> str: The latest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY] + return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY] # type: ignore else: - raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") + raise ClientError( + "Something unexpected went wrong in initialization; saved_search was not populated" + ) @property def cron_schedule(self) -> str: @@ -330,9 +333,11 @@ def cron_schedule(self) -> str: The cron schedule configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY] + return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY] # type: ignore else: - raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") + raise ClientError( + "Something unexpected went wrong in initialization; saved_search was not populated" + ) @property def enabled(self) -> bool: @@ -340,12 +345,14 @@ def enabled(self) -> bool: Whether the saved search is enabled """ if self.saved_search is not None: - if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY]): + if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY]): # type: ignore return False else: return True else: - raise ClientError("Something unexpected went wrong in initialization; saved_search was not populated") + raise ClientError( + "Something unexpected went wrong in initialization; saved_search was not populated" + ) @ property def has_risk_analysis_action(self) -> bool: @@ -387,19 +394,6 @@ def _get_notable_action(content: dict[str, Any]) -> NotableAction | None: return NotableAction.parse_from_dict(content) return None - @staticmethod - def _get_relevant_observables(observables: list[Observable]) -> list[Observable]: - """ - Given a list of observables, identify the subset of those relevant for risk matching - :param observables: the Observable objects to filter - :returns: the filtered list of relevant observables - """ - relevant = [] - for observable in observables: - if not RiskEvent.ignore_observable(observable): - relevant.append(observable) - return relevant - def _parse_risk_and_notable_actions(self) -> None: """Parses the risk/notable metadata we care about from self.saved_search.content @@ -495,7 +489,7 @@ def update_timeframe( if refresh: self.refresh() - def force_run(self, refresh=True) -> None: + def force_run(self, refresh: bool = True) -> None: """Forces a detection run Enables the detection, adjusts the cron schedule to run every 1 minute, and widens the earliest/latest window @@ -506,7 +500,7 @@ def force_run(self, refresh=True) -> None: if not self.enabled: self.enable(refresh=False) else: - self.logger.warn(f"Detection '{self.name}' was already enabled") + self.logger.warning(f"Detection '{self.name}' was already enabled") if refresh: self.refresh() @@ -557,7 +551,7 @@ def get_risk_events(self, force_update: bool = False) -> list[RiskEvent]: if result["index"] == Indexes.RISK_INDEX: try: parsed_raw = json.loads(result["_raw"]) - event = RiskEvent.parse_obj(parsed_raw) + event = RiskEvent.model_validate(parsed_raw) except Exception: self.logger.error(f"Failed to parse RiskEvent from search result: {result}") raise @@ -622,7 +616,7 @@ def get_notable_events(self, force_update: bool = False) -> list[NotableEvent]: if result["index"] == Indexes.NOTABLE_INDEX: try: parsed_raw = json.loads(result["_raw"]) - event = NotableEvent.parse_obj(parsed_raw) + event = NotableEvent.model_validate(parsed_raw) except Exception: self.logger.error(f"Failed to parse NotableEvent from search result: {result}") raise @@ -646,22 +640,26 @@ def validate_risk_events(self) -> None: """Validates the existence of any expected risk events First ensure the risk event exists, and if it does validate its risk message and make sure - any events align with the specified observables. Also adds the risk index to the purge list + any events align with the specified risk object. Also adds the risk index to the purge list if risk events existed :param elapsed_sleep_time: an int representing the amount of time slept thus far waiting to check the risks/notables :returns: an IntegrationTestResult on failure; None on success """ - # Create a mapping of the relevant observables to counters - observables = CorrelationSearch._get_relevant_observables(self.detection.tags.observable) - observable_counts: dict[str, int] = {str(x): 0 for x in observables} + # Ensure the rba object is defined + if self.detection.rba is None: + raise ValidationFailed( + f"Unexpected error: Detection '{self.detection.name}' has no RBA objects associated" + " with it; cannot validate." + ) + risk_object_counts: dict[str, int] = {str(x): 0 for x in self.detection.rba.risk_objects} # NOTE: we intentionally want this to be an error state and not a failure state, as # ultimately this validation should be handled during the build process - if len(observables) != len(observable_counts): + if len(self.detection.rba.risk_objects) != len(risk_object_counts): raise ClientError( - f"At least two observables in '{self.detection.name}' have the same name; " - "each observable for a detection should be unique." + f"At least two risk objects in '{self.detection.name}' have the same name; " + "each risk object for a detection should be unique." ) # Get the risk events; note that we use the cached risk events, expecting they were @@ -678,58 +676,61 @@ def validate_risk_events(self) -> None: ) event.validate_against_detection(self.detection) - # Update observable count based on match - matched_observable = event.get_matched_observable(self.detection.tags.observable) + # Update risk object count based on match + matched_risk_object = event.get_matched_risk_object(self.detection.rba.risk_objects) self.logger.debug( f"Matched risk event (object={event.risk_object}, type={event.risk_object_type}) " - f"to observable (name={matched_observable.name}, type={matched_observable.type}, " - f"role={matched_observable.role}) using the source field " + f"to detection's risk object (name={matched_risk_object.field}, " + f"type={matched_risk_object.type.value} using the source field " f"'{event.source_field_name}'" ) - observable_counts[str(matched_observable)] += 1 + risk_object_counts[str(matched_risk_object)] += 1 - # Report any observables which did not have at least one match to a risk event - for observable in observables: + # Report any risk objects which did not have at least one match to a risk event + for risk_object in self.detection.rba.risk_objects: self.logger.debug( - f"Matched observable (name={observable.name}, type={observable.type}, " - f"role={observable.role}) to {observable_counts[str(observable)]} risk events." + f"Matched risk object (name={risk_object.field}, type={risk_object.type.value} " + f"to {risk_object_counts[str(risk_object)]} risk events." ) - if observable_counts[str(observable)] == 0: + if risk_object_counts[str(risk_object)] == 0: raise ValidationFailed( - f"Observable (name={observable.name}, type={observable.type}, " - f"role={observable.role}) was not matched to any risk events." + f"Risk object (name={risk_object.field}, type={risk_object.type.value} " + "was not matched to any risk events." ) # TODO (#250): Re-enable and refactor code that validates the specific risk counts # Validate risk events in aggregate; we should have an equal amount of risk events for each - # relevant observable, and the total count should match the total number of events + # relevant risk object, and the total count should match the total number of events # individual_count: int | None = None # total_count = 0 - # for observable_str in observable_counts: + # for risk_object_str in risk_object_counts: # self.logger.debug( - # f"Observable <{observable_str}> match count: {observable_counts[observable_str]}" + # f"Risk object <{risk_object_str}> match count: {risk_object_counts[risk_object_str]}" # ) # # Grab the first value encountered if not set yet # if individual_count is None: - # individual_count = observable_counts[observable_str] + # individual_count = risk_object_counts[risk_object_str] # else: - # # Confirm that the count for the current observable matches the count of the others - # if observable_counts[observable_str] != individual_count: + # # Confirm that the count for the current risk object matches the count of the + # # others + # if risk_object_counts[risk_object_str] != individual_count: # raise ValidationFailed( - # f"Count of risk events matching observable <\"{observable_str}\"> " - # f"({observable_counts[observable_str]}) does not match the count of those " - # f"matching other observables ({individual_count})." + # f"Count of risk events matching detection's risk object <\"{risk_object_str}\"> " + # f"({risk_object_counts[risk_object_str]}) does not match the count of those " + # f"matching other risk objects ({individual_count})." # ) - # # Aggregate total count of events matched to observables - # total_count += observable_counts[observable_str] + # # Aggregate total count of events matched to risk objects + # total_count += risk_object_counts[risk_object_str] - # # Raise if the the number of events doesn't match the number of those matched to observables + # # Raise if the the number of events doesn't match the number of those matched to risk + # # objects # if len(events) != total_count: # raise ValidationFailed( # f"The total number of risk events {len(events)} does not match the number of " - # f"risk events we were able to match against observables ({total_count})." + # "risk events we were able to match against risk objects from the detection " + # f"({total_count})." # ) # TODO (PEX-434): implement deeper notable validation @@ -783,11 +784,11 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = Fa ) self.update_pbar(TestingStates.PRE_CLEANUP) if self.risk_event_exists(): - self.logger.warn( + self.logger.warning( f"Risk events matching '{self.name}' already exist; marking for deletion" ) if self.notable_event_exists(): - self.logger.warn( + self.logger.warning( f"Notable events matching '{self.name}' already exist; marking for deletion" ) self.cleanup() @@ -934,11 +935,11 @@ def _search(self, query: str) -> ResultIterator: :param query: the SPL string to run """ self.logger.debug(f"Executing query: `{query}`") - job = self.service.search(query, exec_mode="blocking") + job = self.service.search(query, exec_mode="blocking") # type: ignore # query the results, catching any HTTP status code errors try: - response_reader: ResponseReader = job.results(output_mode="json") + response_reader: ResponseReader = job.results(output_mode="json") # type: ignore except HTTPError as e: # e.g. -> HTTP 400 Bad Request -- b'{"messages":[{"type":"FATAL","text":"Error in \'delete\' command: You # have insufficient privileges to delete events."}]}' @@ -946,7 +947,7 @@ def _search(self, query: str) -> ResultIterator: self.logger.error(message) raise ServerError(message) - return ResultIterator(response_reader) + return ResultIterator(response_reader) # type: ignore def _delete_index(self, index: str) -> None: """Deletes events in a given index @@ -979,7 +980,7 @@ def _delete_index(self, index: str) -> None: message = f"No result returned showing deletion in index {index}" raise ServerError(message) - def cleanup(self, delete_test_index=False) -> None: + def cleanup(self, delete_test_index: bool = False) -> None: """Cleans up after an integration test First, disable the detection; then dump the risk, notable, and (optionally) test indexes. The test index is diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index c30dc453..2a549468 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -42,13 +42,17 @@ class DetectionTags(BaseModel): analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) group: list[str] = [] + + # TODO (cmcginley): should confidence, impact and the risk_score property be removed? confidence: NonNegativeInt = Field(...,le=100) impact: NonNegativeInt = Field(...,le=100) @computed_field @property def risk_score(self) -> int: return round((self.confidence * self.impact)/100) - + + # TODO (cmcginley): @ljstella what's happening w/ severity, given it's use of the top-level + # risk_score? @computed_field @property def severity(self)->RiskSeverity: @@ -69,6 +73,7 @@ def severity(self)->RiskSeverity: mitre_attack_id: List[MITRE_ATTACK_ID_TYPE] = [] nist: list[NistCategory] = [] + # TODO (cmcginley): observable should be removed as well, yes? # TODO (#249): Add pydantic validator to ensure observables are unique within a detection observable: List[Observable] = [] product: list[SecurityContentProductName] = Field(..., min_length=1) @@ -76,7 +81,6 @@ def severity(self)->RiskSeverity: security_domain: SecurityDomain = Field(...) cve: List[CVE_TYPE] = [] atomic_guid: List[AtomicTest] = [] - # enrichment mitre_attack_enrichments: List[MitreAttackEnrichment] = Field([], validate_default=True) @@ -144,6 +148,7 @@ def cis20(self) -> list[Cis18Value]: # ) # return v + # TODO (cmcginley): @ljstella removing risk_score and severity from serialization? @model_serializer def serialize_model(self): # Since this field has no parent, there is no need to call super() serialization function diff --git a/contentctl/objects/drilldown.py b/contentctl/objects/drilldown.py index 3fe41e7c..b5604748 100644 --- a/contentctl/objects/drilldown.py +++ b/contentctl/objects/drilldown.py @@ -23,6 +23,7 @@ class Drilldown(BaseModel): "but it is NOT the default value and must be supplied explicitly.", min_length= 1) + # TODO (cmcginley): @ljstella the drilldowns will need to be updated @classmethod def constructDrilldownsFromDetection(cls, detection: Detection) -> list[Drilldown]: victim_observables = [o for o in detection.tags.observable if o.role[0] == "Victim"] diff --git a/contentctl/objects/observable.py b/contentctl/objects/observable.py index 81b04922..35eb535a 100644 --- a/contentctl/objects/observable.py +++ b/contentctl/objects/observable.py @@ -1,6 +1,7 @@ from pydantic import BaseModel, field_validator, ConfigDict from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, RBA_OBSERVABLE_ROLE_MAPPING +# TODO (cmcginley): should this class be removed? class Observable(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 1f9a6c31..ad25ab46 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -38,6 +38,7 @@ class ThreatObjectType(str, Enum): TLS_HASH = "tls_hash" URL = "url" +# TODO (cmcginley): class names should be capitalized class risk_object(BaseModel): field: str type: RiskObjectType diff --git a/contentctl/objects/risk_event.py b/contentctl/objects/risk_event.py index de98bd0b..cf9f9ea8 100644 --- a/contentctl/objects/risk_event.py +++ b/contentctl/objects/risk_event.py @@ -4,50 +4,10 @@ from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field from contentctl.objects.errors import ValidationFailed from contentctl.objects.detection import Detection -from contentctl.objects.observable import Observable - -# TODO (#259): Map our observable types to more than user/system -# TODO (#247): centralize this mapping w/ usage of SES_OBSERVABLE_TYPE_MAPPING (see -# observable.py) and the ad hoc mapping made in detection_abstract.py (see the risk property func) -TYPE_MAP: dict[str, list[str]] = { - "system": [ - "Hostname", - "IP Address", - "Endpoint" - ], - "user": [ - "User", - "User Name", - "Email Address", - "Email" - ], - "hash_values": [], - "network_artifacts": [], - "host_artifacts": [], - "tools": [], - "other": [ - "Process", - "URL String", - "Unknown", - "Process Name", - "MAC Address", - "File Name", - "File Hash", - "Resource UID", - "Uniform Resource Locator", - "File", - "Geo Location", - "Container", - "Registry Key", - "Registry Value", - "Other" - ] -} - -# Roles that should not generate risks -IGNORE_ROLES: list[str] = ["Attacker"] +from contentctl.objects.rba import risk_object as RiskObject +# TODO (cmcginley): the names below now collide a bit with Lou's new class names class RiskEvent(BaseModel): """Model for risk event in ES""" @@ -79,11 +39,11 @@ class RiskEvent(BaseModel): ) # Contributing events search query (we use this to derive the corresponding field from the - # observables) + # detection's risk object definition) contributing_events_search: str - # Private attribute caching the observable this RiskEvent is mapped to - _matched_observable: Observable | None = PrivateAttr(default=None) + # Private attribute caching the risk object this RiskEvent is mapped to + _matched_risk_object: RiskObject | None = PrivateAttr(default=None) # Allowing fields that aren't explicitly defined to be passed since some of the risk event's # fields vary depending on the SPL which generated them @@ -108,7 +68,7 @@ def _convert_str_value_to_singleton(cls, v: str | list[str]) -> list[str]: def source_field_name(self) -> str: """ A cached derivation of the source field name the risk event corresponds to in the relevant - event(s). Useful for mapping back to an observable in the detection. + event(s). Useful for mapping back to a risk object in the detection. """ pattern = re.compile( r"\| savedsearch \"" + self.search_name + r"\" \| search (?P[^=]+)=.+" @@ -128,13 +88,6 @@ def validate_against_detection(self, detection: Detection) -> None: :param detection: the detection associated w/ this risk event :raises: ValidationFailed """ - # Check risk_score - if self.risk_score != detection.tags.risk_score: - raise ValidationFailed( - f"Risk score observed in risk event ({self.risk_score}) does not match risk score in " - f"detection ({detection.tags.risk_score})." - ) - # Check analyticstories self.validate_analyticstories(detection) @@ -151,8 +104,15 @@ def validate_against_detection(self, detection: Detection) -> None: # Check risk_message self.validate_risk_message(detection) - # Check several conditions against the observables - self.validate_risk_against_observables(detection.tags.observable) + # Ensure the rba object is defined + if detection.rba is None: + raise ValidationFailed( + f"Unexpected error: Detection '{detection.name}' has no RBA objects associated " + "with it; cannot validate." + ) + + # Check several conditions against the detection's risk objects + self.validate_risk_against_risk_objects(detection.rba.risk_objects) def validate_mitre_ids(self, detection: Detection) -> None: """ @@ -180,15 +140,25 @@ def validate_analyticstories(self, detection: Detection) -> None: f" in detection ({detection.tags.analytic_story})." ) + # TODO (cmcginley): all of this type checking is a good use case (potentially) for subtyping + # detections by detection type, instead of having types as enums; could have an EBD subtype + # for any detections that should produce risk so that rba is never None def validate_risk_message(self, detection: Detection) -> None: """ Given the associated detection, validate the risk event's message :param detection: the detection associated w/ this risk event :raises: ValidationFailed """ + # Ensure the rba object is defined + if detection.rba is None: + raise ValidationFailed( + f"Unexpected error: Detection '{detection.name}' has no RBA objects associated " + "with it; cannot validate." + ) + # Extract the field replacement tokens ("$...$") field_replacement_pattern = re.compile(r"\$\S+\$") - tokens = field_replacement_pattern.findall(detection.tags.message) + tokens = field_replacement_pattern.findall(detection.rba.message) # Check for the presence of each token in the message from the risk event for token in tokens: @@ -205,7 +175,7 @@ def validate_risk_message(self, detection: Detection) -> None: escaped_source_message_with_placeholder: str = re.escape( field_replacement_pattern.sub( tmp_placeholder, - detection.tags.message + detection.rba.message ) ) placeholder_replacement_pattern = re.compile(tmp_placeholder) @@ -221,114 +191,91 @@ def validate_risk_message(self, detection: Detection) -> None: raise ValidationFailed( "Risk message in event does not match the pattern set by the detection. Message in " f"risk event: \"{self.risk_message}\". Message in detection: " - f"\"{detection.tags.message}\"." + f"\"{detection.rba.message}\"." ) - def validate_risk_against_observables(self, observables: list[Observable]) -> None: + def validate_risk_against_risk_objects(self, risk_objects: set[RiskObject]) -> None: """ - Given the observables from the associated detection, validate the risk event against those - observables - :param observables: the Observable objects from the detection + Given the risk objects from the associated detection, validate the risk event against those + risk objects + :param risk_objects: the risk objects from the detection :raises: ValidationFailed """ - # Get the matched observable; will raise validation errors if no match can be made or if - # risk is missing values associated w/ observables - matched_observable = self.get_matched_observable(observables) + # Get the matched risk object; will raise validation errors if no match can be made or if + # risk is missing values associated w/ risk objects + matched_risk_object = self.get_matched_risk_object(risk_objects) - # The risk object type should match our mapping of observable types to risk types - expected_type = RiskEvent.observable_type_to_risk_type(matched_observable.type) - if self.risk_object_type != expected_type: + # The risk object type from the risk event should match our mapping of internal risk object + # types + if self.risk_object_type != matched_risk_object.type.value: raise ValidationFailed( - f"The risk object type ({self.risk_object_type}) does not match the expected type " - f"based on the matched observable ({matched_observable.type}->{expected_type}): " - f"risk=(object={self.risk_object}, type={self.risk_object_type}, " - f"source_field_name={self.source_field_name}), " - f"observable=(name={matched_observable.name}, type={matched_observable.type}, " - f"role={matched_observable.role})" + f"The risk object type from the risk event ({self.risk_object_type}) does not match" + " the expected type based on the matched risk object " + f"({matched_risk_object.type.value}): risk event=(object={self.risk_object}, " + f"type={self.risk_object_type}, source_field_name={self.source_field_name}), " + f"risk object=(name={matched_risk_object.field}, " + f"type={matched_risk_object.type.value})" ) - @staticmethod - def observable_type_to_risk_type(observable_type: str) -> str: - """ - Given a string representing the observable type, use our mapping to convert it to the - expected type in the risk event - :param observable_type: the type of the observable - :returns: a string (the risk object type) - :raises ValueError: if the observable type has not yet been mapped to a risk object type - """ - # Iterate over the map and search the lists for a match - for risk_type in TYPE_MAP: - if observable_type in TYPE_MAP[risk_type]: - return risk_type - - raise ValueError( - f"Observable type {observable_type} does not have a mapping to a risk type in TYPE_MAP" - ) + # Check risk_score + if self.risk_score != matched_risk_object.score: + raise ValidationFailed( + f"Risk score observed in risk event ({self.risk_score}) does not match risk score in " + f"matched risk object from detection ({matched_risk_object.score})." + ) - @staticmethod - def ignore_observable(observable: Observable) -> bool: - """ - Given an observable, determine based on its roles if it should be ignored in risk/observable - matching (e.g. Attacker role observables should not generate risk events) - :param observable: the Observable object we are checking the roles of - :returns: a bool indicating whether this observable should be ignored or not + def get_matched_risk_object(self, risk_objects: set[RiskObject]) -> RiskObject: """ - ignore = False - for role in observable.role: - if role in IGNORE_ROLES: - ignore = True - break - return ignore - - def get_matched_observable(self, observables: list[Observable]) -> Observable: - """ - Given a list of observables, return the one this risk event matches - :param observables: the list of Observable objects we are checking against - :returns: the matched Observable object + Given a set of risk objects, return the one this risk event matches + :param risk_objects: the list of risk objects we are checking against + :returns: the matched risk object :raises ValidationFailed: if a match could not be made or if an expected field (based on - one of the observables) could not be found in the risk event + one of the risk objects) could not be found in the risk event """ # Return the cached match if already found - if self._matched_observable is not None: - return self._matched_observable + if self._matched_risk_object is not None: + return self._matched_risk_object - matched_observable: Observable | None = None + matched_risk_object: RiskObject | None = None # Iterate over the obervables and check for a match - for observable in observables: + for risk_object in risk_objects: # TODO (#252): Refactor and re-enable per-field validation of risk events - # Each the field name used in each observable shoud be present in the risk event - # if not hasattr(self, observable.name): + # Each the field name used in each risk object shoud be present in the risk event + # if not hasattr(self, risk_object.field): # raise ValidationFailed( - # f"Observable field \"{observable.name}\" not found in risk event." + # f"Risk object field \"{risk_object.field}\" not found in risk event." # ) - # Try to match the risk_object against a specific observable for the obervables with - # a valid role (some, like Attacker, shouldn't get converted to risk events) - if self.source_field_name == observable.name: - if matched_observable is not None: + # Try to match the risk_object against a specific risk object + if self.source_field_name == risk_object.field: + if matched_risk_object is not None: raise ValueError( "Unexpected conditon: we don't expect the source event field " - "corresponding to an observables field name to be repeated." + "corresponding to an risk object's field name to be repeated." ) - # Report any risk events we find that shouldn't be there - if RiskEvent.ignore_observable(observable): - raise ValidationFailed( - "Risk event matched an observable with an invalid role: " - f"(name={observable.name}, type={observable.type}, role={observable.role})") - # NOTE: we explicitly do not break early as we want to check each observable - matched_observable = observable + # TODO (cmcginley): risk objects and threat objects are now in separate sets, + # so I think we can safely eliminate this check entirely? + # # Report any risk events we find that shouldn't be there + # if RiskEvent.ignore_observable(observable): + # raise ValidationFailed( + # "Risk event matched an observable with an invalid role: " + # f"(name={observable.name}, type={observable.type}, role={observable.role})") + + # NOTE: we explicitly do not break early as we want to check each risk object + matched_risk_object = risk_object - # Ensure we were able to match the risk event to a specific observable - if matched_observable is None: + # Ensure we were able to match the risk event to a specific risk object + if matched_risk_object is None: raise ValidationFailed( f"Unable to match risk event (object={self.risk_object}, type=" - f"{self.risk_object_type}, source_field_name={self.source_field_name}) to an " - "observable; please check for errors in the observable roles/types for this " - "detection, as well as the risk event build process in contentctl." + f"{self.risk_object_type}, source_field_name={self.source_field_name}) to a " + "risk object in the detection; please check for errors in the risk object types for this " + "detection, as well as the risk event build process in contentctl (e.g. threat " + "objects aren't being converted to risk objects somehow)." ) - # Cache and return the matched observable - self._matched_observable = matched_observable - return self._matched_observable + # Cache and return the matched risk object + self._matched_risk_object = matched_risk_object + return self._matched_risk_object From 901415f7853923cf18b9ae6981409b6e02433bf6 Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 16 Jan 2025 12:08:12 -0600 Subject: [PATCH 105/115] Change testing branch --- .github/workflows/test_against_escu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index b3f43434..f29e6a6f 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -35,7 +35,7 @@ jobs: with: path: security_content repository: splunk/security_content - ref: strict_yml_from_rba + ref: rba_migration #Install the given version of Python we will test against - name: Install Required Python Version From da61571948870c8e39343135349cdaba76fddaaf Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 16 Jan 2025 12:59:52 -0600 Subject: [PATCH 106/115] Update template detection --- .../templates/detections/endpoint/anomalous_usage_of_7zip.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml index ad704b35..3eea8300 100644 --- a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +++ b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml @@ -58,8 +58,6 @@ tags: analytic_story: - Cobalt Strike asset_type: Endpoint - confidence: 80 - impact: 80 mitre_attack_id: - T1560.001 - T1560 From 72c51a4c2487c799ad7e859eeb760f1b11cb91b1 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 16 Jan 2025 14:29:06 -0800 Subject: [PATCH 107/115] cleanup; log fixes --- .../detection_abstract.py | 1 - contentctl/objects/correlation_search.py | 36 ++++++++----------- contentctl/objects/detection_tags.py | 8 ++--- contentctl/objects/risk_event.py | 11 ++++-- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index dbaa37e5..8dcf01ae 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -279,7 +279,6 @@ def source(self) -> str: deployment: Deployment = Field({}) - # TODO (cmcginley): @ljstella removing the refs to confidence and impact? @computed_field @property def annotations(self) -> dict[str, Union[List[str], int, str]]: diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index b0cffe4e..b8bc4249 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -31,8 +31,9 @@ from contentctl.objects.notable_event import NotableEvent +# TODO (cmcginley): disable logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = False +ENABLE_LOGGING = True LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" @@ -652,15 +653,8 @@ def validate_risk_events(self) -> None: f"Unexpected error: Detection '{self.detection.name}' has no RBA objects associated" " with it; cannot validate." ) - risk_object_counts: dict[str, int] = {str(x): 0 for x in self.detection.rba.risk_objects} - # NOTE: we intentionally want this to be an error state and not a failure state, as - # ultimately this validation should be handled during the build process - if len(self.detection.rba.risk_objects) != len(risk_object_counts): - raise ClientError( - f"At least two risk objects in '{self.detection.name}' have the same name; " - "each risk object for a detection should be unique." - ) + risk_object_counts: dict[int, int] = {id(x): 0 for x in self.detection.rba.risk_objects} # Get the risk events; note that we use the cached risk events, expecting they were # saved by a prior call to risk_event_exists @@ -681,20 +675,20 @@ def validate_risk_events(self) -> None: self.logger.debug( f"Matched risk event (object={event.risk_object}, type={event.risk_object_type}) " f"to detection's risk object (name={matched_risk_object.field}, " - f"type={matched_risk_object.type.value} using the source field " + f"type={matched_risk_object.type.value}) using the source field " f"'{event.source_field_name}'" ) - risk_object_counts[str(matched_risk_object)] += 1 + risk_object_counts[id(matched_risk_object)] += 1 # Report any risk objects which did not have at least one match to a risk event for risk_object in self.detection.rba.risk_objects: self.logger.debug( f"Matched risk object (name={risk_object.field}, type={risk_object.type.value} " - f"to {risk_object_counts[str(risk_object)]} risk events." + f"to {risk_object_counts[id(risk_object)]} risk events." ) - if risk_object_counts[str(risk_object)] == 0: + if risk_object_counts[id(risk_object)] == 0: raise ValidationFailed( - f"Risk object (name={risk_object.field}, type={risk_object.type.value} " + f"Risk object (name={risk_object.field}, type={risk_object.type.value}) " "was not matched to any risk events." ) @@ -703,26 +697,26 @@ def validate_risk_events(self) -> None: # relevant risk object, and the total count should match the total number of events # individual_count: int | None = None # total_count = 0 - # for risk_object_str in risk_object_counts: + # for risk_object_id in risk_object_counts: # self.logger.debug( - # f"Risk object <{risk_object_str}> match count: {risk_object_counts[risk_object_str]}" + # f"Risk object <{risk_object_id}> match count: {risk_object_counts[risk_object_id]}" # ) # # Grab the first value encountered if not set yet # if individual_count is None: - # individual_count = risk_object_counts[risk_object_str] + # individual_count = risk_object_counts[risk_object_id] # else: # # Confirm that the count for the current risk object matches the count of the # # others - # if risk_object_counts[risk_object_str] != individual_count: + # if risk_object_counts[risk_object_id] != individual_count: # raise ValidationFailed( - # f"Count of risk events matching detection's risk object <\"{risk_object_str}\"> " - # f"({risk_object_counts[risk_object_str]}) does not match the count of those " + # f"Count of risk events matching detection's risk object <\"{risk_object_id}\"> " + # f"({risk_object_counts[risk_object_id]}) does not match the count of those " # f"matching other risk objects ({individual_count})." # ) # # Aggregate total count of events matched to risk objects - # total_count += risk_object_counts[risk_object_str] + # total_count += risk_object_counts[risk_object_id] # # Raise if the the number of events doesn't match the number of those matched to risk # # objects diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index 42800a92..aea02bfe 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -4,8 +4,6 @@ from pydantic import ( BaseModel, Field, - NonNegativeInt, - PositiveInt, computed_field, UUID4, HttpUrl, @@ -34,6 +32,7 @@ from contentctl.objects.atomic import AtomicEnrichment, AtomicTest from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE, CVE_TYPE + class DetectionTags(BaseModel): # detection spec @@ -41,7 +40,7 @@ class DetectionTags(BaseModel): analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) group: list[str] = [] - + mitre_attack_id: List[MITRE_ATTACK_ID_TYPE] = [] nist: list[NistCategory] = [] @@ -84,7 +83,7 @@ def cis20(self) -> list[Cis18Value]: # TODO (#268): Validate manual_test has length > 0 if not None manual_test: Optional[str] = None - + # The following validator is temporarily disabled pending further discussions # @validator('message') # def validate_message(cls,v,values): @@ -117,7 +116,6 @@ def cis20(self) -> list[Cis18Value]: # ) # return v - # TODO (cmcginley): @ljstella removing risk_score and severity from serialization? @model_serializer def serialize_model(self): # Since this field has no parent, there is no need to call super() serialization function diff --git a/contentctl/objects/risk_event.py b/contentctl/objects/risk_event.py index cf9f9ea8..5daa722b 100644 --- a/contentctl/objects/risk_event.py +++ b/contentctl/objects/risk_event.py @@ -137,7 +137,7 @@ def validate_analyticstories(self, detection: Detection) -> None: if sorted(self.analyticstories) != sorted(detection_analytic_story): raise ValidationFailed( f"Analytic stories in risk event ({self.analyticstories}) do not match those" - f" in detection ({detection.tags.analytic_story})." + f" in detection ({[x.name for x in detection.tags.analytic_story]})." ) # TODO (cmcginley): all of this type checking is a good use case (potentially) for subtyping @@ -160,6 +160,9 @@ def validate_risk_message(self, detection: Detection) -> None: field_replacement_pattern = re.compile(r"\$\S+\$") tokens = field_replacement_pattern.findall(detection.rba.message) + # TODO (cmcginley): could expand this to get the field values from the raw events and check + # to see that allexpected strings ARE in the risk message (as opposed to checking only + # that unexpected strings aren't) # Check for the presence of each token in the message from the risk event for token in tokens: if token in self.risk_message: @@ -249,10 +252,12 @@ def get_matched_risk_object(self, risk_objects: set[RiskObject]) -> RiskObject: # Try to match the risk_object against a specific risk object if self.source_field_name == risk_object.field: + # TODO (cmcginley): this should be enforced as part of build validation if matched_risk_object is not None: raise ValueError( - "Unexpected conditon: we don't expect the source event field " - "corresponding to an risk object's field name to be repeated." + "Unexpected conditon: we don't expect multiple risk objects to use the " + "same field name, so we should not be able match the risk event to " + "multiple risk objects." ) # TODO (cmcginley): risk objects and threat objects are now in separate sets, From f72c7962801aaf3b94a45e848ec14dda5ebc0d43 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 16 Jan 2025 14:45:18 -0800 Subject: [PATCH 108/115] resolving some todos --- contentctl/objects/correlation_search.py | 7 ++--- contentctl/objects/rba.py | 1 - contentctl/objects/risk_event.py | 37 +++++++++--------------- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index b8bc4249..9ce66a76 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -31,9 +31,8 @@ from contentctl.objects.notable_event import NotableEvent -# TODO (cmcginley): disable logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = True +ENABLE_LOGGING = False LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" @@ -665,7 +664,7 @@ def validate_risk_events(self) -> None: for event in events: c += 1 self.logger.debug( - f"Validating risk event ({event.risk_object}, {event.risk_object_type}): " + f"Validating risk event ({event.es_risk_object}, {event.es_risk_object_type}): " f"{c}/{len(events)}" ) event.validate_against_detection(self.detection) @@ -673,7 +672,7 @@ def validate_risk_events(self) -> None: # Update risk object count based on match matched_risk_object = event.get_matched_risk_object(self.detection.rba.risk_objects) self.logger.debug( - f"Matched risk event (object={event.risk_object}, type={event.risk_object_type}) " + f"Matched risk event (object={event.es_risk_object}, type={event.es_risk_object_type}) " f"to detection's risk object (name={matched_risk_object.field}, " f"type={matched_risk_object.type.value}) using the source field " f"'{event.source_field_name}'" diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 3823ba8c..96294d9a 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -40,7 +40,6 @@ class ThreatObjectType(str, Enum): TLS_HASH = "tls_hash" URL = "url" -# TODO (cmcginley): class names should be capitalized class risk_object(BaseModel): field: str type: RiskObjectType diff --git a/contentctl/objects/risk_event.py b/contentctl/objects/risk_event.py index 5daa722b..24560d2b 100644 --- a/contentctl/objects/risk_event.py +++ b/contentctl/objects/risk_event.py @@ -7,7 +7,6 @@ from contentctl.objects.rba import risk_object as RiskObject -# TODO (cmcginley): the names below now collide a bit with Lou's new class names class RiskEvent(BaseModel): """Model for risk event in ES""" @@ -15,10 +14,12 @@ class RiskEvent(BaseModel): search_name: str # The subject of the risk event (e.g. a username, process name, system name, account ID, etc.) - risk_object: int | str + # (not to be confused w/ the risk object from the detection) + es_risk_object: int | str - # The type of the risk object (e.g. user, system, or other) - risk_object_type: str + # The type of the risk object from ES (e.g. user, system, or other) (not to be confused w/ + # the risk object from the detection) + es_risk_object_type: str # The level of risk associated w/ the risk event risk_score: int @@ -140,9 +141,6 @@ def validate_analyticstories(self, detection: Detection) -> None: f" in detection ({[x.name for x in detection.tags.analytic_story]})." ) - # TODO (cmcginley): all of this type checking is a good use case (potentially) for subtyping - # detections by detection type, instead of having types as enums; could have an EBD subtype - # for any detections that should produce risk so that rba is never None def validate_risk_message(self, detection: Detection) -> None: """ Given the associated detection, validate the risk event's message @@ -160,7 +158,7 @@ def validate_risk_message(self, detection: Detection) -> None: field_replacement_pattern = re.compile(r"\$\S+\$") tokens = field_replacement_pattern.findall(detection.rba.message) - # TODO (cmcginley): could expand this to get the field values from the raw events and check + # TODO (#346): could expand this to get the field values from the raw events and check # to see that allexpected strings ARE in the risk message (as opposed to checking only # that unexpected strings aren't) # Check for the presence of each token in the message from the risk event @@ -210,12 +208,12 @@ def validate_risk_against_risk_objects(self, risk_objects: set[RiskObject]) -> N # The risk object type from the risk event should match our mapping of internal risk object # types - if self.risk_object_type != matched_risk_object.type.value: + if self.es_risk_object_type != matched_risk_object.type.value: raise ValidationFailed( - f"The risk object type from the risk event ({self.risk_object_type}) does not match" + f"The risk object type from the risk event ({self.es_risk_object_type}) does not match" " the expected type based on the matched risk object " - f"({matched_risk_object.type.value}): risk event=(object={self.risk_object}, " - f"type={self.risk_object_type}, source_field_name={self.source_field_name}), " + f"({matched_risk_object.type.value}): risk event=(object={self.es_risk_object}, " + f"type={self.es_risk_object_type}, source_field_name={self.source_field_name}), " f"risk object=(name={matched_risk_object.field}, " f"type={matched_risk_object.type.value})" ) @@ -252,7 +250,8 @@ def get_matched_risk_object(self, risk_objects: set[RiskObject]) -> RiskObject: # Try to match the risk_object against a specific risk object if self.source_field_name == risk_object.field: - # TODO (cmcginley): this should be enforced as part of build validation + # TODO (#347): enforce that field names are not repeated across risk objects as + # part of build/validate if matched_risk_object is not None: raise ValueError( "Unexpected conditon: we don't expect multiple risk objects to use the " @@ -260,22 +259,14 @@ def get_matched_risk_object(self, risk_objects: set[RiskObject]) -> RiskObject: "multiple risk objects." ) - # TODO (cmcginley): risk objects and threat objects are now in separate sets, - # so I think we can safely eliminate this check entirely? - # # Report any risk events we find that shouldn't be there - # if RiskEvent.ignore_observable(observable): - # raise ValidationFailed( - # "Risk event matched an observable with an invalid role: " - # f"(name={observable.name}, type={observable.type}, role={observable.role})") - # NOTE: we explicitly do not break early as we want to check each risk object matched_risk_object = risk_object # Ensure we were able to match the risk event to a specific risk object if matched_risk_object is None: raise ValidationFailed( - f"Unable to match risk event (object={self.risk_object}, type=" - f"{self.risk_object_type}, source_field_name={self.source_field_name}) to a " + f"Unable to match risk event (object={self.es_risk_object}, type=" + f"{self.es_risk_object_type}, source_field_name={self.source_field_name}) to a " "risk object in the detection; please check for errors in the risk object types for this " "detection, as well as the risk event build process in contentctl (e.g. threat " "objects aren't being converted to risk objects somehow)." From 8293a6d5b90f72f865a01d64e04e0aa9ee7999de Mon Sep 17 00:00:00 2001 From: ljstella Date: Thu, 16 Jan 2025 16:52:44 -0600 Subject: [PATCH 109/115] Class name renaming --- .../detection_abstract.py | 4 ++-- contentctl/objects/rba.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 8dcf01ae..c216eb20 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -36,7 +36,7 @@ from contentctl.objects.integration_test import IntegrationTest from contentctl.objects.data_source import DataSource -from contentctl.objects.rba import rba_object +from contentctl.objects.rba import RBAObject from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.drilldown import Drilldown, DRILLDOWN_SEARCH_PLACEHOLDER @@ -68,7 +68,7 @@ class Detection_Abstract(SecurityContentObject): search: str = Field(...) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) - rba: Optional[rba_object] = Field(default=None) + rba: Optional[RBAObject] = Field(default=None) explanation: None | str = Field( default=None, exclude=True, #Don't serialize this value when dumping the object diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 96294d9a..d1581e0f 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -40,7 +40,7 @@ class ThreatObjectType(str, Enum): TLS_HASH = "tls_hash" URL = "url" -class risk_object(BaseModel): +class RiskObject(BaseModel): field: str type: RiskObjectType score: RiskScoreValue_Type @@ -48,17 +48,17 @@ class risk_object(BaseModel): def __hash__(self): return hash((self.field, self.type, self.score)) -class threat_object(BaseModel): +class ThreatObject(BaseModel): field: str type: ThreatObjectType def __hash__(self): return hash((self.field, self.type)) -class rba_object(BaseModel, ABC): +class RBAObject(BaseModel, ABC): message: str - risk_objects: Annotated[Set[risk_object], Field(min_length=1)] - threat_objects: Set[threat_object] + risk_objects: Annotated[Set[RiskObject], Field(min_length=1)] + threat_objects: Set[ThreatObject] From 51f078032bdfef6a22b799d951e204295f13ada2 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 16 Jan 2025 14:54:00 -0800 Subject: [PATCH 110/115] new class name --- contentctl/objects/risk_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/risk_event.py b/contentctl/objects/risk_event.py index 24560d2b..71ef3ed0 100644 --- a/contentctl/objects/risk_event.py +++ b/contentctl/objects/risk_event.py @@ -4,7 +4,7 @@ from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field from contentctl.objects.errors import ValidationFailed from contentctl.objects.detection import Detection -from contentctl.objects.rba import risk_object as RiskObject +from contentctl.objects.rba import RiskObject class RiskEvent(BaseModel): From c3cc5ab05dad101bb2b543241341afb4a63e33b1 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 16 Jan 2025 15:22:14 -0800 Subject: [PATCH 111/115] little bit more cleanup on lookups. prevent false positively flagging a field named hellolookup=something as a lookup. Also, enforces lookup validation in the search field of baselines. --- contentctl/objects/baseline.py | 25 +++++++++++++++++++++---- contentctl/objects/lookup.py | 13 ++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index f59c7dce..f66b5b2b 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -1,6 +1,9 @@ from __future__ import annotations -from typing import Annotated, List,Any +from typing import Annotated, List,Any, TYPE_CHECKING +if TYPE_CHECKING: + from contentctl.input.director import DirectorOutputDto + from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field from contentctl.objects.deployment import Deployment from contentctl.objects.security_content_object import SecurityContentObject @@ -9,7 +12,7 @@ from contentctl.objects.config import CustomApp - +from contentctl.objects.lookup import Lookup from contentctl.objects.constants import CONTENTCTL_MAX_SEARCH_NAME_LENGTH,CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE class Baseline(SecurityContentObject): @@ -19,10 +22,24 @@ class Baseline(SecurityContentObject): how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) tags: BaselineTags = Field(...) - + lookups: list[Lookup] = Field([], validate_default=True) # enrichment deployment: Deployment = Field({}) - + + + @field_validator('lookups', mode="before") + @classmethod + def getBaselineLookups(cls, v:list[str], info:ValidationInfo) -> list[Lookup]: + ''' + This function has been copied and renamed from the Detection_Abstract class + ''' + director:DirectorOutputDto = info.context.get("output_dto",None) + search: str | None = info.data.get("search",None) + if search is None: + raise ValueError("Search was None - is this file missing the search field?") + + lookups = Lookup.get_lookups(search, director) + return lookups def get_conf_stanza_name(self, app:CustomApp)->str: stanza_name = CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index cba13ab2..59bc5b4c 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -18,6 +18,9 @@ LOOKUPS_TO_IGNORE.add("ut_shannon_lookup") #In the URL toolbox app which is recommended for ESCU LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("cim_corporate_web_domain_lookup") #Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add("cim_corporate_email_domain_lookup") #Shipped with the Enterprise Security +LOOKUPS_TO_IGNORE.add("cim_cloud_domain_lookup") #Shipped with the Enterprise Security + LOOKUPS_TO_IGNORE.add("alexa_lookup_by_str") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("interesting_ports_lookup") #Shipped with the Asset and Identity Framework LOOKUPS_TO_IGNORE.add("asset_lookup_by_str") #Shipped with the Asset and Identity Framework @@ -89,18 +92,18 @@ def match_type_to_conf_format(self)->str: @staticmethod def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: # Comprehensively match all kinds of lookups, including inputlookup and outputlookup - inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s\|]+)', text_field)) - outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s\|]+)',text_field)) - lookupsToGet = set(re.findall(r'(?:(? Date: Thu, 16 Jan 2025 20:05:12 -0600 Subject: [PATCH 112/115] Bump for 0.9.2 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 164a8deb..30df3046 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: detect-private-key - id: forbid-submodules - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.1 + rev: v0.9.2 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index c5d0a4b4..ad40dd17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [tool.poetry.group.dev.dependencies] -ruff = "^0.9.1" +ruff = "^0.9.2" [build-system] requires = ["poetry-core>=1.0.0"] From 9a29afc39014cb27326be220f8edbbdc4c025658 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 17 Jan 2025 13:38:42 -0800 Subject: [PATCH 113/115] Fix an error where a lookup that was not REALLY a lookup could be detected. This occurred in a rare instance where a search was looking for using of the literal 'lookup *' --- contentctl/objects/lookup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 59bc5b4c..8d42c55e 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -92,9 +92,9 @@ def match_type_to_conf_format(self)->str: @staticmethod def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: # Comprehensively match all kinds of lookups, including inputlookup and outputlookup - inputLookupsToGet = set(re.findall(r'[^\w]inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s\|]+)', text_field, re.IGNORECASE)) - outputLookupsToGet = set(re.findall(r'[^\w]outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s\|]+)',text_field,re.IGNORECASE)) - lookupsToGet = set(re.findall(r'[^\w](?:(? Date: Fri, 17 Jan 2025 15:24:28 -0800 Subject: [PATCH 114/115] write all objects to conf file at once instead of one at a time. this reverts the behavior to what it used to be like. also, fix the filename written out in transforms.j2 for a file-backed lookup such that it contains the datetime stamp of the modifications. finally, ruff reformatted the conf_writer.py file, so lots of formatting changes. --- contentctl/output/conf_writer.py | 335 +++++++++++++--------- contentctl/output/templates/transforms.j2 | 4 +- 2 files changed, 203 insertions(+), 136 deletions(-) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 31b776b2..bcfb6d19 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -1,16 +1,17 @@ -from typing import Any, Sequence +import configparser import datetime -import re -import os import json -import configparser -from xmlrpc.client import APPLICATION_ERROR -from jinja2 import Environment, FileSystemLoader, StrictUndefined +import os import pathlib -from contentctl.objects.security_content_object import SecurityContentObject -from contentctl.objects.dashboard import Dashboard -from contentctl.objects.config import build, CustomApp +import re import xml.etree.ElementTree as ET +from typing import Any, Sequence + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from contentctl.objects.config import CustomApp, build +from contentctl.objects.dashboard import Dashboard +from contentctl.objects.security_content_object import SecurityContentObject # This list is not exhaustive of all default conf files, but should be # sufficient for our purposes. @@ -82,59 +83,68 @@ "workload_rules.conf", ] -class ConfWriter(): +class ConfWriter: @staticmethod - def custom_jinja2_enrichment_filter(string:str, object:SecurityContentObject): + def custom_jinja2_enrichment_filter(string: str, object: SecurityContentObject): substitutions = re.findall(r"%[^%]*%", string) updated_string = string for sub in substitutions: - sub_without_percents = sub.replace("%","") + sub_without_percents = sub.replace("%", "") if hasattr(object, sub_without_percents): - updated_string = updated_string.replace(sub, str(getattr(object, sub_without_percents))) - elif hasattr(object,'tags') and hasattr(object.tags, sub_without_percents): - updated_string = updated_string.replace(sub, str(getattr(object.tags, sub_without_percents))) + updated_string = updated_string.replace( + sub, str(getattr(object, sub_without_percents)) + ) + elif hasattr(object, "tags") and hasattr(object.tags, sub_without_percents): + updated_string = updated_string.replace( + sub, str(getattr(object.tags, sub_without_percents)) + ) else: raise Exception(f"Unable to find field {sub} in object {object.name}") - + return updated_string - + @staticmethod - def escapeNewlines(obj:Any): + def escapeNewlines(obj: Any): # Ensure that any newlines that occur in a string are escaped with a \. # Failing to do so will result in an improperly formatted conf files that # cannot be parsed - if isinstance(obj,str): - # Remove leading and trailing characters. Conf parsers may erroneously - # Parse fields if they have leading or trailing newlines/whitespace and we + if isinstance(obj, str): + # Remove leading and trailing characters. Conf parsers may erroneously + # Parse fields if they have leading or trailing newlines/whitespace and we # probably don't want that anyway as it doesn't look good in output - return obj.strip().replace(f"\n"," \\\n") + return obj.strip().replace("\n", " \\\n") else: return obj - @staticmethod - def writeConfFileHeader(app_output_path:pathlib.Path, config: build) -> pathlib.Path: - output = ConfWriter.writeFileHeader(app_output_path, config) - - output_path = config.getPackageDirectoryPath()/app_output_path + def writeConfFileHeader( + app_output_path: pathlib.Path, config: build + ) -> pathlib.Path: + output = ConfWriter.writeFileHeader(app_output_path, config) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "w") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) - #Ensure that the conf file we just generated/update is syntactically valid - ConfWriter.validateConfFile(output_path) + # Ensure that the conf file we just generated/update is syntactically valid + ConfWriter.validateConfFile(output_path) return output_path @staticmethod - def getCustomConfFileStems(config:build)->list[str]: + def getCustomConfFileStems(config: build) -> list[str]: # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if # they are custom conf files - default_path = config.getPackageDirectoryPath()/"default" + default_path = config.getPackageDirectoryPath() / "default" conf_files = default_path.glob("*.conf") - - custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES] + + custom_conf_file_stems = [ + conf_file.stem + for conf_file in conf_files + if conf_file.name not in DEFAULT_CONF_FILES + ] return sorted(custom_conf_file_stems) @staticmethod @@ -145,16 +155,17 @@ def writeServerConf(config: build) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config)) - - output_path = config.getPackageDirectoryPath()/app_output_path + output = template.render( + custom_conf_files=ConfWriter.getCustomConfFileStems(config) + ) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) return output_path - @staticmethod def writeAppConf(config: build) -> pathlib.Path: app_output_path = pathlib.Path("default/app.conf") @@ -163,144 +174,195 @@ def writeAppConf(config: build) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config), - app=config.app) - - output_path = config.getPackageDirectoryPath()/app_output_path + output = template.render( + custom_conf_files=ConfWriter.getCustomConfFileStems(config), app=config.app + ) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) return output_path @staticmethod - def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list[CustomApp]) -> pathlib.Path: + def writeManifestFile( + app_output_path: pathlib.Path, + template_name: str, + config: build, + objects: list[CustomApp], + ) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - - output = template.render(objects=objects, app=config.app, currentDate=datetime.datetime.now(datetime.UTC).date().isoformat()) - - output_path = config.getPackageDirectoryPath()/app_output_path + + output = template.render( + objects=objects, + app=config.app, + currentDate=datetime.datetime.now(datetime.UTC).date().isoformat(), + ) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "w") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) return output_path - - @staticmethod - def writeFileHeader(app_output_path:pathlib.Path, config: build) -> str: - #Do not output microseconds or +00:000 at the end of the datetime string - utc_time = datetime.datetime.now(datetime.UTC).replace(microsecond=0,tzinfo=None).isoformat() - - j2_env = Environment( - loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), - trim_blocks=True) + def writeFileHeader(app_output_path: pathlib.Path, config: build) -> str: + # Do not output microseconds or +00:000 at the end of the datetime string + utc_time = ( + datetime.datetime.now(datetime.UTC) + .replace(microsecond=0, tzinfo=None) + .isoformat() + ) - template = j2_env.get_template('header.j2') - output = template.render(time=utc_time, author=' - '.join([config.app.author_name,config.app.author_company]), author_email=config.app.author_email) - - return output + j2_env = Environment( + loader=FileSystemLoader( + os.path.join(os.path.dirname(__file__), "templates") + ), + trim_blocks=True, + ) + template = j2_env.get_template("header.j2") + output = template.render( + time=utc_time, + author=" - ".join([config.app.author_name, config.app.author_company]), + author_email=config.app.author_email, + ) + return output @staticmethod - def writeXmlFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list[str]) -> None: - - + def writeXmlFile( + app_output_path: pathlib.Path, + template_name: str, + config: build, + objects: list[str], + ) -> None: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - + output = template.render(objects=objects, app=config.app) - - output_path = config.getPackageDirectoryPath()/app_output_path + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) - - #Ensure that the conf file we just generated/update is syntactically valid - ConfWriter.validateXmlFile(output_path) - + # Ensure that the conf file we just generated/update is syntactically valid + ConfWriter.validateXmlFile(output_path) @staticmethod - def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.Path]: - written_files:set[pathlib.Path] = set() + def writeDashboardFiles( + config: build, dashboards: list[Dashboard] + ) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() for dashboard in dashboards: output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config) # Check that the full output path does not exist so that we are not having an # name collision with a file in app_template - if (config.getPackageDirectoryPath()/output_file_path).exists(): - raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") - + if (config.getPackageDirectoryPath() / output_file_path).exists(): + raise FileExistsError( + f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path / 'dashboards'}?" + ) + ConfWriter.writeXmlFileHeader(output_file_path, config) dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config) - ConfWriter.validateXmlFile(config.getPackageDirectoryPath()/output_file_path) + ConfWriter.validateXmlFile( + config.getPackageDirectoryPath() / output_file_path + ) written_files.add(output_file_path) return written_files - @staticmethod - def writeXmlFileHeader(app_output_path:pathlib.Path, config: build) -> None: - output = ConfWriter.writeFileHeader(app_output_path, config) + def writeXmlFileHeader(app_output_path: pathlib.Path, config: build) -> None: + output = ConfWriter.writeFileHeader(app_output_path, config) output_with_xml_comment = f"\n" - output_path = config.getPackageDirectoryPath()/app_output_path + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - output_with_xml_comment = output_with_xml_comment.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "w") as f: + output_with_xml_comment = output_with_xml_comment.encode( + "utf-8", "ignore" + ).decode("utf-8") f.write(output_with_xml_comment) - - # We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now, - # the file is an empty XML document (besides the commented header). This means that it will FAIL validation + # We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now, + # the file is an empty XML document (besides the commented header). This means that it will FAIL validation @staticmethod - def getJ2Environment()->Environment: + def getJ2Environment() -> Environment: j2_env = Environment( - loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + loader=FileSystemLoader( + os.path.join(os.path.dirname(__file__), "templates") + ), trim_blocks=True, - undefined=StrictUndefined) - j2_env.globals.update(objectListToNameList=SecurityContentObject.objectListToNameList) - - - j2_env.filters['custom_jinja2_enrichment_filter'] = ConfWriter.custom_jinja2_enrichment_filter - j2_env.filters['escapeNewlines'] = ConfWriter.escapeNewlines + undefined=StrictUndefined, + ) + j2_env.globals.update( + objectListToNameList=SecurityContentObject.objectListToNameList + ) + + j2_env.filters["custom_jinja2_enrichment_filter"] = ( + ConfWriter.custom_jinja2_enrichment_filter + ) + j2_env.filters["escapeNewlines"] = ConfWriter.escapeNewlines return j2_env @staticmethod - def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : Sequence[SecurityContentObject] | list[CustomApp]) -> pathlib.Path: - output_path = config.getPackageDirectoryPath()/app_output_path + def writeConfFile( + app_output_path: pathlib.Path, + template_name: str, + config: build, + objects: Sequence[SecurityContentObject] | list[CustomApp], + ) -> pathlib.Path: + output_path = config.getPackageDirectoryPath() / app_output_path j2_env = ConfWriter.getJ2Environment() - + template = j2_env.get_template(template_name) - outputs: list[str] = [] - for obj in objects: - try: - outputs.append(template.render(objects=[obj], app=config.app)) - except Exception as e: - raise Exception(f"Failed writing the following object to file:\n" - f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n" - f"Type {type(obj)}: \n" - f"Output File: {app_output_path}\n" - f"Error: {str(e)}\n") - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = ''.join(outputs).encode('utf-8', 'ignore').decode('utf-8') - f.write(output) + + # The following code, which is commented out, serializes one object at a time. + # This is extremely useful from a debugging perspective, because sometimes when + # serializing a large number of objects, exceptions throw in Jinja2 templates can + # be quite hard to diagnose. We leave this code in for use in debugging workflows: + SERIALIZE_ONE_AT_A_TIME = False + if SERIALIZE_ONE_AT_A_TIME: + outputs: list[str] = [] + for obj in objects: + try: + outputs.append(template.render(objects=[obj], app=config.app)) + except Exception as e: + raise Exception( + f"Failed writing the following object to file:\n" + f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n" + f"Type {type(obj)}: \n" + f"Output File: {app_output_path}\n" + f"Error: {str(e)}\n" + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "a") as f: + output = "".join(outputs).encode("utf-8", "ignore").decode("utf-8") + f.write(output) + else: + output = template.render(objects=objects, app=config.app) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") + f.write(output) + return output_path - - + @staticmethod - def validateConfFile(path:pathlib.Path): + def validateConfFile(path: pathlib.Path): """Ensure that the conf file is valid. We will do this by reading back the conf using RawConfigParser to ensure that it does not throw any parsing errors. This is particularly relevant because newlines contained in string fields may break the formatting of the conf file if they have been incorrectly escaped with - the 'ConfWriter.escapeNewlines()' function. + the 'ConfWriter.escapeNewlines()' function. If a conf file failes validation, we will throw an exception @@ -309,7 +371,7 @@ def validateConfFile(path:pathlib.Path): """ return if path.suffix != ".conf": - #there may be some other files built, so just ignore them + # there may be some other files built, so just ignore them return try: _ = configparser.RawConfigParser().read(path) @@ -317,30 +379,35 @@ def validateConfFile(path:pathlib.Path): raise Exception(f"Failed to validate .conf file {str(path)}: {str(e)}") @staticmethod - def validateXmlFile(path:pathlib.Path): + def validateXmlFile(path: pathlib.Path): """Ensure that the XML file is valid XML. Args: path (pathlib.Path): path to the xml file to validate - """ - + """ + try: - with open(path, 'r') as xmlFile: + with open(path, "r") as xmlFile: _ = ET.fromstring(xmlFile.read()) except Exception as e: raise Exception(f"Failed to validate .xml file {str(path)}: {str(e)}") - @staticmethod - def validateManifestFile(path:pathlib.Path): + def validateManifestFile(path: pathlib.Path): """Ensure that the Manifest file is valid JSON. Args: path (pathlib.Path): path to the manifest JSON file to validate - """ + """ return try: - with open(path, 'r') as manifestFile: + with open(path, "r") as manifestFile: _ = json.load(manifestFile) except Exception as e: - raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}") + raise Exception( + f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}" + ) + except Exception as e: + raise Exception( + f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}" + ) diff --git a/contentctl/output/templates/transforms.j2 b/contentctl/output/templates/transforms.j2 index 58011189..e8ec8c4b 100644 --- a/contentctl/output/templates/transforms.j2 +++ b/contentctl/output/templates/transforms.j2 @@ -1,8 +1,8 @@ {% for lookup in objects %} [{{ lookup.name }}] -{% if lookup.filename is defined and lookup.filename != None %} -filename = {{ lookup.filename.name }} +{% if lookup.app_filename is defined and lookup.app_filename != None %} +filename = {{ lookup.app_filename.name }} {% else %} collection = {{ lookup.collection }} external_type = kvstore From 248e4362395de0343fae9722d21e8f7822d16551 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 17 Jan 2025 16:50:52 -0800 Subject: [PATCH 115/115] print giant warning about contentctl 5 being an alpha. change the version in pyproject to 5.0.0-alpha --- contentctl/contentctl.py | 193 +++++++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 115 insertions(+), 80 deletions(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 1bb21174..05d9b952 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -1,31 +1,39 @@ -import traceback +import pathlib import sys +import traceback import warnings -import pathlib + import tyro -from contentctl.actions.initialize import Initialize -from contentctl.objects.config import init, validate, build, new, deploy_acs, test, test_servers, inspect, report, test_common, release_notes -from contentctl.actions.validate import Validate -from contentctl.actions.new_content import NewContent +from contentctl.actions.build import Build, BuildInputDto, DirectorOutputDto +from contentctl.actions.deploy_acs import Deploy from contentctl.actions.detection_testing.GitService import GitService -from contentctl.actions.build import ( - BuildInputDto, - DirectorOutputDto, - Build, -) -from contentctl.actions.test import Test -from contentctl.actions.test import TestInputDto -from contentctl.actions.reporting import ReportingInputDto, Reporting +from contentctl.actions.initialize import Initialize from contentctl.actions.inspect import Inspect -from contentctl.input.yml_reader import YmlReader -from contentctl.actions.deploy_acs import Deploy +from contentctl.actions.new_content import NewContent from contentctl.actions.release_notes import ReleaseNotes +from contentctl.actions.reporting import Reporting, ReportingInputDto +from contentctl.actions.test import Test, TestInputDto +from contentctl.actions.validate import Validate +from contentctl.input.yml_reader import YmlReader +from contentctl.objects.config import ( + build, + deploy_acs, + init, + inspect, + new, + release_notes, + report, + test, + test_common, + test_servers, + validate, +) # def print_ascii_art(): # print( # """ -# Running Splunk Security Content Control Tool (contentctl) +# Running Splunk Security Content Control Tool (contentctl) # ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ # ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢶⠛⡇⠀⠀⠀⠀⠀⠀⣠⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ # ⠀⠀⠀⠀⠀⠀⠀⠀⣀⠼⠖⠛⠋⠉⠉⠓⠢⣴⡻⣾⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ @@ -53,114 +61,137 @@ # ) - - -def init_func(config:test): +def init_func(config: test): Initialize().execute(config) -def validate_func(config:validate)->DirectorOutputDto: +def validate_func(config: validate) -> DirectorOutputDto: validate = Validate() return validate.execute(config) -def report_func(config:report)->None: + +def report_func(config: report) -> None: # First, perform validation. Remember that the validate # configuration is actually a subset of the build configuration director_output_dto = validate_func(config) - - r = Reporting() - return r.execute(ReportingInputDto(director_output_dto=director_output_dto, - config=config)) - -def build_func(config:build)->DirectorOutputDto: + r = Reporting() + return r.execute( + ReportingInputDto(director_output_dto=director_output_dto, config=config) + ) + + +def build_func(config: build) -> DirectorOutputDto: # First, perform validation. Remember that the validate # configuration is actually a subset of the build configuration director_output_dto = validate_func(config) builder = Build() return builder.execute(BuildInputDto(director_output_dto, config)) -def inspect_func(config:inspect)->str: - #Make sure that we have built the most recent version of the app + +def inspect_func(config: inspect) -> str: + # Make sure that we have built the most recent version of the app _ = build_func(config) inspect_token = Inspect().execute(config) return inspect_token - -def release_notes_func(config:release_notes)->None: + +def release_notes_func(config: release_notes) -> None: ReleaseNotes().release_notes(config) -def new_func(config:new): - NewContent().execute(config) +def new_func(config: new): + NewContent().execute(config) -def deploy_acs_func(config:deploy_acs): +def deploy_acs_func(config: deploy_acs): print("Building and inspecting app...") token = inspect_func(config) print("App successfully built and inspected.") print("Deploying app...") Deploy().execute(config, token) -def test_common_func(config:test_common): + +def test_common_func(config: test_common): if type(config) == test: - #construct the container Infrastructure objects + # construct the container Infrastructure objects config.getContainerInfrastructureObjects() - #otherwise, they have already been passed as servers + # otherwise, they have already been passed as servers director_output_dto = build_func(config) - gitServer = GitService(director=director_output_dto,config=config) + gitServer = GitService(director=director_output_dto, config=config) detections_to_test = gitServer.getContent() - - test_input_dto = TestInputDto(detections_to_test, config) - + t = Test() t.filter_tests(test_input_dto) - + if config.plan_only: - #Emit the test plan and quit. Do not actually run the test - config.dumpCICDPlanAndQuit(gitServer.getHash(),test_input_dto.detections) - return - + # Emit the test plan and quit. Do not actually run the test + config.dumpCICDPlanAndQuit(gitServer.getHash(), test_input_dto.detections) + return + success = t.execute(test_input_dto) - + if success: - #Everything passed! + # Everything passed! print("All tests have run successfully or been marked as 'skipped'") return raise Exception("There was at least one unsuccessful test") + +CONTENTCTL_5_WARNING = """ +***************************************************************************** +WARNING - THIS IS AN ALPHA BUILD OF CONTENTCTL 5. +THERE HAVE BEEN NUMEROUS CHANGES IN CONTENTCTL (ESPECIALLY TO YML FORMATS). +YOU ALMOST CERTAINLY DO NOT WANT TO USE THIS BUILD. +IF YOU ENCOUNTER ERRORS, PLEASE USE THE LATEST CURRENTYLY SUPPORTED RELEASE: + +CONTENTCTL==4.4.7 + +YOU HAVE BEEN WARNED! +***************************************************************************** +""" + + def main(): + print(CONTENTCTL_5_WARNING) try: configFile = pathlib.Path("contentctl.yml") - + # We MUST load a config (with testing info) object so that we can # properly construct the command line, including 'contentctl test' parameters. if not configFile.is_file(): - if "init" not in sys.argv and "--help" not in sys.argv and "-h" not in sys.argv: - raise Exception(f"'{configFile}' not found in the current directory.\n" - "Please ensure you are in the correct directory or run 'contentctl init' to create a new content pack.") - + if ( + "init" not in sys.argv + and "--help" not in sys.argv + and "-h" not in sys.argv + ): + raise Exception( + f"'{configFile}' not found in the current directory.\n" + "Please ensure you are in the correct directory or run 'contentctl init' to create a new content pack." + ) + if "--help" in sys.argv or "-h" in sys.argv: - print("Warning - contentctl.yml is missing from this directory. The configuration values showed at the default and are informational only.\n" - "Please ensure that contentctl.yml exists by manually creating it or running 'contentctl init'") + print( + "Warning - contentctl.yml is missing from this directory. The configuration values showed at the default and are informational only.\n" + "Please ensure that contentctl.yml exists by manually creating it or running 'contentctl init'" + ) # Otherwise generate a stub config file. # It will be used during init workflow t = test() config_obj = t.model_dump() - + else: - #The file exists, so load it up! - config_obj = YmlReader().load_file(configFile,add_fields=False) + # The file exists, so load it up! + config_obj = YmlReader().load_file(configFile, add_fields=False) t = test.model_validate(config_obj) except Exception as e: print(f"Error validating 'contentctl.yml':\n{str(e)}") sys.exit(1) - - + # For ease of generating the constructor, we want to allow construction # of an object from default values WITHOUT requiring all fields to be declared # with defaults OR in the config file. As such, we construct the model rather @@ -169,22 +200,19 @@ def main(): models = tyro.extras.subcommand_type_from_defaults( { - "init":init.model_validate(config_obj), + "init": init.model_validate(config_obj), "validate": validate.model_validate(config_obj), "report": report.model_validate(config_obj), - "build":build.model_validate(config_obj), + "build": build.model_validate(config_obj), "inspect": inspect.model_construct(**t.__dict__), - "new":new.model_validate(config_obj), - "test":test.model_validate(config_obj), - "test_servers":test_servers.model_construct(**t.__dict__), + "new": new.model_validate(config_obj), + "test": test.model_validate(config_obj), + "test_servers": test_servers.model_construct(**t.__dict__), "release_notes": release_notes.model_construct(**config_obj), - "deploy_acs": deploy_acs.model_construct(**t.__dict__) + "deploy_acs": deploy_acs.model_construct(**t.__dict__), } ) - - - config = None try: # Since some model(s) were constructed and not model_validated, we have to catch @@ -192,7 +220,6 @@ def main(): with warnings.catch_warnings(action="ignore"): config = tyro.cli(models) - if type(config) == init: t.__dict__.update(config.__dict__) init_func(t) @@ -219,21 +246,29 @@ def main(): print(e) sys.exit(1) except Exception as e: + print(CONTENTCTL_5_WARNING) + if config is None: - print("There was a serious issue where the config file could not be created.\n" - "The entire stack trace is provided below (please include it if filing a bug report).\n") + print( + "There was a serious issue where the config file could not be created.\n" + "The entire stack trace is provided below (please include it if filing a bug report).\n" + ) traceback.print_exc() elif config.verbose: - print("Verbose error logging is ENABLED.\n" - "The entire stack trace has been provided below (please include it if filing a bug report):\n") + print( + "Verbose error logging is ENABLED.\n" + "The entire stack trace has been provided below (please include it if filing a bug report):\n" + ) traceback.print_exc() else: - print("Verbose error logging is DISABLED.\n" - "Please use the --verbose command line argument if you need more context for your error or file a bug report.") + print( + "Verbose error logging is DISABLED.\n" + "Please use the --verbose command line argument if you need more context for your error or file a bug report." + ) print(e) - + sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/pyproject.toml b/pyproject.toml index ad40dd17..7d15db05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contentctl" -version = "5.0.0" +version = "5.0.0-alpha" description = "Splunk Content Control Tool" authors = ["STRT "]