Merge branch 'master' into magma

.github/workflows/publish_docker_image.yml

@@ -21,7 +21,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        with:
+          submodules: recursive
 
       - name: Log in to the Container registry
         uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc

.github/workflows/quality.yml

@@ -21,8 +21,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: 3.7
-            toxenv: py37,style,coverage-ci
           - python-version: 3.8
             toxenv: py38,style,coverage-ci
           - python-version: 3.9

.github/workflows/security.yml

@@ -12,14 +12,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: 3.7
-            toxenv: safety
           - python-version: 3.8
             toxenv: safety
           - python-version: 3.9
             toxenv: safety
           - python-version: 3.10.9
             toxenv: safety
+          - python-version: 3.11
+            toxenv: safety
 
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9

.github/workflows/stale.yml

@@ -22,8 +22,8 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-label: 'no-issue-activity'
         stale-pr-label: 'no-pr-activity'
-        stale-pr-message: 'This issue is stale because it has been open 20 days with no activity. Remove stale label or comment or this will be closed in 5 days'
-        stale-issue-message: 'This issue is stale because it has been open 20 days with no activity. Remove stale label or comment or this will be closed in 5 days'
+        stale-pr-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days'
+        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days'
         exempt-issue-labels: 'feature,keep'
-        days-before-stale: 20
-        days-before-close: 5
+        days-before-stale: 30
+        days-before-close: 7

.readthedocs.yaml

@@ -0,0 +1,29 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: plugins/fieldmanual/sphinx-docs/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: requirements.txt
+
+# Build all formats (incl. pdf, epub)
+formats: all
+
+# Include all submodules
+submodules:
+  include: all
+  recursive: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:latest
+FROM ubuntu:23.04
 SHELL ["/bin/bash", "-c"]
 
 ARG TZ="UTC"
@@ -12,25 +12,31 @@ ADD . .
 RUN if [ -z "$(ls plugins/stockpile)" ]; then echo "stockpile plugin not downloaded - please ensure you recursively cloned the caldera git repository and try again."; exit 1; fi
 
 RUN apt-get update && \
-    apt-get -y install python3 python3-pip git curl
+    apt-get -y install python3 python3-pip python3-venv git curl golang-go
+
 
 #WIN_BUILD is used to enable windows build in sandcat plugin
 ARG WIN_BUILD=false
 RUN if [ "$WIN_BUILD" = "true" ] ; then apt-get -y install mingw-w64; fi
 
+# Set up python virtualenv
+ENV VIRTUAL_ENV=/opt/venv/caldera
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 # Install pip requirements
 RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Set up config file and disable atomic by default
-RUN grep -v "\- atomic" conf/default.yml > conf/local.yml
-
-# Install golang
-RUN curl -L https://go.dev/dl/go1.17.6.linux-amd64.tar.gz -o go1.17.6.linux-amd64.tar.gz
-RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz;
-ENV PATH="${PATH}:/usr/local/go/bin"
-RUN go version;
+RUN python3 -c "import app; import app.utility.config_generator; app.utility.config_generator.ensure_local_config();"; \
+    sed -i '/\- atomic/d' conf/local.yml;
 
 # Compile default sandcat agent binaries, which will download basic golang dependencies.
+
+# Install Go dependencies
+WORKDIR /usr/src/app/plugins/sandcat/gocat
+RUN go mod tidy && go mod download
+
 WORKDIR /usr/src/app/plugins/sandcat
 
 # Fix line ending error that can be caused by cloning the project in a Windows environment
@@ -60,6 +66,15 @@ RUN if [ ! -d "/usr/src/app/plugins/atomic/data/atomic-red-team" ]; then   \
         /usr/src/app/plugins/atomic/data/atomic-red-team;                  \
 fi
 
+WORKDIR /usr/src/app/plugins/emu
+
+# If emu is enabled, complete necessary installation steps
+RUN if [ $(grep -c "\- emu" ../../conf/local.yml)  ]; then \
+    apt-get -y install zlib1g unzip;                \
+    pip3 install -r requirements.txt;               \
+    ./download_payloads.sh;                         \
+fi
+
 WORKDIR /usr/src/app
 
 # Default HTTP port for web interface and agent beacons over HTTP

README.md

@@ -1,5 +1,5 @@
 [![Release](https://img.shields.io/badge/dynamic/json?color=blue&label=Release&query=tag_name&url=https%3A%2F%2Fapi.github.com%2Frepos%2Fmitre%2Fcaldera%2Freleases%2Flatest)](https://github.com/mitre/caldera/releases/latest)
-[![Testing Status](https://github.com/mitre/caldera/actions/workflows/testing.yml/badge.svg?branch=master)](https://github.com/mitre/caldera/actions/workflows/testing.yml?query=branch%3Amaster)
+[![Testing Status](https://github.com/mitre/caldera/actions/workflows/quality.yml/badge.svg?branch=master)](https://github.com/mitre/caldera/actions/workflows/quality.yml?query=branch%3Amaster)
 [![Security Status](https://github.com/mitre/caldera/actions/workflows/security.yml/badge.svg?branch=master)](https://github.com/mitre/caldera/actions/workflows/security.yml?query=branch%3Amaster)
 [![codecov](https://codecov.io/gh/mitre/caldera/branch/master/graph/badge.svg)](https://codecov.io/gh/mitre/caldera)
 [![Documentation Status](https://readthedocs.org/projects/caldera/badge/?version=stable)](http://caldera.readthedocs.io/?badge=stable)
@@ -30,6 +30,7 @@ These plugins are supported and maintained by the Caldera team.
 - **[Access](https://github.com/mitre/access)** (red team initial access tools and techniques)
 - **[Atomic](https://github.com/mitre/atomic)** (Atomic Red Team project TTPs)
 - **[Builder](https://github.com/mitre/builder)** (dynamically compile payloads)
+- **[Caldera for OT](https://github.com/mitre/caldera-ot)** (ICS/OT capabilities for Caldera)
 - **[Compass](https://github.com/mitre/compass)** (ATT&CK visualizations)
 - **[Debrief](https://github.com/mitre/debrief)** (operations insights)
 - **[Emu](https://github.com/mitre/emu)** (CTID emulation plans)
@@ -55,7 +56,7 @@ These plugins are ready to use but are not included by default and are not maint
 These requirements are for the computer running the core framework:
 
 * Any Linux or MacOS
-* Python 3.7+ (with Pip3)
+* Python 3.8+ (with Pip3)
 * Recommended hardware to run on is 8GB+ RAM and 2+ CPUs
 * Recommended: GoLang 1.17+ to dynamically compile GoLang-based agents.
 
@@ -79,7 +80,7 @@ Next, install the PIP requirements:
 ```Bash
 pip3 install -r requirements.txt
 ```
-**Super-power your Caldea server installation! [Install GoLang (1.17+)](https://go.dev/doc/install)**
+**Super-power your Caldera server installation! [Install GoLang (1.19+)](https://go.dev/doc/install)**
 
 Finally, start the server.
 ```Bash

app/api/v2/managers/operation_api_manager.py

@@ -87,6 +87,8 @@ async def update_operation_link(self, operation_id: str, link_id: str, link_data
             if not link.is_valid_status(link_status):
                 raise JsonHttpBadRequest(f'Cannot update link {link_id} due to invalid link status.')
             link.status = link_status
+            if link.can_ignore():
+                operation.add_ignored_link(link.id)
         return link.display
 
     async def create_potential_link(self, operation_id: str, data: dict, access: BaseWorld.Access):

app/contacts/contact_tcp.py

@@ -85,6 +85,7 @@ async def send(self, session_id: int, cmd: str, timeout: int = 60) -> Tuple[int,
         try:
             conn = next(i.connection for i in self.sessions if i.id == int(session_id))
             conn.send(str.encode(' '))
+            time.sleep(0.01)
             conn.send(str.encode('%s\n' % cmd))
             response = await self._attempt_connection(session_id, conn, timeout=timeout)
             response = json.loads(response)

app/objects/c_ability.py

@@ -13,21 +13,21 @@
 
 class AbilitySchema(ma.Schema):
     ability_id = ma.fields.String()
-    tactic = ma.fields.String(missing=None)
-    technique_name = ma.fields.String(missing=None)
-    technique_id = ma.fields.String(missing=None)
-    name = ma.fields.String(missing=None)
-    description = ma.fields.String(missing=None)
+    tactic = ma.fields.String(load_default=None)
+    technique_name = ma.fields.String(load_default=None)
+    technique_id = ma.fields.String(load_default=None)
+    name = ma.fields.String(load_default=None)
+    description = ma.fields.String(load_default=None)
     executors = ma.fields.List(ma.fields.Nested(ExecutorSchema))
-    requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None)
-    privilege = ma.fields.String(missing=None)
-    repeatable = ma.fields.Bool(missing=None)
-    buckets = ma.fields.List(ma.fields.String(), missing=None)
+    requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), load_default=None)
+    privilege = ma.fields.String(load_default=None)
+    repeatable = ma.fields.Bool(load_default=None)
+    buckets = ma.fields.List(ma.fields.String(), load_default=None)
     additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())
-    access = ma.fields.Nested(AccessSchema, missing=None)
-    singleton = ma.fields.Bool(missing=None)
-    plugin = ma.fields.String(missing=None)
-    delete_payload = ma.fields.Bool(missing=None)
+    access = ma.fields.Nested(AccessSchema, load_default=None)
+    singleton = ma.fields.Bool(load_default=None)
+    plugin = ma.fields.String(load_default=None)
+    delete_payload = ma.fields.Bool(load_default=None)
 
     @ma.pre_load
     def fix_id(self, data, **_):

app/objects/c_adversary.py

@@ -18,7 +18,7 @@ class AdversarySchema(ma.Schema):
     objective = ma.fields.String()
     tags = ma.fields.List(ma.fields.String(), allow_none=True)
     has_repeatable_abilities = ma.fields.Boolean(dump_only=True)
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.pre_load
     def fix_id(self, adversary, **_):

app/objects/c_operation.py

@@ -50,7 +50,7 @@ class OperationSchema(ma.Schema):
     visibility = ma.fields.Integer()
     objective = ma.fields.Nested(ObjectiveSchema())
     use_learning_parsers = ma.fields.Boolean()
-    group = ma.fields.String(missing='')
+    group = ma.fields.String(load_default='')
     source = ma.fields.Nested(SourceSchema())
 
     @ma.pre_load()
@@ -234,7 +234,7 @@ async def wait_for_links_completion(self, link_ids):
         for link_id in link_ids:
             link = [link for link in self.chain if link.id == link_id][0]
             if link.can_ignore():
-                self.ignored_links.add(link.id)
+                self.add_ignored_link(link.id)
             member = [member for member in self.agents if member.paw == link.paw][0]
             while not (link.finish or link.can_ignore()):
                 await asyncio.sleep(5)
@@ -256,6 +256,9 @@ async def is_finished(self):
     def link_status(self):
         return -3 if self.autonomous else -1
 
+    def add_ignored_link(self, link_id):
+        self.ignored_links.add(link_id)
+
     async def active_agents(self):
         active = []
         for agent in self.agents:
@@ -272,14 +275,14 @@ async def get_skipped_abilities_by_agent(self, data_svc):
         for agent in self.agents:
             agent_skipped = defaultdict(dict)
             agent_executors = agent.executors
-            agent_ran = set([link.ability.ability_id for link in self.chain if link.paw == agent.paw])
+            agent_ran = set([link.ability.ability_id for link in self.chain if link.paw == agent.paw and link.finish])
             for ab in abilities_by_agent[agent.paw]['all_abilities']:
                 skipped = self._check_reason_skipped(agent=agent, ability=ab, agent_executors=agent_executors,
                                                      op_facts=[f.trait for f in await self.all_facts()],
                                                      state=self.state, agent_ran=agent_ran)
                 if skipped:
                     if agent_skipped[skipped['ability_id']]:
-                        if agent_skipped[skipped['ability_id']]['reason_id'] < skipped['reason_id']:
+                        if agent_skipped[skipped['ability_id']]['reason_id'] > skipped['reason_id']:
                             agent_skipped[skipped['ability_id']] = skipped
                     else:
                         agent_skipped[skipped['ability_id']] = skipped
@@ -438,8 +441,13 @@ async def _unfinished_links_for_agent(self, paw):
 
     async def _get_all_possible_abilities_by_agent(self, data_svc):
         abilities = {'all_abilities': [ab for ab_id in self.adversary.atomic_ordering
-                     for ab in await data_svc.locate('abilities', match=dict(ability_id=ab_id))]}
-        return {a.paw: abilities for a in self.agents}
+                                       for ab in await data_svc.locate('abilities', match=dict(ability_id=ab_id))]}
+        abilities_by_agent = {a.paw: abilities for a in self.agents}
+        for link in self.chain:
+            if link.ability.ability_id not in self.adversary.atomic_ordering:
+                matching_abilities = await data_svc.locate('abilities', match=dict(ability_id=link.ability.ability_id))
+                abilities_by_agent[link.paw]['all_abilities'].extend(matching_abilities)
+        return abilities_by_agent
 
     def _check_reason_skipped(self, agent, ability, op_facts, state, agent_executors, agent_ran):
         if ability.ability_id in agent_ran:
@@ -452,22 +460,34 @@ def _check_reason_skipped(self, agent, ability, op_facts, state, agent_executors
             facts = re.findall(BasePlanningService.re_variable, executor.test) if executor.command else []
             if not facts or all(fact in op_facts for fact in facts):
                 fact_dependency_fulfilled = True
+        associated_links = set([link.id for link in self.chain if link.paw == agent.paw
+                                and link.ability.ability_id == ability.ability_id])
 
-        if not agent.trusted:
-            return dict(reason='Agent untrusted', reason_id=self.Reason.UNTRUSTED.value,
+        if agent.platform == 'unknown':
+            return dict(reason='Platform not available', reason_id=self.Reason.PLATFORM.value,
                         ability_id=ability.ability_id, ability_name=ability.name)
         elif not valid_executors:
-            return dict(reason='Executor not available', reason_id=self.Reason.EXECUTOR.value,
+            return dict(reason='Mismatched ability platform and executor', reason_id=self.Reason.EXECUTOR.value,
+                        ability_id=ability.ability_id, ability_name=ability.name)
+        elif not agent.privileged_to_run(ability):
+            return dict(reason='Ability privilege not fulfilled', reason_id=self.Reason.PRIVILEGE.value,
                         ability_id=ability.ability_id, ability_name=ability.name)
         elif not fact_dependency_fulfilled:
             return dict(reason='Fact dependency not fulfilled', reason_id=self.Reason.FACT_DEPENDENCY.value,
                         ability_id=ability.ability_id, ability_name=ability.name)
-        elif not agent.privileged_to_run(ability):
-            return dict(reason='Ability privilege not fulfilled', reason_id=self.Reason.PRIVILEGE.value,
+        elif not set(associated_links).isdisjoint(self.ignored_links):
+            return dict(reason='Link ignored - highly visible or discarded link',
+                        reason_id=self.Reason.LINK_IGNORED.value, ability_id=ability.ability_id,
+                        ability_name=ability.name)
+        elif not agent.trusted:
+            return dict(reason='Agent not trusted', reason_id=self.Reason.UNTRUSTED.value,
                         ability_id=ability.ability_id, ability_name=ability.name)
         elif state != 'finished':
             return dict(reason='Operation not completed', reason_id=self.Reason.OP_RUNNING.value,
                         ability_id=ability.ability_id, ability_name=ability.name)
+        else:
+            return dict(reason='Other', reason_id=self.Reason.OTHER.value,
+                        ability_id=ability.ability_id, ability_name=ability.name)
 
     def _get_operation_metadata_for_event_log(self):
         return dict(operation_name=self.name,
@@ -524,10 +544,12 @@ async def _get_agent_info_for_event_log(agent_paw, data_svc):
     class Reason(Enum):
         PLATFORM = 0
         EXECUTOR = 1
-        FACT_DEPENDENCY = 2
-        PRIVILEGE = 3
-        OP_RUNNING = 4
+        PRIVILEGE = 2
+        FACT_DEPENDENCY = 3
+        LINK_IGNORED = 4
         UNTRUSTED = 5
+        OP_RUNNING = 6
+        OTHER = 7
 
     class States(Enum):
         RUNNING = 'running'

app/objects/c_planner.py

@@ -16,7 +16,7 @@ class PlannerSchema(ma.Schema):
     stopping_conditions = ma.fields.List(ma.fields.Nested(FactSchema()))
     ignore_enforcement_modules = ma.fields.List(ma.fields.String())
     allow_repeatable_abilities = ma.fields.Boolean()
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.post_load()
     def build_planner(self, data, **kwargs):

app/objects/c_source.py

@@ -33,7 +33,7 @@ class SourceSchema(ma.Schema):
     rules = ma.fields.List(ma.fields.Nested(RuleSchema))
     adjustments = ma.fields.List(ma.fields.Nested(AdjustmentSchema))
     relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema))
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.pre_load
     def fix_adjustments(self, in_data, **_):
@@ -60,17 +60,15 @@ def _fix_loaded_object_origins(input_data):
         :param input_data: A 'source' dictionary
         :return: input_data with updated facts/relationships (patched in place)
         """
-        if 'facts' in input_data:
-            for y in input_data['facts']:
-                y['origin_type'] = OriginType.IMPORTED.name
-                y['source'] = input_data['id']
-        if 'relationships' in input_data:
-            for y in input_data['relationships']:
-                y['source']['origin_type'] = OriginType.IMPORTED.name
-                y['source']['source'] = input_data['id']
-                if 'target' in y:
-                    y['target']['origin_type'] = OriginType.IMPORTED.name
-                    y['target']['source'] = input_data['id']
+        for y in input_data.get('facts', []):
+            y['origin_type'] = OriginType.IMPORTED.name
+            y['source'] = input_data['id']
+        for y in input_data.get('relationships', []):
+            y['source']['origin_type'] = OriginType.IMPORTED.name
+            y['source']['source'] = input_data['id']
+            if y.get('target'):
+                y['target']['origin_type'] = OriginType.IMPORTED.name
+                y['target']['source'] = input_data['id']
 
 
 class Source(FirstClassObjectInterface, BaseObject):