Merge pull request #7 from BuildAndDestroy/may-updates-2024

May updates 2024

.github/workflows/quality.yml

@@ -29,14 +29,16 @@ jobs:
             toxenv: py310,style,coverage-ci
           - python-version: 3.11
             toxenv: py311,style,coverage-ci
+          - python-version: 3.12
+            toxenv: py312,style,coverage-ci
 
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
         with:
           submodules: recursive
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
       - name: Setup python
-        uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -50,7 +52,7 @@ jobs:
       - name: Override Coverage Source Path for Sonar
         run: sed -i "s/<source>\/home\/runner\/work\/caldera\/caldera/<source>\/github\/workspace/g" /home/runner/work/caldera/caldera/coverage.xml
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@5875562561d22a34be0c657405578705a169af6c
+        uses: SonarSource/sonarcloud-github-action@49e6cd3b187936a73b8280d59ffd9da69df63ec9
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/stale.yml

@@ -25,5 +25,5 @@ jobs:
         stale-pr-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days'
         stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days'
         exempt-issue-labels: 'feature,keep'
-        days-before-stale: 30
-        days-before-close: 7
+        days-before-stale: 45
+        days-before-close: 30

.gitmodules

@@ -46,3 +46,6 @@
 [submodule "plugins/emu"]
 	path = plugins/emu
 	url = https://github.com/mitre/emu.git
+[submodule "plugins/magma"]
+	path = plugins/magma
+	url = https://github.com/mitre/magma.git

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
     - repo: https://github.com/PyCQA/flake8
-      rev: 5.0.4
+      rev: 7.0.0
       hooks:
           - id: flake8
             additional_dependencies: [flake8-bugbear]
     - repo: https://github.com/PyCQA/bandit
-      rev: 1.7.4
+      rev: 1.7.7
       hooks:
         - id: bandit
           entry: bandit -ll --exclude=tests/ --skip=B303

CITATION.cff

@@ -3,6 +3,6 @@ message: "If you use this software, please cite it as below."
 authors:
 - name: "MITRE Corporation"
 title: "MITRE Caldera: A Scalable, Automated Adversary Emulation Platform"
-version: 4.2.0
-date-released: 2023-06-19
+version: 5.0.0
+date-released: 2024-02-14
 url: "https://github.com/mitre/caldera"

CONTRIBUTING.md

@@ -40,7 +40,7 @@ python -m pytest
 ```
 This will run all unit tests in your current development environment. Depending on the level of the change, you might need to run the test suite on various versions of Python. The unit testing pipeline will run the entire suite across multiple Python versions that we support when you submit your PR.
 
-We utilize `tox` to test CALDERA in multiple versions of Python. This will only run if the interpreter is present on your system. To run tox, execute:
+We utilize `tox` to test Caldera in multiple versions of Python. This will only run if the interpreter is present on your system. To run tox, execute:
 ```
 tox
 ```

Dockerfile

@@ -97,6 +97,20 @@ fi
 
 WORKDIR /usr/src/app
 
+# Install Node.js, npm, and other build VueJS front-end
+RUN apt-get update && \
+    apt-get install -y nodejs npm && \
+    # Directly use npm to install dependencies and build the application
+    (cd plugins/magma && npm install) && \
+    (cd plugins/magma && npm run build) && \
+    # Remove Node.js, npm, and other unnecessary packages
+    apt-get remove -y nodejs npm && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /usr/src/app
+
 # Default HTTP port for web interface and agent beacons over HTTP
 EXPOSE 8888
 

README.md

@@ -16,7 +16,7 @@ The framework consists of two components:
 an asynchronous command-and-control (C2) server with a REST API and a web interface.
 2) **Plugins**. These repositories expand the core framework capabilities and providing additional functionality. Examples include agents, reporting, collections of TTPs and more.
 
-## Resources and Socials
+## Resources & Socials
 * 📜 [Documentation, training, and use-cases](https://caldera.readthedocs.io/en/latest/)
 * ✍️ [Caldera's blog](https://medium.com/@mitrecaldera/welcome-to-the-official-mitre-caldera-blog-page-f34c2cdfef09)
 * 🌐 [Homepage](https://caldera.mitre.org)
@@ -37,6 +37,7 @@ These plugins are supported and maintained by the Caldera team.
 - **[Fieldmanual](https://github.com/mitre/fieldmanual)** (documentation)
 - **[GameBoard](https://github.com/mitre/gameboard)** (visualize joint red and blue operations)
 - **[Human](https://github.com/mitre/human)** (create simulated noise on an endpoint)
+- **[Magma](https://github.com/mitre/magma)** (VueJS UI for Caldera v5)
 - **[Manx](https://github.com/mitre/manx)** (shell functionality and reverse shell payloads)
 - **[Response](https://github.com/mitre/response)** (incident response)
 - **[Sandcat](https://github.com/mitre/sandcat)** (default agent)
@@ -59,6 +60,7 @@ These requirements are for the computer running the core framework:
 * Python 3.8+ (with Pip3)
 * Recommended hardware to run on is 8GB+ RAM and 2+ CPUs
 * Recommended: GoLang 1.17+ to dynamically compile GoLang-based agents.
+* NodeJS (v16+ recommended for v5 VueJS UI) 
 
 ## Installation
 
@@ -67,7 +69,7 @@ Concise installation steps:
 git clone https://github.com/mitre/caldera.git --recursive
 cd caldera
 pip3 install -r requirements.txt
-python3 server.py --insecure
+python3 server.py --insecure --build
 ```
 
 Full steps:
@@ -84,11 +86,28 @@ pip3 install -r requirements.txt
 
 Finally, start the server.
 ```Bash
-python3 server.py --insecure
+python3 server.py --insecure --build
 ```
-
+The --build flag automatically installs any VueJS UI dependencies, bundles the UI into a dist directory, and is served by the Caldera server. You will only have to use the --build flag again if you add any plugins or make any changes to the UI.
 Once started, log into http://localhost:8888 using the default credentials red/admin. Then go into Plugins -> Training and complete the capture-the-flag style training course to learn how to use Caldera.
 
+If you prefer to not use the new VueJS UI, revert to Caldera v4.2.0. Correspondingly, do not use the `--build` flag for earlier versions as not required.
+
+### User Interface Development
+
+If you'll be developing the UI, there are a few more additional installation steps.
+
+**Requirements**  
+* NodeJS (v16+ recommended)
+
+**Setup**
+
+1. Add the Magma submodule if you haven't already: `git submodule add https://github.com/mitre/magma`
+1. Install NodeJS dependencies: `cd plugins/magma && npm install && cd ..`
+1. Start the Caldera server with an additional flag: `python3 server.py --uidev localhost`
+
+Your Caldera server is available at http://localhost:8888 as usual, but there will now be a hot-reloading development server for the VueJS front-end available at http://localhost:3000. Both logs from the server and the front-end will display in the terminal you launched the server from.
+
 ## Docker Deployment
 To build a Caldera docker image, ensure you have docker installed and perform the following actions:
 ```Bash
@@ -128,4 +147,4 @@ To discuss licensing opportunities, please reach out to [email protected] or dir
 
 ## Caldera Benefactor Program
 
-If you are interested in partnering to support, sustain, and evolve Caldera™'s open source capabilities, please contact us at [email protected].
+If you are interested in partnering to support, sustain, and evolve MITRE Caldera™'s open source capabilities, please contact us at [email protected].

SECURITY.md

@@ -28,14 +28,14 @@ Under this policy, "research" means activities in which you:
 
 ## Reporting a vulnerability
 
-Information submitted under this policy will be used for defensive purposes only, i.e. to mitigate or remediate vulnerabilities. Since CALDERA is run by a not-for-profit and is open source by nature, by
+Information submitted under this policy will be used for defensive purposes only, i.e. to mitigate or remediate vulnerabilities. Since Caldera is run by a not-for-profit and is open source by nature, by
 submitting a vulnerability, you acknowledge that you have no expectation of payment. However, we will ensure that credit is given to the bug finder.  
 
 ## What we would like to see from you
 
 To help us triage and prioritize submissions, please include the following in your report:
 
-- Affected version of CALDERA (committed hash or version number), operating system used, and python version.
+- Affected version of Caldera (committed hash or version number), operating system used, and python version.
 
 - Describe the location the vulnerability was discovered and the potential impact of exploitation.
 
@@ -49,7 +49,7 @@ When you choose to share your contact information with us, we commit to coordina
 
 -   Within ***10 business days***, we will acknowledge that your report has been received.
 
--   After notifying the CALDERA team, we will open reported issues to the public within ***90 days***, or after a fix is released (whichever comes first).
+-   After notifying the Caldera team, we will open reported issues to the public within ***90 days***, or after a fix is released (whichever comes first).
 
 -   To the best of our ability, we will confirm the existence of the vulnerability to you and be as transparent as possible about what steps we are taking during the remediation process, including on issues or challenges that may delay resolution.
 

app/api/packs/advanced.py

@@ -14,6 +14,7 @@ def __init__(self, services):
         self.rest_svc = services.get('rest_svc')
 
     async def enable(self):
+        self.app_svc.application.router._frozen = False
         self.app_svc.application.router.add_route('GET', '/advanced/sources', self._section_sources)
         self.app_svc.application.router.add_route('GET', '/advanced/objectives', self._section_objectives)
         self.app_svc.application.router.add_route('GET', '/advanced/planners', self._section_planners)

app/api/packs/campaign.py

@@ -16,6 +16,7 @@ def __init__(self, services):
         self.rest_svc = services.get('rest_svc')
 
     async def enable(self):
+        self.app_svc.application.router._frozen = False
         self.app_svc.application.router.add_route('GET', '/campaign/agents', self._section_agent)
         self.app_svc.application.router.add_route('GET', '/campaign/abilities', self._section_abilities)
         self.app_svc.application.router.add_route('GET', '/campaign/adversaries', self._section_profiles)

app/api/rest_api.py

@@ -7,7 +7,7 @@
 
 import marshmallow as ma
 from aiohttp import web
-from aiohttp_jinja2 import template, render_template
+from aiohttp_jinja2 import render_template
 
 from app.api.packs.advanced import AdvancedPack
 from app.api.packs.campaign import CampaignPack
@@ -29,44 +29,33 @@ def __init__(self, services):
         asyncio.get_event_loop().create_task(AdvancedPack(services).enable())
 
     async def enable(self):
+        self.app_svc.application.router.add_static('/assets', 'plugins/magma/dist/assets/', append_version=True)
+        # TODO: only serve static files in legacy plugin mode
         self.app_svc.application.router.add_static('/gui', 'static/', append_version=True)
         # unauthorized GUI endpoints
-        self.app_svc.application.router.add_route('*', '/', self.landing)
-        self.app_svc.application.router.add_route('*', '/enter', self.validate_login)
-        self.app_svc.application.router.add_route('*', '/logout', self.logout)
-        self.app_svc.application.router.add_route('GET', '/login', self.login)
+        self.app_svc.application.router.add_route('GET', '/', self.landing)
+        self.app_svc.application.router.add_route('POST', '/enter', self.validate_login)
+        self.app_svc.application.router.add_route('POST', '/logout', self.logout)
         # unauthorized API endpoints
         self.app_svc.application.router.add_route('*', '/file/download', self.download_file)
         self.app_svc.application.router.add_route('POST', '/file/upload', self.upload_file)
         # authorized API endpoints
         self.app_svc.application.router.add_route('*', '/api/rest', self.rest_core)
         self.app_svc.application.router.add_route('GET', '/api/{index}', self.rest_core_info)
         self.app_svc.application.router.add_route('GET', '/file/download_exfil', self.download_exfil_file)
-
-    @template('login.html', status=401)
-    async def login(self, request):
-        return dict()
+        self.app_svc.application.router.add_route('GET', '/{tail:(?!plugin/).*}', self.handle_catch)
 
     async def validate_login(self, request):
         return await self.auth_svc.login_user(request)
 
-    @template('login.html')
     async def logout(self, request):
         await self.auth_svc.logout_user(request)
 
     async def landing(self, request):
-        access = await self.auth_svc.get_permissions(request)
-        if not access:
-            # If user doesn't have access, server will attempt to redirect to login.
-            return await self.auth_svc.login_redirect(request)
-        plugins = await self.data_svc.locate('plugins', {'access': tuple(access), **dict(enabled=True)})
-        data = dict(plugins=[p.display for p in plugins], errors=self.app_svc.errors + self._request_errors(request))
-        template_name = access[0].name
-        if template_name == "RED":
-            template_name = "core_red"
-        elif template_name == "BLUE":
-            template_name = "core_blue"
-        return render_template(f"{template_name}.html", request, data)
+        return render_template("index.html", request, {})
+
+    async def handle_catch(self, request):
+        return render_template("index.html", request, {})
 
     @check_authorization
     async def rest_core(self, request):

app/api/v2/__init__.py

@@ -3,10 +3,11 @@
 
 def make_app(services):
     from .responses import json_request_validation_middleware
-    from .security import authentication_required_middleware_factory
+    from .security import authentication_required_middleware_factory, pass_option_middleware
 
     app = web.Application(
         middlewares=[
+            pass_option_middleware,
             authentication_required_middleware_factory(services['auth_svc']),
             json_request_validation_middleware
         ]
@@ -54,4 +55,7 @@ def make_app(services):
     from .handlers.contact_api import ContactApi
     ContactApi(services).add_routes(app)
 
+    from .handlers.payload_api import PayloadApi
+    PayloadApi(services).add_routes(app)
+
     return app

app/api/v2/handlers/ability_api.py

@@ -48,7 +48,7 @@ async def get_ability_by_id(self, request: web.Request):
         return web.json_response(ability)
 
     @aiohttp_apispec.docs(tags=['abilities'], summary='Creates a new ability.',
-                          description='Creates a new adversary based on the `AbilitySchema`. '
+                          description='Creates a new ability based on the `AbilitySchema`. '
                                       '"name", "tactic", "technique_name", "technique_id" and "executors" are all required fields.')
     @aiohttp_apispec.request_schema(AbilitySchema)
     @aiohttp_apispec.response_schema(AbilitySchema,

app/api/v2/handlers/contact_api.py

@@ -15,6 +15,7 @@ def add_routes(self, app: web.Application):
         router = app.router
         router.add_get('/contacts/{name}', self.get_contact_report)
         router.add_get('/contacts', self.get_available_contact_reports)
+        router.add_get('/contactlist', self.get_contact_list)
 
     @aiohttp_apispec.docs(tags=['contacts'],
                           summary='Retrieve a List of Beacons made by Agents to the specified Contact',
@@ -43,3 +44,7 @@ async def get_contact_report(self, request: web.Request):
     async def get_available_contact_reports(self, request: web.Request):
         contacts = self._api_manager.get_available_contact_reports()
         return web.json_response(contacts)
+
+    async def get_contact_list(self, request: web.Request):
+        contacts = [dict(name=c.name, description=c.description) for c in self._api_manager.contact_svc.contacts]
+        return web.json_response(contacts)

app/api/v2/handlers/health_api.py

@@ -19,15 +19,17 @@ def add_routes(self, app: web.Application):
         router.add_get('/health', security.authentication_exempt(self.get_health_info))
 
     @aiohttp_apispec.docs(tags=['health'],
-                          summary='Health endpoints returns the status of CALDERA',
-                          description='Returns the status of CALDERA and additional details including versions of system components')
+                          summary='Health endpoints returns the status of Caldera',
+                          description='Returns the status of Caldera and additional details including versions of system components')
     @aiohttp_apispec.response_schema(CalderaInfoSchema, 200, description='Includes all loaded plugins and system components.')
     async def get_health_info(self, request):
         loaded_plugins_sorted = sorted(self._app_svc.get_loaded_plugins(), key=operator.attrgetter('name'))
+        access = await self._auth_svc.get_permissions(request)
 
         mapping = {
-            'application': 'CALDERA',
+            'application': 'Caldera',
             'version': app.get_version(),
+            'access': access[0].name,
             'plugins': loaded_plugins_sorted
         }