Merge branch 'master' into enhancement/add-service-to-feeback-table

.github/workflows/enforce-labels.yml

@@ -7,7 +7,7 @@ jobs:
   enforce-label:
     runs-on: ubuntu-latest
     steps:
-      - uses: yogevbd/enforce-label-action@2.1.0
+      - uses: yogevbd/enforce-label-action@2.2.2
         with:
           REQUIRED_LABELS_ANY: "bug,dependencies,documentation,enhancement,feature,skip-changelog,techdebt,tests"
           REQUIRED_LABELS_ANY_DESCRIPTION: "Select at least one label from the following list: bug, dependencies, documentation, enhancement, feature, skip-changelog, techdebt, tests"

requirements-base.txt

@@ -442,7 +442,7 @@ starlette==0.32.0.post1
     #   starlette-testclient
 starlette-testclient==0.2.0
     # via schemathesis
-statsmodels==0.14.3
+statsmodels==0.14.4
     # via -r requirements-base.in
 tabulate==0.9.0
     # via -r requirements-base.in

requirements-dev.txt

@@ -32,7 +32,7 @@ executing==2.0.1
     #   stack-data
 factory-boy==3.3.1
     # via -r requirements-dev.in
-faker==30.0.0
+faker==30.3.0
     # via
     #   -r requirements-dev.in
     #   factory-boy
@@ -42,7 +42,7 @@ identify==2.5.33
     # via pre-commit
 iniconfig==2.0.0
     # via pytest
-ipython==8.27.0
+ipython==8.28.0
     # via -r requirements-dev.in
 jedi==0.19.1
     # via ipython
@@ -86,7 +86,7 @@ python-dateutil==2.9.0.post0
     # via faker
 pyyaml==6.0.1
     # via pre-commit
-ruff==0.6.8
+ruff==0.6.9
     # via -r requirements-dev.in
 six==1.16.0
     # via
@@ -99,10 +99,12 @@ traitlets==5.14.1
     #   ipython
     #   matplotlib-inline
 typing-extensions==4.10.0
-    # via ipython
+    # via
+    #   faker
+    #   ipython
 virtualenv==20.25.0
     # via pre-commit
-vulture==2.12
+vulture==2.13
     # via -r requirements-dev.in
 wcwidth==0.2.12
     # via prompt-toolkit

src/dispatch/cli.py

@@ -5,6 +5,7 @@
 import uvicorn
 
 from dispatch import __version__, config
+from dispatch.config import DISPATCH_UI_URL
 from dispatch.enums import UserRoles
 from dispatch.plugin.models import PluginInstance
 
@@ -636,6 +637,7 @@ def revision_database(
 def dispatch_scheduler():
     """Container for all dispatch scheduler commands."""
     # we need scheduled tasks to be imported
+    from .case.scheduled import case_close_reminder, case_triage_reminder  # noqa
     from .case_cost.scheduled import (
         calculate_cases_response_cost,  # noqa
     )
@@ -656,7 +658,6 @@ def dispatch_scheduler():
     )
     from .term.scheduled import sync_terms  # noqa
     from .workflow.scheduled import sync_workflows  # noqa
-    from .case.scheduled import case_triage_reminder, case_close_reminder  # noqa
 
 
 @dispatch_scheduler.command("list")
@@ -806,10 +807,10 @@ def consume_signals():
         None
     """
     from dispatch.common.utils.cli import install_plugins
-    from dispatch.project import service as project_service
-    from dispatch.plugin import service as plugin_service
+    from dispatch.database.core import get_organization_session, get_session
     from dispatch.organization.service import get_all as get_all_organizations
-    from dispatch.database.core import get_session, get_organization_session
+    from dispatch.plugin import service as plugin_service
+    from dispatch.project import service as project_service
 
     install_plugins()
 
@@ -883,6 +884,125 @@ def process_signals():
             db_session.close()
 
 
+@signals_group.command("perf-test")
+@click.option("--num-instances", default=1, help="Number of signal instances to send.")
+@click.option("--num-workers", default=1, help="Number of threads to use.")
+@click.option(
+    "--api-endpoint",
+    default=f"{DISPATCH_UI_URL}/api/v1/default/signals/instances",
+    required=True,
+    help="API endpoint to send the signal instances to.",
+)
+@click.option(
+    "--api-token",
+    required=True,
+    help="API token to use.",
+)
+@click.option(
+    "--project",
+    default="Test",
+    required=True,
+    help="The Dispatch project to send the instances to.",
+)
+def perf_test(
+    num_instances: int, num_workers: int, api_endpoint: str, api_token: str, project: str
+) -> None:
+    """Performance testing utility for creating signal instances."""
+
+    import concurrent.futures
+    import time
+    import uuid
+
+    import requests
+    from fastapi import status
+
+    NUM_SIGNAL_INSTANCES = num_instances
+    NUM_WORKERS = num_workers
+
+    session = requests.Session()
+    session.headers.update(
+        {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_token}",
+        }
+    )
+    start_time = time.time()
+
+    def _send_signal_instance(
+        api_endpoint: str,
+        api_token: str,
+        session: requests.Session,
+        signal_instance: dict[str, str],
+    ) -> None:
+        try:
+            r = session.post(
+                api_endpoint,
+                json=signal_instance,
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f"Bearer {api_token}",
+                },
+            )
+            log.info(f"Response: {r.json()}")
+            if r.status_code == status.HTTP_401_UNAUTHORIZED:
+                raise PermissionError(
+                    "Unauthorized. Please check your bearer token. You can find it in the Dev Tools under Request Headers -> Authorization."
+                )
+
+            r.raise_for_status()
+
+        except requests.exceptions.RequestException as e:
+            log.error(f"Unable to send finding. Reason: {e} Response: {r.json() if r else 'N/A'}")
+        else:
+            log.info(f"{signal_instance.get('raw', {}).get('id')} created successfully")
+
+    def send_signal_instances(
+        api_endpoint: str, api_token: str, signal_instances: list[dict[str, str]]
+    ):
+        with concurrent.futures.ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
+            futures = [
+                executor.submit(
+                    _send_signal_instance,
+                    api_endpoint=api_endpoint,
+                    api_token=api_token,
+                    session=session,
+                    signal_instance=signal_instance,
+                )
+                for signal_instance in signal_instances
+            ]
+            results = [future.result() for future in concurrent.futures.as_completed(futures)]
+
+        log.info(f"\nSent {len(results)} of {NUM_SIGNAL_INSTANCES} signal instances")
+
+    signal_instances = [
+        {
+            "project": {"name": project},
+            "raw": {
+                "id": str(uuid.uuid4()),
+                "name": "Test Signal",
+                "slug": "test-signal",
+                "canary": False,
+                "events": [
+                    {
+                        "original": {
+                            "dateint": 20240930,
+                            "distinct_lookupkey_count": 95,
+                        },
+                    },
+                ],
+                "created_at": "2024-09-18T19:47:15Z",
+                "quiet_mode": False,
+                "external_id": "4ebbab36-c703-495f-ae47-7051bdc8b3ef",
+            },
+        },
+    ] * NUM_SIGNAL_INSTANCES
+
+    send_signal_instances(api_endpoint, api_token, signal_instances)
+
+    elapsed_time = time.time() - start_time
+    click.echo(f"Elapsed time: {elapsed_time:.2f} seconds")
+
+
 @dispatch_server.command("slack")
 @click.argument("organization")
 @click.argument("project")

src/dispatch/data/alert/service.py

@@ -1,9 +1,10 @@
 from typing import Optional
+
 from pydantic.error_wrappers import ErrorWrapper, ValidationError
 
 from dispatch.exceptions import NotFoundError
 
-from .models import Alert, AlertCreate, AlertUpdate, AlertRead
+from .models import Alert, AlertCreate, AlertRead, AlertUpdate
 
 
 def get(*, db_session, alert_id: int) -> Optional[Alert]:
@@ -16,7 +17,7 @@ def get_by_name(*, db_session, name: str) -> Optional[Alert]:
     return db_session.query(Alert).filter(Alert.name == name).one_or_none()
 
 
-def get_by_name_or_raise(*, db_session, alert_in=AlertRead) -> AlertRead:
+def get_by_name_or_raise(*, db_session, alert_in: AlertRead) -> AlertRead:
     """Returns the alert specified or raises ValidationError."""
     alert = get_by_name(db_session=db_session, name=alert_in.name)
 

src/dispatch/database/revisions/tenant/versions/2024-09-27_f5107ce190fc.py

@@ -0,0 +1,34 @@
+"""Adds custom incident report card fields
+
+Revision ID: f5107ce190fc
+Revises: 32652e0360dd
+Create Date: 2024-09-27 12:33:17.418418
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "f5107ce190fc"
+down_revision = "32652e0360dd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("project", sa.Column("report_incident_instructions", sa.String(), nullable=True))
+    op.add_column("project", sa.Column("report_incident_title_hint", sa.String(), nullable=True))
+    op.add_column(
+        "project", sa.Column("report_incident_description_hint", sa.String(), nullable=True)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("project", "report_incident_description_hint")
+    op.drop_column("project", "report_incident_title_hint")
+    op.drop_column("project", "report_incident_instructions")
+    # ### end Alembic commands ###

src/dispatch/document/flows.py

@@ -8,6 +8,7 @@
 from dispatch.enums import DocumentResourceTypes
 from dispatch.event import service as event_service
 from dispatch.plugin import service as plugin_service
+from dispatch.tag_type import service as tag_type_service
 
 from .models import Document, DocumentCreate
 from .service import create, delete
@@ -170,17 +171,25 @@ def update_document(document: Document, project_id: int, db_session: Session):
         this would be the replaced text. Only create the source replacements if not null.
         For any tag types with multiple selected tags, replace with a comma-separated list.
         """
+        # first ensure all tags types have a placeholder in the document template
+        tag_types = tag_type_service.get_all_by_project(
+            db_session=db_session, project_id=project_id
+        )
+        for tag_type in tag_types:
+            document_kwargs[f"tag_{tag_type.name}"] = "N/A"
+            document_kwargs[f"tag_{tag_type.name}.source"] = "N/A"
+
         # create document template placeholders for tags
         for tag in document.incident.tags:
-            if f"tag_{tag.tag_type.name}" in document_kwargs:
-                document_kwargs[f"tag_{tag.tag_type.name}"] += f", {tag.name}"
-            else:
+            if document_kwargs[f"tag_{tag.tag_type.name}"] == "N/A":
                 document_kwargs[f"tag_{tag.tag_type.name}"] = tag.name
+            else:
+                document_kwargs[f"tag_{tag.tag_type.name}"] += f", {tag.name}"
             if tag.source:
-                if f"tag_{tag.tag_type.name}.source" in document_kwargs:
-                    document_kwargs[f"tag_{tag.tag_type.name}.source"] += f", {tag.source}"
-                else:
+                if document_kwargs[f"tag_{tag.tag_type.name}.source"] == "N/A":
                     document_kwargs[f"tag_{tag.tag_type.name}.source"] = tag.source
+                else:
+                    document_kwargs[f"tag_{tag.tag_type.name}.source"] += f", {tag.source}"
 
     if document.resource_type == DocumentResourceTypes.review:
         document_kwargs["stable_at"] = document.incident.stable_at.strftime("%m/%d/%Y %H:%M:%S")

src/dispatch/organization/service.py

@@ -44,7 +44,7 @@ def get_by_name(*, db_session, name: str) -> Optional[Organization]:
     return db_session.query(Organization).filter(Organization.name == name).one_or_none()
 
 
-def get_by_name_or_raise(*, db_session, organization_in=OrganizationRead) -> Organization:
+def get_by_name_or_raise(*, db_session, organization_in: OrganizationRead) -> Organization:
     """Returns the organization specified or raises ValidationError."""
     organization = get_by_name(db_session=db_session, name=organization_in.name)
 
@@ -67,7 +67,7 @@ def get_by_slug(*, db_session, slug: str) -> Optional[Organization]:
     return db_session.query(Organization).filter(Organization.slug == slug).one_or_none()
 
 
-def get_by_slug_or_raise(*, db_session, organization_in=OrganizationRead) -> Organization:
+def get_by_slug_or_raise(*, db_session, organization_in: OrganizationRead) -> Organization:
     """Returns the organization specified or raises ValidationError."""
     organization = get_by_slug(db_session=db_session, slug=organization_in.slug)
 
@@ -85,7 +85,7 @@ def get_by_slug_or_raise(*, db_session, organization_in=OrganizationRead) -> Org
     return organization
 
 
-def get_by_name_or_default(*, db_session, organization_in=OrganizationRead) -> Organization:
+def get_by_name_or_default(*, db_session, organization_in: OrganizationRead) -> Organization:
     """Returns a organization based on a name or the default if not specified."""
     if organization_in.name:
         return get_by_name_or_raise(db_session=db_session, organization_in=organization_in)