diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 4182895a0d2..87483581d7d 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -22,22 +22,22 @@ StatusEnumField, ) from api.models import ( + ComplianceOverview, Finding, + Invitation, Membership, PermissionChoices, Provider, ProviderGroup, + ProviderSecret, Resource, ResourceTag, + Role, Scan, ScanSummary, SeverityChoices, StateChoices, StatusChoices, - ProviderSecret, - Invitation, - Role, - ComplianceOverview, Task, User, ) @@ -543,3 +543,25 @@ class Meta: "inserted_at": ["date", "gte", "lte"], "region": ["exact", "icontains", "in"], } + + +class ServiceOverviewFilter(ScanSummaryFilter): + muted_findings = None + + def is_valid(self): + # Check if at least one of the inserted_at filters is present + inserted_at_filters = [ + self.data.get("inserted_at"), + self.data.get("inserted_at__gte"), + self.data.get("inserted_at__lte"), + ] + if not any(inserted_at_filters): + raise ValidationError( + { + "inserted_at": [ + "At least one of filter[inserted_at], filter[inserted_at__gte], or " + "filter[inserted_at__lte] is required." + ] + } + ) + return super().is_valid() diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 772d7202321..eca184a1931 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1551,6 +1551,143 @@ paths: schema: $ref: '#/components/schemas/OverviewProviderResponse' description: '' + /api/v1/overviews/services: + get: + operationId: overviews_services_retrieve + description: Retrieve an aggregated summary of findings grouped by service. + The response includes the total count of findings for each service, as long + as there are at least one finding for that service. At least one of the `inserted_at` + filters must be provided. + summary: Get findings data by service + parameters: + - in: query + name: fields[services-overview] + schema: + type: array + items: + type: string + enum: + - id + - total + - fail + - muted + - pass + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__date] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - total + - -total + - fail + - -fail + - muted + - -muted + - pass + - -pass + explode: false + tags: + - Overview + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewServiceResponse' + description: '' /api/v1/provider-groups: get: operationId: provider_groups_list @@ -5996,6 +6133,50 @@ components: type: string enum: - providers-overview + OverviewService: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/OverviewServiceTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + id: + type: string + total: + type: integer + fail: + type: integer + muted: + type: integer + pass: + type: integer + required: + - id + - total + - fail + - muted + - pass + OverviewServiceResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewService' + required: + - data + OverviewServiceTypeEnum: + type: string + enum: + - services-overview OverviewSeverity: type: object required: diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 33c7f3a6f82..794dae71379 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -9,18 +9,18 @@ from rest_framework import status from api.models import ( + Invitation, Membership, Provider, ProviderGroup, ProviderGroupMembership, + ProviderSecret, Role, RoleProviderGroupRelationship, - Invitation, - UserRoleRelationship, - ProviderSecret, Scan, StateChoices, User, + UserRoleRelationship, ) from api.rls import Tenant @@ -3909,7 +3909,37 @@ def test_overview_providers_list( resources_fixture ) - # TODO Add more tests for the rest of overviews + def test_overview_services_list_no_required_filters( + self, authenticated_client, scan_summaries_fixture + ): + response = authenticated_client.get(reverse("overview-services")) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_overview_services_list(self, authenticated_client, scan_summaries_fixture): + response = authenticated_client.get( + reverse("overview-services"), {"filter[inserted_at]": TODAY} + ) + assert response.status_code == status.HTTP_200_OK + # Only two different services + assert len(response.json()["data"]) == 2 + # Fixed data from the fixture, TODO improve this at some point with something more dynamic + service1_data = response.json()["data"][0] + service2_data = response.json()["data"][1] + assert service1_data["id"] == "service1" + assert service2_data["id"] == "service2" + + # TODO fix numbers when muted_findings filter is fixed + assert service1_data["attributes"]["total"] == 3 + assert service2_data["attributes"]["total"] == 1 + + assert service1_data["attributes"]["pass"] == 1 + assert service2_data["attributes"]["pass"] == 1 + + assert service1_data["attributes"]["fail"] == 1 + assert service2_data["attributes"]["fail"] == 0 + + assert service1_data["attributes"]["muted"] == 1 + assert service2_data["attributes"]["muted"] == 0 @pytest.mark.django_db diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 04628350968..d18bfa1cea8 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -14,24 +14,24 @@ from rest_framework_simplejwt.tokens import RefreshToken from api.models import ( + ComplianceOverview, + Finding, + Invitation, + InvitationRoleRelationship, Membership, Provider, ProviderGroup, ProviderGroupMembership, + ProviderSecret, Resource, ResourceTag, - Finding, - ProviderSecret, - Invitation, - InvitationRoleRelationship, Role, RoleProviderGroupRelationship, - UserRoleRelationship, - ComplianceOverview, Scan, StateChoices, Task, User, + UserRoleRelationship, ) from api.rls import Tenant @@ -1655,6 +1655,24 @@ def get_root_meta(self, _resource, _many): return {"version": "v1"} +class OverviewServiceSerializer(serializers.Serializer): + id = serializers.CharField(source="service") + total = serializers.IntegerField() + _pass = serializers.IntegerField() + fail = serializers.IntegerField() + muted = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "services-overview" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["pass"] = self.fields.pop("_pass") + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + # Schedules diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 7ce6cb2e126..600d313ade5 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -8,7 +8,6 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control from drf_spectacular.settings import spectacular_settings -from drf_spectacular_jsonapi.schemas.openapi import JsonApiAutoSchema from drf_spectacular.utils import ( OpenApiParameter, OpenApiResponse, @@ -17,6 +16,7 @@ extend_schema_view, ) from drf_spectacular.views import SpectacularAPIView +from drf_spectacular_jsonapi.schemas.openapi import JsonApiAutoSchema from rest_framework import permissions, status from rest_framework.decorators import action from rest_framework.exceptions import ( @@ -26,10 +26,9 @@ ValidationError, ) from rest_framework.generics import GenericAPIView, get_object_or_404 +from rest_framework.permissions import SAFE_METHODS from rest_framework_json_api.views import RelationshipView, Response from rest_framework_simplejwt.exceptions import InvalidToken, TokenError -from rest_framework.permissions import SAFE_METHODS - from tasks.beat import schedule_provider_scan from tasks.tasks import ( check_provider_connection_task, @@ -50,17 +49,15 @@ ProviderGroupFilter, ProviderSecretFilter, ResourceFilter, + RoleFilter, ScanFilter, ScanSummaryFilter, + ServiceOverviewFilter, TaskFilter, TenantFilter, UserFilter, - RoleFilter, ) from api.models import ( - StatusChoices, - User, - UserRoleRelationship, ComplianceOverview, Finding, Invitation, @@ -69,14 +66,17 @@ ProviderGroup, ProviderGroupMembership, ProviderSecret, + Resource, Role, RoleProviderGroupRelationship, - Resource, Scan, ScanSummary, SeverityChoices, StateChoices, + StatusChoices, Task, + User, + UserRoleRelationship, ) from api.pagination import ComplianceOverviewPagination from api.rbac.permissions import HasPermissions, Permissions @@ -84,12 +84,6 @@ from api.utils import validate_invitation from api.uuid_utils import datetime_to_uuid7 from api.v1.serializers import ( - TokenSerializer, - TokenRefreshSerializer, - UserSerializer, - UserCreateSerializer, - UserUpdateSerializer, - UserRoleRelationshipSerializer, ComplianceOverviewFullSerializer, ComplianceOverviewSerializer, FindingDynamicFilterSerializer, @@ -101,30 +95,36 @@ MembershipSerializer, OverviewFindingSerializer, OverviewProviderSerializer, + OverviewServiceSerializer, OverviewSeveritySerializer, ProviderCreateSerializer, ProviderGroupMembershipSerializer, ProviderGroupSerializer, ProviderGroupUpdateSerializer, - RoleProviderGroupRelationshipSerializer, + ProviderSecretCreateSerializer, + ProviderSecretSerializer, + ProviderSecretUpdateSerializer, ProviderSerializer, ProviderUpdateSerializer, - TenantSerializer, - TaskSerializer, - ScanSerializer, - ScanCreateSerializer, - ScanUpdateSerializer, ResourceSerializer, - ProviderSecretSerializer, - ProviderSecretUpdateSerializer, - ProviderSecretCreateSerializer, - RoleSerializer, RoleCreateSerializer, + RoleProviderGroupRelationshipSerializer, + RoleSerializer, RoleUpdateSerializer, + ScanCreateSerializer, + ScanSerializer, + ScanUpdateSerializer, ScheduleDailyCreateSerializer, + TaskSerializer, + TenantSerializer, + TokenRefreshSerializer, + TokenSerializer, + UserCreateSerializer, + UserRoleRelationshipSerializer, + UserSerializer, + UserUpdateSerializer, ) - CACHE_DECORATOR = cache_control( max_age=django_settings.CACHE_MAX_AGE, stale_while_revalidate=django_settings.CACHE_STALE_WHILE_REVALIDATE, @@ -191,7 +191,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.0.1" + spectacular_settings.VERSION = "1.1.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -1888,6 +1888,15 @@ def list(self, request, *args, **kwargs): ), filters=True, ), + services=extend_schema( + summary="Get findings data by service", + description=( + "Retrieve an aggregated summary of findings grouped by service. The response includes the total count " + "of findings for each service, as long as there are at least one finding for that service. At least " + "one of the `inserted_at` filters must be provided." + ), + filters=True, + ), ) @method_decorator(CACHE_DECORATOR, name="list") class OverviewViewSet(BaseRLSViewSet): @@ -1902,6 +1911,8 @@ def get_queryset(self): return ScanSummary.objects.all() elif self.action == "findings_severity": return ScanSummary.objects.all() + elif self.action == "services": + return ScanSummary.objects.all() else: return super().get_queryset() @@ -1912,6 +1923,8 @@ def get_serializer_class(self): return OverviewFindingSerializer elif self.action == "findings_severity": return OverviewSeveritySerializer + elif self.action == "services": + return OverviewServiceSerializer return super().get_serializer_class() def get_filterset_class(self): @@ -1919,6 +1932,8 @@ def get_filterset_class(self): return None elif self.action in ["findings", "findings_severity"]: return ScanSummaryFilter + elif self.action == "services": + return ServiceOverviewFilter return None @extend_schema(exclude=True) @@ -2064,6 +2079,38 @@ def findings_severity(self, request): serializer = OverviewSeveritySerializer(severity_data) return Response(serializer.data, status=status.HTTP_200_OK) + @action(detail=False, methods=["get"], url_name="services") + def services(self, request): + queryset = self.get_queryset() + filtered_queryset = self.filter_queryset(queryset) + + latest_scan_subquery = ( + Scan.objects.filter( + state=StateChoices.COMPLETED, provider_id=OuterRef("scan__provider_id") + ) + .order_by("-id") + .values("id")[:1] + ) + + annotated_queryset = filtered_queryset.annotate( + latest_scan_id=Subquery(latest_scan_subquery) + ) + + filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id")) + + services_data = ( + filtered_queryset.values("service") + .annotate(_pass=Sum("_pass")) + .annotate(fail=Sum("fail")) + .annotate(muted=Sum("muted")) + .annotate(total=Sum("total")) + .order_by("service") + ) + + serializer = OverviewServiceSerializer(services_data, many=True) + + return Response(serializer.data, status=status.HTTP_200_OK) + @extend_schema(tags=["Schedule"]) @extend_schema_view( diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 273daa6366d..1747c3ae705 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -24,6 +24,7 @@ ResourceTag, Role, Scan, + ScanSummary, StateChoices, Task, User, @@ -762,6 +763,85 @@ def get_api_tokens( ) +@pytest.fixture +def scan_summaries_fixture(tenants_fixture, providers_fixture): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + scan = Scan.objects.create( + name="overview scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + check_id="check1", + service="service1", + severity="high", + region="region1", + _pass=1, + fail=0, + muted=0, + total=1, + new=1, + changed=0, + unchanged=0, + fail_new=0, + fail_changed=0, + pass_new=1, + pass_changed=0, + muted_new=0, + muted_changed=0, + scan=scan, + ) + + ScanSummary.objects.create( + tenant=tenant, + check_id="check1", + service="service1", + severity="high", + region="region2", + _pass=0, + fail=1, + muted=1, + total=2, + new=2, + changed=0, + unchanged=0, + fail_new=1, + fail_changed=0, + pass_new=0, + pass_changed=0, + muted_new=1, + muted_changed=0, + scan=scan, + ) + + ScanSummary.objects.create( + tenant=tenant, + check_id="check2", + service="service2", + severity="critical", + region="region1", + _pass=1, + fail=0, + muted=0, + total=1, + new=1, + changed=0, + unchanged=0, + fail_new=0, + fail_changed=0, + pass_new=1, + pass_changed=0, + muted_new=0, + muted_changed=0, + scan=scan, + ) + + def get_authorization_header(access_token: str) -> dict: return {"Authorization": f"Bearer {access_token}"} @@ -772,10 +852,12 @@ def pytest_collection_modifyitems(items): def pytest_configure(config): - # Apply the mock before the test session starts. This is necessary to avoid admin error when running the 0004_rbac_missing_admin_roles migration + # Apply the mock before the test session starts. This is necessary to avoid admin error when running the + # 0004_rbac_missing_admin_roles migration patch("api.db_router.MainRouter.admin_db", new="default").start() def pytest_unconfigure(config): - # Stop all patches after the test session ends. This is necessary to avoid admin error when running the 0004_rbac_missing_admin_roles migration + # Stop all patches after the test session ends. This is necessary to avoid admin error when running the + # 0004_rbac_missing_admin_roles migration patch.stopall()