Dashboard Parent Ticket (#896)

* Adding free-text-search to related barriers (#845) * Adding free-text-search to related barriers * Rebase * Update and working search endpoint * format * format * Excluding archived barriers from inactive notification alerts (#858) Co-authored-by: santinomolinaro <santino.molinaro@mail> * Deleting cached barrier when creating and updating next steps items so they are reflected instantly on frontend (#861) Co-authored-by: santinomolinaro <santino.molinaro@mail> * TSS-1995: Move set_to_allowed_on into public barrier serializer (#862) * tss-1995: set_to_allowed expected on public_barrier serializer --------- Co-authored-by: abarolo <[email protected]> * updates * format * format * using the full text search on the search page * lint fix --------- Co-authored-by: santinomolinaro <santino.molinaro@mail> Co-authored-by: abarolo <[email protected]> Co-authored-by: abarolo <[email protected]> Co-authored-by: Uka Osim <[email protected]> Co-authored-by: Uka Osim <[email protected]> * feat: lint fix * raise search limit * test fix * TSS 1966 dashboard summary with filters (#885) * hotfix: Reduce Dataworkspace pagination size (#877) * hotfix: Reduce Dataworkspace pagination size --------- Co-authored-by: abarolo <[email protected]> * Basic summary info Basic summary stats exposed with filter set implemented * Fix filtering * Remove comments * Add financial year --------- Co-authored-by: abarolo <[email protected]> Co-authored-by: abarolo <[email protected]> * Tss 1965 dashboard tasks merge branch (#888) * Adding Dashboard Task list view to calculate outstanding tasks for display on new dashboard * Unit tests and edge case fixes * Limiting initial query to 1000 results, and order by last modified on date --------- Co-authored-by: santinomolinaro <santino.molinaro@mail> * Dashboard chart data (#898) * Fix PB100 filter (#890) --------- Co-authored-by: santinomolinaro <santino.molinaro@mail> Co-authored-by: abarolo <[email protected]> Co-authored-by: abarolo <[email protected]> Co-authored-by: Uka Osim <[email protected]> Co-authored-by: Uka Osim <[email protected]> Co-authored-by: Feroze Rub <[email protected]>
uktrade · Sep 16, 2024 · e65f68f · e65f68f
1 parent bd0b941
commit e65f68f
Show file tree

Hide file tree

Showing 18 changed files with 2,563 additions and 209 deletions.
diff --git a/api/barriers/models.py b/api/barriers/models.py
@@ -15,7 +15,8 @@
 from django.core.cache import cache
 from django.core.validators import int_list_validator
 from django.db import models
-from django.db.models import CASCADE, Q, QuerySet
+from django.db.models import CASCADE, CharField, Q, QuerySet
+from django.db.models.functions import Cast
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
 from django_filters.widgets import BooleanWidget
@@ -706,7 +707,7 @@ def current_economic_assessment(self):
     @property
     def current_valuation_assessment(self):
         """
-        Get the current valuration assessment
+        Get the current valuation assessment
 
         Filter in python to avoid another db call if prefetch_related has been used.
         """
@@ -1298,8 +1299,8 @@ class BarrierFilterSet(django_filters.FilterSet):
     combined_priority = django_filters.BaseInFilter(method="combined_priority_filter")
     location = django_filters.BaseInFilter(method="location_filter")
     admin_areas = django_filters.BaseInFilter(method="admin_areas_filter")
-    search = django_filters.Filter(method="text_search")
-    text = django_filters.Filter(method="text_search")
+    search = django_filters.Filter(method="vector_search")
+    text = django_filters.Filter(method="vector_search")
 
     user = django_filters.Filter(method="my_barriers")
     has_action_plan = django_filters.Filter(method="has_action_plan_filter")
@@ -1611,6 +1612,49 @@ def text_search(self, queryset, name, value):
             | Q(combined_company_related_organisation_query)
         )
 
+    def vector_search(self, queryset, name, value):
+        """
+        full text search on multiple fields
+        Args:
+            queryset the queryset to filter
+            name the name of the filter
+            value the value of the filter
+
+        Returns:
+            _type_: the resust is a queryset with a free text search applied to the barrier model
+        """
+
+        from api.related_barriers import manager as handler_manager
+        from api.related_barriers.constants import (
+            SIMILAR_BARRIERS_LIMIT,
+            SIMILARITY_THRESHOLD,
+        )
+
+        if handler_manager.manager is None:
+            handler_manager.init()
+
+        barrier_scores = handler_manager.manager.get_similar_barriers_searched(
+            search_term=value,
+            similarity_threshold=SIMILARITY_THRESHOLD,
+            quantity=SIMILAR_BARRIERS_LIMIT,
+        )
+
+        if not barrier_scores:
+            # If no similar barriers are found, return the queryset with the text search applied
+            # we do this to compensate for the fact that the related barriers handler may not have
+            # emededings and barrier ids to return
+            # this can happend when running the CI tests
+            # or when the related barriers handler is not running
+            return self.text_search(queryset, name, value)
+
+        barrier_ids = [b[0] for b in barrier_scores]
+
+        queryset = queryset.filter(id__in=barrier_ids).annotate(
+            barrier_id=Cast("id", output_field=CharField())
+        )
+
+        return queryset
+
     def my_barriers(self, queryset, name, value):
         if value:
             current_user = self.get_user()

diff --git a/api/barriers/signals/handlers.py b/api/barriers/signals/handlers.py
@@ -234,7 +234,7 @@ def related_barrier_update_embeddings(sender, instance, *args, **kwargs):
             manager.manager.update_barrier(
                 BarrierEntry(
                     id=str(current_barrier_object.id),
-                    barrier_corpus=manager.barrier_to_corpus(current_barrier_object),
+                    barrier_corpus=manager.barrier_to_corpus(instance),
                 )
             )
         except Exception as e:

diff --git a/api/barriers/urls.py b/api/barriers/urls.py
@@ -3,6 +3,7 @@
 
 from api.barriers.views import (
     BarrierActivity,
+    BarrierDashboardSummary,
     BarrierDetail,
     BarrierFullHistory,
     BarrierHibernate,
@@ -170,6 +171,9 @@
         name="unknown-barrier",
     ),
     path("counts", barrier_count, name="barrier-count"),
+    path(
+        "dashboard-summary", BarrierDashboardSummary.as_view(), name="barrier-summary"
+    ),
     path("reports", BarrierReportList.as_view(), name="list-reports"),
     path("reports/<uuid:pk>", BarrierReportDetail.as_view(), name="get-report"),
     path(

diff --git a/api/barriers/views.py b/api/barriers/views.py
@@ -5,7 +5,17 @@
 
 from dateutil.parser import parse
 from django.db import transaction
-from django.db.models import Case, CharField, F, Prefetch, Value, When
+from django.db.models import (
+    Case,
+    CharField,
+    F,
+    IntegerField,
+    Prefetch,
+    Q,
+    Sum,
+    Value,
+    When,
+)
 from django.http import StreamingHttpResponse
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
@@ -49,6 +59,7 @@
 from api.metadata.constants import (
     BARRIER_INTERACTION_TYPE,
     BARRIER_SEARCH_ORDERING_CHOICES,
+    ECONOMIC_ASSESSMENT_IMPACT_MIDPOINTS_NUMERIC_LOOKUP,
     BarrierStatus,
     PublicBarrierStatus,
 )
@@ -178,7 +189,232 @@ class Meta:
         abstract = True
 
 
+class BarrierDashboardSummary(generics.GenericAPIView):
+    """
+    View to return high level stats to the dashboard
+    """
+
+    serializer_class = BarrierListSerializer
+    filterset_class = BarrierFilterSet
+
+    filter_backends = (DjangoFilterBackend,)
+    ordering_fields = (
+        "reported_on",
+        "modified_on",
+        "estimated_resolution_date",
+        "status",
+        "priority",
+        "country",
+    )
+    ordering = ("-reported_on",)
+
+    def get(self, request):
+        filtered_queryset = self.filter_queryset(
+            Barrier.barriers.filter(archived=False)
+        )
+
+        current_user = self.request.user
+
+        # Get current financial year
+        current_year_start = datetime(datetime.now().year, 4, 1)
+        current_year_end = datetime(datetime.now().year + 1, 3, 31)
+        previous_year_start = datetime(datetime.now().year - 1, 4, 1)
+        previous_year_end = datetime(datetime.now().year + 1, 3, 31)
+
+        if not current_user.is_anonymous:
+            user_barrier_count = Barrier.barriers.filter(
+                created_by=current_user
+            ).count()
+            user_report_count = Barrier.reports.filter(created_by=current_user).count()
+            user_open_barrier_count = Barrier.barriers.filter(
+                created_by=current_user, status=2
+            ).count()
+
+        when_assessment = [
+            When(valuation_assessments__impact=k, then=Value(v))
+            for k, v in ECONOMIC_ASSESSMENT_IMPACT_MIDPOINTS_NUMERIC_LOOKUP
+        ]
+
+        # Resolved vs Estimated barriers values chart
+        resolved_valuations = (
+            filtered_queryset.filter(
+                Q(
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ]
+                )
+                | Q(
+                    status_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ]
+                ),
+                Q(status=4) | Q(status=3),
+                valuation_assessments__archived=False,
+            )
+            .annotate(numeric_value=Case(*when_assessment, output_field=IntegerField()))
+            .aggregate(total=Sum("numeric_value"))
+        )
+
+        resolved_barrier_value = resolved_valuations["total"]
+
+        estimated_valuations = (
+            filtered_queryset.filter(
+                Q(
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ]
+                )
+                | Q(
+                    status_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ]
+                ),
+                Q(status=1) | Q(status=2),
+                valuation_assessments__archived=False,
+            )
+            .annotate(numeric_value=Case(*when_assessment, output_field=IntegerField()))
+            .aggregate(total=Sum("numeric_value"))
+        )
+
+        estimated_barrier_value = estimated_valuations["total"]
+
+        # Total resolved barriers vs open barriers value chart
+
+        resolved_barriers = (
+            filtered_queryset.filter(
+                Q(status=4) | Q(status=3),
+                valuation_assessments__archived=False,
+            )
+            .annotate(numeric_value=Case(*when_assessment, output_field=IntegerField()))
+            .aggregate(total=Sum("numeric_value"))
+        )
+
+        total_resolved_barriers = resolved_barriers["total"]
+
+        open_barriers = (
+            filtered_queryset.filter(
+                Q(status=1) | Q(status=2),
+                valuation_assessments__archived=False,
+            )
+            .annotate(numeric_value=Case(*when_assessment, output_field=IntegerField()))
+            .aggregate(total=Sum("numeric_value"))
+        )
+
+        open_barriers_value = open_barriers["total"]
+
+        # Open barriers by status
+        whens = [When(status=k, then=Value(v)) for k, v in BarrierStatus.choices]
+
+        barrier_by_status = (
+            filtered_queryset.filter(
+                valuation_assessments__archived=False,
+            )
+            .annotate(status_display=Case(*whens, output_field=CharField()))
+            .annotate(numeric_value=Case(*when_assessment, output_field=IntegerField()))
+            .values("status_display")
+            .annotate(total=Sum("numeric_value"))
+            .order_by()
+        )
+
+        status_labels = []
+        status_data = []
+
+        for series in barrier_by_status:
+            status_labels.append(series["status_display"])
+            status_data.append(series["total"])
+
+        # TODO for status filter might need to consider status dates as well as ERD
+        counts = {
+            "financial_year": {
+                "current_start": current_year_start,
+                "current_end": current_year_end,
+                "previous_start": previous_year_start,
+                "previous_end": previous_year_end,
+            },
+            "barriers": {
+                "total": filtered_queryset.count(),
+                "open": filtered_queryset.filter(status=2).count(),
+                "paused": filtered_queryset.filter(status=5).count(),
+                "resolved": filtered_queryset.filter(status=4).count(),
+                "pb100": filtered_queryset.filter(
+                    top_priority_status__in=["APPROVED", "REMOVAL_PENDING"]
+                ).count(),
+                "overseas_delivery": filtered_queryset.filter(
+                    priority_level="OVERSEAS"
+                ).count(),
+            },
+            "barriers_current_year": {
+                "total": filtered_queryset.filter(
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ]
+                ).count(),
+                "open": filtered_queryset.filter(
+                    status=2,
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ],
+                ).count(),
+                "paused": filtered_queryset.filter(
+                    status=5,
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ],
+                ).count(),
+                "resolved": filtered_queryset.filter(
+                    status=4,
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ],
+                ).count(),
+                "pb100": filtered_queryset.filter(
+                    top_priority_status__in=["APPROVED", "REMOVAL_PENDING"],
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ],
+                ).count(),
+                "overseas_delivery": filtered_queryset.filter(
+                    priority_level="OVERSEAS",
+                    estimated_resolution_date__range=[
+                        current_year_start,
+                        current_year_end,
+                    ],
+                ).count(),
+            },
+            "user_counts": {
+                "user_barrier_count": user_barrier_count,
+                "user_report_count": user_report_count,
+                "user_open_barrier_count": user_open_barrier_count,
+            },
+            "reports": Barrier.reports.count(),
+            "barrier_value_chart": {
+                "resolved_barriers_value": resolved_barrier_value,
+                "estimated_barriers_value": estimated_barrier_value,
+            },
+            "total_value_chart": {
+                "resolved_barriers_value": total_resolved_barriers,
+                "open_barriers_value": open_barriers_value,
+            },
+            "barriers_by_status_chart": {
+                "series": status_data,
+                "labels": status_labels,
+            },
+        }
+
+        return Response(counts)
+
+
 class BarrierReportList(BarrierReportBase, generics.ListCreateAPIView):
+    # TODO - These report views may now be redundant
     serializer_class = BarrierReportSerializer
     filter_backends = (OrderingFilter,)
     ordering_fields = ("created_on",)