diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5f571ac73b..ead2dab05d 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -3,9 +3,9 @@ import copy from django import forms -from django.db.models.functions import Concat, Coalesce from django.db.models import Value, CharField, Q -from django.http import HttpResponse, HttpResponseRedirect +from django.db.models.functions import Concat, Coalesce +from django.http import HttpResponseRedirect from django.shortcuts import redirect from django_fsm import get_available_FIELD_transitions from django.contrib import admin, messages @@ -16,7 +16,6 @@ from dateutil.relativedelta import relativedelta # type: ignore from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website -from registrar.utility import csv_export from registrar.utility.errors import FSMApplicationError, FSMErrorCodes from registrar.views.utility.mixins import OrderableFieldsMixin from django.contrib.admin.views.main import ORDER_VAR @@ -1465,7 +1464,6 @@ def state_territory(self, obj): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering @@ -1512,56 +1510,6 @@ def changeform_view(self, request, object_id=None, form_url="", extra_context=No return super().changeform_view(request, object_id, form_url, extra_context) - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def get_urls(self): - from django.urls import path - - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - info = self.model._meta.app_label, self.model._meta.model_name - - my_url = [ - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % info, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % info, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % info, - ), - ] - - return my_url + urlpatterns - def response_change(self, request, obj): # Create dictionary of action functions ACTION_FUNCTIONS = { @@ -1693,9 +1641,11 @@ def do_delete_domain(self, request, obj): else: self.message_user( request, - "Error deleting this Domain: " - f"Can't switch from state '{obj.state}' to 'deleted'" - ", must be either 'dns_needed' or 'on_hold'", + ( + "Error deleting this Domain: " + f"Can't switch from state '{obj.state}' to 'deleted'" + ", must be either 'dns_needed' or 'on_hold'" + ), messages.ERROR, ) except Exception: @@ -1707,7 +1657,7 @@ def do_delete_domain(self, request, obj): else: self.message_user( request, - ("Domain %s has been deleted. Thanks!") % obj.name, + "Domain %s has been deleted. Thanks!" % obj.name, ) return HttpResponseRedirect(".") @@ -1749,7 +1699,7 @@ def do_place_client_hold(self, request, obj): else: self.message_user( request, - ("%s is in client hold. This domain is no longer accessible on the public internet.") % obj.name, + "%s is in client hold. This domain is no longer accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") @@ -1778,7 +1728,7 @@ def do_remove_client_hold(self, request, obj): else: self.message_user( request, - ("%s is ready. This domain is accessible on the public internet.") % obj.name, + "%s is ready. This domain is accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 4ed00c33f1..8c60c534f9 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -368,43 +368,6 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, viewLink.setAttribute('title', viewLink.getAttribute('title-template').replace('selected item', elementText)); } -/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, - * attach the seleted start and end dates to a url that'll trigger the view, and finally - * redirect to that url. -*/ -(function (){ - - // Get the current date in the format YYYY-MM-DD - let currentDate = new Date().toISOString().split('T')[0]; - - // Default the value of the start date input field to the current date - let startDateInput =document.getElementById('start'); - - // Default the value of the end date input field to the current date - let endDateInput =document.getElementById('end'); - - let exportGrowthReportButton = document.getElementById('exportLink'); - - if (exportGrowthReportButton) { - startDateInput.value = currentDate; - endDateInput.value = currentDate; - - exportGrowthReportButton.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = document.getElementById('exportLink').dataset.exportUrl; - - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; - }); - } - -})(); - /** An IIFE for admin in DjangoAdmin to listen to changes on the domain request * status select amd to show/hide the rejection reason */ diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js new file mode 100644 index 0000000000..d10cf2dc6c --- /dev/null +++ b/src/registrar/assets/js/get-gov-reports.js @@ -0,0 +1,117 @@ +/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, + * attach the seleted start and end dates to a url that'll trigger the view, and finally + * redirect to that url. + * + * This function also sets the start and end dates to match the url params if they exist +*/ +(function () { + // Function to get URL parameter value by name + function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, '\\$&'); + var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, ' ')); + } + + // Get the current date in the format YYYY-MM-DD + let currentDate = new Date().toISOString().split('T')[0]; + + // Default the value of the start date input field to the current date + let startDateInput = document.getElementById('start'); + + // Default the value of the end date input field to the current date + let endDateInput = document.getElementById('end'); + + let exportButtons = document.querySelectorAll('.exportLink'); + + if (exportButtons.length > 0) { + // Check if start and end dates are present in the URL + let urlStartDate = getParameterByName('start_date'); + let urlEndDate = getParameterByName('end_date'); + + // Set input values based on URL parameters or current date + startDateInput.value = urlStartDate || currentDate; + endDateInput.value = urlEndDate || currentDate; + + exportButtons.forEach((btn) => { + btn.addEventListener('click', function () { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; + + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); + }); + } + +})(); + +document.addEventListener("DOMContentLoaded", function () { + createComparativeColumnChart("myChart1", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Unmanaged domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); + createComparativeColumnChart("myChart6", "All requests", "Start Date", "End Date"); +}); + +function createComparativeColumnChart(canvasId, title, labelOne, labelTwo) { + var canvas = document.getElementById(canvasId); + var ctx = canvas.getContext("2d"); + + var listOne = JSON.parse(canvas.getAttribute('data-list-one')); + var listTwo = JSON.parse(canvas.getAttribute('data-list-two')); + + var data = { + labels: ["Total", "Federal", "Interstate", "State/Territory", "Tribal", "County", "City", "Special District", "School District", "Election Board"], + datasets: [ + { + label: labelOne, + backgroundColor: "rgba(255, 99, 132, 0.2)", + borderColor: "rgba(255, 99, 132, 1)", + borderWidth: 1, + data: listOne, + }, + { + label: labelTwo, + backgroundColor: "rgba(75, 192, 192, 0.2)", + borderColor: "rgba(75, 192, 192, 1)", + borderWidth: 1, + data: listTwo, + }, + ], + }; + + var options = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'top', + }, + title: { + display: true, + text: title + } + }, + scales: { + y: { + beginAtZero: true, + }, + }, + }; + + new Chart(ctx, { + type: "bar", + data: data, + options: options, + }); +} \ No newline at end of file diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 4f715de3c0..18025a9cbb 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -112,7 +112,8 @@ html[data-theme="light"] { .change-list .usa-table thead th, body.dashboard, body.change-list, - body.change-form { + body.change-form, + .analytics { color: var(--body-fg); } } @@ -304,7 +305,36 @@ input.admin-confirm-button { } } -.django-admin-modal .usa-prose ul > li { +.usa-button-group { + margin-left: -0.25rem!important; + padding-left: 0!important; + .usa-button-group__item { + list-style-type: none; + line-height: normal; + } + .button { + display: inline-block; + padding: 10px 8px; + line-height: normal; + } + .usa-icon { + top: 2px; + } + a.button:active, a.button:focus { + text-decoration: none; + } +} + +.module--custom { + a { + font-size: 13px; + font-weight: 600; + border: solid 1px var(--darkened-bg); + background: var(--darkened-bg); + } +} + +.usa-modal--django-admin .usa-prose ul > li { list-style-type: inherit; // Styling based off of the

styling in django admin line-height: 1.5; diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 39074aab2d..646b7298f4 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -330,8 +330,9 @@ # Google analytics requires that we relax our otherwise # strict CSP by allowing scripts to run from their domain -# and inline with a nonce, as well as allowing connections back to their domain -CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"] +# and inline with a nonce, as well as allowing connections back to their domain. +# Note: If needed, we can embed chart.js instead of using the CDN +CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index c743aed0c4..3918fa0871 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,9 +9,16 @@ from django.views.generic import RedirectView from registrar import views - -from registrar.views.admin_views import ExportData - +from registrar.views.admin_views import ( + ExportDataDomainsGrowth, + ExportDataFederal, + ExportDataFull, + ExportDataManagedDomains, + ExportDataRequestsGrowth, + ExportDataType, + ExportDataUnmanagedDomains, + AnalyticsView, +) from registrar.views.domain_request import Step from registrar.views.utility import always_404 @@ -52,7 +59,46 @@ "admin/logout/", RedirectView.as_view(pattern_name="logout", permanent=False), ), - path("export_data/", ExportData.as_view(), name="admin_export_data"), + path( + "admin/analytics/export_data_type/", + ExportDataType.as_view(), + name="export_data_type", + ), + path( + "admin/analytics/export_data_full/", + ExportDataFull.as_view(), + name="export_data_full", + ), + path( + "admin/analytics/export_data_federal/", + ExportDataFederal.as_view(), + name="export_data_federal", + ), + path( + "admin/analytics/export_domains_growth/", + ExportDataDomainsGrowth.as_view(), + name="export_domains_growth", + ), + path( + "admin/analytics/export_requests_growth/", + ExportDataRequestsGrowth.as_view(), + name="export_requests_growth", + ), + path( + "admin/analytics/export_managed_domains/", + ExportDataManagedDomains.as_view(), + name="export_managed_domains", + ), + path( + "admin/analytics/export_unmanaged_domains/", + ExportDataUnmanagedDomains.as_view(), + name="export_unmanaged_domains", + ), + path( + "admin/analytics/", + AnalyticsView.as_view(), + name="analytics", + ), path("admin/", admin.site.urls), path( "domain-request//edit/", diff --git a/src/registrar/public/img/registrar/dotgov_401_illo.svg b/src/registrar/public/img/registrar/dotgov_401_illo.svg deleted file mode 100644 index 71de33eaa5..0000000000 --- a/src/registrar/public/img/registrar/dotgov_401_illo.svg +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/src/registrar/public/img/registrar/dotgov_404_illo.svg b/src/registrar/public/img/registrar/dotgov_404_illo.svg deleted file mode 100644 index 3c9adab7ea..0000000000 --- a/src/registrar/public/img/registrar/dotgov_404_illo.svg +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/src/registrar/public/img/registrar/dotgov_500_illo.svg b/src/registrar/public/img/registrar/dotgov_500_illo.svg deleted file mode 100644 index 6dd5386440..0000000000 --- a/src/registrar/public/img/registrar/dotgov_500_illo.svg +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html new file mode 100644 index 0000000000..e73f22ec51 --- /dev/null +++ b/src/registrar/templates/admin/analytics.html @@ -0,0 +1,196 @@ +{% extends "admin/base_site.html" %} +{% load static %} + +{% block content_title %}

Registrar Analytics

{% endblock %} + +{% block content %} + +
+ +
+
+
+

At a glance

+
+
    +
  • User Count: {{ data.user_count }}
    • +
  • Domain Count: {{ data.domain_count }}
    • +
  • Domains in READY state: {{ data.ready_domain_count }}
    • +
  • Domain applications (last 30 days): {{ data.last_30_days_applications }}
    • +
  • Approved applications (last 30 days): {{ data.last_30_days_approved_applications }}
    • +
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
    • +
+
+
+
+
+
+

Current domains

+
+ +
+
+
+
+ +
+
+
+

Growth reports

+
+ {% comment %} + Inputs of type date suck for accessibility. + We'll need to replace those guys with a django form once we figure out how to hook one onto this page. + See the commit "Review for ticket #999" + {% endcomment %} +
+
+ + +
+
+ + +
+
+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+ +
+
+ +

Chart: Managed domains

+

{{ data.managed_domains_sliced_at_end_date.0 }} managed domains for {{ data.end_date }}

+ +
+
+ +

Chart: Unmanaged domains

+

{{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

+ +
+
+ +
+
+ +

Chart: Deleted domains

+

{{ data.deleted_domains_sliced_at_end_date.0 }} deleted domains for {{ data.end_date }}

+ +
+
+ +

Chart: Ready domains

+

{{ data.ready_domains_sliced_at_end_date.0 }} ready domains for {{ data.end_date }}

+ +
+
+ +
+
+ +

Chart: Submitted requests

+

{{ data.submitted_requests_sliced_at_end_date.0 }} submitted requests for {{ data.end_date }}

+ +
+
+ +

Chart: All requests

+

{{ data.requests_sliced_at_end_date.0 }} requests for {{ data.end_date }}

+ +
+
+ +
+
+
+
+
+{% endblock %} diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index 49df75beb3..49fb59e79e 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -64,6 +64,11 @@ {% endfor %} +
+

Analytics

+ Dashboard +
{% else %}

{% translate 'You don’t have permission to view or edit anything.' %}

-{% endif %} \ No newline at end of file +{% endif %} + diff --git a/src/registrar/templates/admin/base_site.html b/src/registrar/templates/admin/base_site.html index 73e9ba1f0d..58843421af 100644 --- a/src/registrar/templates/admin/base_site.html +++ b/src/registrar/templates/admin/base_site.html @@ -20,7 +20,9 @@ > + + {% endblock %} {% block title %}{% if subtitle %}{{ subtitle }} | {% endif %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} diff --git a/src/registrar/templates/admin/index.html b/src/registrar/templates/admin/index.html deleted file mode 100644 index 04601ef32e..0000000000 --- a/src/registrar/templates/admin/index.html +++ /dev/null @@ -1,33 +0,0 @@ -{% extends "admin/index.html" %} - -{% block content %} -
- {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %} -
-

Reports

-

Domain growth report

- - {% comment %} - Inputs of type date suck for accessibility. - We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. Itdoes NOT like admin/export_data/ - - See the commit "Review for ticket #999" - {% endcomment %} - -
-
- - -
-
- - -
- - -
- -
-
-{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 818522c8dc..44fe6851b9 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -54,7 +54,7 @@ {# Create a modal for the _extend_expiration_date button #}
{# Create a modal for the _on_hold button #}
{# Create a modal for the _remove_domain button #}
-
  • - Export all domain metadata -
    • -
  • - Export current-full.csv -
    • -
  • - Export current-federal.csv -
    • - {% if has_add_permission %} -
  • - - Add domain - -
    • - {% endif %} - -{% endblock %} \ No newline at end of file diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 73bd08b62c..04eb4c82db 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -1,4 +1,3 @@ -import datetime import os import logging @@ -13,6 +12,8 @@ from django.conf import settings from django.contrib.auth import get_user_model, login from django.utils.timezone import make_aware +from datetime import date, datetime, timedelta +from django.utils import timezone from registrar.models import ( Contact, @@ -35,6 +36,7 @@ ErrorCode, responses, ) +from registrar.models.user_domain_role import UserDomainRole from registrar.models.utility.contact_error import ContactError, ContactErrorCodes @@ -492,6 +494,184 @@ def create_full_dummy_domain_object( return domain_request +class MockDb(TestCase): + """Hardcoded mocks make test case assertions straightforward.""" + + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + self.end_date = current_date + timedelta(days=2) + self.start_date = current_date - timedelta(days=2) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True, + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, domain=self.domain_2, organization_type="interstate", is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True, + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True, + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False, + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False, + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False, + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False, + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False, + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False, + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + with less_console_noise(): + self.domain_request_1 = completed_domain_request( + status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" + ) + self.domain_request_2 = completed_domain_request( + status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov" + ) + self.domain_request_3 = completed_domain_request( + status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov" + ) + self.domain_request_4 = completed_domain_request( + status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov" + ) + self.domain_request_5 = completed_domain_request( + status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov" + ) + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() + + def tearDown(self): + super().tearDown() + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainRequest.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() + + def mock_user(): """A simple user.""" user_kwargs = dict( @@ -680,7 +860,7 @@ def dummyInfoContactResultData( self, id, email, - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), pw="thisisnotapassword", ): fake = info.InfoContactResultData( @@ -718,82 +898,82 @@ def dummyInfoContactResultData( mockDataInfoDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meoward.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainAndIPAddress = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoDomainNotSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.subdomainwoip.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataExtensionDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 11, 15), + ex_date=date(2023, 11, 15), ) mockDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "123", "123@mail.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "123", "123@mail.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithContacts = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -818,7 +998,7 @@ def dummyInfoContactResultData( InfoDomainWithDefaultSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultSec", @@ -833,11 +1013,11 @@ def dummyInfoContactResultData( ) mockVerisignDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "defaultVeri", "registrar@dotgov.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "defaultVeri", "registrar@dotgov.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithVerisignSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultVeri", @@ -853,7 +1033,7 @@ def dummyInfoContactResultData( InfoDomainWithDefaultTechnicalContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultTech", @@ -878,14 +1058,14 @@ def dummyInfoContactResultData( infoDomainNoContact = fakedEppObject( "security", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["fake.host.com"], ) infoDomainThreeHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.my-nameserver-1.com", @@ -896,43 +1076,43 @@ def dummyInfoContactResultData( infoDomainNoHost = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[], ) infoDomainTwoHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["ns1.my-nameserver-1.com", "ns1.my-nameserver-2.com"], ) mockDataInfoHosts = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="1.2.3.4"), common.Ip(addr="2.3.4.5")], ) mockDataInfoHosts1IP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoHostsNotSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 26, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 26, 19, 45, 35)), addrs=[], ) mockDataInfoHostsSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 27, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 27, 19, 45, 35)), addrs=[], ) - mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35))) + mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35))) addDsData1 = { "keyTag": 1234, "alg": 3, @@ -964,7 +1144,7 @@ def dummyInfoContactResultData( infoDomainHasIP = fakedEppObject( "nameserverwithip.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -989,7 +1169,7 @@ def dummyInfoContactResultData( justNameserver = fakedEppObject( "justnameserver.com", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -1012,7 +1192,7 @@ def dummyInfoContactResultData( infoDomainCheckHostIPCombo = fakedEppObject( "nameserversubdomain.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.nameserversubdomain.gov", @@ -1022,27 +1202,27 @@ def dummyInfoContactResultData( mockRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockButtonRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2025, 5, 25), + ex_date=date(2025, 5, 25), ) mockDnsNeededRenewedDomainExpDate = fakedEppObject( "fakeneeded.gov", - ex_date=datetime.date(2023, 2, 15), + ex_date=date(2023, 2, 15), ) mockMaximumRenewedDomainExpDate = fakedEppObject( "fakemaximum.gov", - ex_date=datetime.date(2024, 12, 31), + ex_date=date(2024, 12, 31), ) mockRecentRenewedDomainExpDate = fakedEppObject( "waterbutpurple.gov", - ex_date=datetime.date(2024, 11, 15), + ex_date=date(2024, 11, 15), ) def _mockDomainName(self, _name, _avail=False): diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index aa150d55c0..cc4b3f1c7b 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -3,7 +3,7 @@ from registrar.tests.common import create_superuser -class TestViews(TestCase): +class TestAdminViews(TestCase): def setUp(self): self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() @@ -26,7 +26,7 @@ def test_export_data_view(self): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin_export_data") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("export_domains_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 011c60b93d..b91f3bd18d 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -1,18 +1,18 @@ import csv import io -from django.test import Client, RequestFactory, TestCase +from django.test import Client, RequestFactory from io import StringIO -from registrar.models.domain_information import DomainInformation +from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain -from registrar.models.public_contact import PublicContact -from registrar.models.user import User -from django.contrib.auth import get_user_model -from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.common import MockEppLib from registrar.utility.csv_export import ( - write_csv, + export_data_managed_domains_to_csv, + export_data_unmanaged_domains_to_csv, + get_sliced_domains, + get_sliced_requests, + write_domains_csv, get_default_start_date, get_default_end_date, + write_requests_csv, ) from django.core.management import call_command @@ -22,62 +22,19 @@ from botocore.exceptions import ClientError import boto3_mocking from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore -from datetime import date, datetime, timedelta +from datetime import datetime from django.utils import timezone -from .common import less_console_noise +from .common import MockDb, MockEppLib, less_console_noise -class CsvReportsTest(TestCase): +class CsvReportsTest(MockDb): """Tests to determine if we are uploading our reports correctly""" def setUp(self): """Create fake domain data""" + super().setUp() self.client = Client(HTTP_HOST="localhost:8080") self.factory = RequestFactory() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - def tearDown(self): - """Delete all faked data""" - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - super().tearDown() @boto3_mocking.patching def test_generate_federal_report(self): @@ -88,6 +45,7 @@ def test_generate_federal_report(self): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -108,6 +66,7 @@ def test_generate_full_report(self): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -166,6 +125,7 @@ def side_effect(Bucket, Key): @boto3_mocking.patching def test_load_federal_report(self): """Tests the get_current_federal api endpoint""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -199,6 +159,7 @@ def test_load_federal_report(self): @boto3_mocking.patching def test_load_full_report(self): """Tests the current-federal api link""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -231,141 +192,17 @@ def test_load_full_report(self): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockEppLib): +class ExportDataTest(MockDb, MockEppLib): def setUp(self): super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() super().tearDown() def test_export_domains_to_writer_security_emails(self): """Test that export_domains_to_writer returns the expected security email""" + with less_console_noise(): # Add security email information self.domain_1.name = "defaultsecurity.gov" @@ -403,7 +240,7 @@ def test_export_domains_to_writer_security_emails(self): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) @@ -427,10 +264,11 @@ def test_export_domains_to_writer_security_emails(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_csv(self): + def test_write_domains_csv(self): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -462,7 +300,7 @@ def test_write_csv(self): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -486,8 +324,9 @@ def test_write_csv(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_additional(self): + def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -512,7 +351,7 @@ def test_write_body_additional(self): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -535,27 +374,23 @@ def test_write_body_additional(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_with_date_filter_pulls_domains_in_range(self): + def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): """Test that domains that are 1. READY and their first_ready dates are in range 2. DELETED and their deleted dates are in range are pulled when the growth report conditions are applied to export_domains_to_writed. Test that ready domains are sorted by first_ready/deleted dates first, names second. - We considered testing export_data_growth_to_csv which calls write_body + We considered testing export_data_domain_growth_to_csv which calls write_body and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. - TODO: Simplify is created_at is not needed for the report.""" + TODO: Simplify if created_at is not needed for the report.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) - # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -579,19 +414,19 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): "domain__state__in": [ Domain.State.READY, ], - "domain__first_ready__lte": end_date, - "domain__first_ready__gte": start_date, + "domain__first_ready__lte": self.end_date, + "domain__first_ready__gte": self.start_date, } filter_conditions_for_deleted_domains = { "domain__state__in": [ Domain.State.DELETED, ], - "domain__deleted__lte": end_date, - "domain__deleted__gte": start_date, + "domain__deleted__lte": self.end_date, + "domain__deleted__gte": self.start_date, } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -599,7 +434,7 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): get_domain_managers=False, should_write_header=True, ) - write_csv( + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -634,13 +469,13 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): def test_export_domains_to_writer_domain_managers(self): """Test that export_domains_to_writer returns the - expected domain managers""" + expected domain managers.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition - columns = [ "Domain name", "Status", @@ -664,7 +499,7 @@ def test_export_domains_to_writer_domain_managers(self): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True ) @@ -677,11 +512,11 @@ def test_export_domains_to_writer_domain_managers(self): expected_content = ( "Domain name,Status,Expiration date,Domain type,Agency," "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager email 1,Domain manager email 2,\n" + "Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n" "adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n" "cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,info@example.com\n" + ", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" "ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n" ) # Normalize line endings and remove commas, @@ -690,8 +525,132 @@ def test_export_domains_to_writer_domain_managers(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) + def test_export_data_managed_domains_to_csv(self): + """Test get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + export_data_managed_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") + ) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + self.maxDiff = None + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,1\n" + "\n" + "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" + "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + def test_export_data_unmanaged_domains_to_csv(self): + """Test get counts for domains that do not have domain managers for two different dates, + get list of unmanaged domains at end_date.""" + + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + export_data_unmanaged_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") + ) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + self.maxDiff = None + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): + """Test that requests that are + 1. SUBMITTED and their submission_date are in range + are pulled when the growth report conditions are applied to export_requests_to_writed. + Test that requests are sorted by requested domain name. + """ -class HelperFunctions(TestCase): + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + # Define columns, sort fields, and filter condition + # We'll skip submission date because it's dynamic and therefore + # impossible to set in expected_content + columns = [ + "Requested domain", + "Organization type", + ] + sort_fields = [ + "requested_domain__name", + ] + filter_condition = { + "status": DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": self.end_date, + "submission_date__gte": self.start_date, + } + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name + # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name + expected_content = ( + "Requested domain,Organization type\n" + "city3.gov,Federal - Executive\n" + "city4.gov,Federal - Executive\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + +class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" def test_get_default_start_date(self): @@ -704,3 +663,33 @@ def test_get_default_end_date(self): expected_date = timezone.now() actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) + + def test_get_sliced_domains(self): + """Should get fitered domains counts sliced by org type and election office.""" + + with less_console_noise(): + filter_condition = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": self.end_date, + } + # Test with distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True) + expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + + # Test without distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1] + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + + def test_get_sliced_requests(self): + """Should get fitered requests counts sliced by org type and election office.""" + + with less_console_noise(): + filter_condition = { + "status": DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": self.end_date, + } + submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + expected_content = [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 50d580d36f..916e0b0c84 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,7 +1,9 @@ +from collections import Counter import csv import logging from datetime import datetime from registrar.models.domain import Domain +from registrar.models.domain_request import DomainRequest from registrar.models.domain_information import DomainInformation from django.utils import timezone from django.core.paginator import Paginator @@ -19,16 +21,22 @@ def write_header(writer, columns): Receives params from the parent methods and outputs a CSV with a header row. Works with write_header as long as the same writer object is passed. """ - writer.writerow(columns) def get_domain_infos(filter_condition, sort_fields): + """ + Returns DomainInformation objects filtered and sorted based on the provided conditions. + filter_condition -> A dictionary of conditions to filter the objects. + sort_fields -> A list of fields to sort the resulting query set. + returns: A queryset of DomainInformation objects + """ domain_infos = ( DomainInformation.objects.select_related("domain", "authorizing_official") .prefetch_related("domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) + .distinct() ) # Do a mass concat of the first and last name fields for authorizing_official. @@ -45,7 +53,7 @@ def get_domain_infos(filter_condition, sort_fields): return domain_infos_cleaned -def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): +def parse_domain_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" # Domain should never be none when parsing this information @@ -129,7 +137,7 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict -def write_csv( +def write_domains_csv( writer, columns, sort_fields, @@ -138,10 +146,10 @@ def write_csv( should_write_header=True, ): """ - Receives params from the parent methods and outputs a CSV with fltered and sorted domains. - Works with write_header as longas the same writer object is passed. + Receives params from the parent methods and outputs a CSV with filtered and sorted domains. + Works with write_header as long as the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv - should_write_header: Conditional bc export_data_growth_to_csv calls write_body twice + should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ all_domain_infos = get_domain_infos(filter_condition, sort_fields) @@ -175,7 +183,7 @@ def write_csv( columns.append(column_name) try: - row = parse_row(columns, domain_info, security_emails_dict, get_domain_managers) + row = parse_domain_row(columns, domain_info, security_emails_dict, get_domain_managers) rows.append(row) except ValueError: # This should not happen. If it does, just skip this row. @@ -189,6 +197,82 @@ def write_csv( writer.writerows(total_body_rows) +def get_requests(filter_condition, sort_fields): + """ + Returns DomainRequest objects filtered and sorted based on the provided conditions. + filter_condition -> A dictionary of conditions to filter the objects. + sort_fields -> A list of fields to sort the resulting query set. + returns: A queryset of DomainRequest objects + """ + requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() + return requests + + +def parse_request_row(columns, request: DomainRequest): + """Given a set of columns, generate a new row from cleaned column data""" + + requested_domain_name = "No requested domain" + + if request.requested_domain is not None: + requested_domain_name = request.requested_domain.name + + if request.federal_type: + request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" + else: + request_type = request.get_organization_type_display() + + # create a dictionary of fields which can be included in output + FIELDS = { + "Requested domain": requested_domain_name, + "Status": request.get_status_display(), + "Organization type": request_type, + "Agency": request.federal_agency, + "Organization name": request.organization_name, + "City": request.city, + "State": request.state_territory, + "AO email": request.authorizing_official.email if request.authorizing_official else " ", + "Security contact email": request, + "Created at": request.created_at, + "Submission date": request.submission_date, + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + +def write_requests_csv( + writer, + columns, + sort_fields, + filter_condition, + should_write_header=True, +): + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" + + all_requests = get_requests(filter_condition, sort_fields) + + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_requests, 1000) + + for page_num in paginator.page_range: + page = paginator.page(page_num) + rows = [] + for request in page.object_list: + try: + row = parse_request_row(columns, request) + rows.append(row) + except ValueError: + # This should not happen. If it does, just skip this row. + # It indicates that DomainInformation.domain is None. + logger.error("csv_export -> Error when parsing row, domain was None") + continue + + if should_write_header: + write_header(writer, columns) + writer.writerows(rows) + + def export_data_type_to_csv(csv_file): """All domains report with extra columns""" @@ -223,7 +307,9 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True + ) def export_data_full_to_csv(csv_file): @@ -254,7 +340,9 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def export_data_federal_to_csv(csv_file): @@ -286,7 +374,9 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def get_default_start_date(): @@ -299,7 +389,15 @@ def get_default_end_date(): return timezone.now() -def export_data_growth_to_csv(csv_file, start_date, end_date): +def format_start_date(start_date): + return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() + + +def format_end_date(end_date): + return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + + +def export_data_domain_growth_to_csv(csv_file, start_date, end_date): """ Growth report: Receive start and end dates from the view, parse them. @@ -308,16 +406,9 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): the start and end dates. Specify sort params for both lists. """ - start_date_formatted = ( - timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() - ) - - end_date_formatted = ( - timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() - ) - + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) writer = csv.writer(csv_file) - # define columns to include in export columns = [ "Domain name", @@ -353,8 +444,10 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - write_csv( + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -362,3 +455,266 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): get_domain_managers=False, should_write_header=False, ) + + +def get_sliced_domains(filter_condition, distinct=False): + """Get filtered domains counts sliced by org type and election office. + Pass distinct=True when filtering by permissions so we do not to count multiples + when a domain has more that one manager. + """ + + # Round trip 1: Get distinct domain names based on filter condition + domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainInformation.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainInformation.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainInformation.objects.filter(is_election_board=True, **filter_condition).distinct().count() + + return [ + domains_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + + +def get_sliced_requests(filter_condition, distinct=False): + """Get filtered requests counts sliced by org type and election office.""" + + # Round trip 1: Get distinct requests based on filter condition + requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainRequest.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainRequest.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainRequest.objects.filter(is_election_board=True, **filter_condition).distinct().count() + + return [ + requests_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + + +def export_data_managed_domains_to_csv(csv_file, start_date, end_date): + """Get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date, True) + + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date, True) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_managed_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + + +def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): + """Get counts for domains that do not have domain managers for two different dates, + get list of unmanaged domains at end_date.""" + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date, True) + + writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date, True) + + writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_end_date, + get_domain_managers=False, + should_write_header=True, + ) + + +def export_data_requests_growth_to_csv(csv_file, start_date, end_date): + """ + Growth report: + Receive start and end dates from the view, parse them. + Request from write_requests_body SUBMITTED requests that are created between + the start and end dates. Specify sort params. + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + "requested_domain__name", + ] + filter_condition = { + "status": DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + "submission_date__gte": start_date_formatted, + } + + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index f7164663b1..eba8423ed6 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -2,6 +2,12 @@ from django.http import HttpResponse from django.views import View +from django.shortcuts import render +from django.contrib import admin +from django.db.models import Avg, F +from .. import models +import datetime +from django.utils import timezone from registrar.utility import csv_export @@ -10,7 +16,157 @@ logger = logging.getLogger(__name__) -class ExportData(View): +class AnalyticsView(View): + def get(self, request): + thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) + thirty_days_ago = timezone.make_aware(thirty_days_ago) + + last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago) + last_30_days_approved_applications = models.DomainRequest.objects.filter( + created_at__gt=thirty_days_ago, status=models.DomainRequest.DomainRequestStatus.APPROVED + ) + avg_approval_time = last_30_days_approved_applications.annotate( + approval_time=F("approved_domain__created_at") - F("submission_date") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # Format the timedelta to display only days + if avg_approval_time is not None: + avg_approval_time_display = f"{avg_approval_time.days} days" + else: + avg_approval_time_display = "No approvals to use" + + # The start and end dates are passed as url params + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date, True) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date, True) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains( + filter_unmanaged_domains_start_date, True + ) + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date, True) + + filter_ready_domains_start_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + filter_ready_domains_end_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": start_date_formatted, + } + filter_deleted_domains_end_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + filter_requests_start_date = { + "created_at__lte": start_date_formatted, + } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_start_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + filter_submitted_requests_end_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) + submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + + context = dict( + # Generate a dictionary of context variables that are common across all admin templates + # (site_header, site_url, ...), + # include it in the larger context dictionary so it's available in the template rendering context. + # This ensures that the admin interface styling and behavior are consistent with other admin pages. + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.filter(state=models.Domain.State.READY).count(), + last_30_days_applications=last_30_days_applications.count(), + last_30_days_approved_applications=last_30_days_approved_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time_display, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, + ), + ) + return render(request, "admin/analytics.html", context) + + +class ExportDataType(View): + def get(self, request, *args, **kwargs): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + + +class ExportDataFull(View): + def get(self, request, *args, **kwargs): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + + +class ExportDataFederal(View): + def get(self, request, *args, **kwargs): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response + + +class ExportDataDomainsGrowth(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters # #999: not needed if we switch to django forms @@ -19,8 +175,50 @@ def get(self, request, *args, **kwargs): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_growth_to_csv to return the resulting queryset, which we can then use + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use # in context to display this data in the template. - csv_export.export_data_growth_to_csv(response, start_date, end_date) + csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) + + return response + + +class ExportDataRequestsGrowth(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use + # in context to display this data in the template. + csv_export.export_data_requests_growth_to_csv(response, start_date, end_date) + + return response + + +class ExportDataManagedDomains(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' + csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) + + return response + + +class ExportDataUnmanagedDomains(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' + csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response