Merge pull request #154 from uktrade/LTD-2508-Upgrade-Django-to-lates…

…t-LTS

LTD-2508: Upgrade Django to current LTS version 3.2

Pipfile

@@ -17,13 +17,10 @@ pytest-django = "*"
 pytest-cov = "*"
 
 [packages]
-Django = "~=2.2"
 djangorestframework = "~=3.9"
 django-background-tasks = "~=1.2"
 django-environ = "~=0.4"
 django-model-utils = "~=4.0"
-django-jsonfield = "~=1.4"
-elastic-apm = "~=5.8"
 sentry-sdk = "~=0.16"
 typed-ast = "~=1.4.2"
 mohawk = ">=1.1"
@@ -33,6 +30,9 @@ requests = "~=2.21"
 unidecode = "~=1.2"
 django-log-formatter-ecs = "==0.0.5"
 gunicorn = "*"
+django = "==3.2.14"
+elastic-apm = "==6.7.2"
+django-jsonfield = "==1.4"
 
 [requires]
 python_version = "3.7"

conf/tests/test_healthcheck.py

@@ -0,0 +1,65 @@
+from datetime import timedelta
+
+from background_task.models import Task
+from django.conf import settings
+from django.test import testcases
+from django.urls import reverse
+from django.utils import timezone
+from rest_framework import status
+
+from mail.enums import ReceptionStatusEnum, ReplyStatusEnum
+from mail.models import Mail
+from mail.tasks import LICENCE_DATA_TASK_QUEUE, MANAGE_INBOX_TASK_QUEUE
+
+
+class TestHealthcheck(testcases.TestCase):
+    def setUp(self):
+        self.url = reverse("healthcheck")
+
+    def test_healthcheck_return_ok(self):
+        response = self.client.get(self.url)
+        self.assertEqual(response.context["message"], "OK")
+        self.assertEqual(response.context["status"], status.HTTP_200_OK)
+
+    def test_healthcheck_service_unavailable_pending_mail(self):
+        sent_at = timezone.now() - timedelta(seconds=settings.EMAIL_AWAITING_REPLY_TIME)
+        Mail.objects.create(
+            edi_filename="filename",
+            edi_data="1\\fileHeader\\CHIEF\\SPIRE\\",
+            status=ReplyStatusEnum.PENDING,
+            sent_at=sent_at,
+        )
+        response = self.client.get(self.url)
+        self.assertEqual(response.context["message"], "not OK")
+        self.assertEqual(response.context["status"], status.HTTP_503_SERVICE_UNAVAILABLE)
+
+    def test_healthcheck_service_unavailable_rejected_mail(self):
+        sent_at = timezone.now() - timedelta(seconds=settings.EMAIL_AWAITING_CORRECTIONS_TIME)
+        Mail.objects.create(
+            edi_filename="filename",
+            edi_data="1\\fileHeader\\CHIEF\\SPIRE\\",
+            response_data="1\\line1\\rejected\\",
+            status=ReceptionStatusEnum.REPLY_SENT,
+            sent_at=sent_at,
+        )
+        response = self.client.get(self.url)
+        self.assertEqual(response.context["message"], "not OK")
+        self.assertEqual(response.context["status"], status.HTTP_503_SERVICE_UNAVAILABLE)
+
+    def test_healthcheck_service_unavailable_inbox_task_not_responsive(self):
+        run_at = timezone.now() + timedelta(minutes=settings.INBOX_POLL_INTERVAL)
+        task, _ = Task.objects.get_or_create(queue=MANAGE_INBOX_TASK_QUEUE)
+        task.run_at = run_at
+        task.save()
+        response = self.client.get(self.url)
+        self.assertEqual(response.context["message"], "not OK")
+        self.assertEqual(response.context["status"], status.HTTP_503_SERVICE_UNAVAILABLE)
+
+    def test_healthcheck_service_unavailable_licence_update_task_not_responsive(self):
+        run_at = timezone.now() + timedelta(minutes=settings.LITE_LICENCE_DATA_POLL_INTERVAL)
+        task, _ = Task.objects.get_or_create(queue=LICENCE_DATA_TASK_QUEUE)
+        task.run_at = run_at
+        task.save()
+        response = self.client.get(self.url)
+        self.assertEqual(response.context["message"], "not OK")
+        self.assertEqual(response.context["status"], status.HTTP_503_SERVICE_UNAVAILABLE)

conf/urls.py

@@ -22,7 +22,7 @@
 urlpatterns = [
     path("admin/", admin.site.urls),
     path("mail/", include("mail.urls")),
-    path("healthcheck/", HealthCheck.as_view()),
+    path("healthcheck/", HealthCheck.as_view(), name="healthcheck"),
 ]
 
 if settings.ENABLE_MOCK_HMRC_SERVICE:

conf/views.py

@@ -4,7 +4,7 @@
 
 from background_task.models import Task
 from django.conf import settings
-from django.shortcuts import render_to_response
+from django.shortcuts import render
 from django.utils import timezone
 from rest_framework.status import HTTP_200_OK, HTTP_503_SERVICE_UNAVAILABLE
 from rest_framework.views import APIView
@@ -82,10 +82,9 @@ def _get_rejected_mail() -> []:
             ).values_list("id", flat=True)
         )
 
-    @staticmethod
-    def _build_response(status, message, start_time):
+    def _build_response(self, status, message, start_time):
         duration_ms = (time.time() - start_time) * 1000
         response_time = "{:.3f}".format(duration_ms)
-        context = {"message": message, "response_time": response_time}
+        context = {"message": message, "response_time": response_time, "status": status}
 
-        return render_to_response("healthcheck.xml", context, content_type="application/xml", status=status)
+        return render(self.request, "healthcheck.xml", context, content_type="application/xml", status=status)

mail/enums.py

@@ -1,8 +1,7 @@
 import enum
 from types import DynamicClassAttribute
 
-from django.utils.decorators import classproperty
-from django.utils.functional import Promise
+from django.utils.functional import Promise, classproperty
 
 
 # Backported from Django 4.x

mail/migrations/0018_auto_20220714_1127.py

@@ -0,0 +1,63 @@
+# Generated by Django 3.2.14 on 2022-07-14 11:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("mail", "0017_auto_20220517_0830"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="licencepayload",
+            name="data",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="licence_ids",
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="lite_accepted_licences",
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="lite_licences",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="lite_payload",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="lite_rejected_licences",
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="lite_response",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="spire_accepted_licences",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="spire_licences",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name="usagedata",
+            name="spire_rejected_licences",
+            field=models.JSONField(default=dict),
+        ),
+    ]

mail/models.py

@@ -6,7 +6,6 @@
 
 from django.db import IntegrityError, models
 from django.utils import timezone
-from jsonfield import JSONField
 from model_utils.models import TimeStampedModel
 
 from mail.enums import (
@@ -109,21 +108,21 @@ def get_licence_ids(self):
 
 class UsageData(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    licence_ids = JSONField()
+    licence_ids = models.JSONField(default=list)
     mail = models.ForeignKey(Mail, on_delete=models.DO_NOTHING, null=False)
     spire_run_number = models.IntegerField()
     hmrc_run_number = models.IntegerField()
     has_lite_data = models.NullBooleanField(default=None)
     has_spire_data = models.NullBooleanField(default=None)
-    lite_payload = JSONField()
+    lite_payload = models.JSONField(default=dict)
     lite_sent_at = models.DateTimeField(blank=True, null=True)  # When update was sent to LITE API
-    lite_accepted_licences = JSONField()
-    lite_rejected_licences = JSONField()
-    spire_accepted_licences = JSONField()
-    spire_rejected_licences = JSONField()
-    lite_licences = JSONField()
-    spire_licences = JSONField()
-    lite_response = JSONField()
+    lite_accepted_licences = models.JSONField(default=list)
+    lite_rejected_licences = models.JSONField(default=list)
+    spire_accepted_licences = models.JSONField(default=dict)
+    spire_rejected_licences = models.JSONField(default=dict)
+    lite_licences = models.JSONField(default=dict)
+    spire_licences = models.JSONField(default=dict)
+    lite_response = models.JSONField(default=dict)
 
     class Meta:
         ordering = ["mail__created_at"]
@@ -143,7 +142,7 @@ class LicencePayload(models.Model):
     lite_id = models.UUIDField(null=False, blank=False, unique=False)
     reference = models.CharField(null=False, blank=False, max_length=35)
     action = models.CharField(choices=LicenceActionEnum.choices, null=False, blank=False, max_length=6)
-    data = JSONField()
+    data = models.JSONField(default=dict)
     received_at = models.DateTimeField(default=timezone.now)
     is_processed = models.BooleanField(default=False)