ICMSLST-2523 Data Migration - Mailshot E2E Tests

data_migration/tests/test_e2e_users.py

@@ -42,7 +42,9 @@
         (dm.ApprovalRequest, web.ApprovalRequest),
         (dm.ImporterApprovalRequest, web.ImporterApprovalRequest),
         (dm.ExporterApprovalRequest, web.ExporterApprovalRequest),
+        (dm.Mailshot, web.Mailshot),
     ],
+    "file": [(dm.File, web.File)],
 }
 
 
@@ -59,6 +61,14 @@
             QueryModel(queries.exporters, "exporters", dm.Exporter),
             QueryModel(queries.exporter_offices, "exporter_offices", dm.Office),
             QueryModel(queries.access_requests, "access_requests", dm.AccessRequest),
+            QueryModel(queries.mailshots, "mailshot", dm.Mailshot),
+        ],
+        "file_folder": [
+            QueryModel(queries.mailshot_file_folders, "mailshot folders", dm.FileFolder),
+            QueryModel(queries.mailshot_file_targets, "mailshot targets", dm.FileTarget),
+        ],
+        "file": [
+            QueryModel(queries.mailshot_files, "mailshot files", dm.File),
         ],
     },
 )
@@ -71,6 +81,7 @@
             (dm.Office, web.Exporter, "offices"),
             (dm.FurtherInformationRequest, web.AccessRequest, "further_information_requests"),
         ],
+        "file": [(dm.MailshotDoc, web.Mailshot, "documents")],
     },
 )
 @mock.patch.dict(
@@ -95,9 +106,9 @@
 def test_import_user_data(mock_connect, dummy_dm_settings):
     mock_connect.return_value = utils.MockConnect()
 
-    call_command("export_from_v1", "--skip_ia", "--skip_export", "--skip_ref", "--skip_file")
-    call_command("extract_v1_xml", "--skip_ia", "--skip_export", "--skip_ref", "--skip_file")
-    call_command("import_v1_data", "--skip_ia", "--skip_export", "--skip_ref", "--skip_file")
+    call_command("export_from_v1", "--skip_ia", "--skip_export", "--skip_ref")
+    call_command("extract_v1_xml", "--skip_ia", "--skip_export", "--skip_ref")
+    call_command("import_v1_data", "--skip_ia", "--skip_export", "--skip_ref")
 
     assert web.User.objects.filter(groups__isnull=False).count() == 0
 
@@ -363,3 +374,73 @@ def test_import_user_data(mock_connect, dummy_dm_settings):
 
     assert dm.Importer.objects.filter(pk=4).exists()
     assert not web.Importer.objects.filter(pk=4).exists()
+
+    # Check mailshots
+
+    assert web.Mailshot.objects.count() == 3
+
+    assert web.UniqueReference.objects.get(prefix="MAIL", year=None, reference=1)
+    assert web.UniqueReference.objects.get(prefix="MAIL", year=None, reference=2)
+
+    draft_ms = web.Mailshot.objects.get(status="DRAFT")
+    assert draft_ms.is_active is True
+    assert draft_ms.reference is None
+    assert draft_ms.status == "DRAFT"
+    assert draft_ms.title is None
+    assert draft_ms.description is None
+    assert draft_ms.email_subject == "Draft Subject"
+    assert draft_ms.email_body == "Draft Body"
+    assert draft_ms.is_retraction_email is False
+    assert draft_ms.retract_email_subject is None
+    assert draft_ms.retract_email_body is None
+    assert draft_ms.create_datetime == dt.datetime(2022, 11, 14, 8, 47, tzinfo=dt.UTC)
+    assert draft_ms.created_by_id == 2
+    assert draft_ms.published_datetime is None
+    assert draft_ms.published_by_id is None
+    assert draft_ms.retracted_datetime is None
+    assert draft_ms.retracted_by_id is None
+    assert draft_ms.is_to_importers is False
+    assert draft_ms.is_to_exporters is True
+    assert draft_ms.documents.count() == 0
+
+    published_ms = web.Mailshot.objects.get(status="PUBLISHED")
+    assert published_ms.is_active is True
+    assert published_ms.reference == "MAIL/1"
+    assert published_ms.status == "PUBLISHED"
+    assert published_ms.title == "Published Title"
+    assert published_ms.description == "Published Description"
+    assert published_ms.email_subject == "Published Subject"
+    assert published_ms.email_body == "Published Body"
+    assert published_ms.is_retraction_email is False
+    assert published_ms.retract_email_subject is None
+    assert published_ms.retract_email_body is None
+    assert published_ms.create_datetime == dt.datetime(2022, 11, 14, 8, 47, tzinfo=dt.UTC)
+    assert published_ms.created_by_id == 2
+    assert published_ms.published_datetime == dt.datetime(2022, 11, 14, 9, 47, tzinfo=dt.UTC)
+    assert published_ms.published_by_id == 2
+    assert published_ms.retracted_datetime is None
+    assert published_ms.retracted_by_id is None
+    assert published_ms.is_to_importers is True
+    assert published_ms.is_to_exporters is True
+    assert published_ms.documents.count() == 3
+
+    retracted_ms = web.Mailshot.objects.get(status="RETRACTED")
+    assert retracted_ms.is_active is True
+    assert retracted_ms.reference == "MAIL/2"
+    assert retracted_ms.status == "RETRACTED"
+    assert retracted_ms.title == "Retraction Title"
+    assert retracted_ms.description == "Retraction Description"
+    assert retracted_ms.email_subject == "Email Subject"
+    assert retracted_ms.email_body == "Email Body"
+    assert retracted_ms.is_retraction_email is True
+    assert retracted_ms.retract_email_subject == "Retraction Subject"
+    assert retracted_ms.retract_email_body == "Retraction Body"
+    assert retracted_ms.create_datetime == dt.datetime(2022, 11, 14, 8, 47, tzinfo=dt.UTC)
+    assert retracted_ms.created_by_id == 2
+    assert retracted_ms.published_datetime == dt.datetime(2022, 11, 14, 9, 47, tzinfo=dt.UTC)
+    assert retracted_ms.published_by_id == 2
+    assert retracted_ms.retracted_datetime is None
+    assert retracted_ms.retracted_by_id is None
+    assert retracted_ms.is_to_importers is True
+    assert retracted_ms.is_to_exporters is False
+    assert retracted_ms.documents.count() == 1

data_migration/tests/utils/file_data.py

@@ -858,4 +858,99 @@
             ),
         ],
     ),
+    queries.mailshot_file_folders: (
+        FOLDER_COLUMNS,
+        [
+            (2000, "MAILSHOT_DOCUMENTS"),
+            (2001, "MAILSHOT_DOCUMENTS"),
+            (2002, "MAILSHOT_DOCUMENTS"),
+            (2003, "MAILSHOT_DOCUMENTS"),
+        ],
+    ),
+    queries.mailshot_file_targets: (
+        TARGET_COLUMNS,
+        [
+            (
+                2000,  # folder_id
+                "MAILSHOT_DOCUMENT",  # target_type
+                "RECEIVED",  # status
+                20000,  # target_id
+            ),
+            (
+                2000,  # folder_id
+                "MAILSHOT_DOCUMENT",  # target_type
+                "RECEIVED",  # status
+                20001,  # target_id
+            ),
+            (
+                2000,  # folder_id
+                "MAILSHOT_DOCUMENT",  # target_type
+                "RECEIVED",  # status
+                20002,  # target_id
+            ),
+            (
+                2001,  # folder_id
+                "MAILSHOT_DOCUMENT",  # target_type
+                "EMPTY",  # status
+                20003,  # target_id
+            ),
+            (
+                2002,  # folder_id
+                "MAILSHOT_DOCUMENT",  # target_type
+                "DELETED",  # status
+                20004,  # target_id
+            ),
+            (
+                2003,  # folder_id
+                "MAILSHOT_DOCUMENT",  # target_type
+                "RECEIVED",  # status
+                20005,  # target_id
+            ),
+        ],
+    ),
+    queries.mailshot_files: (
+        FILES_COLUMNS,
+        [
+            (
+                20000,  # target_id
+                21000,  # version_id
+                dt.datetime(2022, 4, 27, 12, 23),  # created_date
+                2,  # created_by_id
+                "mailshot/file1",  # path
+                "Test Mailshot 1.pdf",  # filename
+                "pdf",  # content_type
+                12345,  # file_size
+            ),
+            (
+                20001,  # target_id
+                21001,  # version_id
+                dt.datetime(2022, 4, 27, 12, 23),  # created_date
+                2,  # created_by_id
+                "mailshot/file2",  # path
+                "Test Mailshot 2.pdf",  # filename
+                "pdf",  # content_type
+                12345,  # file_size
+            ),
+            (
+                20002,  # target_id
+                21002,  # version_id
+                dt.datetime(2022, 4, 27, 12, 23),  # created_date
+                2,  # created_by_id
+                "mailshot/file3",  # path
+                "Test Mailshot 3.pdf",  # filename
+                "pdf",  # content_type
+                12345,  # file_size
+            ),
+            (
+                20005,  # target_id
+                21005,  # version_id
+                dt.datetime(2022, 4, 27, 12, 23),  # created_date
+                2,  # created_by_id
+                "mailshot/file4",  # path
+                "Test Mailshot 4.pdf",  # filename
+                "pdf",  # content_type
+                12345,  # file_size
+            ),
+        ],
+    ),
 }

data_migration/tests/utils/user_data.py

@@ -586,6 +586,94 @@
             ),
         ],
     ),
+    queries.mailshots: (
+        [
+            ("folder_id",),
+            ("reference",),
+            ("status",),
+            ("title",),
+            ("description",),
+            ("email_subject",),
+            ("email_body",),
+            ("is_retraction_email",),
+            ("retract_email_subject",),
+            ("retract_email_body",),
+            ("created_by_id",),
+            ("create_datetime",),
+            ("published_by_id",),
+            ("published_datetime",),
+            ("retracted_by_id",),
+            ("retracted_datetime",),
+            ("version",),
+            ("is_to_importers",),
+            ("is_to_exporters",),
+        ],
+        [
+            (
+                2000,  # folder_id
+                "MAIL/1",  # reference
+                "PUBLISHED",  # status
+                "Published Title",  # title
+                "Published Description",  # description
+                "Published Subject",  # email_subject
+                "Published Body",  # email_body
+                0,  # is_retraction_email
+                None,  # retract_email_subject
+                None,  # retract_email_body
+                2,  # created_by_id
+                dt.datetime(2022, 11, 14, 8, 47),  # create_datetime
+                2,  # published_by_id
+                dt.datetime(2022, 11, 14, 9, 47),  # published_datetime
+                None,  # retracted_by_id
+                None,  # retracted_datetime
+                1,  # version
+                1,  # is_to_importers
+                1,  # is_to_exporters
+            ),
+            (
+                2001,  # folder_id
+                None,  # reference
+                "DRAFT",  # status
+                None,  # title
+                None,  # description
+                "Draft Subject",  # email_subject
+                "Draft Body",  # email_body
+                0,  # is_retraction_email
+                None,  # retract_email_subject
+                None,  # retract_email_body
+                2,  # created_by_id
+                dt.datetime(2022, 11, 14, 8, 47),  # create_datetime
+                None,  # published_by_id
+                None,  # published_datetime
+                None,  # retracted_by_id
+                None,  # retracted_datetime
+                1,  # version
+                0,  # is_to_importers
+                1,  # is_to_exporters
+            ),
+            (
+                2003,  # folder_id
+                "MAIL/2",  # reference
+                "RETRACTED",  # status
+                "Retraction Title",  # title
+                "Retraction Description",  # description
+                "Email Subject",  # email_subject
+                "Email Body",  # email_body
+                1,  # is_retraction_email
+                "Retraction Subject",  # retract_email_subject
+                "Retraction Body",  # retract_email_body
+                2,  # created_by_id
+                dt.datetime(2022, 11, 14, 8, 47),  # create_datetime
+                2,  # published_by_id
+                dt.datetime(2022, 11, 14, 9, 47),  # published_datetime
+                None,  # retracted_by_id
+                None,  # retracted_datetime
+                1,  # version
+                1,  # is_to_importers
+                0,  # is_to_exporters
+            ),
+        ],
+    ),
     queries.ilb_user_roles: (
         [("username",), ("roles",)],
         [

pii-ner-exclude.txt

@@ -3859,3 +3859,10 @@ the Supplementary Report
 Add New Supplementary Report
 jQuery
 "Check if the date
+Draft Subject
+Published Title
+Published Description
+Published Subject
+Retraction Subject
+MAILSHOT_DOCUMENT
+Test Mailshot 3.pdf