Merge pull request #4776 from freelawproject/4775-failing-case-name-i…

…mport

Add Journal to Citation model type

cl/citations/utils.py

@@ -25,6 +25,7 @@ def map_reporter_db_cite_type(citation_type: str) -> int:
     Citation = apps.get_model("search.Citation")
     citation_map = {
         "specialty": Citation.SPECIALTY,
+        "journal": Citation.JOURNAL,
         "federal": Citation.FEDERAL,
         "state": Citation.STATE,
         "state_regional": Citation.STATE_REGIONAL,

cl/corpus_importer/management/commands/update_casenames_wl_dataset.py

@@ -8,7 +8,7 @@
 from django.db import transaction
 from django.db.models import Q, QuerySet
 from eyecite import get_citations
-from eyecite.models import FullCaseCitation
+from eyecite.models import FullCaseCitation, FullJournalCitation
 from eyecite.tokenizers import HyperscanTokenizer
 from juriscraper.lib.string_utils import harmonize
 
@@ -139,12 +139,16 @@ def parse_citations(citation_strings: list[str]) -> list[dict]:
         # We find all the citations that could match a cluster to update the case name
         found_cites = get_citations(cite_str, tokenizer=HYPERSCAN_TOKENIZER)
         if not found_cites:
+            logger.info("Unable to parse %s", cite_str)
             continue
-
         citation = found_cites[0]
+        if len(citation.all_editions) > 1:
+            # In case we have two editions which could have different types
+            logger.info("Unable to disambiguate citation: %s", cite_str)
+            continue
 
         # Ensure we have valid citations to process
-        if isinstance(citation, FullCaseCitation):
+        if isinstance(citation, (FullCaseCitation, FullJournalCitation)):
             volume = citation.groups.get("volume")
 
             # Validate the volume

cl/corpus_importer/tests.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from queue import Queue
 from random import randint
-from unittest.mock import call, patch
+from unittest.mock import MagicMock, call, patch
 
 import eyecite
 import pytest
@@ -64,6 +64,7 @@
 )
 from cl.corpus_importer.management.commands.update_casenames_wl_dataset import (
     check_case_names_match,
+    parse_citations,
 )
 from cl.corpus_importer.signals import (
     handle_update_latest_case_id_and_schedule_iquery_sweep,
@@ -4083,6 +4084,30 @@ def test_probe_iquery_pages_daemon_court_got_stuck(
         self.assertEqual(int(court_empty_attempts), 0)
 
 
+class WestCitationImportTest(TestCase):
+    def test_parse_citation(self) -> None:
+        """Test parse citation for federal and journal citations"""
+        correct_response = [
+            {
+                "volume": "238",
+                "reporter": "F.3d",
+                "page": "273",
+                "type": Citation.FEDERAL,
+            },
+            {
+                "volume": "72",
+                "reporter": "Soc. Serv. Rev.",
+                "page": "318",
+                "type": Citation.JOURNAL,
+            },
+        ]
+        citation_strings = ["238 F.3d 273", "72 Soc.Sec.Rep.Serv. 318"]
+        valid_citations = parse_citations(citation_strings)
+        self.assertEqual(
+            valid_citations, correct_response, msg="Citations incorrect parsed"
+        )
+
+
 class CaseNamesTest(SimpleTestCase):
     def test_check_case_names_match(self) -> None:
         """Can we check if the case names match?"""

cl/search/migrations/0037_alter_citation_type_noop.py

@@ -0,0 +1,79 @@
+# Generated by Django 5.1.2 on 2024-12-03 17:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("search", "0036_add_searchquery"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="citation",
+            name="type",
+            field=models.SmallIntegerField(
+                choices=[
+                    (1, "A federal reporter citation (e.g. 5 F. 55)"),
+                    (
+                        2,
+                        "A citation in a state-based reporter (e.g. Alabama Reports)",
+                    ),
+                    (
+                        3,
+                        "A citation in a regional reporter (e.g. Atlantic Reporter)",
+                    ),
+                    (
+                        4,
+                        "A citation in a specialty reporter (e.g. Lawyers' Edition)",
+                    ),
+                    (
+                        5,
+                        "A citation in an early SCOTUS reporter (e.g. 5 Black. 55)",
+                    ),
+                    (6, "A citation in the Lexis system (e.g. 5 LEXIS 55)"),
+                    (7, "A citation in the WestLaw system (e.g. 5 WL 55)"),
+                    (8, "A vendor neutral citation (e.g. 2013 FL 1)"),
+                    (
+                        9,
+                        "A law journal citation within a scholarly or professional legal periodical (e.g. 95 Yale L.J. 5; 72 Soc.Sec.Rep.Serv. 318)",
+                    ),
+                ],
+                help_text="The type of citation that this is.",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="citationevent",
+            name="type",
+            field=models.SmallIntegerField(
+                choices=[
+                    (1, "A federal reporter citation (e.g. 5 F. 55)"),
+                    (
+                        2,
+                        "A citation in a state-based reporter (e.g. Alabama Reports)",
+                    ),
+                    (
+                        3,
+                        "A citation in a regional reporter (e.g. Atlantic Reporter)",
+                    ),
+                    (
+                        4,
+                        "A citation in a specialty reporter (e.g. Lawyers' Edition)",
+                    ),
+                    (
+                        5,
+                        "A citation in an early SCOTUS reporter (e.g. 5 Black. 55)",
+                    ),
+                    (6, "A citation in the Lexis system (e.g. 5 LEXIS 55)"),
+                    (7, "A citation in the WestLaw system (e.g. 5 WL 55)"),
+                    (8, "A vendor neutral citation (e.g. 2013 FL 1)"),
+                    (
+                        9,
+                        "A law journal citation within a scholarly or professional legal periodical (e.g. 95 Yale L.J. 5; 72 Soc.Sec.Rep.Serv. 318)",
+                    ),
+                ],
+                help_text="The type of citation that this is.",
+            ),
+        ),
+    ]

cl/search/migrations/0037_alter_citation_type_noop.sql

@@ -0,0 +1,10 @@
+BEGIN;
+--
+-- Alter field type on citation
+--
+-- (no-op)
+--
+-- Alter field type on citationevent
+--
+-- (no-op)
+COMMIT;

cl/search/models.py

@@ -3260,6 +3260,7 @@ class Citation(models.Model):
     LEXIS = 6
     WEST = 7
     NEUTRAL = 8
+    JOURNAL = 9
     CITATION_TYPES = (
         (FEDERAL, "A federal reporter citation (e.g. 5 F. 55)"),
         (
@@ -3281,6 +3282,12 @@ class Citation(models.Model):
         (LEXIS, "A citation in the Lexis system (e.g. 5 LEXIS 55)"),
         (WEST, "A citation in the WestLaw system (e.g. 5 WL 55)"),
         (NEUTRAL, "A vendor neutral citation (e.g. 2013 FL 1)"),
+        (
+            JOURNAL,
+            "A law journal citation within a scholarly or professional "
+            "legal periodical (e.g. 95 Yale L.J. 5; "
+            "72 Soc.Sec.Rep.Serv. 318)",
+        ),
     )
     cluster = models.ForeignKey(
         OpinionCluster,