4432 Fixed phrase search queries with §

cl/lib/utils.py

@@ -251,7 +251,10 @@ def cleanup_main_query(query_string: str) -> str:
     cleaned_items = []
     # Replace smart quotes with standard double quotes for consistency.
     query_string = re.sub(r"[“”]", '"', query_string)
-    for item in re.split(r'([^a-zA-Z0-9_\-^~":]+)', query_string):
+    # Tweaks to the following regex for special characters exceptions
+    # like §, $, %, and ¶ should also be applied to type_table in
+    # custom_word_delimiter_filter.
+    for item in re.split(r'([^a-zA-Z0-9_\-^~":§$%¶]+)', query_string):
         if not item:
             continue
 

cl/search/tests/tests.py

@@ -422,27 +422,42 @@ def setUpTestData(cls):
             sub_opinions=RelatedFactory(
                 OpinionWithChildrenFactory,
                 factory_related_name="cluster",
-                html_columbia="<p>Code, &#167; 1-815</p>",
+                html_columbia="<p>Code, &#167; 1-815 Lorem §247 $247 %247 ¶247</p>",
             ),
             precedential_status=PRECEDENTIAL_STATUS.PUBLISHED,
         )
         OpinionClusterFactoryWithChildrenAndParents(
             case_name="Strickland v. Lorem.",
             docket=DocketFactory(court=cls.court, docket_number="123456"),
+            sub_opinions=RelatedFactory(
+                OpinionWithChildrenFactory,
+                factory_related_name="cluster",
+                plain_text="Random plain_text",
+            ),
             precedential_status=PRECEDENTIAL_STATUS.PUBLISHED,
         )
         OpinionClusterFactoryWithChildrenAndParents(
             case_name="America vs Bank",
             docket=DocketFactory(
                 court=cls.child_court_1, docket_number="34-2535"
             ),
+            sub_opinions=RelatedFactory(
+                OpinionWithChildrenFactory,
+                factory_related_name="cluster",
+                plain_text="Lorem 247",
+            ),
             precedential_status=PRECEDENTIAL_STATUS.PUBLISHED,
         )
         OpinionClusterFactoryWithChildrenAndParents(
             case_name="Johnson v. National",
             docket=DocketFactory(
                 court=cls.child_court_2_2, docket_number="36-2000"
             ),
+            sub_opinions=RelatedFactory(
+                OpinionWithChildrenFactory,
+                factory_related_name="cluster",
+                plain_text="Random plain_text",
+            ),
             precedential_status=PRECEDENTIAL_STATUS.PUBLISHED,
         )
 
@@ -451,6 +466,11 @@ def setUpTestData(cls):
             docket=DocketFactory(
                 court=cls.child_gand_2, docket_number="38-1000"
             ),
+            sub_opinions=RelatedFactory(
+                OpinionWithChildrenFactory,
+                factory_related_name="cluster",
+                plain_text="Random plain_text",
+            ),
             precedential_status=PRECEDENTIAL_STATUS.PUBLISHED,
         )
         call_command(
@@ -838,6 +858,59 @@ def test_raise_forbidden_error_on_depth_pagination(self) -> None:
         )
         self.assertEqual(r.status_code, HTTPStatus.FORBIDDEN)
 
+    async def test_avoid_splitting_terms_on_special_chars(self) -> None:
+        """Can we avoid splitting words in queries such as §247 and phrases
+        like "§247"?
+        """
+
+        special_chars_exceptions = ["§", "$", "%", "¶"]
+        # A search for phrase "§247" shouldn't match "247"
+        for special_char in special_chars_exceptions:
+            with self.subTest(
+                special_char=special_char, msg="Phrase query and special char."
+            ):
+                r = await self.async_client.get(
+                    reverse("show_results"), {"q": f'"{special_char}247"'}
+                )
+                actual = self.get_article_count(r)
+                self.assertEqual(
+                    actual, 1, msg="Didn't get the right number of results"
+                )
+                self.assertIn("1:21-cv-1234", r.content.decode())
+
+        # A search for phrase "247" shouldn't match "§247"
+        r = await self.async_client.get(
+            reverse("show_results"), {"q": '"247"'}
+        )
+        actual = self.get_article_count(r)
+        self.assertEqual(
+            actual, 1, msg="Didn't get the right number of results"
+        )
+        self.assertIn("34-2535", r.content.decode())
+
+        # A search for §247 shouldn't match 247
+        for special_char in special_chars_exceptions:
+            with self.subTest(
+                special_char=special_char,
+                msg="Non-phrase query and special char.",
+            ):
+                r = await self.async_client.get(
+                    reverse("show_results"), {"q": f"{special_char}247"}
+                )
+                actual = self.get_article_count(r)
+                self.assertEqual(
+                    actual, 1, msg="Didn't get the right number of results"
+                )
+                self.assertIn("1:21-cv-1234", r.content.decode())
+
+        # A search for 247 shouldn't match §247
+        r = await self.async_client.get(reverse("show_results"), {"q": "247"})
+        actual = self.get_article_count(r)
+        self.assertEqual(
+            actual, 1, msg="Didn't get the right number of results"
+        )
+        self.assertIn("34-2535", r.content.decode())
+
 
 class SearchAPIV4CommonTest(ESIndexTestCase, TestCase):
     """Common tests for the Search API V4 endpoints."""
@@ -991,6 +1064,10 @@ def test_query_cleanup_function(self) -> None:
                 '"this is a test" ~2 and "net neutrality" ~5 and 22cv3332',
                 '"this is a test"~2 and "net neutrality"~5 and docketNumber:"22-cv-3332"~1',
             ),
+            ("§242", "§242"),
+            ("$242", "$242"),
+            ("%242", "%242"),
+            ("¶242", "¶242"),
         )
         for q, a in q_a:
             print("Does {q} --> {a} ? ".format(**{"q": q, "a": a}))

cl/settings/third_party/elasticsearch.py

@@ -125,6 +125,12 @@
         "filter": {
             "custom_word_delimiter_filter": {
                 "type": "word_delimiter",
+                "type_table": [
+                    "§ => ALPHANUM",
+                    "$ => ALPHANUM",
+                    "% => ALPHANUM",
+                    "¶ => ALPHANUM",
+                ],
                 "split_on_numerics": False,
                 "preserve_original": True,
             },