tests: Fix test examples in test_reduced_vocabulary_with_rare_tokens

dottxt-ai · Jan 7, 2025 · dbec88d · dbec88d
1 parent 3c66f72
commit dbec88d
Showing 1 changed file with 7 additions and 5 deletions.
diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py
@@ -542,12 +542,14 @@ def convert_token_to_string(self, token):
         "�",
         "��",
         "�.",
-        "�..",
+        ".�",
+        ".�.",
         "▁�",
-        "▁▁�",
-        "▁�.",
-        "▁�.",
-        "▁▁�..",
+        "�▁",
+        "▁�▁",
+        "?�",
+        "�?",
+        "?�?",
     ],
 )
 def test_reduced_vocabulary_with_rare_tokens(rare_token):