Merge pull request #168 from macrocosm-os/prompt_fix

Improve prompts for forbidden keywords and bullet count.

finetune/eval/if_eval/bullet_count.py

@@ -9,13 +9,13 @@ def __init__(self, count: int):
 
         if count < 1:
             raise ValueError(
-                f"BulletFrequencyRule must expect at least 1 bullet point."
+                f"BulletFrequencyRule must expect at least 1 '*' bullet point."
             )
         self.count = count
 
     def get_prompt(self, index: int = -1) -> str:
         bullet = "bullet point" if self.count == 1 else "bullet points"
-        return f"The response must contain exactly {self.count} {bullet} in markdown format."
+        return f"The response must contain exactly {self.count} '*' {bullet} in markdown format."
 
     def matches(self, text: str, index: int = -1) -> bool:
         return (

finetune/eval/if_eval/keywords.py

@@ -142,7 +142,9 @@ def __init__(self, keywords: List[str]):
     def get_prompt(self, index: int) -> str:
         if index >= len(self.keywords):
             raise ValueError("Index is out of range for keywords.")
-        return f'The word "{self.keywords[index]}" must not be the response.'
+        return (
+            f'The word "{self.keywords[index]}" must not be included in the response.'
+        )
 
     def get_keywords(self) -> List[str]:
         return self.keywords

tests/finetune/eval/if_eval/test_bullet_count.py

@@ -32,14 +32,14 @@ def test_get_prompt_frequency_one(self):
         rule = BulletFrequencyRule(count=1)
         self.assertEqual(
             rule.get_prompt(),
-            "The response must contain exactly 1 bullet point in markdown format.",
+            "The response must contain exactly 1 '*' bullet point in markdown format.",
         )
 
     def test_get_prompt_frequency_two(self):
         rule = BulletFrequencyRule(count=2)
         self.assertEqual(
             rule.get_prompt(),
-            "The response must contain exactly 2 bullet points in markdown format.",
+            "The response must contain exactly 2 '*' bullet points in markdown format.",
         )
 
 

tests/finetune/eval/if_eval/test_keywords.py

@@ -74,10 +74,11 @@ class TestForbiddenKeyword(unittest.TestCase):
     def test_get_prompt(self):
         rule = KeywordForbiddenRule(["test", "example"])
         self.assertEqual(
-            rule.get_prompt(0), 'The word "test" must not be the response.'
+            rule.get_prompt(0), 'The word "test" must not be included in the response.'
         )
         self.assertEqual(
-            rule.get_prompt(1), 'The word "example" must not be the response.'
+            rule.get_prompt(1),
+            'The word "example" must not be included in the response.',
         )
 
     def test_matches(self):