bugfix: chat turns instead of repeating the conversation in preproces…

…s_data_with_chat_template.py (#1258) * bugfix: chat turns instead of repeating the conversation * pre-commit
EleutherAI · Aug 15, 2024 · 0ef2c07 · 0ef2c07
1 parent c1ea2a1
commit 0ef2c07
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tools/datasets/preprocess_data_with_chat_template.py b/tools/datasets/preprocess_data_with_chat_template.py
@@ -104,7 +104,7 @@ def build_chat(
         )
         chat_tokens = tokenizer.apply_chat_template(
             chat[: i + 1], add_generation_prompt=add_gen
-        )
+        )[len(tokens) :]
         # remove previous stuff...
         tokens.extend(chat_tokens)
         if only_last_turn and (i != len(chat) - 1):