add padding

GRAAL-Research · Nov 14, 2023 · 29ab854 · 29ab854
1 parent 66721af
commit 29ab854
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/training/few_shot_training.py b/src/training/few_shot_training.py
@@ -69,7 +69,7 @@
 
 
 def tokenize_function(example):
-    return tokenizer(example["original"], example["simplification"], truncation=True)
+    return tokenizer(example["original"], example["simplification"], truncation=True, padding=True)
 
 
 tokenized_csmd_dataset = csmd_dataset.map(tokenize_function, batched=True)