Fix preprocess_data.py link

EleutherAI · Oct 24, 2023 · 66d0666 · 66d0666
1 parent b02d989
commit 66d0666
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tools/datasets/corpora.py b/tools/datasets/corpora.py
@@ -141,7 +141,7 @@ def tokenize(self):
             [os.path.join(parent_folder, os.path.basename(url)) for url in self.urls]
         )
 
-        cmd = f"python tools/preprocess_data.py \
+        cmd = f"python tools/datasets/preprocess_data.py \
             --input {jsonl_filepath} \
             --output-prefix {parent_folder}/{self.name} \
             --vocab {self.vocab_file} \