Link to gia-dataset for oscar (instead of Clément's repo) (#140)

huggingface · Nov 22, 2023 · f7ca594 · f7ca594
1 parent 5b36e2f
commit f7ca594
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 8 deletions.
diff --git a/scripts/download_all_datasets.py b/scripts/download_all_datasets.py
@@ -19,8 +19,5 @@
     print(f"Loading {task}...")
     cache_path = f"{HF_DATASETS_CACHE}/gia-project/gia-dataset/{task}"
     if not os.path.exists(cache_path):
-        if task == "oscar":
-            dataset = load_dataset("ClementRomac/cleaned_deduplicated_oscar")
-        else:
-            dataset = load_dataset("gia-project/gia-dataset", task)
+        dataset = load_dataset("gia-project/gia-dataset", task)
         dataset.save_to_disk(cache_path)
diff --git a/scripts/train_gia.py b/scripts/train_gia.py
@@ -139,10 +139,7 @@ def main():
             dataset_dict[task] = {s: d.to_iterable_dataset() for s, d in dataset.items()}
     else:
         for task in tasks:
-            if task == "oscar":
-                dataset_dict[task] = load_dataset("ClementRomac/cleaned_deduplicated_oscar", streaming=True)
-            else:
-                dataset_dict[task] = load_dataset("gia-project/gia-dataset", task, streaming=True)
+            dataset_dict[task] = load_dataset("gia-project/gia-dataset", task, streaming=True)
 
     # Preprocess the dataset
     for task in dataset_dict.keys():