Fix tools paths for megatron imports

EleutherAI · Oct 2, 2023 · a76d767 · a76d767
1 parent 42c4774
commit a76d767
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 6 deletions.
diff --git a/tools/ckpts/convert_hf_to_sequential.py b/tools/ckpts/convert_hf_to_sequential.py
@@ -14,7 +14,7 @@
 from transformers import GPTNeoXForCausalLM, GPTNeoXConfig
 
 sys.path.append(
-    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 )
 from megatron.neox_arguments import NeoXArgs
 from megatron.training import get_model, get_optimizer, get_learning_rate_scheduler

diff --git a/tools/ckpts/convert_module_to_hf.py b/tools/ckpts/convert_module_to_hf.py
@@ -25,7 +25,7 @@
 
 
 sys.path.append(
-    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 )
 from megatron.tokenizer import build_tokenizer
 

diff --git a/tools/ckpts/convert_sequential_to_hf.py b/tools/ckpts/convert_sequential_to_hf.py
@@ -25,7 +25,7 @@
 from typing import List
 
 sys.path.append(
-    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 )
 from megatron.tokenizer import build_tokenizer
 

diff --git a/tools/datasets/merge_datasets.py b/tools/datasets/merge_datasets.py
@@ -4,7 +4,7 @@
 import argparse
 
 sys.path.append(
-    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 )
 
 from megatron.data import indexed_dataset

diff --git a/tools/datasets/preprocess_data.py b/tools/datasets/preprocess_data.py
@@ -26,7 +26,7 @@
 import numpy as np
 
 sys.path.append(
-    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 )
 import time
 import tqdm

diff --git a/tools/datasets/preprocess_data_with_mask.py b/tools/datasets/preprocess_data_with_mask.py
@@ -62,7 +62,7 @@
 import numpy as np
 
 sys.path.append(
-    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 )
 import time
 import tqdm