add assert for missing tokenizer_type in config

EleutherAI · Sep 4, 2024 · d975955 · d975955
1 parent c786367
commit d975955
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -31,6 +31,8 @@ def build_tokenizer(args):
     """Initialize tokenizer."""
     if args.rank == 0:
         print("> building {} tokenizer ...".format(args.tokenizer_type), flush=True)
+
+    assert args.tokenizer_type is not None, "tokenizer_type must be specified in the .yml config"
 
     # Select and instantiate the tokenizer.
     if args.tokenizer_type.lower() == "GPT2BPETokenizer".lower():