Update neox_args.py

EleutherAI · Nov 25, 2023 · 4808ced · 4808ced
1 parent aaa000c
commit 4808ced
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/megatron/neox_arguments/neox_args.py b/megatron/neox_arguments/neox_args.py
@@ -254,6 +254,11 @@ class NeoXArgsModel(NeoXArgsTemplate):
     """
     Enable bias and dropout fusion.
     """
+
+    rope_fusion: bool = False
+    """
+    Enable rotary embedding fusion.
+    """
 
     fp16_lm_cross_entropy: bool = False
     """