Update 125M_fused_rope.yml

EleutherAI · Dec 4, 2023 · 491806b · 491806b
1 parent 29f50ae
commit 491806b
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/configs/125M_fused_rope.yml b/configs/125M_fused_rope.yml
@@ -20,6 +20,7 @@
    # these should provide some speedup but takes a while to build, set to true if desired
    "scaled_upper_triang_masked_softmax_fusion": false,
    "bias_gelu_fusion": false,
+   "rope_fusion": true,
 
    # init methods
    "init_method": "small_init",