diff --git a/megatron/initialize.py b/megatron/initialize.py
index bc4032649..dd752659b 100644
--- a/megatron/initialize.py
+++ b/megatron/initialize.py
@@ -57,6 +57,7 @@ def finish_mpu_init():
     if (
         neox_args.scaled_upper_triang_masked_softmax_fusion
         or neox_args.scaled_masked_softmax_fusion
+        or neox_args.rope_fusion
     ):
         fused_kernels.load_fused_kernels()