hotfix activation typo from #1212

EleutherAI · Sep 9, 2024 · 3169c84 · 3169c84
1 parent 836aefa
commit 3169c84
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/megatron/model/transformer.py b/megatron/model/transformer.py
@@ -979,6 +979,7 @@ def __init__(
         self.gpt_j_tied = neox_args.gpt_j_tied
         self.mlp_type = neox_args.mlp_type
         self.moe_type = neox_args.moe_type
+        self.activation = neox_args.activation
 
         if self.gpt_j_residual:
             # GPT-J style layers allow us to defer the reduction of results across TP ranks until the end of the two sublayers.