Fix RoPE theta in config (#102)

google-deepmind · Dec 15, 2024 · e23bfed · e23bfed
1 parent fda6cd1
commit e23bfed
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/penzai/models/transformer/variants/llamalike_common.py b/penzai/models/transformer/variants/llamalike_common.py
@@ -596,7 +596,7 @@ def llamalike_from_huggingface_model(
       num_decoder_blocks=hf_config.num_hidden_layers,
       vocab_size=hf_config.vocab_size,
       mlp_variant="swiglu",
-      rope_wavelength=10_000,
+      rope_wavelength=hf_config.rope_theta,
       tie_embedder_and_logits=False,
       attention_type=attention_type,
       rms_norm_eps=hf_config.rms_norm_eps,