multipack support for phi moe

axolotl-ai-cloud · Nov 16, 2024 · cb8bfab · cb8bfab
1 parent 0c8b1d8
commit cb8bfab
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/src/axolotl/common/architectures.py b/src/axolotl/common/architectures.py
@@ -10,6 +10,7 @@
         "JetMoeMoE",
     ],
     "mixtral": "MixtralSparseMoeBlock",
+    "phimoe": "PhiMoESparseMoeBlock",
     "qwen2_moe": "Qwen2MoeSparseMoeBlock",
     "deepseek_v2": "DeepseekV2MoE",
 }
diff --git a/src/axolotl/monkeypatch/multipack.py b/src/axolotl/monkeypatch/multipack.py
@@ -20,6 +20,7 @@
     "falcon",
     "phi",
     "phi3",
+    "phimoe",
     "gemma",
     "gemma2",
     "gemmoe",