From cb8bfab9ccac309e03c96431874f8929e090738a Mon Sep 17 00:00:00 2001
From: Wing Lian <wing@axolotl.ai>
Date: Fri, 15 Nov 2024 22:56:25 -0500
Subject: [PATCH] multipack support for phi moe

---
 src/axolotl/common/architectures.py  | 1 +
 src/axolotl/monkeypatch/multipack.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/axolotl/common/architectures.py b/src/axolotl/common/architectures.py
index 827a63c070..00046ca4bb 100644
--- a/src/axolotl/common/architectures.py
+++ b/src/axolotl/common/architectures.py
@@ -10,6 +10,7 @@
         "JetMoeMoE",
     ],
     "mixtral": "MixtralSparseMoeBlock",
+    "phimoe": "PhiMoESparseMoeBlock",
     "qwen2_moe": "Qwen2MoeSparseMoeBlock",
     "deepseek_v2": "DeepseekV2MoE",
 }
diff --git a/src/axolotl/monkeypatch/multipack.py b/src/axolotl/monkeypatch/multipack.py
index 3ee89d2e5c..265f088183 100644
--- a/src/axolotl/monkeypatch/multipack.py
+++ b/src/axolotl/monkeypatch/multipack.py
@@ -20,6 +20,7 @@
     "falcon",
     "phi",
     "phi3",
+    "phimoe",
     "gemma",
     "gemma2",
     "gemmoe",