change inter_pp

intel-analytics · Nov 7, 2024 · 48f33c3 · 48f33c3
1 parent a7b6668
commit 48f33c3
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/python/llm/src/ipex_llm/transformers/npu_models/convert_mp.py b/python/llm/src/ipex_llm/transformers/npu_models/convert_mp.py
@@ -337,7 +337,15 @@ def optimize_llm(
         if intra_pp is None:
             intra_pp = 2
         if inter_pp is None:
-            inter_pp = 2 if group_size == 0 else 8
+            if group_size == 0:
+                inter_pp = 2
+            elif model.config.intermediate_size == 8192:
+                if model.config.hidden_size == 2048:
+                    inter_pp = 1
+                else:
+                    inter_pp = 2
+            else:
+                inter_pp = 8
         convert_llama(model,
                       max_output_len=max_context_len,
                       max_prompt_len=max_prompt_len,