use batch kernel for fp8e5

intel-analytics · Dec 12, 2024 · c0225e9 · c0225e9
1 parent 478fcdf
commit c0225e9
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/python/llm/src/ipex_llm/transformers/low_bit_linear.py b/python/llm/src/ipex_llm/transformers/low_bit_linear.py
@@ -405,7 +405,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int):
             or (device in ["arc", "flex"] and qtype in [SYM_INT8, FP4])
             or (device in ["arc", "flex", "mtl"] and qtype in [FP8E4])
             or (device in ["lnl"] and qtype in [SYM_INT4] and x.shape[1] % 512 == 0)
-            or (device in ["bmg"] and qtype in [SYM_INT4])
+            or (device in ["bmg"] and qtype in [SYM_INT4, FP8E5])
         )
     return False