From 3a72f5f1512b9172890f58197ad9150d2ecd6340 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:33:00 +0800 Subject: [PATCH 1/5] Update get_xpu_device_type() to support bmg --- python/llm/src/ipex_llm/transformers/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/llm/src/ipex_llm/transformers/utils.py b/python/llm/src/ipex_llm/transformers/utils.py index 2ec0dcf456f..72bc46aef4d 100644 --- a/python/llm/src/ipex_llm/transformers/utils.py +++ b/python/llm/src/ipex_llm/transformers/utils.py @@ -174,6 +174,8 @@ def get_xpu_device_type(x): name = torch.xpu.get_device_name(x.device.index) if name.startswith("Intel(R) Arc(TM) A"): return "arc" + if name.startswith("Intel(R) Graphics [0xe20b]"): + return "bmg" elif name.startswith("Intel(R) Arc(TM)"): if 'V' in name: return "lnl" From b696335e33a0763426085ac791b56e88491c7f36 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:08:19 +0800 Subject: [PATCH 2/5] enable use_batch_forward for bmg --- python/llm/src/ipex_llm/transformers/low_bit_linear.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/llm/src/ipex_llm/transformers/low_bit_linear.py b/python/llm/src/ipex_llm/transformers/low_bit_linear.py index ed44140d708..4892d33ad4a 100644 --- a/python/llm/src/ipex_llm/transformers/low_bit_linear.py +++ b/python/llm/src/ipex_llm/transformers/low_bit_linear.py @@ -405,6 +405,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int): or (device in ["arc", "flex"] and qtype in [SYM_INT8, FP4]) or (device in ["arc", "flex", "mtl"] and qtype in [FP8E4]) or (device in ["lnl"] and qtype in [SYM_INT4] and x.shape[1] % 512 == 0) + or device in ["bmg"] ) return False From 56247e67b0abfd672f3def9d9ff0349bc32242d1 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:26:48 +0800 Subject: [PATCH 3/5] Update low_bit_linear.py --- python/llm/src/ipex_llm/transformers/low_bit_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/llm/src/ipex_llm/transformers/low_bit_linear.py b/python/llm/src/ipex_llm/transformers/low_bit_linear.py index 4892d33ad4a..bf4ee265f72 100644 --- a/python/llm/src/ipex_llm/transformers/low_bit_linear.py +++ b/python/llm/src/ipex_llm/transformers/low_bit_linear.py @@ -405,7 +405,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int): or (device in ["arc", "flex"] and qtype in [SYM_INT8, FP4]) or (device in ["arc", "flex", "mtl"] and qtype in [FP8E4]) or (device in ["lnl"] and qtype in [SYM_INT4] and x.shape[1] % 512 == 0) - or device in ["bmg"] + or (device in ["bmg"] and qtype in [SYM_INT4]) ) return False From 478fcdf16ad6104949be18cd038ac717f156f294 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:28:03 +0800 Subject: [PATCH 4/5] Update utils.py --- python/llm/src/ipex_llm/transformers/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/llm/src/ipex_llm/transformers/utils.py b/python/llm/src/ipex_llm/transformers/utils.py index 72bc46aef4d..b2ae0ca374d 100644 --- a/python/llm/src/ipex_llm/transformers/utils.py +++ b/python/llm/src/ipex_llm/transformers/utils.py @@ -174,7 +174,7 @@ def get_xpu_device_type(x): name = torch.xpu.get_device_name(x.device.index) if name.startswith("Intel(R) Arc(TM) A"): return "arc" - if name.startswith("Intel(R) Graphics [0xe20b]"): + elif name.startswith("Intel(R) Graphics [0xe20b]"): return "bmg" elif name.startswith("Intel(R) Arc(TM)"): if 'V' in name: From c0225e90078ab56ad5c83e511a52c03e4f9c5118 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:25:57 +0800 Subject: [PATCH 5/5] use batch kernel for fp8e5 --- python/llm/src/ipex_llm/transformers/low_bit_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/llm/src/ipex_llm/transformers/low_bit_linear.py b/python/llm/src/ipex_llm/transformers/low_bit_linear.py index bf4ee265f72..3d328812c7d 100644 --- a/python/llm/src/ipex_llm/transformers/low_bit_linear.py +++ b/python/llm/src/ipex_llm/transformers/low_bit_linear.py @@ -405,7 +405,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int): or (device in ["arc", "flex"] and qtype in [SYM_INT8, FP4]) or (device in ["arc", "flex", "mtl"] and qtype in [FP8E4]) or (device in ["lnl"] and qtype in [SYM_INT4] and x.shape[1] % 512 == 0) - or (device in ["bmg"] and qtype in [SYM_INT4]) + or (device in ["bmg"] and qtype in [SYM_INT4, FP8E5]) ) return False