diff --git a/python/llm/dev/benchmark/all-in-one/run-stress-test.py b/python/llm/dev/benchmark/all-in-one/run-stress-test.py index 05b10f020cc..295ae1221a3 100644 --- a/python/llm/dev/benchmark/all-in-one/run-stress-test.py +++ b/python/llm/dev/benchmark/all-in-one/run-stress-test.py @@ -148,7 +148,7 @@ def run_transformer_int4_gpu(repo_id, num_beams, low_bit): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoTokenizer, LlamaTokenizer import intel_extension_for_pytorch as ipex reserved_mem_list = [] model_path = get_model_path(repo_id, local_model_hub) @@ -170,9 +170,6 @@ def run_transformer_int4_gpu(repo_id, trust_remote_code=True, use_cache=True).eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) model = model.to('xpu') - if isinstance(model, GPTJForCausalLM): - # For gpt-j model family, this optimization can provide a better performance. - model = ipex.optimize(model.eval(), inplace=True) end = time.perf_counter() print(">> loading of model costs {}s".format(end - st)) reserved_mem_list.append(torch.xpu.memory.memory_reserved()/(1024**3)) @@ -227,7 +224,7 @@ def run_transformer_int4_gpu(repo_id, today = date.today() if 'exclude' in conf: excludes = conf['exclude'] - + import pandas as pd for api in conf.test_api: for model in conf.repo_id: @@ -240,7 +237,7 @@ def run_transformer_int4_gpu(repo_id, run_model(model, api, in_out_pairs, conf['local_model_hub'], conf['warm_up'], conf['num_trials'], conf['num_beams'], conf['low_bit'], conf['cpu_embedding']) df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)', - 'input/output tokens', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding', + 'input/output tokens', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding', 'peak mem (GB)']) df.to_csv(f'{current_dir}/{api}-results-{today}.csv') diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index 39a82eed584..50cec772426 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -138,8 +138,8 @@ def preprocess_prompt(tokenizer, in_len, task): elif in_len == 4096: input_str = open(f"prompt/QA/orca_497.txt", 'r', encoding='utf-8').read() else: - raise ValueError("No corresponding prompt available now, will be added later.") - input_ids = tokenizer.encode(input_str, return_tensors="pt") + raise ValueError("No corresponding prompt available now, will be added later.") + input_ids = tokenizer.encode(input_str, return_tensors="pt") return input_ids def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True, group_size=64): @@ -222,7 +222,7 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, streaming if 'win' in test_api else 'N/A', use_fp16_torch_dtype if 'pipeline_parallel_gpu' in test_api else 'N/A', group_size if any(keyword in test_api for keyword in ['transformers_int4_npu_win', 'transformers_int4_npu_pipeline_win']) else 'N/A'], - ) + ) def get_model_path(repo_id, local_model_hub): @@ -475,7 +475,7 @@ def run_transformer_int4_gpu(repo_id, lookahead=False, task='continuation'): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoTokenizer, LlamaTokenizer model_path = get_model_path(repo_id, local_model_hub) # Load model in 4 bit, # which convert the relevant layers in the model into INT4 format @@ -490,7 +490,7 @@ def run_transformer_int4_gpu(repo_id, model = AutoModel.load_low_bit(model_path, optimize_model=True, trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding, - torch_dtype=torch_dtype).eval() + torch_dtype=torch_dtype).eval() else: model = AutoModel.from_pretrained(model_path, load_in_low_bit=low_bit, optimize_model=True, trust_remote_code=True, use_cache=True, @@ -507,7 +507,7 @@ def run_transformer_int4_gpu(repo_id, model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit, _attn_implementation="eager", modules_to_not_convert=["vision_embed_tokens"], - trust_remote_code=True, use_cache=True, + trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) model = model.to('xpu') @@ -632,14 +632,14 @@ def transformers_int4_npu_win(repo_id, st = time.perf_counter() if repo_id in MINICPM_V_IDS: model = AutoModel.from_pretrained(model_path, load_in_low_bit=low_bit, optimize_model=optimize_model, - trust_remote_code=True, use_cache=True, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), + trust_remote_code=True, use_cache=True, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), quantization_group_size=npu_group_size, transpose_value_cache=transpose_value_cache, save_directory=save_directory, attn_implementation="eager", torch_dtype=torch.float16).eval() model = model.llm tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) else: model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit=low_bit, trust_remote_code=True, torch_dtype=torch.float16, - optimize_model=optimize_model, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), + optimize_model=optimize_model, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), quantization_group_size=npu_group_size, transpose_value_cache=transpose_value_cache, save_directory=save_directory, use_cache=True, attn_implementation="eager").eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) @@ -707,7 +707,7 @@ def transformers_int4_npu_pipeline_win(repo_id, st = time.perf_counter() model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit=low_bit, trust_remote_code=True, pipeline=True, torch_dtype=torch.float16, - optimize_model=optimize_model, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), + optimize_model=optimize_model, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), quantization_group_size=npu_group_size, transpose_value_cache=transpose_value_cache, use_cache=True, attn_implementation="eager", save_directory=save_directory).eval() @@ -843,7 +843,7 @@ def run_transformers_openvino(repo_id, ov_config = {"PERFORMANCE_HINT": "LATENCY", "NUM_STREAMS": "1", "CACHE_DIR": ""} - config_dict = dict(pretrained_model_name_or_path=model_path, + config_dict = dict(pretrained_model_name_or_path=model_path, trust_remote_code=True, use_cache=True, low_cpu_mem_usage=True) @@ -906,7 +906,7 @@ def run_optimize_model_gpu(repo_id, num_beams, low_bit, batch_size): - from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, LlamaTokenizer from ipex_llm import optimize_model model_path = get_model_path(repo_id, local_model_hub) # Load model in 4 bit, @@ -986,7 +986,7 @@ def run_ipex_fp16_gpu(repo_id, num_beams, batch_size): from transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoTokenizer, LlamaTokenizer model_path = get_model_path(repo_id, local_model_hub) st = time.perf_counter() if repo_id in CHATGLM_IDS: @@ -1051,7 +1051,7 @@ def run_bigdl_fp16_gpu(repo_id, num_beams, batch_size): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoTokenizer, LlamaTokenizer model_path = get_model_path(repo_id, local_model_hub) st = time.perf_counter() if repo_id in CHATGLM_IDS: @@ -1209,7 +1209,7 @@ def run_transformer_int4_gpu_win(repo_id, batch_size, streaming): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer, TextStreamer + from transformers import AutoTokenizer, LlamaTokenizer, TextStreamer model_path = get_model_path(repo_id, local_model_hub) # Load model in 4 bit, # which convert the relevant layers in the model into INT4 format @@ -1338,7 +1338,7 @@ def run_transformer_int4_fp16_gpu_win(repo_id, batch_size, streaming): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer, TextStreamer + from transformers import AutoTokenizer, LlamaTokenizer, TextStreamer model_path = get_model_path(repo_id, local_model_hub) # Load model in 4 bit, # which convert the relevant layers in the model into INT4 format @@ -1475,7 +1475,7 @@ def run_transformer_int4_loadlowbit_gpu_win(repo_id, batch_size, streaming): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer, TextStreamer + from transformers import AutoTokenizer, LlamaTokenizer, TextStreamer model_path = get_model_path(repo_id, local_model_hub) # Load BigDL-LLM optimized low bit model st = time.perf_counter() @@ -1585,7 +1585,7 @@ def run_transformer_int4_fp16_loadlowbit_gpu_win(repo_id, batch_size, streaming): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer, TextStreamer + from transformers import AutoTokenizer, LlamaTokenizer, TextStreamer model_path = get_model_path(repo_id, local_model_hub) # Load BigDL-LLM optimized low bit model st = time.perf_counter() @@ -1972,7 +1972,7 @@ def get_int_from_env(env_keys, default): os.environ["WORLD_SIZE"] = str(world_size) os.environ["MASTER_PORT"] = os.environ.get("MASTER_PORT", "29500") - from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, LlamaTokenizer from ipex_llm import optimize_model import deepspeed from deepspeed.accelerator.cpu_accelerator import CPU_Accelerator @@ -2013,7 +2013,7 @@ def get_int_from_env(env_keys, default): # Move model back to xpu model = model.to(f'xpu:{local_rank}') - # Modify backend related settings + # Modify backend related settings if world_size > 1: get_accelerator().set_device(local_rank) dist_backend = get_accelerator().communication_backend_name() @@ -2215,7 +2215,7 @@ def run_pipeline_parallel_gpu(repo_id, cpu_embedding, fp16=False): from ipex_llm.transformers import AutoModel, AutoModelForCausalLM, init_pipeline_parallel - from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer + from transformers import AutoTokenizer, LlamaTokenizer init_pipeline_parallel() model_path = get_model_path(repo_id, local_model_hub) pipeline_parallel_stages = torch.distributed.get_world_size() @@ -2311,7 +2311,7 @@ def run_pipeline_parallel_gpu(repo_id, transpose_value_cache = True if 'transpose_value_cache' in conf: transpose_value_cache = conf['transpose_value_cache'] - + import pandas as pd for api in conf.test_api: global csv_name diff --git a/python/llm/src/ipex_llm/transformers/convert.py b/python/llm/src/ipex_llm/transformers/convert.py index e2c0c0cb8e5..6f78b9a8a0f 100644 --- a/python/llm/src/ipex_llm/transformers/convert.py +++ b/python/llm/src/ipex_llm/transformers/convert.py @@ -680,18 +680,9 @@ def _replace_with_low_bit_linear(model, qtype, modules_to_not_convert=None, optimize_lm_head=optimize_lm_head ) device = module.weight.data.device - from ipex_llm.transformers.utils import get_ipex_version - if get_ipex_version() < "2.1.10+xpu": - new_linear._parameters['weight'] = nn.Parameter(module.weight) - else: - # only from 2.1, ipex provides matmul_bias_out - # so we need to transpose weight - new_weight = module.weight.transpose(0, 1).contiguous() - new_linear._parameters['weight'] = nn.Parameter(new_weight) - new_linear.weight_type = 2 + new_linear._parameters['weight'] = nn.Parameter(module.weight) if module.bias is not None: - new_linear._parameters['bias'] = nn.Parameter(module.bias.data)\ - .to(device) + new_linear._parameters['bias'] = nn.Parameter(module.bias.data).to(device) elif qtype == ggml_tensor_qtype["bf16"]: module.to(torch.bfloat16) if _USE_VLLM: @@ -1452,21 +1443,6 @@ def _optimize_post(model): module.MultiheadAttention, mpt_multihead_attention_forward ) - elif "gptj" in model.config.model_type: - # dolly-v1-6b - modeling_module_name = model.__class__.__module__ - module = importlib.import_module(modeling_module_name) - from ipex_llm.transformers.models.gptj import gptj_attention_forward, gptj_model_forward,\ - gptj_block_forward - convert_forward(model, - module.GPTJAttention, - gptj_attention_forward) - convert_forward(model, - module.GPTJModel, - gptj_model_forward) - convert_forward(model, - module.GPTJBlock, - gptj_block_forward) elif "bloom" in model.config.model_type: modeling_module_name = model.__class__.__module__ module = importlib.import_module(modeling_module_name) diff --git a/python/llm/src/ipex_llm/transformers/loader.py b/python/llm/src/ipex_llm/transformers/loader.py index 7eaf679cc81..59a354dfcc5 100644 --- a/python/llm/src/ipex_llm/transformers/loader.py +++ b/python/llm/src/ipex_llm/transformers/loader.py @@ -22,7 +22,7 @@ from datetime import date import argparse from ipex_llm.utils.common import invalidInputError -from transformers import AutoTokenizer, GPTJForCausalLM, LlamaTokenizer +from transformers import AutoTokenizer, LlamaTokenizer LLAMA_IDS = ['llama', 'vicuna', 'merged-baize'] diff --git a/python/llm/src/ipex_llm/transformers/low_bit_linear.py b/python/llm/src/ipex_llm/transformers/low_bit_linear.py index 59d03c97850..317a103bf59 100644 --- a/python/llm/src/ipex_llm/transformers/low_bit_linear.py +++ b/python/llm/src/ipex_llm/transformers/low_bit_linear.py @@ -759,9 +759,9 @@ def __init__(self, input_features, output_features, bias=True, self.weight_length = self.out_len * self.in_len self.qtype = ggml_tensor_qtype["fp16"] self.mp_group = mp_group - # weigh_type = 1 means original weight - # weigh_type = 2 means weight has been transposed - # weigh_type = 3 means weight has been transposed by esimd method + # weight_type = 1 means original weight + # weight_type = 2 means weight has been transposed + # weight_type = 3 means weight has been transposed by esimd method self.weight_type = 1 self.optimize_lm_head = optimize_lm_head self.disable_fp16_opt = False @@ -775,28 +775,14 @@ def forward(self, x: torch.Tensor): x = x.to(torch.float16) if self.bias is not None and self.bias.dtype != x.dtype: - self.bias.data = self.bias.data.to(x.dtype) + self.bias.data = self.bias.data.to(x.dtype) if self.weight is not None and self.weight.dtype != x.dtype: self.weight.data = self.weight.data.to(x.dtype) if not self.use_esimd_kernel(x): - if ( - get_ipex_version() < "2.1.10+xpu" - or get_xpu_device_name(x.device) not in ["arc", "pvc"] - or self.disable_fp16_opt - ): - if self.weight_type == 2: - self.weight = torch.nn.Parameter(self.weight.transpose(0, 1).contiguous(), - requires_grad=False) - self.weight_type = 1 - result = F.linear(x, self.weight, self.bias) - else: - if self.weight_type == 1: - self.weight = torch.nn.Parameter(self.weight.transpose(0, 1).contiguous(), - requires_grad=False) - self.weight_type = 2 - result = torch.ops.torch_ipex.matmul_bias_out(x.contiguous(), - self.weight, self.bias) + invalidInputError(self.weight_type == 1, "weight_type should be 1") + result = F.linear(x, self.weight, self.bias) + if self.mp_group is not None: if get_use_vllm(): result = self.mp_group.all_reduce(result) @@ -852,7 +838,7 @@ def use_esimd_kernel(self, x): if self.disable_fp16_opt: return False # esimd kernel can only be used for Arc and Flex - if gpu_type not in ["arc", "flex"]: + if gpu_type not in ["arc"]: return False # now esimd kernel can only be used for specific cases (llama2-7b shape) if self.in_len == 11008 and self.out_features == 4096: diff --git a/python/llm/src/ipex_llm/transformers/model.py b/python/llm/src/ipex_llm/transformers/model.py index 7411afde974..182b1a83372 100644 --- a/python/llm/src/ipex_llm/transformers/model.py +++ b/python/llm/src/ipex_llm/transformers/model.py @@ -103,12 +103,6 @@ def save_low_bit(self, *args, **kwargs): self.to(origin_device) -def _load_pre(): - from transformers import GPTJModel - from ipex_llm.transformers.models.gptj import gptj_model_new_init - GPTJModel.__init__ = gptj_model_new_init - - class _BaseAutoModelClass: HF_MODEL = None @@ -495,7 +489,6 @@ def load_convert(cls, q_k, optimize_model, *args, **kwargs): else: if quant_config is not None: kwargs["quantization_config"] = quant_config - _load_pre() try: # To handle the input CUDA setting (such as 'device_map={"":0}'), ignore it kwargs.pop('device_map', None) diff --git a/python/llm/src/ipex_llm/transformers/models/gptj.py b/python/llm/src/ipex_llm/transformers/models/gptj.py deleted file mode 100644 index 20af48c8863..00000000000 --- a/python/llm/src/ipex_llm/transformers/models/gptj.py +++ /dev/null @@ -1,441 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This file is adapted from -# https://github.com/huggingface/transformers/blob/main/src/transformers/models/gptj/modeling_gptj.py -# - -import torch -from typing import Optional, Tuple, Union -from ipex_llm.transformers.models.utils import init_kv_cache, extend_kv_cache, \ - apply_rotary_pos_emb, append_kv_cache, apply_ipex_rotate_every_two -from transformers.utils.import_utils import is_torch_fx_proxy -from transformers.modeling_outputs import BaseModelOutputWithPast -from transformers.models.gptj.modeling_gptj import GPTJModel -from ipex_llm.utils.common import invalidInputError - -import os - -KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)) - - -def _get_embed_positions(self, position_ids): - embed_positions = self.embed_positions - if embed_positions.device != position_ids.device: - embed_positions = embed_positions.to(position_ids.device) - self.embed_positions = embed_positions - return embed_positions.repeat(position_ids.shape[0], 1, 1) - - -def _attn( - self, - query, - key, - value, - attention_mask=None, - head_mask=None, -): - # compute causal mask from causal mask buffer - query_length, key_length = query.size(-2), key.size(-2) - causal_mask = self.bias[:, :, key_length - query_length: key_length, :key_length] - - # Keep the attention weights computation in fp32 to avoid overflow issues - query = query.to(torch.float32) - key = key.to(torch.float32) - - attn_weights = torch.matmul(query, key.transpose(-1, -2)) - - mask_value = torch.finfo(attn_weights.dtype).min - # Need to be a tensor, otherwise we get error: - # `RuntimeError: expected scalar type float but found double`. - # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device` - mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device) - attn_weights = torch.where(causal_mask, attn_weights, mask_value) - - attn_weights = attn_weights / self.scale_attn - - if attention_mask is not None: - # Apply the attention mask - attn_weights = attn_weights + attention_mask - - attn_weights = nn.functional.softmax(attn_weights, dim=-1) - attn_weights = attn_weights.to(value.dtype) - attn_weights = self.attn_dropout(attn_weights) - - # Mask heads if we want to - if head_mask is not None: - attn_weights = attn_weights * head_mask - - attn_output = torch.matmul(attn_weights, value) - - return attn_output, attn_weights - - -def gptj_attention_forward( - self, - hidden_states: torch.FloatTensor, - layer_past: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - head_mask: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = False, - rotary_emb: Optional[Tuple]=None, - output_attentions: Optional[bool] = False, -) -> Union[ - Tuple[torch.Tensor, Tuple[torch.Tensor]], - Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]], -]: - query = self.q_proj(hidden_states) - key = self.k_proj(hidden_states) - value = self.v_proj(hidden_states) - - query = self._split_heads(query, self.num_attention_heads, self.head_dim, True) - key = self._split_heads(key, self.num_attention_heads, self.head_dim, True) - value = self._split_heads(value, self.num_attention_heads, self.head_dim, False) - - sin, cos = rotary_emb - use_fuse_rope = hidden_states.device.type == "xpu" and not self.training - - if self.rotary_dim is not None: - k_rot = key[:, :, :, : self.rotary_dim] - q_rot = query[:, :, :, : self.rotary_dim] - - if use_fuse_rope: - apply_ipex_rotate_every_two(q_rot, k_rot, cos, sin) - else: - k_pass = key[:, :, :, self.rotary_dim:] - q_pass = query[:, :, :, self.rotary_dim:] - q_rot, k_rot = apply_rotary_pos_emb(q_rot, k_rot, cos, sin, position_ids, "gptj") - key = torch.cat([k_rot, k_pass], dim=-1) - query = torch.cat([q_rot, q_pass], dim=-1) - else: - if use_fuse_rope: - apply_ipex_rotate_every_two(query, key, cos, sin) - else: - query, key = apply_rotary_pos_emb(query, key, cos, sin, position_ids, "gptj") - - batch_size, q_len, _ = hidden_states.size() - - key = key.permute(0, 2, 1, 3).contiguous() - query = query.permute(0, 2, 1, 3).contiguous() - - kv_seq_len = key.size(-2) - device = hidden_states.device - - if layer_past is not None: - kv_seq_len += layer_past[0].size(2) - - if layer_past is not None: - cache_k = layer_past[0] - cache_v = layer_past[1] - past_length = cache_k.size(2) - if cache_k.stride()[1] < kv_seq_len * cache_k.size(3): - new_cache_k, new_cache_v = extend_kv_cache(batch_size, - self.num_attention_heads, - self.head_dim, - past_length, - kv_seq_len + KV_CACHE_ALLOC_BLOCK_LENGTH, - dtype=cache_v.dtype, - device=device) - new_cache_k[:] = cache_k - new_cache_v[:] = cache_v - cache_k = new_cache_k - cache_v = new_cache_v - key, value = append_kv_cache(cache_k, cache_v, key, value) - - elif use_cache: - key_cache, value_cache = init_kv_cache(batch_size, - self.num_attention_heads, - self.head_dim, - kv_seq_len, - kv_seq_len + KV_CACHE_ALLOC_BLOCK_LENGTH, - dtype=value.dtype, - device=device) - key_cache[:] = key - value_cache[:] = value - key = key_cache - value = value_cache - - if use_cache is True: - present = (key, value) - else: - present = None - - # compute self-attention: V x Softmax(QK^T) - attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask) - - attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim) - attn_output = self.out_proj(attn_output) - attn_output = self.resid_dropout(attn_output) - - outputs = (attn_output, present) - if output_attentions: - outputs += (attn_weights,) - - return outputs # a, present, (attentions) - - -def gptj_block_forward( - self, - hidden_states: Optional[torch.FloatTensor], - layer_past: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - head_mask: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = False, - rotary_emb: Optional[Tuple]=None, - output_attentions: Optional[bool] = False, -) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]: - residual = hidden_states - hidden_states = self.ln_1(hidden_states) - attn_outputs = self.attn( - hidden_states=hidden_states, - layer_past=layer_past, - attention_mask=attention_mask, - position_ids=position_ids, - head_mask=head_mask, - use_cache=use_cache, - rotary_emb=rotary_emb, - output_attentions=output_attentions, - ) - attn_output = attn_outputs[0] # output_attn: a, present, (attentions) - outputs = attn_outputs[1:] - - feed_forward_hidden_states = self.mlp(hidden_states) - hidden_states = attn_output + feed_forward_hidden_states + residual - - if use_cache: - outputs = (hidden_states,) + outputs - else: - outputs = (hidden_states,) + outputs[1:] - - return outputs # hidden_states, present, (attentions) - - -def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: - inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim)) - sinusoid_inp = torch.einsum("i , j -> i j", - torch.arange(num_pos, dtype=torch.float), inv_freq).float() - return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) - - -old_init = GPTJModel.__init__ - - -def gptj_model_new_init(self, config): - old_init(self, config) - embed_dim = config.hidden_size - rotary_dim = config.rotary_dim - pos_embd_dim = rotary_dim or embed_dim - max_positions = config.max_position_embeddings - self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim) - - -def get_new_embed_positions(position_ids, prev_embed_positions): - embed_positions = prev_embed_positions - if embed_positions.device != position_ids.device: - embed_positions = embed_positions.to(position_ids.device) - prev_embed_positions = embed_positions - return embed_positions.repeat(position_ids.shape[0], 1, 1), prev_embed_positions - - -def gptj_model_forward( - self, - input_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, - attention_mask: Optional[torch.FloatTensor] = None, - token_type_ids: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - head_mask: Optional[torch.FloatTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, -) -> Union[Tuple, BaseModelOutputWithPast]: - output_attentions = output_attentions if output_attentions is not None \ - else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None - else self.config.output_hidden_states - ) - use_cache = use_cache if use_cache is not None else self.config.use_cache - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - if input_ids is not None and inputs_embeds is not None: - invalidInputError(False, - "You cannot specify both input_ids and inputs_embeds at the same time") - elif input_ids is not None: - self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask) - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_shape[-1]) - batch_size = input_ids.shape[0] - elif inputs_embeds is not None: - input_shape = inputs_embeds.size()[:-1] - batch_size = inputs_embeds.shape[0] - else: - invalidInputError(False, "You have to specify either input_ids or inputs_embeds") - - device = input_ids.device if input_ids is not None else inputs_embeds.device - - if token_type_ids is not None: - token_type_ids = token_type_ids.view(-1, input_shape[-1]) - - if past_key_values is None: - past_length = 0 - past_key_values = tuple([None] * len(self.h)) - else: - past_length = past_key_values[0][0].size(-2) - - if position_ids is None: - position_ids = torch.arange(past_length, input_shape[-1] + past_length, - dtype=torch.long, device=device) - position_ids = position_ids.unsqueeze(0) - - # Attention mask. - if attention_mask is not None: - if batch_size <= 0: - invalidInputError(False, "batch_size has to be defined and > 0") - attention_mask = attention_mask.view(batch_size, -1) - # We create a 3D attention mask from a 2D tensor mask. - # Sizes are [batch_size, 1, 1, to_seq_length] - # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] - # this attention mask is more simple than the triangular masking of causal attention - # used in OpenAI GPT, we just need to prepare the broadcast dimension here. - attention_mask = attention_mask[:, None, None, :] - - # Since attention_mask is 1.0 for positions we want to attend and 0.0 for - # masked positions, this operation will create a tensor which is 0.0 for - # positions we want to attend and the dtype's smallest value for masked positions. - # Since we are adding it to the raw scores before the softmax, this is - # effectively the same as removing these entirely. - attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility - attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min - - # Prepare head mask if needed - # 1.0 in head_mask indicate we keep the head - # attention_probs has shape bsz x num_attention_heads x N x N - # head_mask has shape n_layer x batch x num_attention_heads x N x N - head_mask = self.get_head_mask(head_mask, self.config.n_layer) - - if inputs_embeds is None: - inputs_embeds = self.wte(input_ids) - - hidden_states = inputs_embeds - - if token_type_ids is not None: - token_type_embeds = self.wte(token_type_ids) - hidden_states = hidden_states + token_type_embeds - - hidden_states = self.drop(hidden_states) - - output_shape = (-1,) + input_shape[1:] + (hidden_states.size(-1),) - - if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing." - "Setting `use_cache=False`..." - ) - use_cache = False - - presents = () if use_cache else None - all_self_attentions = () if output_attentions else None - all_hidden_states = () if output_hidden_states else None - - # Repeat cos sin here, call only once for each token. - # If put this to attension forward, it will generate too many times. - if is_torch_fx_proxy(position_ids) or torch.jit.is_tracing(): - # The logic to conditionally copy to GPU could not be traced, so we do this - # every time in the torch.fx case - embed_positions = get_embed_positions(self.embed_positions, position_ids) - else: - embed_positions, self.embed_positions = get_new_embed_positions(position_ids, - self.embed_positions) - - repeated_position_ids = position_ids.unsqueeze(-1).repeat(1, 1, embed_positions.shape[-1]) - sincos = torch.gather(embed_positions, 1, repeated_position_ids) - sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1) - sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3) - cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3) - - for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): - # Model parallel - if self.model_parallel: - torch.cuda.set_device(hidden_states.device) - # Ensure layer_past is on same device as hidden_states (might not be correct) - if layer_past is not None: - layer_past = tuple(past_state.to(hidden_states.device) for past_state in layer_past) - # Ensure that attention_mask is always on the same device as hidden_states - if attention_mask is not None: - attention_mask = attention_mask.to(hidden_states.device) - if isinstance(head_mask, torch.Tensor): - head_mask = head_mask.to(hidden_states.device) - if output_hidden_states: - all_hidden_states = all_hidden_states + (hidden_states,) - - if self.gradient_checkpointing and self.training: - outputs = self._gradient_checkpointing_func( - block.__call__, - hidden_states, - None, - attention_mask, - position_ids, - head_mask[i], - use_cache, - output_attentions, - ) - else: - outputs = block( - hidden_states=hidden_states, - layer_past=layer_past, - attention_mask=attention_mask, - position_ids=position_ids, - head_mask=head_mask[i], - use_cache=use_cache, - rotary_emb=(sin, cos), - output_attentions=output_attentions, - ) - - hidden_states = outputs[0] - if use_cache is True: - presents = presents + (outputs[1],) - - if output_attentions: - all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],) - - # Model Parallel: If it's the last layer for that device, put things on the next device - if self.model_parallel: - for k, v in self.device_map.items(): - if i == v[-1] and "cuda:" + str(k) != self.last_device: - hidden_states = hidden_states.to("cuda:" + str(k + 1)) - - hidden_states = self.ln_f(hidden_states) - - hidden_states = hidden_states.view(output_shape) - # Add last hidden state - if output_hidden_states: - all_hidden_states = all_hidden_states + (hidden_states,) - - if not return_dict: - return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] - if v is not None) - - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=presents, - hidden_states=all_hidden_states, - attentions=all_self_attentions, - ) diff --git a/python/llm/src/ipex_llm/transformers/models/utils.py b/python/llm/src/ipex_llm/transformers/models/utils.py index cd16b71be23..2a4d2f518d4 100644 --- a/python/llm/src/ipex_llm/transformers/models/utils.py +++ b/python/llm/src/ipex_llm/transformers/models/utils.py @@ -168,7 +168,7 @@ def should_use_fuse_rope(hidden_states, position_ids, training): def apply_rotary_pos_emb(q, k, cos, sin, position_ids, model_family): if model_family in ["llama", "baichuan", "internlm", "aquila", "gpt_neox", "mistral", - "mixtral", "qwen2", "yuan", "stablelm", "qwen2_moe"]: + "qwen2", "yuan", "stablelm", "qwen2_moe"]: # The first two dimensions of cos and sin are always 1, so we can `squeeze` them. cos = cos.squeeze(1).squeeze(0) # [seq_len, dim] sin = sin.squeeze(1).squeeze(0) # [seq_len, dim] @@ -183,7 +183,7 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids, model_family): q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed - elif model_family in ["gptj", "chatglm"]: + elif model_family in ["chatglm"]: q_embed = (q * cos) + (rotate_every_two(q) * sin) k_embed = (k * cos) + (rotate_every_two(k) * sin) return q_embed, k_embed @@ -192,19 +192,6 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids, model_family): f"{model_family} is not supported.") -def apply_ipex_rotate_every_two(q, k, cos, sin): - # ipex's apply_rotary_embedding_two_qk can change the origin storage, - # so q/k will get the result directly. - from ipex_llm.transformers.utils import get_ipex_version - if get_ipex_version() >= "2.1.10+xpu": - torch.ops.torch_ipex.apply_rotary_embedding_two_qk( - q, k, sin, cos, q, k - ) - else: - torch.ops.torch_ipex.apply_rotary_embedding(q, sin, cos, q) - torch.ops.torch_ipex.apply_rotary_embedding(k, sin, cos, k) - - def is_enough_kv_cache_room_4_36(past_key_value, idx, seq_len=1): # to determinate if is enough kv cache room in transformers==4.36 # seq_len for current seq len diff --git a/python/llm/src/ipex_llm/transformers/speculative.py b/python/llm/src/ipex_llm/transformers/speculative.py index 1d3107c8df9..d0c90b1053a 100644 --- a/python/llm/src/ipex_llm/transformers/speculative.py +++ b/python/llm/src/ipex_llm/transformers/speculative.py @@ -432,8 +432,7 @@ def _check_and_extend_kv_cache(past_key_values, max_step_draft, kv_alloc_block_l from ipex_llm.transformers.models.utils import is_enough_kv_cache_room_4_31, \ extend_kv_cache enough_kv_room = True - if model_type not in ["chatglm", "qwen", "baichuan", "llama", "mistral", - "gptj", "opt"]: + if model_type not in ["chatglm", "qwen", "baichuan", "llama", "mistral", "opt"]: return past_key_values, False cache_k = past_key_values[0][0] if model_type == "chatglm": @@ -527,7 +526,7 @@ def _crop_past_key_values(self, past_key_values, new_cache_size, _enable_ipex=Fa v[:-(new_cache_size), :, :, :]) for k, v in past_key_values ] - elif self.config.model_type in ["baichuan", "gptj"]: + elif self.config.model_type in ["baichuan"]: past_key_values = [ (k[:, :, :-(new_cache_size), :], v[:, :, :-(new_cache_size), :]) @@ -796,13 +795,6 @@ def _non_cpu_ipex_verify(self, verify_input_ids, past_key_values, cur_attention_ device=verify_input_ids.device) position_ids = position_ids.unsqueeze(0).repeat(1, 1) + past_key_value_len forward_args["position_ids"] = position_ids - elif self.config.model_type == "gptj": - past_length = past_key_values[0][0].size(2) - input_len = verify_input_ids.shape[1] - position_ids = torch.arange(past_length, input_len + past_length, - dtype=torch.long, device=verify_input_ids.device) - position_ids = position_ids.unsqueeze(0).view(-1, input_len) - forward_args["position_ids"] = position_ids return self(**forward_args) @@ -971,10 +963,6 @@ def speculative_generate(self, past_key_value_len = past_key_values[0][0].shape[0] position_ids = torch.Tensor([[past_key_value_len + step_draft]]).long() forward_args["position_ids"] = position_ids - elif self.config.model_type == "gptj": - past_length = draft_past_key_values[0][0].size(2) - position_ids = torch.Tensor([[past_length]]).long().to(self.device) - forward_args["position_ids"] = position_ids if _enable_ipex: if any(keyword in self.config.model_type