From fa261b8af1502e6597b2a073096d41dd0816c777 Mon Sep 17 00:00:00 2001 From: Heyang Sun <60865256+Uxito-Ada@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:47:04 +0800 Subject: [PATCH] torch 2.3 inference docker (#12517) * torch 2.3 inference docker * Update README.md * add convert code * rename image * remove 2.1 and add graph example * Update README.md --- docker/llm/inference/xpu/docker/Dockerfile | 25 +- python/llm/example/GPU/GraphMode/README.md | 48 ++++ .../convert-model-textgen-to-classfication.py | 57 +++++ .../GraphMode/gpt2-graph-mode-benchmark.py | 228 ++++++++++++++++++ 4 files changed, 352 insertions(+), 6 deletions(-) create mode 100644 python/llm/example/GPU/GraphMode/README.md create mode 100644 python/llm/example/GPU/GraphMode/convert-model-textgen-to-classfication.py create mode 100644 python/llm/example/GPU/GraphMode/gpt2-graph-mode-benchmark.py diff --git a/docker/llm/inference/xpu/docker/Dockerfile b/docker/llm/inference/xpu/docker/Dockerfile index 9c717556da5..0a6221f14aa 100644 --- a/docker/llm/inference/xpu/docker/Dockerfile +++ b/docker/llm/inference/xpu/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04 +FROM intel/oneapi:2024.2.1-0-devel-ubuntu22.04 ARG http_proxy ARG https_proxy @@ -6,7 +6,7 @@ ARG https_proxy ENV TZ=Asia/Shanghai ENV PYTHONUNBUFFERED=1 -# When cache is enabled SYCL runtime will try to cache and reuse JIT-compiled binaries. +# When cache is enabled SYCL runtime will try to cache and reuse JIT-compiled binaries. ENV SYCL_CACHE_PERSISTENT=1 COPY chat.py /llm/chat.py @@ -29,6 +29,19 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO env DEBIAN_FRONTEND=noninteractive apt-get update && \ # add-apt-repository requires gnupg, gpg-agent, software-properties-common apt-get install -y --no-install-recommends gnupg gpg-agent software-properties-common && \ + export PRE_DIR=$(pwd) && \ + # Install Compute Runtime + mkdir -p /tmp/neo && \ + cd /tmp/neo && \ + wget https://github.com/oneapi-src/level-zero/releases/download/v1.18.5/level-zero_1.18.5+u22.04_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb && \ + dpkg -i *.deb && \ + rm -rf /tmp/neo && \ + cd $PRE_DIR && \ # Add Python 3.11 PPA repository add-apt-repository ppa:deadsnakes/ppa -y && \ apt-get install -y --no-install-recommends python3.11 git curl wget && \ @@ -41,13 +54,12 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO python3 get-pip.py && \ rm get-pip.py && \ pip install --upgrade requests argparse urllib3 && \ - pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ && \ + pip install --pre --upgrade ipex-llm[xpu_arc] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ && \ + pip install --pre pytorch-triton-xpu==3.0.0+1b2f15840e --index-url https://download.pytorch.org/whl/nightly/xpu && \ # Fix Trivy CVE Issues - pip install transformers==4.36.2 && \ pip install transformers_stream_generator einops tiktoken && \ # Install opencl-related repos apt-get update && \ - apt-get install -y --no-install-recommends intel-opencl-icd=23.35.27191.42-775~22.04 intel-level-zero-gpu=1.3.27191.42-775~22.04 level-zero=1.14.0-744~22.04 && \ # Install related libary of chat.py pip install --upgrade colorama && \ # Download all-in-one benchmark and examples @@ -71,7 +83,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO # Download Deepspeed-AutoTP cp -r ./ipex-llm/python/llm/example/GPU/Deepspeed-AutoTP/ ./Deepspeed-AutoTP && \ # Install related library of Deepspeed-AutoTP - pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ && \ + pip install oneccl_bind_pt==2.3.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ && \ pip install git+https://github.com/microsoft/DeepSpeed.git@ed8aed5 && \ pip install git+https://github.com/intel/intel-extension-for-deepspeed.git@0eb734b && \ pip install mpi4py && \ @@ -82,3 +94,4 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO WORKDIR /llm/ +ENV BIGDL_CHECK_DUPLICATE_IMPORT=0 diff --git a/python/llm/example/GPU/GraphMode/README.md b/python/llm/example/GPU/GraphMode/README.md new file mode 100644 index 00000000000..c87b8118a10 --- /dev/null +++ b/python/llm/example/GPU/GraphMode/README.md @@ -0,0 +1,48 @@ +# Torch Graph Mode + +Here, we provide how to run [torch graph mode](https://pytorch.org/blog/optimizing-production-pytorch-performance-with-graph-transformations/) on Intel Arc™ A-Series Graphics with ipex-llm, and [gpt2-medium](https://huggingface.co/openai-community/gpt2-medium) for classification task is used as illustration: + +### 1. Install +```bash +conda create -n ipex-llm python=3.11 +conda activate ipex-llm +pip install --pre --upgrade ipex-llm[xpu_arc] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ +pip install --pre pytorch-triton-xpu==3.0.0+1b2f15840e --index-url https://download.pytorch.org/whl/nightly/xpu +conda install -c conda-forge libstdcxx-ng +unset OCL_ICD_VENDORS +``` + +### 2. Configures OneAPI environment variables + +> [!NOTE] +> Skip this step if you are running on Windows. + +This is a required step on Linux for APT or offline installed oneAPI. Skip this step for PIP-installed oneAPI. + +```bash +source /opt/intel/oneapi/setvars.sh +``` + +### 3. Run + +Convert text-generating GPT2-Medium to the classification: + + ```bash + # The convert step needs to access the internet + export http_proxy=http://your_proxy_url + export https_proxy=http://your_proxy_url + + # This will yield gpt2-medium-classification under /llm/models in the container + python convert-model-textgen-to-classfication.py --model-path MODEL_PATH + ``` + +This will yield a mode directory ends with '-classification' neart your input model path. + +Benchmark GPT2-Medium's performance with IPEX-LLM engine: + + ``` sbash + ipexrun xpu gpt2-graph-mode-benchmark.py --device xpu --engine ipex-llm --batch 16 --model-path MODEL_PATH + + # You will see the key output like: + # Average time taken (excluding the first two loops): xxxx seconds, Classification per seconds is xxxx + ``` diff --git a/python/llm/example/GPU/GraphMode/convert-model-textgen-to-classfication.py b/python/llm/example/GPU/GraphMode/convert-model-textgen-to-classfication.py new file mode 100644 index 00000000000..d94364dacb2 --- /dev/null +++ b/python/llm/example/GPU/GraphMode/convert-model-textgen-to-classfication.py @@ -0,0 +1,57 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This is modified from https://github.com/intel-sandbox/customer-ai-test-code/blob/main/convert-model-textgen-to-classfication.py +# +import torch +from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoConfig, AutoModelForCausalLM +import argparse + +parser = argparse.ArgumentParser(description='Process some integers.') +parser.add_argument('--model_path', type=str, help='an string for the device') +args = parser.parse_args() +model_path = args.model_path + +dtype=torch.bfloat16 +num_labels = 5 + +model_name=model_path + +save_directory = model_name + "-classification" + +# Initialize the tokenizer +# Need padding from the left and padding to 1024 +tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) +# tokenizer.padding_side = "left" +tokenizer.pad_token = tokenizer.eos_token +tokenizer.save_pretrained(save_directory) + + +model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=dtype, pad_token_id=tokenizer.eos_token_id,) +config = AutoConfig.from_pretrained(model_name) +print("text gen model") +print(model) +print(config) + + +model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels, torch_dtype=dtype) +save_directory = model_name + "-classification" +model.save_pretrained(save_directory) + + +model = AutoModelForSequenceClassification.from_pretrained(save_directory, torch_dtype=dtype, pad_token_id=tokenizer.eos_token_id) +config = AutoConfig.from_pretrained(save_directory) +print("text classification model") +print(model) +print(config) diff --git a/python/llm/example/GPU/GraphMode/gpt2-graph-mode-benchmark.py b/python/llm/example/GPU/GraphMode/gpt2-graph-mode-benchmark.py new file mode 100644 index 00000000000..dd903bea8b6 --- /dev/null +++ b/python/llm/example/GPU/GraphMode/gpt2-graph-mode-benchmark.py @@ -0,0 +1,228 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This is modified from https://github.com/intel-sandbox/customer-ai-test-code/blob/main/gpt2-benchmark-for-sangfor.py +# +import torch +import time +import argparse +from transformers import GPT2ForSequenceClassification, AutoTokenizer, AutoModelForSequenceClassification, AutoConfig, Qwen2ForSequenceClassification +from torch.profiler import profile, record_function, ProfilerActivity, schedule + + +# Get the batch size and device +parser = argparse.ArgumentParser(description='Process some integers.') +parser.add_argument('--batch_size', type=int, default=1, help='an integer for the batch size') +parser.add_argument('--device', type=str, default='cpu', help='an string for the device') +parser.add_argument('--profile', type=bool, default=False, help='enable protch profiler for CPU/XPU') +parser.add_argument('--engine', type=str, default='ipex-llm', help='an string for the device') +parser.add_argument('--model_path', type=str, help='an string for the device') +args = parser.parse_args() +enable_profile=args.profile +batch_size = args.batch_size +device = args.device +engine = args.engine +model_path = args.model_path +print(f"The batch size is: {batch_size}, device is {device}") + + +###################################################################################### +# PyTorch Profiling with IPEX +# export IPEX_ZE_TRACING=1 +# export ZE_ENABLE_TRACING_LAYER=1 +import contextlib +def profiler_setup(profiling=False, *args, **kwargs): + if profiling: + return torch.profiler.profile(*args, **kwargs) + else: + return contextlib.nullcontext() + +my_schedule = schedule( + skip_first=6, + wait=1, + warmup=1, + active=1 + ) + +# also define a handler for outputing results +def trace_handler(p): + if(device == 'xpu'): + print(p.key_averages().table(sort_by="self_xpu_time_total", row_limit=20)) + print(p.key_averages().table(sort_by="cpu_time_total", row_limit=20)) + # p.export_chrome_trace("./trace_" + str(p.step_num) + ".json") +####################################################################################### + + + +dtype = torch.bfloat16 if device == 'cpu' else torch.float16 +num_labels = 5 + +model_name = model_path + +model_name = model_name + "-classification" +model_name_ov = model_name + "-ov" +model_name_ov = model_name_ov + "-fp16" + +if (engine == 'ipex') : + import torch + import intel_extension_for_pytorch as ipex + # Need padding from the left and padding to 1024 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + tokenizer.padding_side = "left" + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForSequenceClassification.from_pretrained(model_name, torch_dtype=dtype, + pad_token_id=tokenizer.eos_token_id, + low_cpu_mem_usage=True + ).eval().to(device) +elif (engine == 'ipex-llm'): + from ipex_llm.transformers import AutoModelForSequenceClassification + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(model_name,trust_remote_code=True) + tokenizer.padding_side = "left" + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForSequenceClassification.from_pretrained(model_name, + torch_dtype=dtype, + load_in_low_bit="fp16", + pad_token_id=tokenizer.eos_token_id, + low_cpu_mem_usage=True).to(device) + model = torch.compile(model, backend='inductor') + print(model) +else: + from optimum.intel import OVModelForSequenceClassification + tokenizer = AutoTokenizer.from_pretrained(model_name_ov, trust_remote_code=True) + tokenizer.padding_side = "left" + tokenizer.pad_token = tokenizer.eos_token + model = OVModelForSequenceClassification.from_pretrained(model_name_ov, torch_dtype=dtype).to(device) + + + +# Intel(R) Extension for PyTorch* +if engine == 'ipex': + if device == 'cpu': + # model = ipex.llm.optimize(model, dtype=dtype, inplace=True, deployment_mode=True) + # ############## TorchDynamo ############### + model = ipex.optimize(model, dtype=torch.bfloat16, weights_prepack=False) + model = torch.compile(model, backend='ipex') + # ########################################## + else: # Intel XPU + #model = ipex.llm.optimize(model, dtype=dtype, device="xpu", inplace=True) + model = ipex.optimize(model, dtype=dtype, inplace=True) + + model=torch.compile(model, backend="inductor") + print(model) + + # # #######calulate the total num of parameters######## + # def model_size(model): + # return sum(t.numel() for t in model.parameters()) + # print(f"GPT2 size: {model_size(model)/1000**2:.1f}M parameters") + # # # #######print model information ################### + # print(model) + + # ########Enable the BetterTransformer ################### + # only Better Transformer only support GPT2, not support Qwen2 + # model = BetterTransformer.transform(model) +#elif engine == 'ipex-llm': +# model = ipex.optimize(model, dtype=dtype, inplace=True) +# model=torch.compile(model) #backend="inductor") +elif engine == 'ov': + print("OV inference") + + +prompt = ["this is the first prompt"] +prompts = prompt * batch_size +#print(prompts) + +# Tokenize the batch of prompts +inputs = tokenizer(prompts, return_tensors="pt", padding="max_length", max_length=1024, truncation=True) +# print(inputs) + +if engine == 'ipex' or engine == 'ipex-llm': + #ipex need move the inputs to device, but OV doesn't need + inputs.to(device) + + # Initialize an empty list to store elapsed times + elapsed_times = [] + + # Loop for batch processing 10 times and calculate the time for every loop + with profiler_setup(profiling=enable_profile, activities=[ProfilerActivity.CPU, ProfilerActivity.XPU], + schedule=my_schedule, + on_trace_ready=trace_handler, + # on_trace_ready=torch.profiler.tensorboard_trace_handler('./log/gpt2'), + record_shapes=True, + with_stack=True + ) as prof: + + for i in range(10): + start_time = time.time() + + # Perform inference + with torch.inference_mode(): + # logits = model(**inputs).logits + outputs = model(**inputs) + logits = outputs.logits + + # Get the predicted class for each input in the batch + predicted_class_ids = logits.argmax(dim=1).tolist() + + end_time = time.time() + elapsed_time = end_time - start_time + + # Save the elapsed time in the list + elapsed_times.append(elapsed_time) + + if(enable_profile): + prof.step() + + # print(outputs) + # print(type(outputs)) + # print("logits.shape is " + str(logits.shape)) + # print(logits) + + # print(predicted_class_ids) + +elif engine == 'ov': + print("OV inference") + # Initialize an empty list to store elapsed times + elapsed_times = [] + + # Loop for batch processing 10 times and calculate the time for every loop + for i in range(10): + start_time = time.time() + + outputs = model(**inputs) + logits = outputs.logits + + # Get the predicted class for each input in the batch + predicted_class_ids = logits.argmax(dim=1).tolist() + + end_time = time.time() + elapsed_time = end_time - start_time + + # Save the elapsed time in the list + elapsed_times.append(elapsed_time) + + # print(outputs) + # print(type(outputs)) + # print("logits.shape is " + str(logits.shape)) + # print(logits) + + # print(predictions) + #print(predicted_class_ids) + + +# Skip the first two values and calculate the average of the remaining elapsed times +average_elapsed_time = sum(elapsed_times[2:]) / len(elapsed_times[2:]) +classfication_per_second = batch_size/average_elapsed_time +print(f"Average time taken (excluding the first two loops): {average_elapsed_time:.4f} seconds, Classification per seconds is {classfication_per_second:.4f}")