Test MiniCPM performance on iGPU in a more stable way (#11573)

* Test MiniCPM performance on iGPU in a more stable way * small fix --------- Co-authored-by: ATMxsp01 <[email protected]>
intel-analytics · Jul 12, 2024 · 13a72dc · 13a72dc
1 parent 0981b72
commit 13a72dc
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 12 deletions.
diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
@@ -623,13 +623,18 @@ jobs:
 
           sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
 
-      - name: Add extra warmup for chatglm3-6b int4+fp32 for more stable results
+      - name: Add extra warmup for chatglm3-6b int4+fp32 & MiniCPM int4+fp16 int4+fp32 for more stable results
         shell: bash
         run: |
           sed -i '/^\s*result = run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
-                  if repo_id in ["THUDM/chatglm3-6b", "THUDM/glm-4-9b-chat"]:\
+                  if repo_id in ["THUDM/chatglm3-6b", "THUDM/glm-4-9b-chat", "openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
                       run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
           ' python/llm/dev/benchmark/all-in-one/run.py
+          
+          sed -i '/^\s*result = run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
+                  if repo_id in ["openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
+                      run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
+          ' python/llm/dev/benchmark/all-in-one/run.py
 
       # 32-32 int4+fp16
       - name: Prepare igpu perf test (32-32 int4+fp16)

diff --git a/python/llm/test/benchmark/igpu-perf/1024-128.yaml b/python/llm/test/benchmark/igpu-perf/1024-128.yaml
@@ -1,6 +1,4 @@
 repo_id:
-  - 'openbmb/MiniCPM-1B-sft-bf16'
-  - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'THUDM/chatglm3-6b'
   - 'THUDM/glm-4-9b-chat'
   - 'baichuan-inc/Baichuan2-7B-Chat'
@@ -13,6 +11,8 @@ repo_id:
   - 'RWKV/v5-Eagle-7B-HF'
   - '01-ai/Yi-6B-Chat'
   - 'Qwen/Qwen-VL-Chat'
+  - 'openbmb/MiniCPM-1B-sft-bf16'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
@@ -1,6 +1,4 @@
 repo_id:
-  - 'openbmb/MiniCPM-1B-sft-bf16'
-  - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'THUDM/chatglm3-6b'
   - 'THUDM/glm-4-9b-chat'
   - 'baichuan-inc/Baichuan2-7B-Chat'
@@ -12,6 +10,8 @@ repo_id:
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
   - 'Qwen/Qwen-VL-Chat'
+  - 'openbmb/MiniCPM-1B-sft-bf16'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml
@@ -1,6 +1,4 @@
 repo_id:
-  - 'openbmb/MiniCPM-1B-sft-bf16'
-  - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'THUDM/chatglm3-6b'
   - 'THUDM/glm-4-9b-chat'
   - 'baichuan-inc/Baichuan2-7B-Chat'
@@ -12,6 +10,8 @@ repo_id:
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
   - 'Qwen/Qwen-VL-Chat'
+  - 'openbmb/MiniCPM-1B-sft-bf16'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml b/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml
@@ -1,6 +1,4 @@
 repo_id:
-  - 'openbmb/MiniCPM-1B-sft-bf16'
-  - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'THUDM/chatglm3-6b'
   - 'THUDM/glm-4-9b-chat'
   - 'baichuan-inc/Baichuan2-7B-Chat'
@@ -12,6 +10,8 @@ repo_id:
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
   - 'Qwen/Qwen-VL-Chat'
+  - 'openbmb/MiniCPM-1B-sft-bf16'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml b/python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml
@@ -1,6 +1,4 @@
 repo_id:
-  - 'openbmb/MiniCPM-1B-sft-bf16'
-  - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'THUDM/chatglm3-6b'
   - 'THUDM/glm-4-9b-chat'
   - 'baichuan-inc/Baichuan2-7B-Chat'
@@ -12,6 +10,8 @@ repo_id:
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
   - 'Qwen/Qwen-VL-Chat'
+  - 'openbmb/MiniCPM-1B-sft-bf16'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
 local_model_hub: 'path to your local model hub'
 warm_up: 3
 num_trials: 5