remove models we no longer need in benchmark. (#11492)

Co-authored-by: ATMxsp01 <[email protected]>
intel-analytics · Jul 2, 2024 · 52519e0 · 52519e0
1 parent 6a0134a
commit 52519e0
Show file tree

Hide file tree

Showing 6 changed files with 0 additions and 57 deletions.
diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
@@ -1,21 +1,9 @@
 repo_id:
   - 'meta-llama/Llama-2-7b-chat-hf'
   - 'meta-llama/Llama-2-13b-chat-hf'
-  - 'THUDM/chatglm2-6b'
   - 'THUDM/chatglm3-6b-4bit'
-  - 'tiiuae/falcon-7b-instruct-with-patch'
-  - 'mosaicml/mpt-7b-chat'
-  - 'redpajama/gptneox-7b-redpajama-bf16'
-  - 'bigcode/starcoder-15.5b-4bit'
-  - 'databricks/dolly-v1-6b'
-  - 'databricks/dolly-v2-7b'
-  # - 'databricks/dolly-v2-12b'
-  - 'internlm/internlm-chat-7b'
-  - 'Qwen/Qwen-7B-Chat'
-  - 'BAAI/AquilaChat-7B'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'bigscience/bloomz-7b1'
 #  - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
   - 'mistralai/Mistral-7B-v0.1'
 local_model_hub: '/mnt/disk1/models'
@@ -32,8 +20,5 @@ test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
-  - 'bigcode/starcoder-15.5b-4bit:2048'
-  # - 'databricks/dolly-v2-12b:2048'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-  - 'bigscience/bloomz-7b1:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-test-batch4.yaml b/python/llm/test/benchmark/arc-perf-test-batch4.yaml
@@ -1,21 +1,9 @@
 repo_id:
   - 'meta-llama/Llama-2-7b-chat-hf'
   - 'meta-llama/Llama-2-13b-chat-hf'
-  - 'THUDM/chatglm2-6b'
   - 'THUDM/chatglm3-6b-4bit'
-  - 'tiiuae/falcon-7b-instruct-with-patch'
-  - 'mosaicml/mpt-7b-chat'
-  - 'redpajama/gptneox-7b-redpajama-bf16'
-  - 'bigcode/starcoder-15.5b-4bit'
-  - 'databricks/dolly-v1-6b'
-  - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
-  - 'internlm/internlm-chat-7b' 
-  - 'Qwen/Qwen-7B-Chat'
-  - 'BAAI/AquilaChat-7B'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'bigscience/bloomz-7b1'
 #  - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
   - 'mistralai/Mistral-7B-v0.1' #mwj: need to check
 local_model_hub: '/mnt/disk1/models'
@@ -33,23 +21,9 @@ test_api:
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
   - 'meta-llama/Llama-2-13b-chat-hf:2048'
-  - 'tiiuae/falcon-7b-instruct-with-patch:2048'
-  - 'mosaicml/mpt-7b-chat:2048'
-  - 'redpajama/gptneox-7b-redpajama-bf16:2048'
-  - 'bigcode/starcoder-15.5b-4bit:1024'
-  - 'bigcode/starcoder-15.5b-4bit:2048' 
-  - 'databricks/dolly-v1-6b:2048' 
-  - 'databricks/dolly-v2-7b:2048' 
-  - 'databricks/dolly-v2-12b:1024' 
-  - 'databricks/dolly-v2-12b:2048' 
-  - 'internlm/internlm-chat-7b:2048' 
-  - 'Qwen/Qwen-7B-Chat:2048'
-  - 'BAAI/AquilaChat-7B:2048'
   - 'baichuan-inc/Baichuan2-7B-Chat:2048'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:1024'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-  - 'bigscience/bloomz-7b1:1024'
-  - 'bigscience/bloomz-7b1:2048' 
 #  - 'fnlp/moss-moon-003-sft-4bit:1024'
 #  - 'fnlp/moss-moon-003-sft-4bit:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
@@ -1,21 +1,9 @@
 repo_id:
   - 'meta-llama/Llama-2-7b-chat-hf'
   - 'meta-llama/Llama-2-13b-chat-hf'
-  - 'THUDM/chatglm2-6b'
   - 'THUDM/chatglm3-6b-4bit'
-  - 'tiiuae/falcon-7b-instruct-with-patch'
-  - 'mosaicml/mpt-7b-chat'
-  - 'redpajama/gptneox-7b-redpajama-bf16'
-  - 'bigcode/starcoder-15.5b-4bit'
-  - 'databricks/dolly-v1-6b'
-  - 'databricks/dolly-v2-7b'
-  # - 'databricks/dolly-v2-12b'
-  - 'internlm/internlm-chat-7b'
-  - 'Qwen/Qwen-7B-Chat'
-  - 'BAAI/AquilaChat-7B'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'bigscience/bloomz-7b1'
 #  - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
   - 'mistralai/Mistral-7B-v0.1'
 local_model_hub: '/mnt/disk1/models'
@@ -35,5 +23,4 @@ exclude:
 #  - 'fnlp/moss-moon-003-sft-4bit:1024'
 #  - 'fnlp/moss-moon-003-sft-4bit:2048'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-  - 'bigscience/bloomz-7b1:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml
@@ -1,7 +1,6 @@
 # For the models that require transformers 4.37.0
 repo_id:
   - 'Qwen/Qwen1.5-7B-Chat'
-  - 'microsoft/phi-2'
   - 'microsoft/Phi-3-mini-4k-instruct'
   - 'meta-llama/Meta-Llama-3-8B-Instruct'
 local_model_hub: '/mnt/disk1/models'

diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml
@@ -1,7 +1,6 @@
 # For the models that require transformers 4.37.0
 repo_id:
   - 'Qwen/Qwen1.5-7B-Chat'
-  - 'microsoft/phi-2'
   - 'microsoft/Phi-3-mini-4k-instruct'
   - 'meta-llama/Meta-Llama-3-8B-Instruct' # mwj: need to test
 local_model_hub: '/mnt/disk1/models'

diff --git a/python/llm/test/benchmark/arc-perf-transformers-437.yaml b/python/llm/test/benchmark/arc-perf-transformers-437.yaml
@@ -1,7 +1,6 @@
 # For the models that require transformers 4.37.0
 repo_id:
   - 'Qwen/Qwen1.5-7B-Chat'
-  - 'microsoft/phi-2'
   - 'microsoft/Phi-3-mini-4k-instruct'
   - 'meta-llama/Meta-Llama-3-8B-Instruct'
 local_model_hub: '/mnt/disk1/models'