From 52519e07dfc4c3a981f4648deee214b54b18c750 Mon Sep 17 00:00:00 2001 From: "Xu, Shuo" <100334393+ATMxsp01@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:20:48 +0800 Subject: [PATCH] remove models we no longer need in benchmark. (#11492) Co-authored-by: ATMxsp01 --- .../test/benchmark/arc-perf-test-batch2.yaml | 15 ----------- .../test/benchmark/arc-perf-test-batch4.yaml | 26 ------------------- python/llm/test/benchmark/arc-perf-test.yaml | 13 ---------- .../arc-perf-transformers-437-batch2.yaml | 1 - .../arc-perf-transformers-437-batch4.yaml | 1 - .../benchmark/arc-perf-transformers-437.yaml | 1 - 6 files changed, 57 deletions(-) diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml index 68e58aadaaa..b8b81338190 100644 --- a/python/llm/test/benchmark/arc-perf-test-batch2.yaml +++ b/python/llm/test/benchmark/arc-perf-test-batch2.yaml @@ -1,21 +1,9 @@ repo_id: - 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf' - - 'THUDM/chatglm2-6b' - 'THUDM/chatglm3-6b-4bit' - - 'tiiuae/falcon-7b-instruct-with-patch' - - 'mosaicml/mpt-7b-chat' - - 'redpajama/gptneox-7b-redpajama-bf16' - - 'bigcode/starcoder-15.5b-4bit' - - 'databricks/dolly-v1-6b' - - 'databricks/dolly-v2-7b' - # - 'databricks/dolly-v2-12b' - - 'internlm/internlm-chat-7b' - - 'Qwen/Qwen-7B-Chat' - - 'BAAI/AquilaChat-7B' - 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat-4bit' - - 'bigscience/bloomz-7b1' # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ - 'mistralai/Mistral-7B-v0.1' local_model_hub: '/mnt/disk1/models' @@ -32,8 +20,5 @@ test_api: - "transformer_int4_fp16_gpu" # on Intel GPU cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api) exclude: - - 'bigcode/starcoder-15.5b-4bit:2048' - # - 'databricks/dolly-v2-12b:2048' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - - 'bigscience/bloomz-7b1:2048' task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' \ No newline at end of file diff --git a/python/llm/test/benchmark/arc-perf-test-batch4.yaml b/python/llm/test/benchmark/arc-perf-test-batch4.yaml index b74d3833387..fe855d4ba93 100644 --- a/python/llm/test/benchmark/arc-perf-test-batch4.yaml +++ b/python/llm/test/benchmark/arc-perf-test-batch4.yaml @@ -1,21 +1,9 @@ repo_id: - 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf' - - 'THUDM/chatglm2-6b' - 'THUDM/chatglm3-6b-4bit' - - 'tiiuae/falcon-7b-instruct-with-patch' - - 'mosaicml/mpt-7b-chat' - - 'redpajama/gptneox-7b-redpajama-bf16' - - 'bigcode/starcoder-15.5b-4bit' - - 'databricks/dolly-v1-6b' - - 'databricks/dolly-v2-7b' - - 'databricks/dolly-v2-12b' - - 'internlm/internlm-chat-7b' - - 'Qwen/Qwen-7B-Chat' - - 'BAAI/AquilaChat-7B' - 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat-4bit' - - 'bigscience/bloomz-7b1' # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ - 'mistralai/Mistral-7B-v0.1' #mwj: need to check local_model_hub: '/mnt/disk1/models' @@ -33,23 +21,9 @@ test_api: cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api) exclude: - 'meta-llama/Llama-2-13b-chat-hf:2048' - - 'tiiuae/falcon-7b-instruct-with-patch:2048' - - 'mosaicml/mpt-7b-chat:2048' - - 'redpajama/gptneox-7b-redpajama-bf16:2048' - - 'bigcode/starcoder-15.5b-4bit:1024' - - 'bigcode/starcoder-15.5b-4bit:2048' - - 'databricks/dolly-v1-6b:2048' - - 'databricks/dolly-v2-7b:2048' - - 'databricks/dolly-v2-12b:1024' - - 'databricks/dolly-v2-12b:2048' - - 'internlm/internlm-chat-7b:2048' - - 'Qwen/Qwen-7B-Chat:2048' - - 'BAAI/AquilaChat-7B:2048' - 'baichuan-inc/Baichuan2-7B-Chat:2048' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:1024' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - - 'bigscience/bloomz-7b1:1024' - - 'bigscience/bloomz-7b1:2048' # - 'fnlp/moss-moon-003-sft-4bit:1024' # - 'fnlp/moss-moon-003-sft-4bit:2048' task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml index 4ad6ccde1e6..d467132a0cb 100644 --- a/python/llm/test/benchmark/arc-perf-test.yaml +++ b/python/llm/test/benchmark/arc-perf-test.yaml @@ -1,21 +1,9 @@ repo_id: - 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf' - - 'THUDM/chatglm2-6b' - 'THUDM/chatglm3-6b-4bit' - - 'tiiuae/falcon-7b-instruct-with-patch' - - 'mosaicml/mpt-7b-chat' - - 'redpajama/gptneox-7b-redpajama-bf16' - - 'bigcode/starcoder-15.5b-4bit' - - 'databricks/dolly-v1-6b' - - 'databricks/dolly-v2-7b' - # - 'databricks/dolly-v2-12b' - - 'internlm/internlm-chat-7b' - - 'Qwen/Qwen-7B-Chat' - - 'BAAI/AquilaChat-7B' - 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat-4bit' - - 'bigscience/bloomz-7b1' # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ - 'mistralai/Mistral-7B-v0.1' local_model_hub: '/mnt/disk1/models' @@ -35,5 +23,4 @@ exclude: # - 'fnlp/moss-moon-003-sft-4bit:1024' # - 'fnlp/moss-moon-003-sft-4bit:2048' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - - 'bigscience/bloomz-7b1:2048' task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml index b7d771619a4..5db92de72b9 100644 --- a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml +++ b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml @@ -1,7 +1,6 @@ # For the models that require transformers 4.37.0 repo_id: - 'Qwen/Qwen1.5-7B-Chat' - - 'microsoft/phi-2' - 'microsoft/Phi-3-mini-4k-instruct' - 'meta-llama/Meta-Llama-3-8B-Instruct' local_model_hub: '/mnt/disk1/models' diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml index e09869164f0..602a553e82b 100644 --- a/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml +++ b/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml @@ -1,7 +1,6 @@ # For the models that require transformers 4.37.0 repo_id: - 'Qwen/Qwen1.5-7B-Chat' - - 'microsoft/phi-2' - 'microsoft/Phi-3-mini-4k-instruct' - 'meta-llama/Meta-Llama-3-8B-Instruct' # mwj: need to test local_model_hub: '/mnt/disk1/models' diff --git a/python/llm/test/benchmark/arc-perf-transformers-437.yaml b/python/llm/test/benchmark/arc-perf-transformers-437.yaml index 10c4e07a051..a7173e05930 100644 --- a/python/llm/test/benchmark/arc-perf-transformers-437.yaml +++ b/python/llm/test/benchmark/arc-perf-transformers-437.yaml @@ -1,7 +1,6 @@ # For the models that require transformers 4.37.0 repo_id: - 'Qwen/Qwen1.5-7B-Chat' - - 'microsoft/phi-2' - 'microsoft/Phi-3-mini-4k-instruct' - 'meta-llama/Meta-Llama-3-8B-Instruct' local_model_hub: '/mnt/disk1/models'