diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index dea84f46ceb..e4f7281f177 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -87,57 +87,57 @@ jobs: source /opt/intel/oneapi/setvars.sh bash python/llm/test/run-llm-install-tests.sh - # - name: Test on xpu(transformers==4.36.2) - # shell: bash - # run: | - # date_for_test_version=$(date -d yesterday +%Y-%m-%d) - # sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py - # source /opt/intel/oneapi/setvars.sh - # export USE_XETLA=OFF - # export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 - # cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml - # cd python/llm/dev/benchmark/all-in-one - # mkdir test_batch1 - # mkdir test_batch2 - # # batch_size 1 - # # hide time info - # sed -i 's/str(end - st)/"xxxxxx"/g' run.py - # # change csv name - # sed -i 's/{today}/{today}_test1_batch1/g' run.py - # python run.py - # mv *.csv test_batch1 - # # batch_size 2 - # cd ../../../../../ - # cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml - # cd python/llm/dev/benchmark/all-in-one - # # change csv name - # sed -i 's/batch1/batch2/g' run.py - # python run.py - # mv *.csv test_batch2 - - # - name: Test on xpu(transformers==4.37.0) - # shell: bash - # run: | - # source /opt/intel/oneapi/setvars.sh - # export USE_XETLA=OFF - # export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 - # # upgrade transformers for model Qwen/Qwen1.5-7B-Chat - # python -m pip install transformers==4.37.0 - # # batch_size 1 - # cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml - # cd python/llm/dev/benchmark/all-in-one - # # change csv name - # sed -i 's/test1_batch2/test2_batch1/g' run.py - # python run.py - # mv *.csv test_batch1 - # # batch_size 2 - # cd ../../../../../ - # cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml - # cd python/llm/dev/benchmark/all-in-one - # # change csv name - # sed -i 's/batch1/batch2/g' run.py - # python run.py - # mv *.csv test_batch2 + - name: Test on xpu(transformers==4.36.2) + shell: bash + run: | + date_for_test_version=$(date -d yesterday +%Y-%m-%d) + sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py + source /opt/intel/oneapi/setvars.sh + export USE_XETLA=OFF + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + mkdir test_batch1 + mkdir test_batch2 + # batch_size 1 + # hide time info + sed -i 's/str(end - st)/"xxxxxx"/g' run.py + # change csv name + sed -i 's/{today}/{today}_test1_batch1/g' run.py + python run.py + mv *.csv test_batch1 + # batch_size 2 + cd ../../../../../ + cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + # change csv name + sed -i 's/batch1/batch2/g' run.py + python run.py + mv *.csv test_batch2 + + - name: Test on xpu(transformers==4.37.0) + shell: bash + run: | + source /opt/intel/oneapi/setvars.sh + export USE_XETLA=OFF + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + # upgrade transformers for model Qwen/Qwen1.5-7B-Chat + python -m pip install transformers==4.37.0 + # batch_size 1 + cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + # change csv name + sed -i 's/test1_batch2/test2_batch1/g' run.py + python run.py + mv *.csv test_batch1 + # batch_size 2 + cd ../../../../../ + cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + # change csv name + sed -i 's/batch1/batch2/g' run.py + python run.py + mv *.csv test_batch2 - name: Concat csv and generate html shell: bash diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml index 00b2e4c1a53..a12dbfa44ff 100644 --- a/python/llm/test/benchmark/arc-perf-test-batch2.yaml +++ b/python/llm/test/benchmark/arc-perf-test-batch2.yaml @@ -1,23 +1,23 @@ repo_id: - 'meta-llama/Llama-2-7b-chat-hf' - - 'meta-llama/Llama-2-13b-chat-hf' - - 'THUDM/chatglm2-6b' - - 'THUDM/chatglm3-6b-4bit' - - 'tiiuae/falcon-7b-instruct-with-patch' - - 'mosaicml/mpt-7b-chat' - - 'redpajama/gptneox-7b-redpajama-bf16' - - 'bigcode/starcoder-15.5b-4bit' - - 'databricks/dolly-v1-6b' - - 'databricks/dolly-v2-7b' - - 'databricks/dolly-v2-12b' - - 'internlm/internlm-chat-7b' - - 'Qwen/Qwen-7B-Chat' - - 'BAAI/AquilaChat-7B' - - 'baichuan-inc/Baichuan2-7B-Chat' - - 'baichuan-inc/Baichuan2-13B-Chat-4bit' - - 'bigscience/bloomz-7b1' + # - 'meta-llama/Llama-2-13b-chat-hf' + # - 'THUDM/chatglm2-6b' + # - 'THUDM/chatglm3-6b-4bit' + # - 'tiiuae/falcon-7b-instruct-with-patch' + # - 'mosaicml/mpt-7b-chat' + # - 'redpajama/gptneox-7b-redpajama-bf16' + # - 'bigcode/starcoder-15.5b-4bit' + # - 'databricks/dolly-v1-6b' + # - 'databricks/dolly-v2-7b' + # - 'databricks/dolly-v2-12b' + # - 'internlm/internlm-chat-7b' + # - 'Qwen/Qwen-7B-Chat' + # - 'BAAI/AquilaChat-7B' + # - 'baichuan-inc/Baichuan2-7B-Chat' + # - 'baichuan-inc/Baichuan2-13B-Chat-4bit' + # - 'bigscience/bloomz-7b1' # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ - - 'mistralai/Mistral-7B-v0.1' + # - 'mistralai/Mistral-7B-v0.1' local_model_hub: '/mnt/disk1/models' warm_up: 1 num_trials: 3 @@ -32,7 +32,7 @@ test_api: - "transformer_int4_gpu" # on Intel GPU cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api) exclude: - - 'bigcode/starcoder-15.5b-4bit:2048' - - 'databricks/dolly-v2-12b:2048' - - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - - 'bigscience/bloomz-7b1:2048' \ No newline at end of file + # - 'bigcode/starcoder-15.5b-4bit:2048' + # - 'databricks/dolly-v2-12b:2048' + # - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' + # - 'bigscience/bloomz-7b1:2048' \ No newline at end of file diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml index 895588ce4e4..0117e1d7deb 100644 --- a/python/llm/test/benchmark/arc-perf-test.yaml +++ b/python/llm/test/benchmark/arc-perf-test.yaml @@ -1,23 +1,23 @@ repo_id: - 'meta-llama/Llama-2-7b-chat-hf' - - 'meta-llama/Llama-2-13b-chat-hf' - - 'THUDM/chatglm2-6b' - - 'THUDM/chatglm3-6b-4bit' - - 'tiiuae/falcon-7b-instruct-with-patch' - - 'mosaicml/mpt-7b-chat' - - 'redpajama/gptneox-7b-redpajama-bf16' - - 'bigcode/starcoder-15.5b-4bit' - - 'databricks/dolly-v1-6b' - - 'databricks/dolly-v2-7b' - - 'databricks/dolly-v2-12b' - - 'internlm/internlm-chat-7b' - - 'Qwen/Qwen-7B-Chat' - - 'BAAI/AquilaChat-7B' - - 'baichuan-inc/Baichuan2-7B-Chat' - - 'baichuan-inc/Baichuan2-13B-Chat-4bit' - - 'bigscience/bloomz-7b1' + # - 'meta-llama/Llama-2-13b-chat-hf' + # - 'THUDM/chatglm2-6b' + # - 'THUDM/chatglm3-6b-4bit' + # - 'tiiuae/falcon-7b-instruct-with-patch' + # - 'mosaicml/mpt-7b-chat' + # - 'redpajama/gptneox-7b-redpajama-bf16' + # - 'bigcode/starcoder-15.5b-4bit' + # - 'databricks/dolly-v1-6b' + # - 'databricks/dolly-v2-7b' + # - 'databricks/dolly-v2-12b' + # - 'internlm/internlm-chat-7b' + # - 'Qwen/Qwen-7B-Chat' + # - 'BAAI/AquilaChat-7B' + # - 'baichuan-inc/Baichuan2-7B-Chat' + # - 'baichuan-inc/Baichuan2-13B-Chat-4bit' + # - 'bigscience/bloomz-7b1' # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ - - 'mistralai/Mistral-7B-v0.1' + # - 'mistralai/Mistral-7B-v0.1' local_model_hub: '/mnt/disk1/models' warm_up: 1 num_trials: 3 @@ -34,5 +34,5 @@ cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu w exclude: # - 'fnlp/moss-moon-003-sft-4bit:1024' # - 'fnlp/moss-moon-003-sft-4bit:2048' - - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - - 'bigscience/bloomz-7b1:2048' + # - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' + # - 'bigscience/bloomz-7b1:2048' diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml index c9644dc905c..f26969b4ec4 100644 --- a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml +++ b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml @@ -1,9 +1,9 @@ # For the models that require transformers 4.37.0 repo_id: - 'Qwen/Qwen1.5-7B-Chat' - - 'microsoft/phi-2' - - 'microsoft/Phi-3-mini-4k-instruct' - - 'meta-llama/Meta-Llama-3-8B-Instruct' + # - 'microsoft/phi-2' + # - 'microsoft/Phi-3-mini-4k-instruct' + # - 'meta-llama/Meta-Llama-3-8B-Instruct' local_model_hub: '/mnt/disk1/models' warm_up: 1 num_trials: 3 diff --git a/python/llm/test/benchmark/arc-perf-transformers-437.yaml b/python/llm/test/benchmark/arc-perf-transformers-437.yaml index c9cc5ce82a5..77dc132263b 100644 --- a/python/llm/test/benchmark/arc-perf-transformers-437.yaml +++ b/python/llm/test/benchmark/arc-perf-transformers-437.yaml @@ -1,9 +1,9 @@ # For the models that require transformers 4.37.0 repo_id: - 'Qwen/Qwen1.5-7B-Chat' - - 'microsoft/phi-2' - - 'microsoft/Phi-3-mini-4k-instruct' - - 'meta-llama/Meta-Llama-3-8B-Instruct' + # - 'microsoft/phi-2' + # - 'microsoft/Phi-3-mini-4k-instruct' + # - 'meta-llama/Meta-Llama-3-8B-Instruct' local_model_hub: '/mnt/disk1/models' warm_up: 1 num_trials: 3