use one model per comfig

intel-analytics · Jun 12, 2024 · 33a8001 · 33a8001
1 parent 698a2da
commit 33a8001
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 97 deletions.
diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
@@ -87,57 +87,57 @@ jobs:
           source /opt/intel/oneapi/setvars.sh
           bash python/llm/test/run-llm-install-tests.sh
 
-      # - name: Test on xpu(transformers==4.36.2)
-      #   shell: bash
-      #   run: |
-      #     date_for_test_version=$(date -d yesterday +%Y-%m-%d)
-      #     sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
-      #     source /opt/intel/oneapi/setvars.sh
-      #     export USE_XETLA=OFF
-      #     export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
-      #     cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
-      #     cd python/llm/dev/benchmark/all-in-one
-      #     mkdir test_batch1
-      #     mkdir test_batch2
-      #     # batch_size 1
-      #     # hide time info
-      #     sed -i 's/str(end - st)/"xxxxxx"/g' run.py
-      #     # change csv name
-      #     sed -i 's/{today}/{today}_test1_batch1/g' run.py
-      #     python run.py
-      #     mv *.csv test_batch1
-      #     # batch_size 2
-      #     cd ../../../../../ 
-      #     cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
-      #     cd python/llm/dev/benchmark/all-in-one
-      #     # change csv name
-      #     sed -i 's/batch1/batch2/g' run.py
-      #     python run.py
-      #     mv *.csv test_batch2
-
-      # - name: Test on xpu(transformers==4.37.0)
-      #   shell: bash
-      #   run: |
-      #     source /opt/intel/oneapi/setvars.sh
-      #     export USE_XETLA=OFF
-      #     export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
-      #     # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
-      #     python -m pip install transformers==4.37.0
-      #     # batch_size 1
-      #     cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
-      #     cd python/llm/dev/benchmark/all-in-one
-      #     # change csv name
-      #     sed -i 's/test1_batch2/test2_batch1/g' run.py
-      #     python run.py
-      #     mv *.csv test_batch1
-      #     # batch_size 2
-      #     cd ../../../../../ 
-      #     cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
-      #     cd python/llm/dev/benchmark/all-in-one
-      #     # change csv name 
-      #     sed -i 's/batch1/batch2/g' run.py
-      #     python run.py
-      #     mv *.csv test_batch2
+      - name: Test on xpu(transformers==4.36.2)
+        shell: bash
+        run: |
+          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
+          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
+          source /opt/intel/oneapi/setvars.sh
+          export USE_XETLA=OFF
+          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+          cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          mkdir test_batch1
+          mkdir test_batch2
+          # batch_size 1
+          # hide time info
+          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
+          # change csv name
+          sed -i 's/{today}/{today}_test1_batch1/g' run.py
+          python run.py
+          mv *.csv test_batch1
+          # batch_size 2
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name
+          sed -i 's/batch1/batch2/g' run.py
+          python run.py
+          mv *.csv test_batch2
+
+      - name: Test on xpu(transformers==4.37.0)
+        shell: bash
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          export USE_XETLA=OFF
+          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+          # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
+          python -m pip install transformers==4.37.0
+          # batch_size 1
+          cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name
+          sed -i 's/test1_batch2/test2_batch1/g' run.py
+          python run.py
+          mv *.csv test_batch1
+          # batch_size 2
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch1/batch2/g' run.py
+          python run.py
+          mv *.csv test_batch2
 
       - name: Concat csv and generate html
         shell: bash

diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
@@ -1,23 +1,23 @@
 repo_id:
   - 'meta-llama/Llama-2-7b-chat-hf'
-  - 'meta-llama/Llama-2-13b-chat-hf'
-  - 'THUDM/chatglm2-6b'
-  - 'THUDM/chatglm3-6b-4bit'
-  - 'tiiuae/falcon-7b-instruct-with-patch'
-  - 'mosaicml/mpt-7b-chat'
-  - 'redpajama/gptneox-7b-redpajama-bf16'
-  - 'bigcode/starcoder-15.5b-4bit'
-  - 'databricks/dolly-v1-6b'
-  - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
-  - 'internlm/internlm-chat-7b'
-  - 'Qwen/Qwen-7B-Chat'
-  - 'BAAI/AquilaChat-7B'
-  - 'baichuan-inc/Baichuan2-7B-Chat'
-  - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'bigscience/bloomz-7b1'
+  # - 'meta-llama/Llama-2-13b-chat-hf'
+  # - 'THUDM/chatglm2-6b'
+  # - 'THUDM/chatglm3-6b-4bit'
+  # - 'tiiuae/falcon-7b-instruct-with-patch'
+  # - 'mosaicml/mpt-7b-chat'
+  # - 'redpajama/gptneox-7b-redpajama-bf16'
+  # - 'bigcode/starcoder-15.5b-4bit'
+  # - 'databricks/dolly-v1-6b'
+  # - 'databricks/dolly-v2-7b'
+  # - 'databricks/dolly-v2-12b'
+  # - 'internlm/internlm-chat-7b'
+  # - 'Qwen/Qwen-7B-Chat'
+  # - 'BAAI/AquilaChat-7B'
+  # - 'baichuan-inc/Baichuan2-7B-Chat'
+  # - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
+  # - 'bigscience/bloomz-7b1'
 #  - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
-  - 'mistralai/Mistral-7B-v0.1'
+  # - 'mistralai/Mistral-7B-v0.1'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -32,7 +32,7 @@ test_api:
   - "transformer_int4_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
-  - 'bigcode/starcoder-15.5b-4bit:2048'
-  - 'databricks/dolly-v2-12b:2048'
-  - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-  - 'bigscience/bloomz-7b1:2048'
+  # - 'bigcode/starcoder-15.5b-4bit:2048'
+  # - 'databricks/dolly-v2-12b:2048'
+  # - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
+  # - 'bigscience/bloomz-7b1:2048'
diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
@@ -1,23 +1,23 @@
 repo_id:
   - 'meta-llama/Llama-2-7b-chat-hf'
-  - 'meta-llama/Llama-2-13b-chat-hf'
-  - 'THUDM/chatglm2-6b'
-  - 'THUDM/chatglm3-6b-4bit'
-  - 'tiiuae/falcon-7b-instruct-with-patch'
-  - 'mosaicml/mpt-7b-chat'
-  - 'redpajama/gptneox-7b-redpajama-bf16'
-  - 'bigcode/starcoder-15.5b-4bit'
-  - 'databricks/dolly-v1-6b'
-  - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
-  - 'internlm/internlm-chat-7b'
-  - 'Qwen/Qwen-7B-Chat'
-  - 'BAAI/AquilaChat-7B'
-  - 'baichuan-inc/Baichuan2-7B-Chat'
-  - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'bigscience/bloomz-7b1'
+  # - 'meta-llama/Llama-2-13b-chat-hf'
+  # - 'THUDM/chatglm2-6b'
+  # - 'THUDM/chatglm3-6b-4bit'
+  # - 'tiiuae/falcon-7b-instruct-with-patch'
+  # - 'mosaicml/mpt-7b-chat'
+  # - 'redpajama/gptneox-7b-redpajama-bf16'
+  # - 'bigcode/starcoder-15.5b-4bit'
+  # - 'databricks/dolly-v1-6b'
+  # - 'databricks/dolly-v2-7b'
+  # - 'databricks/dolly-v2-12b'
+  # - 'internlm/internlm-chat-7b'
+  # - 'Qwen/Qwen-7B-Chat'
+  # - 'BAAI/AquilaChat-7B'
+  # - 'baichuan-inc/Baichuan2-7B-Chat'
+  # - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
+  # - 'bigscience/bloomz-7b1'
 #  - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
-  - 'mistralai/Mistral-7B-v0.1'
+  # - 'mistralai/Mistral-7B-v0.1'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -34,5 +34,5 @@ cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu w
 exclude:
 #  - 'fnlp/moss-moon-003-sft-4bit:1024'
 #  - 'fnlp/moss-moon-003-sft-4bit:2048'
-  - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-  - 'bigscience/bloomz-7b1:2048'
+  # - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
+  # - 'bigscience/bloomz-7b1:2048'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml
@@ -1,9 +1,9 @@
 # For the models that require transformers 4.37.0
 repo_id:
   - 'Qwen/Qwen1.5-7B-Chat'
-  - 'microsoft/phi-2'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'meta-llama/Meta-Llama-3-8B-Instruct'
+  # - 'microsoft/phi-2'
+  # - 'microsoft/Phi-3-mini-4k-instruct'
+  # - 'meta-llama/Meta-Llama-3-8B-Instruct'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/arc-perf-transformers-437.yaml b/python/llm/test/benchmark/arc-perf-transformers-437.yaml
@@ -1,9 +1,9 @@
 # For the models that require transformers 4.37.0
 repo_id:
   - 'Qwen/Qwen1.5-7B-Chat'
-  - 'microsoft/phi-2'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'meta-llama/Meta-Llama-3-8B-Instruct'
+  # - 'microsoft/phi-2'
+  # - 'microsoft/Phi-3-mini-4k-instruct'
+  # - 'meta-llama/Meta-Llama-3-8B-Instruct'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3