add tasks

intel-analytics · Jun 18, 2024 · b2f5eb7 · b2f5eb7
1 parent c357007
commit b2f5eb7
Show file tree

Hide file tree

Showing 6 changed files with 8 additions and 3 deletions.
diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
@@ -35,4 +35,5 @@ exclude:
   # - 'bigcode/starcoder-15.5b-4bit:2048'
   # - 'databricks/dolly-v2-12b:2048'
   # - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-  # - 'bigscience/bloomz-7b1:2048'
+  # - 'bigscience/bloomz-7b1:2048'
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-test-batch4.yaml b/python/llm/test/benchmark/arc-perf-test-batch4.yaml
@@ -13,4 +13,4 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
-  
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
@@ -36,3 +36,4 @@ exclude:
 #  - 'fnlp/moss-moon-003-sft-4bit:2048'
   # - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
   # - 'bigscience/bloomz-7b1:2048'
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml
@@ -17,3 +17,4 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml
@@ -13,4 +13,5 @@ in_out_pairs:
   - '2048-256'
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
-cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437.yaml b/python/llm/test/benchmark/arc-perf-transformers-437.yaml
@@ -17,3 +17,4 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'