Add pre-generated prompts option for benchmark

benchmark/python/benchmark_e2e.py

@@ -83,6 +83,14 @@
         generator.generate_next_token()
     return tokenizer.decode(generator.get_sequence(0))
 
+# Use prompt length to get pre-defined prompt
+def get_prompt_by_length(prompt_length):
+    json_path = "prompts.json"
+    f = open(json_path)
+    data = json.load(f)
+
+    return data[f"{prompt_length}"]
+
 def get_target_pip_package_version(target_pip_package_name_list):
     # get package name and version
     import pkg_resources
@@ -219,7 +227,7 @@
     num_repetitions = args.repetitions
     temperature = 1.0
 
     # Get tokenizer, and model
     if args.verbose: print("Loading model... ")
     model=og.Model(f'{args.input_folder}')
     if args.verbose: print("Model loaded")
@@ -232,6 +240,9 @@
         # use random tokens instead of generating a prompt using the model and then tokenizing it
         tokens = np.random.randint(100, size=(batch_size, prompt_length))
         prompt = [tokenizer.decode(tokens[0])] * batch_size
+    elif args.use_prompt_set:
+        prompt = get_prompt_by_length(prompt_length)
+        tokens = tokenizer.encode_batch(prompt)
     else:
         prompt = [generate_prompt(model, tokenizer, prompt_length, args.use_graph_capture)] * batch_size
         tokens = tokenizer.encode_batch(prompt)
@@ -424,6 +435,7 @@
     parser.add_argument('-mn', '--model_name', type=str, default='model_name', help='Model name defined by users')
     parser.add_argument('-pr', '--precision', type=str, default='fp16', help='Model precision for metrics info')
     parser.add_argument('--use_random_tokens', action='store_true', help='Use random tokens instead of generating a prompt')
+    parser.add_argument('--use_prompt_set', action='store_true', help='Use pre-generated prompt set instead of generating a prompt')
     args = parser.parse_args()
 
     # check max_lengths

benchmark/python/prompts.json

@@ -0,0 +1,7 @@
+{
+    "16": "How are astronauts launched into space quickly on those rockets? ",
+    "64": "",
+    "256": "",
+    "1024": "",
+    "2048": ""
+}