fix

intel · Jun 24, 2024 · 4cd78b8 · 4cd78b8
1 parent be1906c
commit 4cd78b8
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/python/llm/example/GPU/Pipeline-Parallel-FastAPI/pipeline_models.py b/python/llm/example/GPU/Pipeline-Parallel-FastAPI/pipeline_models.py
@@ -90,7 +90,7 @@ def load_model(self, model_path, my_rank, my_size, low_bit='sym_int4'):
                                                     optimize_model=True,
                                                     trust_remote_code=True,
                                                     use_cache=True,
-                                                    pipeline_parallel_stages=my_size)
+                                                    pipeline_parallel_stages=my_size).eval()
         # print(model)
 
         # config_class = type(model.config).__name__