add lowbit_path for generate.py, fix npu_model (#12077)

* add `lowbit_path` for `generate.py`, fix `npu_model`

* update `README.md`

python/llm/example/NPU/HF-Transformers-AutoModels/Multimodal/README.md

@@ -58,6 +58,7 @@ python ./generate.py
 
 Arguments info:
 - `--repo-id-or-model-path REPO_ID_OR_MODEL_PATH`: argument defining the huggingface repo id for the Phi-3-vision model (e.g. `microsoft/Phi-3-vision-128k-instruct`) to be downloaded, or the path to the huggingface checkpoint folder. It is default to be `'microsoft/Phi-3-vision-128k-instruct'`, and more verified models please see the list in [Verified Models](#verified-models).
+- `--lowbit-path LOWBIT_MODEL_PATH`: argument defining the path to save/load lowbit version of the model. If it is an empty string, the original pretrained model specified by `REPO_ID_OR_MODEL_PATH` will be loaded. If it is an existing path, the lowbit model in `LOWBIT_MODEL_PATH` will be loaded. If it is a non-existing path, the original pretrained model specified by `REPO_ID_OR_MODEL_PATH` will be loaded, and the converted lowbit version will be saved into `LOWBIT_MODEL_PATH`. It is default to be `''`, i.e. an empty string.
 - `--image-url-or-path IMAGE_URL_OR_PATH`: argument defining the image to be infered. It is default to be `'http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg'`.
 - `--prompt PROMPT`: argument defining the prompt to be infered (with integrated prompt format for chat). It is default to be `'What is in the image?'`.
 - `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `32`.

python/llm/example/NPU/HF-Transformers-AutoModels/Multimodal/generate.py

@@ -29,6 +29,11 @@
     parser.add_argument('--repo-id-or-model-path', type=str, default="microsoft/Phi-3-vision-128k-instruct",
                         help='The huggingface repo id for the phi-3-vision model to be downloaded'
                              ', or the path to the huggingface checkpoint folder')
+    parser.add_argument("--lowbit-path", type=str,
+        default="",
+        help='The path to the lowbit model folder, leave blank if you do not want to save. \
+            If path not exists, lowbit model will be saved there. \
+            Else, lowbit model will be loaded.')
     parser.add_argument('--image-url-or-path', type=str,
                         default="http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg",
                         help='The URL or path to the image to infer')
@@ -49,12 +54,27 @@
     # You could also try `'sym_int8'` for INT8
     # `_attn_implementation="eager"` is required for phi-3-vision
     # `modules_to_not_convert=["vision_embed_tokens"]` and `model = model.half()` are for acceleration and are optional
-    model = AutoModelForCausalLM.from_pretrained(model_path,
-                                                 trust_remote_code=True,
-                                                 load_in_low_bit=args.load_in_low_bit,
-                                                 _attn_implementation="eager",
-                                                 modules_to_not_convert=["vision_embed_tokens"])
-
+
+    if not args.lowbit_path or not os.path.exists(args.lowbit_path):
+        model = AutoModelForCausalLM.from_pretrained(
+            model_path,
+            trust_remote_code=True,
+            load_in_low_bit=args.load_in_low_bit,
+            _attn_implementation="eager",
+            modules_to_not_convert=["vision_embed_tokens"]
+        )
+    else:
+        model = AutoModelForCausalLM.load_low_bit(
+            args.lowbit_path,
+            trust_remote_code=True,
+            bigdl_transformers_low_bit=args.load_in_low_bit,
+            attn_implementation="eager",
+            modules_to_not_convert=["vision_embed_tokens"]
+        )
+
+    if args.lowbit_path and not os.path.exists(args.lowbit_path):
+        model.save_low_bit(args.lowbit_path)
+
     # Load processor
     processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
 

python/llm/src/ipex_llm/transformers/npu_model.py

@@ -207,7 +207,6 @@ def load_low_bit(cls, pretrained_model_name_or_path: str, *model_args, **kwargs)
         ignore_argument(kwargs, "lightweight_bmm")
         ignore_argument(kwargs, "cpu_embedding")
         ignore_argument(kwargs, "embedding_qtype")
-        ignore_argument(kwargs, "modules_to_not_convert")
         ignore_argument(kwargs, "speculative")
         ignore_argument(kwargs, "pipeline_parallel_stages")
         optimize_model = kwargs.pop("optimize_model", False)