update script match recipe

Signed-off-by: yaoyu-33 <[email protected]>
NVIDIA · Nov 22, 2024 · 6b2b5e9 · 6b2b5e9
1 parent ec1ebac
commit 6b2b5e9
Showing 1 changed file with 9 additions and 7 deletions.
diff --git a/scripts/vlm/mllama_finetune.py b/scripts/vlm/mllama_finetune.py
@@ -155,14 +155,16 @@ def main(args):
     # PEFT setup
     if args.peft == 'lora':
         peft = vlm.peft.LoRA(
+            freeze_vision_model=True,
             target_modules=[
-                "*.language_model.*.linear_qkv",
-                "*.language_model.*.linear_q",
-                "*.language_model.*.linear_kv",
-                "*.language_model.*.linear_proj",
-                "*.language_model.*.linear_fc1",
-                "*.language_model.*.linear_fc2",
-            ]
+                "linear_qkv",
+                "linear_q",
+                "linear_kv",
+            ],
+            dim=8,
+            alpha=32,
+            dropout=0.05,
+            dropout_position="pre",
         )
     else:
         peft = None