From 29572ddd655817dd8b6be3dd085870cd1ef3494b Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 22 Nov 2024 11:29:36 -0800 Subject: [PATCH] update recipes Signed-off-by: yaoyu-33 --- nemo/collections/vlm/recipes/mllama_11b.py | 2 ++ nemo/collections/vlm/recipes/mllama_90b.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/nemo/collections/vlm/recipes/mllama_11b.py b/nemo/collections/vlm/recipes/mllama_11b.py index ce956c7c8135..4b08606900e3 100644 --- a/nemo/collections/vlm/recipes/mllama_11b.py +++ b/nemo/collections/vlm/recipes/mllama_11b.py @@ -134,6 +134,8 @@ def finetune_recipe( recipe.trainer.strategy.tensor_model_parallel_size = 2 recipe.optim.config.lr = 2e-05 elif peft_scheme.lower() == 'lora': + # pylint: disable=line-too-long + """Adapted from https://github.com/meta-llama/llama-recipes/blob/main/src/llama_recipes/configs/peft.py""" recipe.peft = run.Config( vlm.LoRA, freeze_vision_model=True, diff --git a/nemo/collections/vlm/recipes/mllama_90b.py b/nemo/collections/vlm/recipes/mllama_90b.py index 28a6ff7ff9a6..787cc54483ec 100644 --- a/nemo/collections/vlm/recipes/mllama_90b.py +++ b/nemo/collections/vlm/recipes/mllama_90b.py @@ -131,17 +131,20 @@ def finetune_recipe( if peft_scheme is None or peft_scheme.lower() == 'none': raise ValueError("Full finetuning recipe for Llama-3.2-90B model will be supported soon.") elif peft_scheme.lower() == 'lora': + # pylint: disable=line-too-long + """Adapted from https://github.com/meta-llama/llama-recipes/blob/main/src/llama_recipes/configs/peft.py""" recipe.peft = run.Config( vlm.LoRA, - freeze_vision_model=False, + freeze_vision_model=True, target_modules=[ - "*.language_model.*.linear_qkv", - "*.language_model.*.linear_q", - "*.language_model.*.linear_kv", - "*.language_model.*.linear_proj", - "*.language_model.*.linear_fc1", - "*.language_model.*.linear_fc2", + "linear_qkv", + "linear_q", + "linear_kv", ], + dim=8, + alpha=32, + dropout=0.05, + dropout_position="pre", ) recipe.optim.config.lr = 1e-4 else: