fix masking bug

mlcommons · Feb 1, 2024 · 87992ca · 87992ca
1 parent 6a6ca47
commit 87992ca
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/llm_finetune/scripts/train.py b/llm_finetune/scripts/train.py
@@ -106,7 +106,7 @@ class ScriptArguments:
         default=6, metadata={"help": "Log every X updates steps."}
     )
     target_eval_loss: float = field(
-        default=1.19, metadata={"help": "target eval loss - NOT FINAL."}
+        default=0.92, metadata={"help": "target eval loss - NOT FINAL."}
     )
     output_dir: str = field(
         default="results", metadata={"help": "Where to store the final model."}

diff --git a/llm_finetune/scripts/utils.py b/llm_finetune/scripts/utils.py
@@ -143,7 +143,8 @@ def group_texts(examples, block_size):
         k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
         for k, t in concatenated_examples.items()
     }
-    result["labels"] = result["input_ids"].copy()
+    if 'labels' not in result:
+        result["labels"] = result["input_ids"].copy()
     return result