debug loss mask

stanford-crfm · Jan 12, 2025 · 5530adf · 5530adf
1 parent bb708c4
commit 5530adf
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/src/levanter/data/packing.py b/src/levanter/data/packing.py
@@ -106,6 +106,11 @@ def pack_prompt_completions(
     for sequence in sequences:
         loss_mask = np.arange(len(sequence.ids)) >= sequence.prompt_length - 1
         loss_mask[-1] = 0
+
+        if not np.any(loss_mask):
+            logger.error(f"Invalid loss_mask: all zeros for sequence with prompt_length={sequence.prompt_length} and len(ids)={len(sequence.ids)}")
+            raise ValueError("Loss mask has no active tokens for loss computation.")
+
         assert np.any(loss_mask)
 
         for packer in packers: