Fix training data shifting bug

delphi-suite · Apr 27, 2024 · 161c3f0 · 161c3f0
1 parent 3281227
commit 161c3f0
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/src/delphi/train/utils.py b/src/delphi/train/utils.py
@@ -167,7 +167,8 @@ def get_xy_batch(
     end = (batch_num + 1) * batch_size
     batch_indices = indices[start:end]
     data = dataset[batch_indices][feature_name].to(device)
-    return data[:, :-1], data[:, 1:]
+    # *ForCausalLM models do shifting internally, so input and labels are the same
+    return data, data
 
 
 def gen_minibatches(