seq2seq: revert per-batch language assignment

gaussalgo · Oct 17, 2024 · a56b937 · a56b937
1 parent c7a581d
commit a56b937
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 4 deletions.
diff --git a/adaptor/objectives/CLM.py b/adaptor/objectives/CLM.py
@@ -135,7 +135,6 @@ def _get_seq2seq_collated_iterator(self,
             # yield last nonempty residual batch
             yield self.collator(features_batch)
 
-
     def _compute_loss(self,
                       logit_outputs: torch.FloatTensor,
                       labels: torch.LongTensor,

diff --git a/adaptor/objectives/seq2seq.py b/adaptor/objectives/seq2seq.py
@@ -34,10 +34,9 @@ def _get_seq2seq_collated_iterator(self,
         :return: Iterator of encoded batches.
         """
         features_batch = []
-        self.tokenizer.src_lang = self.source_lang_id
-        self.tokenizer.tgt_lang = self.target_lang_id
-
         for source_text, target_text in zip(source_texts, target_texts):
+            self.tokenizer.src_lang = self.source_lang_id
+            self.tokenizer.tgt_lang = self.target_lang_id
             sample_features = self.tokenizer(source_text, truncation=True)
 
             with self.tokenizer.as_target_tokenizer():