fix small bug

OptimalScale · Jul 17, 2023 · 78159d0 · 78159d0
1 parent 868a124
commit 78159d0
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/src/lmflow/pipeline/finetuner.py b/src/lmflow/pipeline/finetuner.py
@@ -149,7 +149,8 @@ def group_text(self, tokenized_datasets, model_max_length):
                         f"({model_max_length})."
                         f"Using block_size={data_args.block_size}.")
                     block_size = data_args.block_size
-
+            else:
+                block_size = data_args.block_size
         # Main data processing function that will concatenate all texts from
         # our dataset and generate chunks of block_size.
         def group_texts(examples):