From 9b3f98dc478694a87070c6275d96f181dd7a8a32 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 5 Nov 2024 14:48:07 -0800 Subject: [PATCH] pre-commit --- src/levanter/data/text.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/levanter/data/text.py b/src/levanter/data/text.py index 7fb844d8f..f2bea44b2 100644 --- a/src/levanter/data/text.py +++ b/src/levanter/data/text.py @@ -575,6 +575,7 @@ class LMSupervisedDatasetConfig: validation_urls: List[str] = () # type:ignore + def preprocess_supervised_example( batch, tokenizer: PreTrainedTokenizerBase, input_field: str, output_field: str ) -> dict: