bring back the kiloshard thing

stanford-crfm · Oct 2, 2024 · 455459d · 455459d
1 parent 0e9b78f
commit 455459d
Show file tree

Hide file tree

Showing 3 changed files with 210 additions and 79 deletions.
diff --git a/src/levanter/data/text.py b/src/levanter/data/text.py
@@ -673,7 +673,6 @@ def build_or_load_cache(
             split_cache_dir,
             source,
             bt,
-            await_finished=split == "validation",
             monitors=monitors,
         )
-Original file line number
+Diff line change
@@ Expand Up / @@ -673,7 +673,6 @@ def build_or_load_cache( @@
                 split_cache_dir,
                 source,
                 bt,
-                await_finished=split == "validation",
                 monitors=monitors,
             )
@@ Expand Down @@