diff --git a/src/levanter/data/text.py b/src/levanter/data/text.py
index ad03f6d01..7e25c88ef 100644
--- a/src/levanter/data/text.py
+++ b/src/levanter/data/text.py
@@ -600,7 +600,7 @@ def train_set(
 
         if self.shuffle is True:
             ds = ds.shuffle(key)
-        elif isinstance(self.shuffle, int):
+        elif isinstance(self.shuffle, int) and self.shuffle > 0:
             ds = ds.era_shuffle(self.shuffle, key=key)
 
         return ds  # type: ignore