add token counter

stanford-crfm · Nov 18, 2024 · 824b63c · 824b63c
1 parent 03aa29d
commit 824b63c
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/examples/count_tokens.py b/examples/count_tokens.py
@@ -0,0 +1,5 @@
+from levanter.store import JaggedArrayStore
+
+a = JaggedArrayStore.open("gs://marin-us-central2/tokenized/dolma/algebraic-stack-cc00cf/train/input_ids", dtype=int)
+
+a.data_size