WIP

stanford-crfm · Nov 20, 2024 · 02fd529 · 02fd529
1 parent 824b63c
commit 02fd529
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/examples/count_tokens.py b/examples/count_tokens.py
@@ -2,4 +2,6 @@
 
 a = JaggedArrayStore.open("gs://marin-us-central2/tokenized/dolma/algebraic-stack-cc00cf/train/input_ids", dtype=int)
 
-a.data_size
+a.data_size
+
+150,849,275