c4 dataset

stanford-crfm · Apr 23, 2024 · 451173c · 451173c
1 parent a98c70b
commit 451173c
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/config/llama2_7b_continued.yaml b/config/llama2_7b_continued.yaml
@@ -1,9 +1,6 @@
 data:
-  cache_dir: "gs://levanter-data/tokenized/git-llama2/"
-  train_urls:
-    - gs://levanter-data/pile-domains/github/{00..29}.jsonl.zst
-  validation_urls:
-    - gs://levanter-data/pile-domains/github/val.jsonl.zst
+  id: allenai/c4
+  cache_dir: "gs://levanter-data/tokenized/llama2_c4/"
   tokenizer: "meta-llama/Llama-2-70b-hf"
 model:
   type: llama