Baseline data

configs/baselines/decontamination/c4.yaml

@@ -0,0 +1,20 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/*.gz
+
+dedupe:
+  name: perplexity_suite_v3_option2_redo
+  paragraphs:
+    attribute_name: bff_duplicate_paragraph_spans_decontamination
+  skip_empty: true
+
+bloom_filter:
+  read_only: true
+  estimated_doc_count: 488541
+  size_in_bytes: 33554432  # 100 MB; smaller causes too many FPs
+  file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin
+
+processes: 224
+
+work_dir:
+  input: /mnt/tank/dolma_tmp/c4_input
+  output: /mnt/tank/dolma_tmp/c4_output

configs/baselines/decontamination/falcon-refinedweb.yaml

@@ -0,0 +1,20 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/*.gz
+
+dedupe:
+  name: perplexity_suite_v3_option2
+  paragraphs:
+    attribute_name: bff_duplicate_paragraph_spans_decontamination
+  skip_empty: true
+
+bloom_filter:
+  read_only: true
+  estimated_doc_count: 488541
+  size_in_bytes: 33554432  # 100 MB; smaller causes too many FPs
+  file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin
+
+processes: 224
+
+work_dir:
+  input: /mnt/tank/dolma_tmp/falcon_input
+  output: /mnt/tank/dolma_tmp/falcon_output

configs/baselines/decontamination/mc4.yaml

@@ -0,0 +1,20 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/mc4/en_wimbd_splits/documents/train/*.gz
+
+dedupe:
+  name: perplexity_suite_v3_option2
+  paragraphs:
+    attribute_name: bff_duplicate_paragraph_spans_decontamination
+  skip_empty: true
+
+bloom_filter:
+  read_only: true
+  estimated_doc_count: 488541
+  size_in_bytes: 33554432  # 100 MB; smaller causes too many FPs
+  file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin
+
+processes: 224
+
+work_dir:
+  input: /mnt/tank/dolma_tmp/mc4_input
+  output: /mnt/tank/dolma_tmp/mc4_output

configs/baselines/decontamination/pile.yaml

@@ -0,0 +1,20 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/pile/v0/documents/train/*.gz
+
+dedupe:
+  name: perplexity_suite_v3_option2
+  paragraphs:
+    attribute_name: bff_duplicate_paragraph_spans_decontamination
+  skip_empty: true
+
+bloom_filter:
+  read_only: true
+  estimated_doc_count: 488541
+  size_in_bytes: 33554432  # 100 MB; smaller causes too many FPs
+  file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin
+
+processes: 224
+
+work_dir:
+  input: /mnt/tank/dolma_tmp/pile_input
+  output: /mnt/tank/dolma_tmp/pile_output

configs/baselines/decontamination/redpajama.yaml

@@ -0,0 +1,25 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=arxiv/*.gz
+  - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=book/*.gz
+  - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=c4/*.gz
+  - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=common_crawl/*.gz
+  - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=stackexchange/*.gz
+  - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=wikipedia/*.gz
+
+dedupe:
+  name: perplexity_suite_v3_option2
+  paragraphs:
+    attribute_name: bff_duplicate_paragraph_spans_decontamination
+  skip_empty: true
+
+bloom_filter:
+  read_only: true
+  estimated_doc_count: 488541
+  size_in_bytes: 33554432  # 100 MB; smaller causes too many FPs
+  file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin
+
+processes: 224
+
+work_dir:
+  input: /mnt/tank/dolma_tmp/rp_input
+  output: /mnt/tank/dolma_tmp/rp_output

configs/baselines/mixing/c4.json

@@ -0,0 +1,27 @@
+{
+    "streams": [
+      {
+        "name": "c4",
+        "documents": [
+          "s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/*.gz"
+        ],
+        "output": {
+          "path": "s3://ai2-llm/pretraining-data/sources/c4/v0_decon_ppl_suite_v3",
+          "max_size_in_bytes": 1000000000
+        },
+        "attributes": [
+          "perplexity_suite_v3_option2_redo"
+        ],
+        "filter": {
+          "exclude": [
+            "[email protected][?(@.bff_duplicate_paragraph_spans_decontamination && @.bff_duplicate_paragraph_spans_decontamination[0] && @.bff_duplicate_paragraph_spans_decontamination[0][2] >= 1.0)]"
+          ]
+        }
+      }
+    ],
+    "work_dir": {
+        "input" : "/mnt/tank/dolma_tmp/c4_input_mix",
+        "output" : "/mnt/tank/dolma_tmp/c4_output_mix"
+    },
+    "processes": 1
+  }

configs/baselines/mixing/falcon-refinedweb.json

@@ -0,0 +1,27 @@
+{
+    "streams": [
+      {
+        "name": "falcon-refinedweb",
+        "documents": [
+          "s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/*.gz"
+        ],
+        "output": {
+          "path": "s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement_decon_ppl_suite_v3",
+          "max_size_in_bytes": 1000000000
+        },
+        "attributes": [
+          "perplexity_suite_v3_option2"
+        ],
+        "filter": {
+          "exclude": [
+            "[email protected][?(@.bff_duplicate_paragraph_spans_decontamination && @.bff_duplicate_paragraph_spans_decontamination[0] && @.bff_duplicate_paragraph_spans_decontamination[0][2] >= 1.0)]"
+          ]
+        }
+      }
+    ],
+    "work_dir": {
+        "input" : "/mnt/tank/dolma_tmp/falcon_input_mix",
+        "output" : "/mnt/tank/dolma_tmp/falcon_output_mix"
+    },
+    "processes": 1
+  }

configs/baselines/mixing/mc4.json

@@ -0,0 +1,27 @@
+{
+    "streams": [
+      {
+        "name": "mc4",
+        "documents": [
+          "s3://ai2-llm/pretraining-data/sources/mc4/en_wimbd_splits/documents/train/*.gz"
+        ],
+        "output": {
+          "path": "s3://ai2-llm/pretraining-data/sources/mc4/en_wimbd_splits_decon_ppl_suite_v3/",
+          "max_size_in_bytes": 1000000000
+        },
+        "attributes": [
+          "perplexity_suite_v3_option2"
+        ],
+        "filter": {
+          "exclude": [
+            "[email protected][?(@.bff_duplicate_paragraph_spans_decontamination && @.bff_duplicate_paragraph_spans_decontamination[0] && @.bff_duplicate_paragraph_spans_decontamination[0][2] >= 1.0)]"
+          ]
+        }
+      }
+    ],
+    "work_dir": {
+        "input" : "/mnt/tank/dolma_tmp/mc4_input_mix",
+        "output" : "/mnt/tank/dolma_tmp/mc4_output_mix"
+    },
+    "processes": 1
+  }

configs/baselines/mixing/pile.json

@@ -0,0 +1,27 @@
+{
+    "streams": [
+      {
+        "name": "pile",
+        "documents": [
+          "s3://ai2-llm/pretraining-data/sources/pile/v0/documents/train/*.gz"
+        ],
+        "output": {
+          "path": "s3://ai2-llm/pretraining-data/sources/pile/v0_decon_ppl_suite_v3",
+          "max_size_in_bytes": 1000000000
+        },
+        "attributes": [
+          "perplexity_suite_v3_option2"
+        ],
+        "filter": {
+          "exclude": [
+            "[email protected][?(@.bff_duplicate_paragraph_spans_decontamination && @.bff_duplicate_paragraph_spans_decontamination[0] && @.bff_duplicate_paragraph_spans_decontamination[0][2] >= 1.0)]"
+          ]
+        }
+      }
+    ],
+    "work_dir": {
+        "input" : "/mnt/tank/dolma_tmp/pile_input_mix",
+        "output" : "/mnt/tank/dolma_tmp/pile_output_mix"
+    },
+    "processes": 1
+  }

configs/baselines/mixing/redpajama.json

@@ -0,0 +1,32 @@
+{
+    "streams": [
+      {
+        "name": "redpajama",
+        "documents": [
+          "s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=arxiv/*.gz",
+          "s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=book/*.gz",
+          "s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=c4/*.gz",
+          "s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=common_crawl/*.gz",
+          "s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=stackexchange/*.gz",
+          "s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=wikipedia/*.gz"
+        ],
+        "output": {
+          "path": "s3://ai2-llm/pretraining-data/sources/redpajama/v1_decon_ppl_suite_v3",
+          "max_size_in_bytes": 1000000000
+        },
+        "attributes": [
+          "perplexity_suite_v3_option2"
+        ],
+        "filter": {
+          "exclude": [
+            "[email protected][?(@.bff_duplicate_paragraph_spans_decontamination && @.bff_duplicate_paragraph_spans_decontamination[0] && @.bff_duplicate_paragraph_spans_decontamination[0][2] >= 1.0)]"
+          ]
+        }
+      }
+    ],
+    "work_dir": {
+        "input" : "/mnt/tank/dolma_tmp/rp_input_mix",
+        "output" : "/mnt/tank/dolma_tmp/rp_output_mix"
+    },
+    "processes": 1
+  }

configs/baselines/tokenization/c4.yaml

@@ -0,0 +1,9 @@
+destination: s3://ai2-llm/preprocessed/c4/v0_decon_ppl_suite_v3/gpt-neox-20b-pii-special
+documents:
+- s3://ai2-llm/pretraining-data/sources/c4/v0_decon_ppl_suite_v3/*.json.gz
+processes: 224
+seed: 3920
+tokenizer_name_or_path: allenai/eleuther-ai-gpt-neox-20b-pii-special
+work_dir:
+  input: /mnt/tank/dolma_tmp/c4_input_tokenized
+  output: /mnt/tank/dolma_tmp/c4_output_tokenized

configs/baselines/tokenization/dolma_v1_5_cc_only.yaml

@@ -0,0 +1,11 @@
+destination: s3://ai2-llm/preprocessed/olmo-mix/v1_5_cc_only/gpt-neox-20b-pii-special/
+documents:
+- s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5/documents/cc_en_head/*.json.gz
+- s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5/documents/cc_en_middle/*.json.gz
+- s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5/documents/cc_en_tail/*.json.gz
+processes: 224
+seed: 3920
+tokenizer_name_or_path: allenai/eleuther-ai-gpt-neox-20b-pii-special
+work_dir:
+  input: /mnt/tank/dolma_tmp/v1_5_cc_only_input_tokenized
+  output: /mnt/tank/dolma_tmp/v1_5_cc_only_output_tokenized

configs/baselines/tokenization/falcon-refinedweb.yaml

@@ -0,0 +1,9 @@
+destination: s3://ai2-llm/preprocessed/falcon-refinedweb/v0-0.05-heldout-complement_decon_ppl_suite_v3/gpt-neox-20b-pii-special
+documents:
+- s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement_decon_ppl_suite_v3/*.json.gz
+processes: 224
+seed: 3920
+tokenizer_name_or_path: allenai/eleuther-ai-gpt-neox-20b-pii-special
+work_dir:
+  input: /mnt/tank/dolma_tmp/falcon_input_tokenized
+  output: /mnt/tank/dolma_tmp/falcon_output_tokenized

configs/baselines/tokenization/mc4.yaml

@@ -0,0 +1,9 @@
+destination: s3://ai2-llm/preprocessed/mc4/en_wimbd_splits_decon_ppl_suite_v3/gpt-neox-20b-pii-special
+documents:
+- s3://ai2-llm/pretraining-data/sources/mc4/en_wimbd_splits_decon_ppl_suite_v3/*.json.gz
+processes: 224
+seed: 3920
+tokenizer_name_or_path: allenai/eleuther-ai-gpt-neox-20b-pii-special
+work_dir:
+  input: /mnt/tank/dolma_tmp/mc4_input_tokenized
+  output: /mnt/tank/dolma_tmp/mc4_output_tokenized

configs/baselines/tokenization/pile.yaml

@@ -0,0 +1,9 @@
+destination: s3://ai2-llm/preprocessed/pile/v0_decon_ppl_suite_v3_fixed/gpt-neox-20b-pii-special
+documents:
+- s3://ai2-llm/pretraining-data/sources/pile/v0_decon_ppl_suite_v3/*.json.gz
+processes: 150
+seed: 3920
+tokenizer_name_or_path: allenai/eleuther-ai-gpt-neox-20b-pii-special
+work_dir:
+  input: /mnt/tank/tmp/pile_v0_decon_ppl_suite_v3_fixed_input
+  output: /mnt/tank/tmp/pile_v0_decon_ppl_suite_v3_fixed_output

configs/baselines/tokenization/redpajama.yaml

@@ -0,0 +1,9 @@
+destination: s3://ai2-llm/preprocessed/redpajama/v1_decon_ppl_suite_v3/gpt-neox-20b-pii-special
+documents:
+- s3://ai2-llm/pretraining-data/sources/redpajama/v1_decon_ppl_suite_v3/*.json.gz
+processes: 224
+seed: 3920
+tokenizer_name_or_path: allenai/eleuther-ai-gpt-neox-20b-pii-special
+work_dir:
+  input: /mnt/tank/dolma_tmp/rp_input_tokenized
+  output: /mnt/tank/dolma_tmp/rp_output_tokenized

configs/dolma-v1_5/README.md

@@ -0,0 +1,3 @@
+# Dolma 1.5
+
+This directory