reduce ram

allenai · Oct 29, 2024 · e897c55 · e897c55
1 parent 39ca09a
commit e897c55
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/configs/peteish-anneal/mix-fw25.yaml b/configs/peteish-anneal/mix-fw25.yaml
@@ -36,7 +36,7 @@ streams:
         # Remove repetitions
         - >-
           (.attributes.HuggingFaceFW_fineweb_edu_classifier[0][-1] * 0.8) +
-          (.attributes.random_number_v1__random_number_v1__random[0][-1] * 5 * 0.2) >= 2.0
+          (.attributes.random_number_v1__random_number_v1__random[0][-1] * 5 * 0.2) >= 1.7
       syntax: jq
     span_replacement: []
 

diff --git a/configs/peteish-anneal/stackexchange/fuzzy-dedupe.yaml b/configs/peteish-anneal/stackexchange/fuzzy-dedupe.yaml
@@ -17,6 +17,6 @@ bloom_filter:
   read_only: false
   # set to of words
   estimated_doc_count: 10_000_000_000
-  desired_false_positive_rate: 0.001
+  desired_false_positive_rate: 0.01
 
 processes: 16