Make split_every configurable (#58)

pyproject.toml

@@ -102,3 +102,4 @@ ignore = "D105"
 
 [tool.ruff]
 ignore = ["E501"]
+per-file-ignores = {"__init__.py" = ["E402", "F401"]}

src/dask_histogram/__init__.py

@@ -1,4 +1,4 @@
-"""Histogramming with Dask collections."""
+from dask_histogram import config  # isort:skip; load dask-histogram config
 
 import boost_histogram.axis as _axis
 import boost_histogram.storage as _storage

src/dask_histogram/config.py

@@ -0,0 +1,12 @@
+import os
+
+import dask.config
+import yaml
+
+config = dask.config.config
+
+fn = os.path.join(os.path.dirname(__file__), "histogram.yaml")
+with open(fn) as f:
+    defaults = yaml.safe_load(f)
+
+dask.config.update_defaults(defaults)

src/dask_histogram/core.py

@@ -649,7 +649,7 @@ def _reduction(
     from dask.bag.core import empty_safe_aggregate
 
     if split_every is None:
-        split_every = 4
+        split_every = dask.config.get("histogram.aggregation.split_every", 8)
     if split_every is False:
         split_every = ph.npartitions
 

src/dask_histogram/histogram.yaml

@@ -0,0 +1,10 @@
+histogram:
+
+  # Aggregation specific configuration table
+  aggregation:
+
+    # When converting from a partitioned histogram to a single
+    # aggregated histogram this parameter controls how the tree
+    # reduction is handled; this number of nodes will be combined at a
+    # time as a new dask task.
+    split_every: 8