Merge pull request #109 from quickwit-oss/ddelemeny/shard-size

Reduce default terms agg size, add shard_size
quickwit-oss · Mar 20, 2024 · 9647a98 · 9647a98
2 parents 016e2e8 + f27e45e
commit 9647a98
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 2 deletions.
diff --git a/pkg/quickwit/client/models.go b/pkg/quickwit/client/models.go
@@ -260,6 +260,7 @@ type FiltersAggregation struct {
 type TermsAggregation struct {
 	Field       string                 `json:"field"`
 	Size        int                    `json:"size"`
+	ShardSize   int                    `json:"shard_size"`
 	Order       map[string]interface{} `json:"order"`
 	MinDocCount *int                   `json:"min_doc_count,omitempty"`
 	Missing     *string                `json:"missing,omitempty"`

diff --git a/pkg/quickwit/data_query.go b/pkg/quickwit/data_query.go
@@ -229,6 +229,11 @@ func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*Metr
 		} else {
 			a.Size = stringToIntWithDefaultValue(bucketAgg.Settings.Get("size").MustString(), defaultSize)
 		}
+		if shard_size, err := bucketAgg.Settings.Get("shard_size").Int(); err == nil {
+			a.ShardSize = shard_size
+		} else {
+			a.ShardSize = stringToIntWithDefaultValue(bucketAgg.Settings.Get("shard_size").MustString(), defaultSize)
+		}
 
 		if minDocCount, err := bucketAgg.Settings.Get("min_doc_count").Int(); err == nil {
 			a.MinDocCount = &minDocCount

diff --git a/src/QueryBuilder/elastic.ts b/src/QueryBuilder/elastic.ts
@@ -11,6 +11,7 @@ type OrderByType =  '_key' | '_term' | '_count'
 function getTermsAgg(
   fieldName: string,
   size: number,
+  shard_size: number,
   orderBy: OrderByType = "_key",
   order: 'asc'|'desc' = 'asc'
   ): BucketAggregation {
@@ -20,6 +21,7 @@ function getTermsAgg(
     field: fieldName,
     settings:{
       size: size.toString(),
+      shard_size: shard_size.toString(),
       order: order,
       orderBy: orderBy,
     }
@@ -31,7 +33,8 @@ export function getDataQuery(queryDef: TermsQuery, refId: string): Elasticsearch
     {id:"count1", type:'count'}
   ];
 
-  // Default behaviour is to order results by { _key: asc }
+  // Default behaviour is to order results by { _key: asc } and get 100 terms,
+  // shard_size is set to 100 to limit terms fetched by shard (by default it's 100 * 10).
   // queryDef.order allows selection of asc/desc
   // queryDef.orderBy allows selection of doc_count ordering (defaults desc)
 
@@ -55,7 +58,7 @@ export function getDataQuery(queryDef: TermsQuery, refId: string): Elasticsearch
 
   const bucketAggs: BucketAggregation[] = [];
   if (queryDef.field) {
-    bucketAggs.push(getTermsAgg(queryDef.field, 500, orderBy, order))
+    bucketAggs.push(getTermsAgg(queryDef.field, 100, 100, orderBy, order))
   }
 
   return {

diff --git a/src/dataquery.gen.ts b/src/dataquery.gen.ts
@@ -71,6 +71,7 @@ export interface Terms extends BucketAggregationWithField {
   settings?: {
     order?: TermsOrder;
     size?: string;
+    shard_size?: string;
     min_doc_count?: string;
     orderBy?: string;
     missing?: string;