From 2b39607c8c5f7d9997ed58022c54668b5955bab3 Mon Sep 17 00:00:00 2001 From: talktovikas Date: Thu, 12 Dec 2024 17:11:29 +0530 Subject: [PATCH] knife fix when the rows in wild search exceeds 10k. Signed-off-by: talktovikas fixing omnibus issue. Signed-off-by: talktovikas fixing pedant issue. Signed-off-by: talktovikas binding at runtime. Signed-off-by: talktovikas Doc Changes for knife index. Signed-off-by: talktovikas adding default to toml file. Signed-off-by: talktovikas --- .../config_rb_server_optional_settings.md | 5 +++ .../spec/running_configs/basic_config_spec.rb | 4 +++ .../infra-server/attributes/default.rb | 1 + .../templates/default/oc_erchef.config.erb | 1 + .../apps/chef_index/src/chef_index_query.erl | 1 + .../apps/chef_index/src/chef_opensearch.erl | 33 +++++++++++++++++++ .../test/chef_index_query_tests.erl | 2 ++ src/oc_erchef/habitat/config/sys.config | 1 + src/oc_erchef/habitat/default.toml | 1 + src/oc_erchef/include/chef_solr.hrl | 1 + 10 files changed, 50 insertions(+) diff --git a/docs-chef-io/content/server/config_rb_server_optional_settings.md b/docs-chef-io/content/server/config_rb_server_optional_settings.md index 310c6217b9..34941dd053 100644 --- a/docs-chef-io/content/server/config_rb_server_optional_settings.md +++ b/docs-chef-io/content/server/config_rb_server_optional_settings.md @@ -2023,6 +2023,11 @@ This configuration file has the following settings for `opscode-erchef`: Default value: `127.0.0.1`. +`opscode_erchef['track_total_hits']` + +: Whether to return how many documents matched the query. + + Default value: `false`. ### opscode-expander diff --git a/oc-chef-pedant/spec/running_configs/basic_config_spec.rb b/oc-chef-pedant/spec/running_configs/basic_config_spec.rb index 8b3ed84604..5995b96e3a 100644 --- a/oc-chef-pedant/spec/running_configs/basic_config_spec.rb +++ b/oc-chef-pedant/spec/running_configs/basic_config_spec.rb @@ -142,6 +142,10 @@ expect(config['opscode-erchef']['solr_http_max_age'].to_s).not_to eq '' end + it "opscode-erchef/track_total_hits" do + expect(config['opscode-erchef']['track_total_hits'].to_s).not_to eq '' + end + it "opscode-erchef/solr_http_max_connection_duration" do expect(config['opscode-erchef']['solr_http_max_connection_duration'].to_s).not_to eq '' end diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb b/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb index 77f85dcdfe..422e86b306 100755 --- a/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb +++ b/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb @@ -344,6 +344,7 @@ default['private_chef']['opscode-erchef']['search_batch_max_size'] = '5000000' default['private_chef']['opscode-erchef']['search_batch_max_wait'] = '10' default['private_chef']['opscode-erchef']['search_auth_username'] = 'opensearch_user' +default['private_chef']['opscode-erchef']['track_total_hits'] = false # default['private_chef']['opscode-erchef']['search_auth_password'] = "admin" # solr_service configuration for erchef. These are used to configure an opscoderl_httpc pool # of HTTP connecton workers. diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb index c020307c0f..74978e9d95 100755 --- a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb +++ b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb @@ -189,6 +189,7 @@ {solr_elasticsearch_major_version, <%= @solr_elasticsearch_major_version %>}, {search_auth_username, "<%= node['private_chef']['opscode-erchef']['search_auth_username'] %>"}, {search_auth_password, "<%= @helper.search_auth_password() %>"}, + {track_total_hits, <%= node['private_chef']['opscode-erchef']['track_total_hits'] -%>}, {solr_service, [ {root_url, "<%= @helper.search_engine_url() %>"}, {timeout, <%= @solr_timeout %>}, diff --git a/src/oc_erchef/apps/chef_index/src/chef_index_query.erl b/src/oc_erchef/apps/chef_index/src/chef_index_query.erl index 3b68518fe8..8f0b9e07ce 100644 --- a/src/oc_erchef/apps/chef_index/src/chef_index_query.erl +++ b/src/oc_erchef/apps/chef_index/src/chef_index_query.erl @@ -30,6 +30,7 @@ from_params(Provider, ObjType, QueryString, Start, Rows) -> search_provider = Provider, start = decode({nonneg_int, "start"}, Start, 0), rows = decode({nonneg_int, "rows"}, Rows, 1000), + track_total_hits = envy:get(chef_index, track_total_hits, false, boolean), sort = "X_CHEF_id_CHEF_X asc", index = index_type(ObjType)}. diff --git a/src/oc_erchef/apps/chef_index/src/chef_opensearch.erl b/src/oc_erchef/apps/chef_index/src/chef_opensearch.erl index 3d6eec7bc3..22d52686bf 100644 --- a/src/oc_erchef/apps/chef_index/src/chef_opensearch.erl +++ b/src/oc_erchef/apps/chef_index/src/chef_opensearch.erl @@ -103,21 +103,54 @@ query_body(#chef_solr_query{ query_string = Query, filter_query = undefined, start = Start, + track_total_hits = true, rows = Rows}) -> jiffy:encode({[{fields_tag(), <<"_id">>}, {<<"from">>, Start}, {<<"size">>, Rows}, + {<<"track_total_hits">>,true}, {<<"query">>, {[query_string_query_ejson(Query)]}} ]}); +query_body(#chef_solr_query{ + query_string = Query, + filter_query = undefined, + start = Start, + track_total_hits = false, + rows = Rows}) -> + jiffy:encode({[{fields_tag(), <<"_id">>}, + {<<"from">>, Start}, + {<<"size">>, Rows}, + {<<"query">>, {[query_string_query_ejson(Query)]}} + ]}); + +query_body(#chef_solr_query{ + query_string = Query, + filter_query = FilterQuery, + start = Start, + track_total_hits = false, + rows = Rows}) -> +chef_index_query:assert_org_id_filter(FilterQuery), +jiffy:encode({[{ fields_tag(), <<"_id">>}, + {<<"from">>, Start}, + {<<"size">>, Rows}, + {<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]}, + {<<"query">>, {[ + {<<"bool">>,{[ + {<<"must">>, {[query_string_query_ejson(Query)]}}, + {<<"filter">>, {[query_string_query_ejson(FilterQuery)]}} + ]}}]} + }]}); query_body(#chef_solr_query{ query_string = Query, filter_query = FilterQuery, start = Start, + track_total_hits = true, rows = Rows}) -> chef_index_query:assert_org_id_filter(FilterQuery), jiffy:encode({[{ fields_tag(), <<"_id">>}, {<<"from">>, Start}, {<<"size">>, Rows}, + {<<"track_total_hits">>,true}, {<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]}, {<<"query">>, {[ {<<"bool">>,{[ diff --git a/src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl b/src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl index d82a7b9249..96a53f04cb 100644 --- a/src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl +++ b/src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl @@ -40,6 +40,7 @@ query_from_params_test_() -> sort = "X_CHEF_id_CHEF_X asc", start = 2, rows = 5, + track_total_hits=false, index = node}, ?assertEqual(Expect, Query) end}, @@ -57,6 +58,7 @@ query_from_params_test_() -> sort = "X_CHEF_id_CHEF_X asc", start = 0, rows = 1000, + track_total_hits=false, index = role}, ?assertEqual(Expect, Query) end}, diff --git a/src/oc_erchef/habitat/config/sys.config b/src/oc_erchef/habitat/config/sys.config index 9d75dc2cd3..ce47422f9a 100644 --- a/src/oc_erchef/habitat/config/sys.config +++ b/src/oc_erchef/habitat/config/sys.config @@ -173,6 +173,7 @@ {search_batch_max_wait, 10}, {reindex_sleep_min_ms, 500}, {reindex_sleep_max_ms, 2000}, + {track_total_hits, {{track_total_hits}} }, {reindex_item_retries, 3}, {solr_elasticsearch_major_version, 5}, {solr_service, [ diff --git a/src/oc_erchef/habitat/default.toml b/src/oc_erchef/habitat/default.toml index 918fab190c..deb21b86bf 100644 --- a/src/oc_erchef/habitat/default.toml +++ b/src/oc_erchef/habitat/default.toml @@ -30,6 +30,7 @@ keygen_timeout=5000 [chef_db] [chef_index] +track_total_hits=false [chef_objects] diff --git a/src/oc_erchef/include/chef_solr.hrl b/src/oc_erchef/include/chef_solr.hrl index f926962cd9..58e824f261 100644 --- a/src/oc_erchef/include/chef_solr.hrl +++ b/src/oc_erchef/include/chef_solr.hrl @@ -21,6 +21,7 @@ search_provider = solr :: 'solr' | 'elasticsearch' | 'opensearch', start :: integer() | undefined, rows :: integer() | undefined, + track_total_hits :: boolean() |undefined, sort :: string() | undefined, index :: 'node' | 'role'