Skip to content

Commit

Permalink
knife fix when the rows in wild search exceeds 10k.
Browse files Browse the repository at this point in the history
Signed-off-by: talktovikas <[email protected]>

fixing omnibus issue.

Signed-off-by: talktovikas <[email protected]>

fixing pedant issue.

Signed-off-by: talktovikas <[email protected]>

binding at runtime.

Signed-off-by: talktovikas <[email protected]>

Doc Changes for knife index.

Signed-off-by: talktovikas <[email protected]>

adding default to toml file.

Signed-off-by: talktovikas <[email protected]>
  • Loading branch information
talktovikas committed Dec 23, 2024
1 parent b49d612 commit 2b39607
Show file tree
Hide file tree
Showing 10 changed files with 50 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2023,6 +2023,11 @@ This configuration file has the following settings for `opscode-erchef`:

Default value: `127.0.0.1`.

`opscode_erchef['track_total_hits']`

: Whether to return how many documents matched the query.

Default value: `false`.

### opscode-expander

Expand Down
4 changes: 4 additions & 0 deletions oc-chef-pedant/spec/running_configs/basic_config_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,10 @@
expect(config['opscode-erchef']['solr_http_max_age'].to_s).not_to eq ''
end

it "opscode-erchef/track_total_hits" do
expect(config['opscode-erchef']['track_total_hits'].to_s).not_to eq ''
end

it "opscode-erchef/solr_http_max_connection_duration" do
expect(config['opscode-erchef']['solr_http_max_connection_duration'].to_s).not_to eq ''
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@
default['private_chef']['opscode-erchef']['search_batch_max_size'] = '5000000'
default['private_chef']['opscode-erchef']['search_batch_max_wait'] = '10'
default['private_chef']['opscode-erchef']['search_auth_username'] = 'opensearch_user'
default['private_chef']['opscode-erchef']['track_total_hits'] = false
# default['private_chef']['opscode-erchef']['search_auth_password'] = "admin"
# solr_service configuration for erchef. These are used to configure an opscoderl_httpc pool
# of HTTP connecton workers.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@
{solr_elasticsearch_major_version, <%= @solr_elasticsearch_major_version %>},
{search_auth_username, "<%= node['private_chef']['opscode-erchef']['search_auth_username'] %>"},
{search_auth_password, "<%= @helper.search_auth_password() %>"},
{track_total_hits, <%= node['private_chef']['opscode-erchef']['track_total_hits'] -%>},
{solr_service, [
{root_url, "<%= @helper.search_engine_url() %>"},
{timeout, <%= @solr_timeout %>},
Expand Down
1 change: 1 addition & 0 deletions src/oc_erchef/apps/chef_index/src/chef_index_query.erl
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ from_params(Provider, ObjType, QueryString, Start, Rows) ->
search_provider = Provider,
start = decode({nonneg_int, "start"}, Start, 0),
rows = decode({nonneg_int, "rows"}, Rows, 1000),
track_total_hits = envy:get(chef_index, track_total_hits, false, boolean),
sort = "X_CHEF_id_CHEF_X asc",
index = index_type(ObjType)}.

Expand Down
33 changes: 33 additions & 0 deletions src/oc_erchef/apps/chef_index/src/chef_opensearch.erl
Original file line number Diff line number Diff line change
Expand Up @@ -103,21 +103,54 @@ query_body(#chef_solr_query{
query_string = Query,
filter_query = undefined,
start = Start,
track_total_hits = true,
rows = Rows}) ->
jiffy:encode({[{fields_tag(), <<"_id">>},
{<<"from">>, Start},
{<<"size">>, Rows},
{<<"track_total_hits">>,true},
{<<"query">>, {[query_string_query_ejson(Query)]}}
]});
query_body(#chef_solr_query{
query_string = Query,
filter_query = undefined,
start = Start,
track_total_hits = false,
rows = Rows}) ->
jiffy:encode({[{fields_tag(), <<"_id">>},
{<<"from">>, Start},
{<<"size">>, Rows},
{<<"query">>, {[query_string_query_ejson(Query)]}}
]});

query_body(#chef_solr_query{
query_string = Query,
filter_query = FilterQuery,
start = Start,
track_total_hits = false,
rows = Rows}) ->
chef_index_query:assert_org_id_filter(FilterQuery),
jiffy:encode({[{ fields_tag(), <<"_id">>},
{<<"from">>, Start},
{<<"size">>, Rows},
{<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]},
{<<"query">>, {[
{<<"bool">>,{[
{<<"must">>, {[query_string_query_ejson(Query)]}},
{<<"filter">>, {[query_string_query_ejson(FilterQuery)]}}
]}}]}
}]});
query_body(#chef_solr_query{
query_string = Query,
filter_query = FilterQuery,
start = Start,
track_total_hits = true,
rows = Rows}) ->
chef_index_query:assert_org_id_filter(FilterQuery),
jiffy:encode({[{ fields_tag(), <<"_id">>},
{<<"from">>, Start},
{<<"size">>, Rows},
{<<"track_total_hits">>,true},
{<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]},
{<<"query">>, {[
{<<"bool">>,{[
Expand Down
2 changes: 2 additions & 0 deletions src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ query_from_params_test_() ->
sort = "X_CHEF_id_CHEF_X asc",
start = 2,
rows = 5,
track_total_hits=false,
index = node},
?assertEqual(Expect, Query)
end},
Expand All @@ -57,6 +58,7 @@ query_from_params_test_() ->
sort = "X_CHEF_id_CHEF_X asc",
start = 0,
rows = 1000,
track_total_hits=false,
index = role},
?assertEqual(Expect, Query)
end},
Expand Down
1 change: 1 addition & 0 deletions src/oc_erchef/habitat/config/sys.config
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@
{search_batch_max_wait, 10},
{reindex_sleep_min_ms, 500},
{reindex_sleep_max_ms, 2000},
{track_total_hits, {{track_total_hits}} },
{reindex_item_retries, 3},
{solr_elasticsearch_major_version, 5},
{solr_service, [
Expand Down
1 change: 1 addition & 0 deletions src/oc_erchef/habitat/default.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ keygen_timeout=5000
[chef_db]

[chef_index]
track_total_hits=false

[chef_objects]

Expand Down
1 change: 1 addition & 0 deletions src/oc_erchef/include/chef_solr.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
search_provider = solr :: 'solr' | 'elasticsearch' | 'opensearch',
start :: integer() | undefined,
rows :: integer() | undefined,
track_total_hits :: boolean() |undefined,
sort :: string() | undefined,
index :: 'node'
| 'role'
Expand Down

0 comments on commit 2b39607

Please sign in to comment.