diff --git a/operator/opensearchcontroller/observe.go b/operator/opensearchcontroller/observe.go index 529e05fc..d04f16c9 100644 --- a/operator/opensearchcontroller/observe.go +++ b/operator/opensearchcontroller/observe.go @@ -66,9 +66,15 @@ func (p *pipeline) Observe(ctx context.Context, mg resource.Managed) (managed.Ex return managed.ExternalObservation{}, fmt.Errorf("cannot parse parameters: %w", err) } + currentParams, err := setSettingsDefaults(ctx, p.exo, &openSearchInstance.Spec.ForProvider) + if err != nil { + log.Error(err, "unable to set opensearch settings schema") + currentParams = &openSearchInstance.Spec.ForProvider + } + return managed.ExternalObservation{ ResourceExists: true, - ResourceUpToDate: isUpToDate(&openSearchInstance.Spec.ForProvider, params, log), + ResourceUpToDate: isUpToDate(currentParams, params, log), ConnectionDetails: connDetails, }, nil } diff --git a/operator/opensearchcontroller/settings.go b/operator/opensearchcontroller/settings.go new file mode 100644 index 00000000..c638e907 --- /dev/null +++ b/operator/opensearchcontroller/settings.go @@ -0,0 +1,41 @@ +package opensearchcontroller + +import ( + "context" + + exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" + + "github.com/exoscale/egoscale/v2/oapi" + "github.com/vshn/provider-exoscale/internal/settings" +) + +type settingsFetcher interface { + GetDbaasSettingsOpensearchWithResponse(ctx context.Context, reqEditors ...oapi.RequestEditorFn) (*oapi.GetDbaasSettingsOpensearchResponse, error) +} + +func setSettingsDefaults(ctx context.Context, f settingsFetcher, in *exoscalev1.OpenSearchParameters) (*exoscalev1.OpenSearchParameters, error) { + s, err := fetchSettingSchema(ctx, f) + if err != nil { + return nil, err + } + res := in.DeepCopy() + + res.OpenSearchSettings, err = s.SetDefaults("opensearch", res.OpenSearchSettings) + if err != nil { + return nil, err + } + + return res, nil +} + +func fetchSettingSchema(ctx context.Context, f settingsFetcher) (settings.Schemas, error) { + resp, err := f.GetDbaasSettingsOpensearchWithResponse(ctx) + if err != nil { + return nil, err + } + schemas, err := settings.ParseSchemas(resp.Body) + if err != nil { + return nil, err + } + return schemas, nil +} diff --git a/operator/opensearchcontroller/settings_test.go b/operator/opensearchcontroller/settings_test.go new file mode 100644 index 00000000..39685e78 --- /dev/null +++ b/operator/opensearchcontroller/settings_test.go @@ -0,0 +1,52 @@ +package opensearchcontroller + +import ( + "context" + "testing" + + "github.com/exoscale/egoscale/v2/oapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" + "github.com/vshn/provider-exoscale/operator/mapper" + "k8s.io/apimachinery/pkg/runtime" +) + +type fakeSettingsFetcher struct{} + +func (fakeSettingsFetcher) GetDbaasSettingsOpensearchWithResponse(ctx context.Context, reqEditors ...oapi.RequestEditorFn) (*oapi.GetDbaasSettingsOpensearchResponse, error) { + return &oapi.GetDbaasSettingsOpensearchResponse{ + Body: rawResponse, + }, nil +} + +func mustToRawExt(t *testing.T, set map[string]interface{}) runtime.RawExtension { + res, err := mapper.ToRawExtension(&set) + require.NoError(t, err, "failed to parse input setting") + return res +} + +func TestDefaultSettings(t *testing.T) { + found := exoscalev1.OpenSearchParameters{ + Maintenance: exoscalev1.MaintenanceSpec{}, + Zone: "gva-2", + DBaaSParameters: exoscalev1.DBaaSParameters{ + TerminationProtection: false, + Size: exoscalev1.SizeSpec{ + Plan: "startup-4", + }, + }, + OpenSearchSettings: mustToRawExt(t, map[string]interface{}{ + "thread_pool_search_throttled_size": 42, + }), + } + + withDefaults, err := setSettingsDefaults(context.Background(), fakeSettingsFetcher{}, &found) + require.NoError(t, err, "failed to set defaults") + setingsWithDefaults, err := mapper.ToMap(withDefaults.OpenSearchSettings) + require.NoError(t, err, "failed to parse set defaults") + assert.EqualValues(t, 42, setingsWithDefaults["thread_pool_search_throttled_size"]) + assert.Len(t, setingsWithDefaults, 1) +} + +var rawResponse = []byte(`{"settings":{"opensearch":{"properties":{"thread_pool_search_throttled_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"search_throttled thread pool size","minimum":1},"thread_pool_analyze_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"analyze thread pool size","minimum":1},"thread_pool_get_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"get thread pool size","minimum":1},"thread_pool_get_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"get thread pool queue size","minimum":10},"indices_recovery_max_concurrent_file_chunks":{"description":"Number of file chunks sent in parallel for each recovery. Defaults to 2.","maximum":5,"type":"integer","title":"indices.recovery.max_concurrent_file_chunks","minimum":2},"indices_queries_cache_size":{"description":"Percentage value. Default is 10%. Maximum amount of heap used for query cache. This is an expert setting. Too low value will decrease query performance and increase performance for other operations; too high value will cause issues with other OpenSearch functionality.","maximum":40,"type":"integer","title":"indices.queries.cache.size","minimum":3},"thread_pool_search_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"search thread pool size","minimum":1},"indices_recovery_max_bytes_per_sec":{"description":"Limits total inbound and outbound recovery traffic for each node. Applies to both peer recoveries as well as snapshot recoveries (i.e., restores from a snapshot). Defaults to 40mb","maximum":400,"type":"integer","title":"indices.recovery.max_bytes_per_sec","minimum":40},"http_max_initial_line_length":{"description":"The max length of an HTTP URL, in bytes","maximum":65536,"type":"integer","title":"http.max_initial_line_length","minimum":1024,"example":4096},"thread_pool_write_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"write thread pool queue size","minimum":10},"script_max_compilations_rate":{"description":"Script compilation circuit breaker limits the number of inline script compilations within a period of time. Default is use-context","type":"string","title":"Script max compilation rate - circuit breaker to prevent/minimize OOMs","maxLength":1024,"example":"75/5m"},"search_max_buckets":{"description":"Maximum number of aggregation buckets allowed in a single response. OpenSearch default value is used when this is not defined.","maximum":20000,"type":["integer","null"],"title":"search.max_buckets","minimum":1,"example":10000},"reindex_remote_whitelist":{"description":"Whitelisted addresses for reindexing. Changing this value will cause all OpenSearch instances to restart.","type":["array","null"],"title":"reindex_remote_whitelist","items":{"type":["string","null"],"title":"Address (hostname:port or IP:port)","maxLength":261,"example":"anotherservice.aivencloud.com:12398"},"maxItems":32},"override_main_response_version":{"description":"Compatibility mode sets OpenSearch to report its version as 7.10 so clients continue to work. Default is false","type":"boolean","title":"compatibility.override_main_response_version","example":true},"http_max_header_size":{"description":"The max size of allowed headers, in bytes","maximum":262144,"type":"integer","title":"http.max_header_size","minimum":1024,"example":8192},"email_sender_name":{"description":"This should be identical to the Sender name defined in Opensearch dashboards","type":["string"],"user_error":"Must consist of lower-case alpha-numeric characters and dashes, max 40 characters","title":"Sender email name placeholder to be used in Opensearch Dashboards and Opensearch keystore","maxLength":40,"example":"alert-sender","pattern":"^[a-zA-Z0-9-_]+$"},"indices_fielddata_cache_size":{"description":"Relative amount. Maximum amount of heap memory used for field data cache. This is an expert setting; decreasing the value too much will increase overhead of loading field data; too much memory used for field data cache will decrease amount of heap available for other operations.","default":null,"maximum":100,"type":["integer","null"],"title":"indices.fielddata.cache.size","minimum":3},"action_destructive_requires_name":{"type":["boolean","null"],"title":"Require explicit index names when deleting","example":true},"email_sender_username":{"type":["string"],"user_error":"Must be a valid email address","title":"Sender email address for Opensearch alerts","maxLength":320,"example":"jane@example.com","pattern":"^[A-Za-z0-9_\\-\\.+\\'&]+@(([\\da-zA-Z])([_\\w-]{,62})\\.){,127}(([\\da-zA-Z])[_\\w-]{,61})?([\\da-zA-Z]\\.((xn\\-\\-[a-zA-Z\\d]+)|([a-zA-Z\\d]{2,})))$"},"indices_memory_index_buffer_size":{"description":"Percentage value. Default is 10%. Total amount of heap used for indexing buffer, before writing segments to disk. This is an expert setting. Too low value will slow down indexing; too high value will increase indexing performance but causes performance issues for query performance.","maximum":40,"type":"integer","title":"indices.memory.index_buffer_size","minimum":3},"thread_pool_force_merge_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"force_merge thread pool size","minimum":1},"cluster_routing_allocation_node_concurrent_recoveries":{"description":"How many concurrent incoming/outgoing shard recoveries (normally replicas) are allowed to happen on a node. Defaults to 2.","maximum":16,"type":"integer","title":"Concurrent incoming/outgoing shard recoveries per node","minimum":2},"email_sender_password":{"description":"Sender email password for Opensearch alerts to authenticate with SMTP server","type":["string"],"title":"Sender email password for Opensearch alerts to authenticate with SMTP server","maxLength":1024,"example":"very-secure-mail-password","pattern":"^[^\\x00-\\x1F]+$"},"thread_pool_analyze_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"analyze thread pool queue size","minimum":10},"action_auto_create_index_enabled":{"description":"Explicitly allow or block automatic creation of indices. Defaults to true","type":"boolean","title":"action.auto_create_index","example":false},"http_max_content_length":{"description":"Maximum content length for HTTP requests to the OpenSearch HTTP API, in bytes.","maximum":2147483647,"type":"integer","title":"http.max_content_length","minimum":1},"thread_pool_write_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"write thread pool size","minimum":1},"thread_pool_search_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"search thread pool queue size","minimum":10},"indices_query_bool_max_clause_count":{"description":"Maximum number of clauses Lucene BooleanQuery can have. The default value (1024) is relatively high, and increasing it may cause performance issues. Investigate other approaches first before increasing this value.","maximum":4096,"type":"integer","title":"indices.query.bool.max_clause_count","minimum":64},"thread_pool_search_throttled_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"search_throttled thread pool queue size","minimum":10},"cluster_max_shards_per_node":{"description":"Controls the number of shards allowed in the cluster per data node","maximum":10000,"type":"integer","title":"cluster.max_shards_per_node","minimum":100,"example":1000}},"additionalProperties":false,"type":"object","title":"OpenSearch settings","dependencies":{"email_sender_name":{"required":["email_sender_username","email_sender_password"]},"email_sender_username":{"required":["email_sender_name","email_sender_password"]},"email_sender_password":{"required":["email_sender_username","email_sender_name"]}}}}}`)