Add default settings to opensearch

operator/opensearchcontroller/observe.go

@@ -66,9 +66,15 @@ func (p *pipeline) Observe(ctx context.Context, mg resource.Managed) (managed.Ex
 		return managed.ExternalObservation{}, fmt.Errorf("cannot parse parameters: %w", err)
 	}
 
+	currentParams, err := setSettingsDefaults(ctx, p.exo, &openSearchInstance.Spec.ForProvider)
+	if err != nil {
+		log.Error(err, "unable to set opensearch settings schema")
+		currentParams = &openSearchInstance.Spec.ForProvider
+	}
+
 	return managed.ExternalObservation{
 		ResourceExists:    true,
-		ResourceUpToDate:  isUpToDate(&openSearchInstance.Spec.ForProvider, params, log),
+		ResourceUpToDate:  isUpToDate(currentParams, params, log),
 		ConnectionDetails: connDetails,
 	}, nil
 }

operator/opensearchcontroller/settings.go

@@ -0,0 +1,41 @@
+package opensearchcontroller
+
+import (
+	"context"
+
+	exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1"
+
+	"github.com/exoscale/egoscale/v2/oapi"
+	"github.com/vshn/provider-exoscale/internal/settings"
+)
+
+type settingsFetcher interface {
+	GetDbaasSettingsOpensearchWithResponse(ctx context.Context, reqEditors ...oapi.RequestEditorFn) (*oapi.GetDbaasSettingsOpensearchResponse, error)
+}
+
+func setSettingsDefaults(ctx context.Context, f settingsFetcher, in *exoscalev1.OpenSearchParameters) (*exoscalev1.OpenSearchParameters, error) {
+	s, err := fetchSettingSchema(ctx, f)
+	if err != nil {
+		return nil, err
+	}
+	res := in.DeepCopy()
+
+	res.OpenSearchSettings, err = s.SetDefaults("opensearch", res.OpenSearchSettings)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func fetchSettingSchema(ctx context.Context, f settingsFetcher) (settings.Schemas, error) {
+	resp, err := f.GetDbaasSettingsOpensearchWithResponse(ctx)
+	if err != nil {
+		return nil, err
+	}
+	schemas, err := settings.ParseSchemas(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return schemas, nil
+}

operator/opensearchcontroller/settings_test.go

@@ -0,0 +1,52 @@
+package opensearchcontroller
+
+import (
+	"context"
+	"testing"
+
+	"github.com/exoscale/egoscale/v2/oapi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1"
+	"github.com/vshn/provider-exoscale/operator/mapper"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type fakeSettingsFetcher struct{}
+
+func (fakeSettingsFetcher) GetDbaasSettingsOpensearchWithResponse(ctx context.Context, reqEditors ...oapi.RequestEditorFn) (*oapi.GetDbaasSettingsOpensearchResponse, error) {
+	return &oapi.GetDbaasSettingsOpensearchResponse{
+		Body: rawResponse,
+	}, nil
+}
+
+func mustToRawExt(t *testing.T, set map[string]interface{}) runtime.RawExtension {
+	res, err := mapper.ToRawExtension(&set)
+	require.NoError(t, err, "failed to parse input setting")
+	return res
+}
+
+func TestDefaultSettings(t *testing.T) {
+	found := exoscalev1.OpenSearchParameters{
+		Maintenance: exoscalev1.MaintenanceSpec{},
+		Zone:        "gva-2",
+		DBaaSParameters: exoscalev1.DBaaSParameters{
+			TerminationProtection: false,
+			Size: exoscalev1.SizeSpec{
+				Plan: "startup-4",
+			},
+		},
+		OpenSearchSettings: mustToRawExt(t, map[string]interface{}{
+			"thread_pool_search_throttled_size": 42,
+		}),
+	}
+
+	withDefaults, err := setSettingsDefaults(context.Background(), fakeSettingsFetcher{}, &found)
+	require.NoError(t, err, "failed to set defaults")
+	setingsWithDefaults, err := mapper.ToMap(withDefaults.OpenSearchSettings)
+	require.NoError(t, err, "failed to parse set defaults")
+	assert.EqualValues(t, 42, setingsWithDefaults["thread_pool_search_throttled_size"])
+	assert.Len(t, setingsWithDefaults, 1)
+}
+
+var rawResponse = []byte(`{"settings":{"opensearch":{"properties":{"thread_pool_search_throttled_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"search_throttled thread pool size","minimum":1},"thread_pool_analyze_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"analyze thread pool size","minimum":1},"thread_pool_get_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"get thread pool size","minimum":1},"thread_pool_get_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"get thread pool queue size","minimum":10},"indices_recovery_max_concurrent_file_chunks":{"description":"Number of file chunks sent in parallel for each recovery. Defaults to 2.","maximum":5,"type":"integer","title":"indices.recovery.max_concurrent_file_chunks","minimum":2},"indices_queries_cache_size":{"description":"Percentage value. Default is 10%. Maximum amount of heap used for query cache. This is an expert setting. Too low value will decrease query performance and increase performance for other operations; too high value will cause issues with other OpenSearch functionality.","maximum":40,"type":"integer","title":"indices.queries.cache.size","minimum":3},"thread_pool_search_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"search thread pool size","minimum":1},"indices_recovery_max_bytes_per_sec":{"description":"Limits total inbound and outbound recovery traffic for each node. Applies to both peer recoveries as well as snapshot recoveries (i.e., restores from a snapshot). Defaults to 40mb","maximum":400,"type":"integer","title":"indices.recovery.max_bytes_per_sec","minimum":40},"http_max_initial_line_length":{"description":"The max length of an HTTP URL, in bytes","maximum":65536,"type":"integer","title":"http.max_initial_line_length","minimum":1024,"example":4096},"thread_pool_write_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"write thread pool queue size","minimum":10},"script_max_compilations_rate":{"description":"Script compilation circuit breaker limits the number of inline script compilations within a period of time. Default is use-context","type":"string","title":"Script max compilation rate - circuit breaker to prevent/minimize OOMs","maxLength":1024,"example":"75/5m"},"search_max_buckets":{"description":"Maximum number of aggregation buckets allowed in a single response. OpenSearch default value is used when this is not defined.","maximum":20000,"type":["integer","null"],"title":"search.max_buckets","minimum":1,"example":10000},"reindex_remote_whitelist":{"description":"Whitelisted addresses for reindexing. Changing this value will cause all OpenSearch instances to restart.","type":["array","null"],"title":"reindex_remote_whitelist","items":{"type":["string","null"],"title":"Address (hostname:port or IP:port)","maxLength":261,"example":"anotherservice.aivencloud.com:12398"},"maxItems":32},"override_main_response_version":{"description":"Compatibility mode sets OpenSearch to report its version as 7.10 so clients continue to work. Default is false","type":"boolean","title":"compatibility.override_main_response_version","example":true},"http_max_header_size":{"description":"The max size of allowed headers, in bytes","maximum":262144,"type":"integer","title":"http.max_header_size","minimum":1024,"example":8192},"email_sender_name":{"description":"This should be identical to the Sender name defined in Opensearch dashboards","type":["string"],"user_error":"Must consist of lower-case alpha-numeric characters and dashes, max 40 characters","title":"Sender email name placeholder to be used in Opensearch Dashboards and Opensearch keystore","maxLength":40,"example":"alert-sender","pattern":"^[a-zA-Z0-9-_]+$"},"indices_fielddata_cache_size":{"description":"Relative amount. Maximum amount of heap memory used for field data cache. This is an expert setting; decreasing the value too much will increase overhead of loading field data; too much memory used for field data cache will decrease amount of heap available for other operations.","default":null,"maximum":100,"type":["integer","null"],"title":"indices.fielddata.cache.size","minimum":3},"action_destructive_requires_name":{"type":["boolean","null"],"title":"Require explicit index names when deleting","example":true},"email_sender_username":{"type":["string"],"user_error":"Must be a valid email address","title":"Sender email address for Opensearch alerts","maxLength":320,"example":"[email protected]","pattern":"^[A-Za-z0-9_\\-\\.+\\'&]+@(([\\da-zA-Z])([_\\w-]{,62})\\.){,127}(([\\da-zA-Z])[_\\w-]{,61})?([\\da-zA-Z]\\.((xn\\-\\-[a-zA-Z\\d]+)|([a-zA-Z\\d]{2,})))$"},"indices_memory_index_buffer_size":{"description":"Percentage value. Default is 10%. Total amount of heap used for indexing buffer, before writing segments to disk. This is an expert setting. Too low value will slow down indexing; too high value will increase indexing performance but causes performance issues for query performance.","maximum":40,"type":"integer","title":"indices.memory.index_buffer_size","minimum":3},"thread_pool_force_merge_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"force_merge thread pool size","minimum":1},"cluster_routing_allocation_node_concurrent_recoveries":{"description":"How many concurrent incoming/outgoing shard recoveries (normally replicas) are allowed to happen on a node. Defaults to 2.","maximum":16,"type":"integer","title":"Concurrent incoming/outgoing shard recoveries per node","minimum":2},"email_sender_password":{"description":"Sender email password for Opensearch alerts to authenticate with SMTP server","type":["string"],"title":"Sender email password for Opensearch alerts to authenticate with SMTP server","maxLength":1024,"example":"very-secure-mail-password","pattern":"^[^\\x00-\\x1F]+$"},"thread_pool_analyze_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"analyze thread pool queue size","minimum":10},"action_auto_create_index_enabled":{"description":"Explicitly allow or block automatic creation of indices. Defaults to true","type":"boolean","title":"action.auto_create_index","example":false},"http_max_content_length":{"description":"Maximum content length for HTTP requests to the OpenSearch HTTP API, in bytes.","maximum":2147483647,"type":"integer","title":"http.max_content_length","minimum":1},"thread_pool_write_size":{"description":"Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.","maximum":128,"type":"integer","title":"write thread pool size","minimum":1},"thread_pool_search_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"search thread pool queue size","minimum":10},"indices_query_bool_max_clause_count":{"description":"Maximum number of clauses Lucene BooleanQuery can have. The default value (1024) is relatively high, and increasing it may cause performance issues. Investigate other approaches first before increasing this value.","maximum":4096,"type":"integer","title":"indices.query.bool.max_clause_count","minimum":64},"thread_pool_search_throttled_queue_size":{"description":"Size for the thread pool queue. See documentation for exact details.","maximum":2000,"type":"integer","title":"search_throttled thread pool queue size","minimum":10},"cluster_max_shards_per_node":{"description":"Controls the number of shards allowed in the cluster per data node","maximum":10000,"type":"integer","title":"cluster.max_shards_per_node","minimum":100,"example":1000}},"additionalProperties":false,"type":"object","title":"OpenSearch settings","dependencies":{"email_sender_name":{"required":["email_sender_username","email_sender_password"]},"email_sender_username":{"required":["email_sender_name","email_sender_password"]},"email_sender_password":{"required":["email_sender_username","email_sender_name"]}}}}}`)