Skip to content

Commit

Permalink
[databases]: add support for Opensearch advanced configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
@loosla
loosla committed Sep 20, 2024
1 parent 8348cdd commit 2205c1a
Show file tree
Hide file tree
Showing 2 changed files with 213 additions and 0 deletions.
79 changes: 79 additions & 0 deletions databases.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,12 @@ type DatabasesService interface {
GetRedisConfig(context.Context, string) (*RedisConfig, *Response, error)
GetMySQLConfig(context.Context, string) (*MySQLConfig, *Response, error)
GetMongoDBConfig(context.Context, string) (*MongoDBConfig, *Response, error)
GetOpensearchConfig(context.Context, string) (*OpensearchConfig, *Response, error)
UpdatePostgreSQLConfig(context.Context, string, *PostgreSQLConfig) (*Response, error)
UpdateRedisConfig(context.Context, string, *RedisConfig) (*Response, error)
UpdateMySQLConfig(context.Context, string, *MySQLConfig) (*Response, error)
UpdateMongoDBConfig(context.Context, string, *MongoDBConfig) (*Response, error)
UpdateOpensearchConfig(context.Context, string, *OpensearchConfig) (*Response, error)
ListOptions(todo context.Context) (*DatabaseOptions, *Response, error)
UpgradeMajorVersion(context.Context, string, *UpgradeVersionRequest) (*Response, error)
ListTopics(context.Context, string, *ListOptions) ([]DatabaseTopic, *Response, error)
Expand Down Expand Up @@ -703,6 +705,10 @@ type databaseMongoDBConfigRoot struct {
Config *MongoDBConfig `json:"config"`
}

type databaseOpensearchConfigRoot struct {
Config *OpensearchConfig `json:"config"`
}

type databaseBackupsRoot struct {
Backups []DatabaseBackup `json:"backups"`
}
Expand Down Expand Up @@ -1546,6 +1552,38 @@ func (svc *DatabasesServiceOp) UpdateMongoDBConfig(ctx context.Context, database
return resp, nil
}

// GetOpensearchConfig retrieves the config for a Opensearch database cluster.
func (svc *DatabasesServiceOp) GetOpensearchConfig(ctx context.Context, databaseID string) (*OpensearchConfig, *Response, error) {
path := fmt.Sprintf(databaseConfigPath, databaseID)
req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, nil, err
}
root := new(databaseOpensearchConfigRoot)
resp, err := svc.client.Do(ctx, req, root)
if err != nil {
return nil, resp, err
}
return root.Config, resp, nil
}

// UpdateOpensearchConfig updates the config for a Opensearch database cluster.
func (svc *DatabasesServiceOp) UpdateOpensearchConfig(ctx context.Context, databaseID string, config *OpensearchConfig) (*Response, error) {
path := fmt.Sprintf(databaseConfigPath, databaseID)
root := &databaseOpensearchConfigRoot{
Config: config,
}
req, err := svc.client.NewRequest(ctx, http.MethodPatch, path, root)
if err != nil {
return nil, err
}
resp, err := svc.client.Do(ctx, req, nil)
if err != nil {
return resp, err
}
return resp, nil
}

// ListOptions gets the database options available.
func (svc *DatabasesServiceOp) ListOptions(ctx context.Context) (*DatabaseOptions, *Response, error) {
root := new(databaseOptionsRoot)
Expand Down Expand Up @@ -1817,3 +1855,44 @@ func (svc *DatabasesServiceOp) DeleteLogsink(ctx context.Context, databaseID, lo
}
return resp, nil
}

// OpensearchConfig holds advanced configurations for Opensearch database clusters.
type OpensearchConfig struct {
HttpMaxContentLengthBytes *int `json:"http_max_content_length_bytes,omitempty"`
HttpMaxHeaderSizeBytes *int `json:"http_max_header_size_bytes,omitempty"`
HttpMaxInitialLineLengthBytes *int `json:"http_max_initial_line_length_bytes,omitempty"`
IndicesQueryBoolMaxClauseCount *int `json:"indices_query_bool_max_clause_count,omitempty"`
IndicesFielddataCacheSizePercentage *int `json:"indices_fielddata_cache_size_percentage,omitempty"`
IndicesMemoryIndexBufferSizePercentage *int `json:"indices_memory_index_buffer_size_percentage,omitempty"`
IndicesMemoryMinIndexBufferSizeMb *int `json:"indices_memory_min_index_buffer_size_mb,omitempty"`
IndicesMemoryMaxIndexBufferSizeMb *int `json:"indices_memory_max_index_buffer_size_mb,omitempty"`
IndicesQueriesCacheSizePercentage *int `json:"indices_queries_cache_size_percentage,omitempty"`
IndicesRecoveryMaxMbPerSec *int `json:"indices_recovery_max_mb_per_sec,omitempty"`
IndicesRecoveryMaxConcurrentFileChunks *int `json:"indices_recovery_max_concurrent_file_chunks,omitempty"`
ThreadPoolSearchSize *int `json:"thread_pool_search_size,omitempty"`
ThreadPoolSearchThrottledSize *int `json:"thread_pool_search_throttled_size,omitempty"`
ThreadPoolGetSize *int `json:"thread_pool_get_size,omitempty"`
ThreadPoolAnalyzeSize *int `json:"thread_pool_analyze_size,omitempty"`
ThreadPoolWriteSize *int `json:"thread_pool_write_size,omitempty"`
ThreadPoolForceMergeSize *int `json:"thread_pool_force_merge_size,omitempty"`
ThreadPoolSearchQueueSize *int `json:"thread_pool_search_queue_size,omitempty"`
ThreadPoolSearchThrottledQueueSize *int `json:"thread_pool_search_throttled_queue_size,omitempty"`
ThreadPoolGetQueueSize *int `json:"thread_pool_get_queue_size,omitempty"`
ThreadPoolAnalyzeQueueSize *int `json:"thread_pool_analyze_queue_size,omitempty"`
ThreadPoolWriteQueueSize *int `json:"thread_pool_write_queue_size,omitempty"`
IsmEnabled *bool `json:"ism_enabled,omitempty"`
IsmHistoryEnabled *bool `json:"ism_history_enabled,omitempty"`
IsmHistoryMaxAgeHours *int `json:"ism_history_max_age_hours,omitempty"`
IsmHistoryMaxDocs *uint64 `json:"ism_history_max_docs,omitempty"`
IsmHistoryRolloverCheckPeriodHours *int `json:"ism_history_rollover_check_period_hours,omitempty"`
IsmHistoryRolloverRetentionPeriodDays *int `json:"ism_history_rollover_retention_period_days,omitempty"`
SearchMaxBuckets *int `json:"search_max_buckets,omitempty"`
ActionAutoCreateIndexEnabled *bool `json:"action_auto_create_index_enabled,omitempty"`
EnableSecurityAudit *bool `json:"enable_security_audit,omitempty"`
ActionDestructiveRequiresName *bool `json:"action_destructive_requires_name,omitempty"`
ClusterMaxShardsPerNode *int `json:"cluster_max_shards_per_node,omitempty"`
OverrideMainResponseVersion *bool `json:"override_main_response_version,omitempty"`
ScriptMaxCompilationsRate *string `json:"script_max_compilations_rate,omitempty"`
ClusterRoutingAllocationNodeConcurrentRecoveries *int `json:"cluster_routing_allocation_node_concurrent_recoveries,omitempty"`
ReindexRemoteWhitelist []string `json:"reindex_remote_whitelist,omitempty"`
}
134 changes: 134 additions & 0 deletions databases_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3064,6 +3064,140 @@ func TestDatabases_UpdateConfigMongoDB(t *testing.T) {
require.NoError(t, err)
}

func TestDatabases_GetConfigOpensearch(t *testing.T) {
setup()
defer teardown()

var (
dbSvc = client.Databases
dbID = "da4e0206-d019-41d7-b51f-deadbeefbb8f"
path = fmt.Sprintf("/v2/databases/%s/config", dbID)

opensearchConfigJSON = `{
"config": {
"ism_enabled": true,
"ism_history_enabled": true,
"ism_history_max_age_hours": 24,
"ism_history_max_docs": 2500000,
"ism_history_rollover_check_period_hours": 8,
"ism_history_rollover_retention_period_days": 30,
"http_max_content_length_bytes": 100000000,
"http_max_header_size_bytes": 8192,
"http_max_initial_line_length_bytes": 4096,
"indices_query_bool_max_clause_count": 1024,
"search_max_buckets": 10000,
"indices_fielddata_cache_size_percentage": 0,
"indices_memory_index_buffer_size_percentage": 10,
"indices_memory_min_index_buffer_size_mb": 48,
"indices_memory_max_index_buffer_size_mb": 0,
"indices_queries_cache_size_percentage": 10,
"indices_recovery_max_mb_per_sec": 40,
"indices_recovery_max_concurrent_file_chunks": 2,
"action_auto_create_index_enabled": true,
"action_destructive_requires_name": false,
"plugins_alerting_filter_by_backend_roles_enabled": false,
"enable_security_audit": false,
"thread_pool_search_size": 0,
"thread_pool_search_throttled_size": 0,
"thread_pool_search_throttled_queue_size": 0,
"thread_pool_search_queue_size": 0,
"thread_pool_get_size": 0,
"thread_pool_get_queue_size": 0,
"thread_pool_analyze_size": 0,
"thread_pool_analyze_queue_size": 0,
"thread_pool_write_size": 0,
"thread_pool_write_queue_size": 0,
"thread_pool_force_merge_size": 0,
"override_main_response_version": false,
"script_max_compilations_rate": "use-context",
"cluster_max_shards_per_node": 0,
"cluster_routing_allocation_node_concurrent_recoveries": 2
}
}`

kafkaConfig = OpensearchConfig{
HttpMaxContentLengthBytes: PtrTo(100000000),
HttpMaxHeaderSizeBytes: PtrTo(8192),
HttpMaxInitialLineLengthBytes: PtrTo(4096),
IndicesQueryBoolMaxClauseCount: PtrTo(1024),
IndicesFielddataCacheSizePercentage: PtrTo(0),
IndicesMemoryIndexBufferSizePercentage: PtrTo(10),
IndicesMemoryMinIndexBufferSizeMb: PtrTo(48),
IndicesMemoryMaxIndexBufferSizeMb: PtrTo(0),
IndicesQueriesCacheSizePercentage: PtrTo(10),
IndicesRecoveryMaxMbPerSec: PtrTo(40),
IndicesRecoveryMaxConcurrentFileChunks: PtrTo(2),
ThreadPoolSearchSize: PtrTo(0),
ThreadPoolSearchThrottledSize: PtrTo(0),
ThreadPoolGetSize: PtrTo(0),
ThreadPoolAnalyzeSize: PtrTo(0),
ThreadPoolWriteSize: PtrTo(0),
ThreadPoolForceMergeSize: PtrTo(0),
ThreadPoolSearchQueueSize: PtrTo(0),
ThreadPoolSearchThrottledQueueSize: PtrTo(0),
ThreadPoolGetQueueSize: PtrTo(0),
ThreadPoolAnalyzeQueueSize: PtrTo(0),
ThreadPoolWriteQueueSize: PtrTo(0),
IsmEnabled: PtrTo(true),
IsmHistoryEnabled: PtrTo(true),
IsmHistoryMaxAgeHours: PtrTo(24),
IsmHistoryMaxDocs: PtrTo(uint64(2500000)),
IsmHistoryRolloverCheckPeriodHours: PtrTo(8),
IsmHistoryRolloverRetentionPeriodDays: PtrTo(30),
SearchMaxBuckets: PtrTo(10000),
ActionAutoCreateIndexEnabled: PtrTo(true),
EnableSecurityAudit: PtrTo(false),
ActionDestructiveRequiresName: PtrTo(false),
ClusterMaxShardsPerNode: PtrTo(0),
OverrideMainResponseVersion: PtrTo(false),
ScriptMaxCompilationsRate: PtrTo("use-context"),
ClusterRoutingAllocationNodeConcurrentRecoveries: PtrTo(2),
ReindexRemoteWhitelist: nil,
}
)

mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
testMethod(t, r, http.MethodGet)
fmt.Fprint(w, opensearchConfigJSON)
})

got, _, err := dbSvc.GetOpensearchConfig(ctx, dbID)
require.NoError(t, err)
require.Equal(t, &kafkaConfig, got)
}

func TestDatabases_UpdateConfigOpensearch(t *testing.T) {
setup()
defer teardown()

var (
dbID = "deadbeef-dead-4aa5-beef-deadbeef347d"
path = fmt.Sprintf("/v2/databases/%s/config", dbID)
opensearchConfig = &OpensearchConfig{
HttpMaxContentLengthBytes: PtrTo(1),
HttpMaxHeaderSizeBytes: PtrTo(0),
}
)

mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
testMethod(t, r, http.MethodPatch)

var b databaseOpensearchConfigRoot
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(&b)
require.NoError(t, err)

assert.Equal(t, b.Config, opensearchConfig)
assert.Equal(t, 0, *b.Config.HttpMaxHeaderSizeBytes, "pointers to zero value should be sent")
assert.Nil(t, b.Config.HttpMaxInitialLineLengthBytes, "excluded value should not be sent")

w.WriteHeader(http.StatusNoContent)
})

_, err := client.Databases.UpdateOpensearchConfig(ctx, dbID, opensearchConfig)
require.NoError(t, err)
}

func TestDatabases_UpgradeMajorVersion(t *testing.T) {
setup()
defer teardown()
Expand Down

0 comments on commit 2205c1a

Please sign in to comment.