From 76eec6124cf2f11a1a9c113545039a48705bae7a Mon Sep 17 00:00:00 2001 From: Bohdan Siryk Date: Wed, 24 Jan 2024 16:00:00 +0200 Subject: [PATCH] spec refactor --- apis/clusters/v1beta1/opensearch_types.go | 123 ++++++++++++++---- apis/clusters/v1beta1/opensearch_webhook.go | 8 +- apis/clusters/v1beta1/structs.go | 35 ++--- ...clusters.instaclustr.com_opensearches.yaml | 7 +- controllers/clusters/cadence_controller.go | 2 +- .../clusters/datatest/opensearch_v1beta1.yaml | 2 +- controllers/clusters/opensearch_controller.go | 9 +- 7 files changed, 122 insertions(+), 64 deletions(-) diff --git a/apis/clusters/v1beta1/opensearch_types.go b/apis/clusters/v1beta1/opensearch_types.go index 2d943634c..ba849f4db 100644 --- a/apis/clusters/v1beta1/opensearch_types.go +++ b/apis/clusters/v1beta1/opensearch_types.go @@ -40,16 +40,16 @@ type OpenSearchSpec struct { Cluster `json:",inline"` DataCentres []*OpenSearchDataCentre `json:"dataCentres,omitempty"` DataNodes []*OpenSearchDataNodes `json:"dataNodes,omitempty"` + Dashboards []*OpenSearchDashboards `json:"opensearchDashboards,omitempty"` + ClusterManagerNodes []*ClusterManagerNodes `json:"clusterManagerNodes,omitempty"` ICUPlugin bool `json:"icuPlugin,omitempty"` AsynchronousSearchPlugin bool `json:"asynchronousSearchPlugin,omitempty"` KNNPlugin bool `json:"knnPlugin,omitempty"` - Dashboards []*OpenSearchDashboards `json:"opensearchDashboards,omitempty"` ReportingPlugin bool `json:"reportingPlugin,omitempty"` SQLPlugin bool `json:"sqlPlugin,omitempty"` NotificationsPlugin bool `json:"notificationsPlugin,omitempty"` AnomalyDetectionPlugin bool `json:"anomalyDetectionPlugin,omitempty"` LoadBalancer bool `json:"loadBalancer,omitempty"` - ClusterManagerNodes []*ClusterManagerNodes `json:"clusterManagerNodes,omitempty"` IndexManagementPlugin bool `json:"indexManagementPlugin,omitempty"` AlertingPlugin bool `json:"alertingPlugin,omitempty"` BundledUseOnly bool `json:"bundledUseOnly,omitempty"` @@ -60,6 +60,90 @@ type OpenSearchSpec struct { IngestNodes []*OpenSearchIngestNodes `json:"ingestNodes,omitempty"` } +func (s *OpenSearchSpec) FromModel(model *models.OpenSearchCluster) { + s.Cluster.FromModel(&model.GenericClusterFields) + + s.Version = model.OpenSearchVersion + s.ICUPlugin = model.ICUPlugin + s.AsynchronousSearchPlugin = model.AsynchronousSearchPlugin + s.KNNPlugin = model.KNNPlugin + s.ReportingPlugin = model.ReportingPlugin + s.SQLPlugin = model.SQLPlugin + s.NotificationsPlugin = model.NotificationsPlugin + s.AnomalyDetectionPlugin = model.AnomalyDetectionPlugin + s.LoadBalancer = model.LoadBalancer + s.IndexManagementPlugin = model.IndexManagementPlugin + s.AlertingPlugin = model.AlertingPlugin + s.BundledUseOnly = model.BundledUseOnly + + s.dcsFromModel(model.DataCentres) + s.dataNodesFromModel(model.DataNodes) + s.dashboardsFromModel(model.OpenSearchDashboards) + s.ingestNodesFromModel(model.IngestNodes) + s.clusterManagerNodesFromModel(model.ClusterManagerNodes) +} + +func (s *OpenSearchSpec) dcsFromModel(dcModels []*models.OpenSearchDataCentre) { + s.DataCentres = make([]*OpenSearchDataCentre, 0, len(dcModels)) + for _, model := range dcModels { + dc := &OpenSearchDataCentre{} + dc.FromModel(model) + s.DataCentres = append(s.DataCentres, dc) + } +} + +func (dc *OpenSearchDataCentre) FromModel(model *models.OpenSearchDataCentre) { + dc.PrivateLink = model.PrivateLink + dc.Name = model.Name + dc.Region = model.Region + dc.CloudProvider = model.CloudProvider + dc.ProviderAccountName = model.ProviderAccountName + dc.Network = model.Network + dc.Tags = tagsFromInstAPI(model.Tags) + dc.CloudProviderSettings = cloudProviderSettingsFromInstAPI(&model.GenericDataCentreFields) +} + +func (s *OpenSearchSpec) dataNodesFromModel(dataNodeModels []*models.OpenSearchDataNodes) { + s.DataNodes = make([]*OpenSearchDataNodes, 0, len(dataNodeModels)) + for _, model := range dataNodeModels { + s.DataNodes = append(s.DataNodes, &OpenSearchDataNodes{ + NodeSize: model.NodeSize, + NodesNumber: model.NodeCount, + }) + } +} + +func (s *OpenSearchSpec) dashboardsFromModel(dashboardsModels []*models.OpenSearchDashboards) { + s.Dashboards = make([]*OpenSearchDashboards, 0, len(dashboardsModels)) + for _, model := range dashboardsModels { + s.Dashboards = append(s.Dashboards, &OpenSearchDashboards{ + NodeSize: model.NodeSize, + OIDCProvider: model.OIDCProvider, + Version: model.Version, + }) + } +} + +func (s *OpenSearchSpec) ingestNodesFromModel(ingestNodeModels []*models.OpenSearchIngestNodes) { + s.IngestNodes = make([]*OpenSearchIngestNodes, 0, len(ingestNodeModels)) + for _, model := range ingestNodeModels { + s.IngestNodes = append(s.IngestNodes, &OpenSearchIngestNodes{ + NodeSize: model.NodeSize, + NodeCount: model.NodeCount, + }) + } +} + +func (s *OpenSearchSpec) clusterManagerNodesFromModel(clusterManagerNodeModels []*models.ClusterManagerNodes) { + s.ClusterManagerNodes = make([]*ClusterManagerNodes, 0, len(clusterManagerNodeModels)) + for _, model := range clusterManagerNodeModels { + s.ClusterManagerNodes = append(s.ClusterManagerNodes, &ClusterManagerNodes{ + NodeSize: model.NodeSize, + DedicatedManager: model.DedicatedManager, + }) + } +} + type OpenSearchDataCentre struct { PrivateLink bool `json:"privateLink,omitempty"` Name string `json:"name,omitempty"` @@ -69,9 +153,7 @@ type OpenSearchDataCentre struct { CloudProviderSettings []*CloudProviderSettings `json:"cloudProviderSettings,omitempty"` Network string `json:"network"` Tags map[string]string `json:"tags,omitempty"` - - // ReplicationFactor is a number of racks to use when allocating data nodes. - ReplicationFactor int `json:"replicationFactor"` + NumberOfRacks int `json:"numberOfRacks,omitempty"` } type OpenSearchDataNodes struct { @@ -144,7 +226,7 @@ func (oss *OpenSearchSpec) dcsToInstAPI() (iDCs []*models.OpenSearchDataCentre) ProviderAccountName: dc.ProviderAccountName, }, PrivateLink: dc.PrivateLink, - NumberOfRacks: dc.ReplicationFactor, + NumberOfRacks: dc.NumberOfRacks, }) } @@ -251,6 +333,11 @@ func (oss *OpenSearch) FromInstAPI(iData []byte) (*OpenSearch, error) { return o, nil } +func (oss *OpenSearch) FromModel(model *models.OpenSearchCluster) { + oss.Spec.FromModel(model) + oss.Status.FromModel(model) +} + func (oss *OpenSearchSpec) FromInstAPI(iOpenSearch *models.OpenSearchCluster) OpenSearchSpec { return OpenSearchSpec{ Cluster: Cluster{ @@ -307,11 +394,11 @@ func (oss *OpenSearchSpec) DCsFromInstAPI(iDCs []*models.OpenSearchDataCentre) ( Region: iDC.Region, CloudProvider: iDC.CloudProvider, ProviderAccountName: iDC.ProviderAccountName, - CloudProviderSettings: cloudProviderSettingsFromInstAPI(iDC), + CloudProviderSettings: cloudProviderSettingsFromInstAPI(&iDC.GenericDataCentreFields), Network: iDC.Network, Tags: tagsFromInstAPI(iDC.Tags), PrivateLink: iDC.PrivateLink, - ReplicationFactor: iDC.NumberOfRacks, + NumberOfRacks: iDC.NumberOfRacks, }) } @@ -326,11 +413,7 @@ func tagsFromInstAPI(iTags []*models.Tag) map[string]string { return newTags } -func cloudProviderSettingsFromInstAPI(iDC *models.OpenSearchDataCentre) (settings []*CloudProviderSettings) { - if isCloudProviderSettingsProvided(iDC.GenericDataCentreFields) { - return nil - } - +func cloudProviderSettingsFromInstAPI(iDC *models.GenericDataCentreFields) (settings []*CloudProviderSettings) { switch iDC.CloudProvider { case models.AWSVPC: for _, awsSetting := range iDC.AWSSettings { @@ -435,7 +518,7 @@ func (oss *OpenSearchSpec) areDCsEqual(b []*OpenSearchDataCentre) bool { a[i].Network != b[i].Network && areTagsEqual(a[i].Tags, b[i].Tags) && a[i].PrivateLink != b[i].PrivateLink || - a[i].ReplicationFactor != b[i].ReplicationFactor { + a[i].NumberOfRacks != b[i].NumberOfRacks { return false } } @@ -538,14 +621,12 @@ func (oss *OpenSearchStatus) FromModel(model *models.OpenSearchCluster) { oss.IngestNodesLoadBalancerConnectionURL = model.IngestNodesLoadBalancerConnectionURL oss.LoadBalancerConnectionURL = model.LoadBalancerConnectionURL - dcs := make([]OpenSearchDataCentreStatus, 0, len(model.DataCentres)) + oss.DataCentres = make([]OpenSearchDataCentreStatus, 0, len(model.DataCentres)) for _, dc := range model.DataCentres { d := OpenSearchDataCentreStatus{} d.fromModel(dc) - dcs = append(dcs, d) + oss.DataCentres = append(oss.DataCentres, d) } - - oss.DataCentres = dcs } type OpenSearchDataCentreStatus struct { @@ -563,14 +644,12 @@ func (s *OpenSearchDataCentreStatus) fromModel(model *models.OpenSearchDataCentr s.PrivateLinkEndpointServiceName = model.PrivateLinkEndpointServiceName s.Status = model.Status - nodes := make([]OpenSearchNodeStatus, 0, len(model.Nodes)) + s.Nodes = make([]OpenSearchNodeStatus, 0, len(model.Nodes)) for _, node := range model.Nodes { n := OpenSearchNodeStatus{} n.FromModel(&node) - nodes = append(nodes, n) + s.Nodes = append(s.Nodes, n) } - - s.Nodes = nodes } type OpenSearchNodeStatus struct { diff --git a/apis/clusters/v1beta1/opensearch_webhook.go b/apis/clusters/v1beta1/opensearch_webhook.go index 074649392..c8b9d3861 100644 --- a/apis/clusters/v1beta1/opensearch_webhook.go +++ b/apis/clusters/v1beta1/opensearch_webhook.go @@ -121,7 +121,7 @@ func (osv *openSearchValidator) ValidateCreate(ctx context.Context, obj runtime. return err } - err = validateReplicationFactor(models.OpenSearchReplicationFactors, dc.ReplicationFactor) + err = validateReplicationFactor(models.OpenSearchReplicationFactors, dc.NumberOfRacks) if err != nil { return err } @@ -318,7 +318,7 @@ func (oss *OpenSearchDataCentre) newImmutableFields() *immutableOpenSearchDCFiel }, specificOpenSearchDC{ PrivateLink: oss.PrivateLink, - ReplicationFactor: oss.ReplicationFactor, + ReplicationFactor: oss.NumberOfRacks, }, } } @@ -396,8 +396,8 @@ func (oss *OpenSearchSpec) validateImmutableDataCentresUpdate(oldDCs []*OpenSear func (dc *OpenSearchDataCentre) validateDataNode(nodes []*OpenSearchDataNodes) error { for _, node := range nodes { - if node.NodesNumber%dc.ReplicationFactor != 0 { - return fmt.Errorf("number of data nodes must be a multiple of replication factor: %v", dc.ReplicationFactor) + if node.NodesNumber%dc.NumberOfRacks != 0 { + return fmt.Errorf("number of data nodes must be a multiple of replication factor: %v", dc.NumberOfRacks) } } diff --git a/apis/clusters/v1beta1/structs.go b/apis/clusters/v1beta1/structs.go index 28621e957..d29590e2d 100644 --- a/apis/clusters/v1beta1/structs.go +++ b/apis/clusters/v1beta1/structs.go @@ -93,6 +93,14 @@ type Cluster struct { Description string `json:"description,omitempty"` } +func (c *Cluster) FromModel(model *models.GenericClusterFields) { + c.Name = model.Name + c.PCICompliance = model.PCIComplianceMode + c.PrivateNetworkCluster = model.PrivateNetworkCluster + c.SLATier = model.SLATier + c.Description = model.Description +} + type ClusterStatus struct { ID string `json:"id,omitempty"` State string `json:"state,omitempty"` @@ -660,33 +668,6 @@ func isCloudProviderSettingsEmpty(iDC models.DataCentre) bool { return true } -func isCloudProviderSettingsProvided(iDC models.GenericDataCentreFields) bool { - var empty bool - - for i := range iDC.AWSSettings { - empty = *iDC.AWSSettings[i] == models.AWSSetting{} - if !empty { - return false - } - } - - for i := range iDC.AzureSettings { - empty = *iDC.AzureSettings[i] == models.AzureSetting{} - if !empty { - return false - } - } - - for i := range iDC.GCPSettings { - empty = *iDC.GCPSettings[i] == models.GCPSetting{} - if !empty { - return false - } - } - - return true -} - func (cs *ClusterStatus) NodesFromInstAPI(iNodes []*models.Node) (nodes []*Node) { for _, iNode := range iNodes { nodes = append(nodes, &Node{ diff --git a/config/crd/bases/clusters.instaclustr.com_opensearches.yaml b/config/crd/bases/clusters.instaclustr.com_opensearches.yaml index 21406280f..608cad626 100644 --- a/config/crd/bases/clusters.instaclustr.com_opensearches.yaml +++ b/config/crd/bases/clusters.instaclustr.com_opensearches.yaml @@ -90,14 +90,12 @@ spec: type: string network: type: string + numberOfRacks: + type: integer privateLink: type: boolean region: type: string - replicationFactor: - description: ReplicationFactor is a number of racks to use when - allocating data nodes. - type: integer tags: additionalProperties: type: string @@ -106,7 +104,6 @@ spec: - cloudProvider - network - region - - replicationFactor type: object type: array dataNodes: diff --git a/controllers/clusters/cadence_controller.go b/controllers/clusters/cadence_controller.go index ce4b61f91..202f80521 100644 --- a/controllers/clusters/cadence_controller.go +++ b/controllers/clusters/cadence_controller.go @@ -1205,7 +1205,7 @@ func (r *CadenceReconciler) newOpenSearchSpec(c *v1beta1.Cadence, oldestOpenSear CloudProvider: cloudProvider, ProviderAccountName: providerAccountName, Network: osNetwork, - ReplicationFactor: osReplicationFactor, + NumberOfRacks: osReplicationFactor, }, } spec := v1beta1.OpenSearchSpec{ diff --git a/controllers/clusters/datatest/opensearch_v1beta1.yaml b/controllers/clusters/datatest/opensearch_v1beta1.yaml index 2d861038d..6d40fbf88 100644 --- a/controllers/clusters/datatest/opensearch_v1beta1.yaml +++ b/controllers/clusters/datatest/opensearch_v1beta1.yaml @@ -19,7 +19,7 @@ spec: - cloudProvider: AWS_VPC name: AWS_VPC_US_EAST_1 network: 10.0.0.0/16 - replicationFactor: 3 + numberOfRacks: 3 privateLink: false region: US_EAST_1 # dataNodes: diff --git a/controllers/clusters/opensearch_controller.go b/controllers/clusters/opensearch_controller.go index 05fdd0ede..44297ef60 100644 --- a/controllers/clusters/opensearch_controller.go +++ b/controllers/clusters/opensearch_controller.go @@ -644,14 +644,15 @@ func (r *OpenSearchReconciler) newWatchStatusJob(o *v1beta1.OpenSearch) schedule return err } - iO, err := o.FromInstAPI(iData) + opensearchModel := &models.OpenSearchCluster{} + err = json.Unmarshal(iData, opensearchModel) if err != nil { - l.Error(err, "Cannot convert OpenSearch cluster from the Instaclustr API", - "cluster ID", o.Status.ID) - return err } + iO := v1beta1.OpenSearch{} + iO.FromModel(opensearchModel) + if !iO.Status.Equals(&o.Status) { l.Info("Updating OpenSearch cluster status", "old", o.Status, "new", iO.Status)