From c2ce2d6719533cb79af8d3c87e6eff65d9d28886 Mon Sep 17 00:00:00 2001 From: "Md. Ishtiaq Islam" Date: Fri, 26 Jan 2024 16:54:03 +0600 Subject: [PATCH] Add os dashboard backup and restore support Signed-off-by: Md. Ishtiaq Islam --- go.mod | 2 +- go.sum | 4 +- pkg/backup.go | 11 +- pkg/restore.go | 8 +- pkg/utils.go | 11 +- .../elasticsearchdashboard/client.go | 4 +- .../elasticsearchdashboard/ed_client_v7.go | 2 +- .../elasticsearchdashboard/ed_client_v8.go | 2 +- .../kubedb-client-builder.go | 19 ++- .../elasticsearchdashboard/os_client.go | 146 ++++++++++++++++++ vendor/modules.txt | 2 +- 11 files changed, 189 insertions(+), 22 deletions(-) create mode 100644 vendor/kubedb.dev/db-client-go/elasticsearchdashboard/os_client.go diff --git a/go.mod b/go.mod index 4e138cd07..596379867 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( kmodules.xyz/custom-resources v0.29.0 kmodules.xyz/offshoot-api v0.29.0 kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c - kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f + kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92 sigs.k8s.io/controller-runtime v0.16.3 stash.appscode.dev/apimachinery v0.32.1-0.20240118085630-4c06ed8c04a7 ) diff --git a/go.sum b/go.sum index e4e02b456..3cd8794f2 100644 --- a/go.sum +++ b/go.sum @@ -581,8 +581,8 @@ kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg= kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo= kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c h1:QdnEBPdmd/Z3JOFJKMQ3nCFiJYiWxsuIAqMLgXep880= kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c/go.mod h1:IqIwU4I/UfmcMi4X7G01M7XAEFUPehtiFAKZNKwH8Ek= -kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f h1:0DaQjz9awpbSCkIvsXLQGUzgRmTVJEL1rD9zk88KAEM= -kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f/go.mod h1:tCtLrUfxNHHTGKN9znhJo7vVLX+SHAY0nhCTXIDvo4o= +kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92 h1:EmIulT/VTbJSPfuW1TPuB/mfUac1rp93jW1vbJJCHx4= +kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92/go.mod h1:tCtLrUfxNHHTGKN9znhJo7vVLX+SHAY0nhCTXIDvo4o= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/gateway-api v0.8.0 h1:isQQ3Jx2qFP7vaA3ls0846F0Amp9Eq14P08xbSwVbQg= sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM= diff --git a/pkg/backup.go b/pkg/backup.go index a66de75e1..ead7e9826 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "path/filepath" "time" @@ -246,7 +247,7 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res } if err = opt.dumpDashboardObjects(appBinding); err != nil { - return nil, fmt.Errorf("failed to dump kibana dashboard %w", err) + return nil, fmt.Errorf("failed to dump dashboard objects %w", err) } // dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash. @@ -311,10 +312,14 @@ func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) er return err } - data, err := io.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { return err } - return os.WriteFile(filepath.Join(opt.interimDataDir, "kibana.ndjson"), data, os.ModePerm) + if response.Code != http.StatusOK { + return fmt.Errorf("failed to export dashboard saved objects %s", string(body)) + } + + return os.WriteFile(filepath.Join(opt.interimDataDir, DashboardObjectsFile), body, os.ModePerm) } diff --git a/pkg/restore.go b/pkg/restore.go index c24522326..76223bd3b 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -222,11 +222,11 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re } if err = opt.restoreDashboardObjects(appBinding); err != nil { - return nil, fmt.Errorf("failed to restore kibana dashboard %w", err) + return nil, fmt.Errorf("failed to restore dashboard objects %w", err) } // delete the metadata file as it is not required for restoring the dumps - if err := clearFile(filepath.Join(opt.interimDataDir, "kibana.ndjson")); err != nil { + if err := clearFile(filepath.Join(opt.interimDataDir, DashboardObjectsFile)); err != nil { return nil, err } @@ -261,7 +261,7 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding) return err } - response, err := dashboardClient.ImportSavedObjects(filepath.Join(opt.interimDataDir, "kibana.ndjson")) + response, err := dashboardClient.ImportSavedObjects(filepath.Join(opt.interimDataDir, DashboardObjectsFile)) if err != nil { return err } @@ -272,7 +272,7 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding) } if response.Code != http.StatusOK { - return fmt.Errorf("failed to import kibana saved objects %s", string(body)) + return fmt.Errorf("failed to import dashboard saved objects %s", string(body)) } return nil diff --git a/pkg/utils.go b/pkg/utils.go index 0082bea25..013bd55cf 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -50,11 +50,12 @@ import ( ) const ( - ESUser = "ADMIN_USERNAME" - ESPassword = "ADMIN_PASSWORD" - MultiElasticDumpCMD = "multielasticdump" - ESCACertFile = "root.pem" - ESAuthFile = "auth.txt" + ESUser = "ADMIN_USERNAME" + ESPassword = "ADMIN_PASSWORD" + MultiElasticDumpCMD = "multielasticdump" + ESCACertFile = "root.pem" + ESAuthFile = "auth.txt" + DashboardObjectsFile = "dashboard.ndjson" ) type esOptions struct { diff --git a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/client.go b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/client.go index b604c196d..ffea20b9c 100644 --- a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/client.go +++ b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/client.go @@ -30,9 +30,11 @@ import ( ) const ( - SavedObjectsReqBody = `{"type": ["dashboard", "config", "index-pattern", "url", "query", "tag", "canvas-element", "canvas-workpad", "action", "alert", "visualization", + SavedObjectsReqBodyES = `{"type": ["dashboard", "config", "index-pattern", "url", "query", "tag", "canvas-element", "canvas-workpad", "action", "alert", "visualization", "graph-workspace", "map", "lens", "cases", "search", "osquery-saved-query", "osquery-pack", "uptime-dynamic-settings", "infrastructure-ui-source", "metrics-explorer-view", "inventory-view", "apm-indices"]}` + SavedObjectsReqBodyOS = `{"type": ["config", "url", "index-pattern", "query", "dashboard", "visualization", "visualization-visbuilder", "augment-vis", "map", +"observability-panel", "observability-visualization", "search"]}` SavedObjectsExportURL = "/api/saved_objects/_export" SavedObjectsImportURL = "/api/saved_objects/_import" ) diff --git a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v7.go b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v7.go index 2029b64e5..91dda15e9 100644 --- a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v7.go +++ b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v7.go @@ -114,7 +114,7 @@ func (h *EDClientV7) ExportSavedObjects() (*Response, error) { "Content-Type": "application/json", "kbn-xsrf": "true", }). - SetBody([]byte(SavedObjectsReqBody)) + SetBody([]byte(SavedObjectsReqBodyES)) res, err := req.Post(SavedObjectsExportURL) if err != nil { klog.Error(err, "Failed to send http request") diff --git a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v8.go b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v8.go index b4df628d7..f2a47fdd8 100644 --- a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v8.go +++ b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/ed_client_v8.go @@ -112,7 +112,7 @@ func (h *EDClientV8) ExportSavedObjects() (*Response, error) { "Content-Type": "application/json", "kbn-xsrf": "true", }). - SetBody([]byte(SavedObjectsReqBody)) + SetBody([]byte(SavedObjectsReqBodyES)) res, err := req.Post(SavedObjectsExportURL) if err != nil { klog.Error(err, "Failed to send http request") diff --git a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/kubedb-client-builder.go b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/kubedb-client-builder.go index 631f0322d..6c4daca7b 100644 --- a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/kubedb-client-builder.go +++ b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/kubedb-client-builder.go @@ -170,9 +170,8 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error) } switch { - // for Elasticsearch 7.x.x and OpenSearch 1.x.x - case (config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() <= 7) || - (config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginOpenSearch && (version.Major() == 1 || version.Major() == 2)): + // for Elasticsearch 7.x.x + case config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() == 7: newClient := resty.New() newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host) newClient.SetHeader("Accept", "application/json") @@ -199,6 +198,20 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error) Config: &config, }, }, nil + + case config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginOpenSearch: + newClient := resty.New() + newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host) + newClient.SetHeader("Accept", "application/json") + newClient.SetBasicAuth(config.username, config.password) + newClient.SetTimeout(time.Second * 30) + + return &Client{ + &OSClient{ + Client: newClient, + Config: &config, + }, + }, nil } return nil, fmt.Errorf("unknown version: %s", config.dbVersionInfo.Name) diff --git a/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/os_client.go b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/os_client.go new file mode 100644 index 000000000..abb78d3d9 --- /dev/null +++ b/vendor/kubedb.dev/db-client-go/elasticsearchdashboard/os_client.go @@ -0,0 +1,146 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elasticsearchdashboard + +import ( + "encoding/json" + "io" + "strings" + + esapi "kubedb.dev/apimachinery/apis/elasticsearch/v1alpha1" + + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" + "k8s.io/klog/v2" +) + +type OSClient struct { + Client *resty.Client + Config *Config +} + +func (h *OSClient) GetHealthStatus() (*Health, error) { + req := h.Client.R().SetDoNotParseResponse(true) + res, err := req.Get(h.Config.api) + if err != nil { + klog.Error(err, "Failed to send http request") + return nil, err + } + + statesList := make(map[string]string) + + healthStatus := &Health{ + ConnectionResponse: Response{ + Code: res.StatusCode(), + header: res.Header(), + Body: res.RawBody(), + }, + StateFailedReason: statesList, + } + + return healthStatus, nil +} + +// GetStateFromHealthResponse parse health response in json from server and +// return overall status of the server +func (h *OSClient) GetStateFromHealthResponse(health *Health) (esapi.DashboardServerState, error) { + resStatus := health.ConnectionResponse + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + err1 := errors.Wrap(err, "failed to parse response body") + if err1 != nil { + return + } + return + } + }(resStatus.Body) + + var responseBody ResponseBody + body, _ := io.ReadAll(resStatus.Body) + err := json.Unmarshal(body, &responseBody) + if err != nil { + return "", errors.Wrap(err, "Failed to parse response body") + } + + if overallStatus, ok := responseBody.Status["overall"].(map[string]interface{}); ok { + if overallState, ok := overallStatus["state"].(string); ok { + health.OverallState = overallState + } else { + return "", errors.New("Failed to parse overallState") + } + } else { + return "", errors.New("Failed to parse overallStatus") + } + + // get the statuses for plugins stored, + // so that the plugins which are not available or ready can be shown from condition message + if statuses, ok := responseBody.Status["statuses"].([]interface{}); ok { + for _, sts := range statuses { + if curr, ok := sts.(map[string]interface{}); ok { + if curr["state"].(string) != string(esapi.StateGreen) { + health.StateFailedReason[curr["id"].(string)] = strings.Join([]string{curr["state"].(string), curr["message"].(string)}, ",") + } + } else { + return "", errors.New("Failed to convert statuses to map[string]interface{}") + } + } + } else { + return "", errors.New("Failed to convert statuses to []interface{}") + } + + return esapi.DashboardServerState(health.OverallState), nil +} + +func (h *OSClient) ExportSavedObjects() (*Response, error) { + req := h.Client.R(). + SetDoNotParseResponse(true). + SetHeaders(map[string]string{ + "Content-Type": "application/json", + "osd-xsrf": "true", + }). + SetBody([]byte(SavedObjectsReqBodyOS)) + res, err := req.Post(SavedObjectsExportURL) + if err != nil { + klog.Error(err, "Failed to send http request") + return nil, err + } + + return &Response{ + Code: res.StatusCode(), + Body: res.RawBody(), + }, nil +} + +func (h *OSClient) ImportSavedObjects(filepath string) (*Response, error) { + req := h.Client.R(). + SetDoNotParseResponse(true). + SetHeader("osd-xsrf", "true"). + SetFile("file", filepath). + SetQueryParam("overwrite", "true") + res, err := req.Post(SavedObjectsImportURL) + if err != nil { + klog.Error(err, "Failed to send http request") + return nil, err + } + + return &Response{ + Code: res.StatusCode(), + Body: res.RawBody(), + }, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5b38ea15e..e0da4106e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -754,7 +754,7 @@ kubedb.dev/apimachinery/apis/kubedb kubedb.dev/apimachinery/apis/kubedb/v1alpha2 kubedb.dev/apimachinery/crds kubedb.dev/apimachinery/pkg/validator -# kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f +# kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92 ## explicit; go 1.21.5 kubedb.dev/db-client-go/elasticsearchdashboard # sigs.k8s.io/controller-runtime v0.16.3 => github.com/kmodules/controller-runtime v0.16.1-0.20231224083233-bead154270db