Skip to content

Commit

Permalink
Add os dashboard backup and restore support
Browse files Browse the repository at this point in the history
Signed-off-by: Md. Ishtiaq Islam <[email protected]>
  • Loading branch information
ishtiaqhimel committed Jan 26, 2024
1 parent 964fa06 commit c2ce2d6
Show file tree
Hide file tree
Showing 11 changed files with 189 additions and 22 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
kmodules.xyz/custom-resources v0.29.0
kmodules.xyz/offshoot-api v0.29.0
kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c
kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f
kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92
sigs.k8s.io/controller-runtime v0.16.3
stash.appscode.dev/apimachinery v0.32.1-0.20240118085630-4c06ed8c04a7
)
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -581,8 +581,8 @@ kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg=
kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo=
kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c h1:QdnEBPdmd/Z3JOFJKMQ3nCFiJYiWxsuIAqMLgXep880=
kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c/go.mod h1:IqIwU4I/UfmcMi4X7G01M7XAEFUPehtiFAKZNKwH8Ek=
kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f h1:0DaQjz9awpbSCkIvsXLQGUzgRmTVJEL1rD9zk88KAEM=
kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f/go.mod h1:tCtLrUfxNHHTGKN9znhJo7vVLX+SHAY0nhCTXIDvo4o=
kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92 h1:EmIulT/VTbJSPfuW1TPuB/mfUac1rp93jW1vbJJCHx4=
kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92/go.mod h1:tCtLrUfxNHHTGKN9znhJo7vVLX+SHAY0nhCTXIDvo4o=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/gateway-api v0.8.0 h1:isQQ3Jx2qFP7vaA3ls0846F0Amp9Eq14P08xbSwVbQg=
sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM=
Expand Down
11 changes: 8 additions & 3 deletions pkg/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"time"
Expand Down Expand Up @@ -246,7 +247,7 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
}

if err = opt.dumpDashboardObjects(appBinding); err != nil {
return nil, fmt.Errorf("failed to dump kibana dashboard %w", err)
return nil, fmt.Errorf("failed to dump dashboard objects %w", err)
}

// dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash.
Expand Down Expand Up @@ -311,10 +312,14 @@ func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) er
return err
}

data, err := io.ReadAll(response.Body)
body, err := io.ReadAll(response.Body)
if err != nil {
return err
}

return os.WriteFile(filepath.Join(opt.interimDataDir, "kibana.ndjson"), data, os.ModePerm)
if response.Code != http.StatusOK {
return fmt.Errorf("failed to export dashboard saved objects %s", string(body))
}

return os.WriteFile(filepath.Join(opt.interimDataDir, DashboardObjectsFile), body, os.ModePerm)
}
8 changes: 4 additions & 4 deletions pkg/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,11 +222,11 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
}

if err = opt.restoreDashboardObjects(appBinding); err != nil {
return nil, fmt.Errorf("failed to restore kibana dashboard %w", err)
return nil, fmt.Errorf("failed to restore dashboard objects %w", err)
}

// delete the metadata file as it is not required for restoring the dumps
if err := clearFile(filepath.Join(opt.interimDataDir, "kibana.ndjson")); err != nil {
if err := clearFile(filepath.Join(opt.interimDataDir, DashboardObjectsFile)); err != nil {
return nil, err
}

Expand Down Expand Up @@ -261,7 +261,7 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding)
return err
}

response, err := dashboardClient.ImportSavedObjects(filepath.Join(opt.interimDataDir, "kibana.ndjson"))
response, err := dashboardClient.ImportSavedObjects(filepath.Join(opt.interimDataDir, DashboardObjectsFile))
if err != nil {
return err
}
Expand All @@ -272,7 +272,7 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding)
}

if response.Code != http.StatusOK {
return fmt.Errorf("failed to import kibana saved objects %s", string(body))
return fmt.Errorf("failed to import dashboard saved objects %s", string(body))
}

return nil
Expand Down
11 changes: 6 additions & 5 deletions pkg/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,12 @@ import (
)

const (
ESUser = "ADMIN_USERNAME"
ESPassword = "ADMIN_PASSWORD"
MultiElasticDumpCMD = "multielasticdump"
ESCACertFile = "root.pem"
ESAuthFile = "auth.txt"
ESUser = "ADMIN_USERNAME"
ESPassword = "ADMIN_PASSWORD"
MultiElasticDumpCMD = "multielasticdump"
ESCACertFile = "root.pem"
ESAuthFile = "auth.txt"
DashboardObjectsFile = "dashboard.ndjson"
)

type esOptions struct {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ import (
)

const (
SavedObjectsReqBody = `{"type": ["dashboard", "config", "index-pattern", "url", "query", "tag", "canvas-element", "canvas-workpad", "action", "alert", "visualization",
SavedObjectsReqBodyES = `{"type": ["dashboard", "config", "index-pattern", "url", "query", "tag", "canvas-element", "canvas-workpad", "action", "alert", "visualization",
"graph-workspace", "map", "lens", "cases", "search", "osquery-saved-query", "osquery-pack", "uptime-dynamic-settings", "infrastructure-ui-source", "metrics-explorer-view",
"inventory-view", "apm-indices"]}`
SavedObjectsReqBodyOS = `{"type": ["config", "url", "index-pattern", "query", "dashboard", "visualization", "visualization-visbuilder", "augment-vis", "map",
"observability-panel", "observability-visualization", "search"]}`
SavedObjectsExportURL = "/api/saved_objects/_export"
SavedObjectsImportURL = "/api/saved_objects/_import"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func (h *EDClientV7) ExportSavedObjects() (*Response, error) {
"Content-Type": "application/json",
"kbn-xsrf": "true",
}).
SetBody([]byte(SavedObjectsReqBody))
SetBody([]byte(SavedObjectsReqBodyES))
res, err := req.Post(SavedObjectsExportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func (h *EDClientV8) ExportSavedObjects() (*Response, error) {
"Content-Type": "application/json",
"kbn-xsrf": "true",
}).
SetBody([]byte(SavedObjectsReqBody))
SetBody([]byte(SavedObjectsReqBodyES))
res, err := req.Post(SavedObjectsExportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,8 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error)
}

switch {
// for Elasticsearch 7.x.x and OpenSearch 1.x.x
case (config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() <= 7) ||
(config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginOpenSearch && (version.Major() == 1 || version.Major() == 2)):
// for Elasticsearch 7.x.x
case config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() == 7:
newClient := resty.New()
newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host)
newClient.SetHeader("Accept", "application/json")
Expand All @@ -199,6 +198,20 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error)
Config: &config,
},
}, nil

case config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginOpenSearch:
newClient := resty.New()
newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host)
newClient.SetHeader("Accept", "application/json")
newClient.SetBasicAuth(config.username, config.password)
newClient.SetTimeout(time.Second * 30)

return &Client{
&OSClient{
Client: newClient,
Config: &config,
},
}, nil
}

return nil, fmt.Errorf("unknown version: %s", config.dbVersionInfo.Name)
Expand Down
146 changes: 146 additions & 0 deletions vendor/kubedb.dev/db-client-go/elasticsearchdashboard/os_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
/*
Copyright AppsCode Inc. and Contributors
Licensed under the AppsCode Free Trial License 1.0.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package elasticsearchdashboard

import (
"encoding/json"
"io"
"strings"

esapi "kubedb.dev/apimachinery/apis/elasticsearch/v1alpha1"

"github.com/go-resty/resty/v2"
"github.com/pkg/errors"
"k8s.io/klog/v2"
)

type OSClient struct {
Client *resty.Client
Config *Config
}

func (h *OSClient) GetHealthStatus() (*Health, error) {
req := h.Client.R().SetDoNotParseResponse(true)
res, err := req.Get(h.Config.api)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

statesList := make(map[string]string)

healthStatus := &Health{
ConnectionResponse: Response{
Code: res.StatusCode(),
header: res.Header(),
Body: res.RawBody(),
},
StateFailedReason: statesList,
}

return healthStatus, nil
}

// GetStateFromHealthResponse parse health response in json from server and
// return overall status of the server
func (h *OSClient) GetStateFromHealthResponse(health *Health) (esapi.DashboardServerState, error) {
resStatus := health.ConnectionResponse

defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
err1 := errors.Wrap(err, "failed to parse response body")
if err1 != nil {
return
}
return
}
}(resStatus.Body)

var responseBody ResponseBody
body, _ := io.ReadAll(resStatus.Body)
err := json.Unmarshal(body, &responseBody)
if err != nil {
return "", errors.Wrap(err, "Failed to parse response body")
}

if overallStatus, ok := responseBody.Status["overall"].(map[string]interface{}); ok {
if overallState, ok := overallStatus["state"].(string); ok {
health.OverallState = overallState
} else {
return "", errors.New("Failed to parse overallState")
}
} else {
return "", errors.New("Failed to parse overallStatus")
}

// get the statuses for plugins stored,
// so that the plugins which are not available or ready can be shown from condition message
if statuses, ok := responseBody.Status["statuses"].([]interface{}); ok {
for _, sts := range statuses {
if curr, ok := sts.(map[string]interface{}); ok {
if curr["state"].(string) != string(esapi.StateGreen) {
health.StateFailedReason[curr["id"].(string)] = strings.Join([]string{curr["state"].(string), curr["message"].(string)}, ",")
}
} else {
return "", errors.New("Failed to convert statuses to map[string]interface{}")
}
}
} else {
return "", errors.New("Failed to convert statuses to []interface{}")
}

return esapi.DashboardServerState(health.OverallState), nil
}

func (h *OSClient) ExportSavedObjects() (*Response, error) {
req := h.Client.R().
SetDoNotParseResponse(true).
SetHeaders(map[string]string{
"Content-Type": "application/json",
"osd-xsrf": "true",
}).
SetBody([]byte(SavedObjectsReqBodyOS))
res, err := req.Post(SavedObjectsExportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

return &Response{
Code: res.StatusCode(),
Body: res.RawBody(),
}, nil
}

func (h *OSClient) ImportSavedObjects(filepath string) (*Response, error) {
req := h.Client.R().
SetDoNotParseResponse(true).
SetHeader("osd-xsrf", "true").
SetFile("file", filepath).
SetQueryParam("overwrite", "true")
res, err := req.Post(SavedObjectsImportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

return &Response{
Code: res.StatusCode(),
Body: res.RawBody(),
}, nil
}
2 changes: 1 addition & 1 deletion vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,7 @@ kubedb.dev/apimachinery/apis/kubedb
kubedb.dev/apimachinery/apis/kubedb/v1alpha2
kubedb.dev/apimachinery/crds
kubedb.dev/apimachinery/pkg/validator
# kubedb.dev/db-client-go v0.0.9-0.20240125100355-dd2b92a0910f
# kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92
## explicit; go 1.21.5
kubedb.dev/db-client-go/elasticsearchdashboard
# sigs.k8s.io/controller-runtime v0.16.3 => github.com/kmodules/controller-runtime v0.16.1-0.20231224083233-bead154270db
Expand Down

0 comments on commit c2ce2d6

Please sign in to comment.