From 00581c7898eaac3811e2cdba0db47f5a6222cf68 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Thu, 15 Apr 2021 15:53:01 +0800 Subject: [PATCH] fixed DeployController.DeployCk to store clusters --- controller/clickhouse.go | 30 ++++++-------- controller/deploy.go | 8 +++- go.mod | 5 ++- go.sum | 13 ++++++ service/clickhouse/clickhouse_service.go | 52 +++++++++++------------- service/nacos/nacos.go | 7 ++-- 6 files changed, 63 insertions(+), 52 deletions(-) diff --git a/controller/clickhouse.go b/controller/clickhouse.go index ba3bc117..91c2f17b 100644 --- a/controller/clickhouse.go +++ b/controller/clickhouse.go @@ -42,13 +42,18 @@ func NewClickHouseController(nacosClient *nacos.NacosClient) *ClickHouseControll } func (ck *ClickHouseController) syncDownClusters(c *gin.Context) (err error) { - data, err := ck.nacosClient.GetConfig() + var data string + data, err = ck.nacosClient.GetConfig() if err != nil { model.WrapMsg(c, model.GET_NACOS_CONFIG_FAIL, model.GetMsg(c, model.GET_NACOS_CONFIG_FAIL), err) return } if data != "" { - clickhouse.UpdateLocalCkClusterConfig([]byte(data)) + var updated bool + if updated, err = clickhouse.UpdateLocalCkClusterConfig([]byte(data)); err == nil && updated { + buf, _ := clickhouse.MarshalClusters() + clickhouse.WriteClusterConfigFile(buf) + } } return } @@ -524,6 +529,7 @@ func (ck *ClickHouseController) DestroyCluster(c *gin.Context) { var conf model.CKManClickHouseConfig clusterName := c.Param(ClickHouseClusterPath) + var err error con, ok := clickhouse.CkClusters.Load(clusterName) if !ok { model.WrapMsg(c, model.CLUSTER_NOT_EXIST, model.GetMsg(c, model.CLUSTER_NOT_EXIST), @@ -538,27 +544,17 @@ func (ck *ClickHouseController) DestroyCluster(c *gin.Context) { return } - clickhouse.CkServices.Delete(clusterName) - err := deploy.DestroyCkCluster(&conf) - if err != nil { + if err = deploy.DestroyCkCluster(&conf); err != nil { model.WrapMsg(c, model.DESTROY_CK_CLUSTER_FAIL, model.GetMsg(c, model.DESTROY_CK_CLUSTER_FAIL), err) return } - - data, err := ck.nacosClient.GetConfig() - if err != nil { - model.WrapMsg(c, model.GET_NACOS_CONFIG_FAIL, model.GetMsg(c, model.GET_NACOS_CONFIG_FAIL), err) + if err = ck.syncDownClusters(c); err != nil { return } - if data != "" { - clickhouse.UpdateLocalCkClusterConfig([]byte(data)) + clickhouse.CkServices.Delete(clusterName) + if err = ck.syncUpClusters(c); err != nil { + return } - clickhouse.CkClusters.Delete(clusterName) - clickhouse.AddCkClusterConfigVersion() - buf, _ := clickhouse.MarshalClusters() - clickhouse.WriteClusterConfigFile(buf) - ck.nacosClient.PublishConfig(string(buf)) - model.WrapMsg(c, model.SUCCESS, model.GetMsg(c, model.SUCCESS), nil) } diff --git a/controller/deploy.go b/controller/deploy.go index c7632abd..b5407fd2 100644 --- a/controller/deploy.go +++ b/controller/deploy.go @@ -65,7 +65,11 @@ func (d *DeployController) syncDownClusters(c *gin.Context) (err error) { return } if data != "" { - clickhouse.UpdateLocalCkClusterConfig([]byte(data)) + var updated bool + if updated, err = clickhouse.UpdateLocalCkClusterConfig([]byte(data)); err == nil && updated { + buf, _ := clickhouse.MarshalClusters() + clickhouse.WriteClusterConfigFile(buf) + } } return } @@ -135,7 +139,7 @@ func (d *DeployController) DeployCk(c *gin.Context) { return } clickhouse.CkClusters.Store(req.ClickHouse.ClusterName, conf) - if err = d.syncDownClusters(c); err != nil { + if err = d.syncUpClusters(c); err != nil { return } model.WrapMsg(c, model.SUCCESS, model.GetMsg(c, model.SUCCESS), nil) diff --git a/go.mod b/go.mod index ea201055..8486f6f1 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/pkg/sftp v1.12.0 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.18.0 + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.8.0 github.com/spf13/cobra v1.1.3 github.com/stretchr/testify v1.6.1 @@ -34,8 +35,8 @@ require ( github.com/txn2/txeh v1.3.0 go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 - golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 // indirect - golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 // indirect + golang.org/x/net v0.0.0-20210414194228-064579744ee0 // indirect + golang.org/x/sys v0.0.0-20210415045647-66c3f260301c // indirect golang.org/x/tools v0.1.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/sevlyar/go-daemon.v0 v0.1.5 diff --git a/go.sum b/go.sum index 1cbf645f..82c281f4 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,10 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -112,6 +114,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= @@ -452,11 +455,15 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -520,7 +527,9 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/ugorji/go/codec v1.1.13 h1:013LbFhocBoIqgHeIHKlV4JWYhqogATYWZhIcH0WHn4= github.com/ugorji/go/codec v1.1.13/go.mod h1:oNVt3Dq+FO91WNQ/9JnHKQP2QJxTzoN7wCBFCq1OeuU= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -612,6 +621,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 h1:4qWs8cYYH6PoEFy4dfhDFgoMGkwAcETd+MmPdCPMzUc= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210414194228-064579744ee0 h1:iqW3Mjl/6IP9cHJC/wdiIu3lyBDMUfDElRMyFlqbtiQ= +golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -659,6 +670,8 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210415045647-66c3f260301c h1:6L+uOeS3OQt/f4eFHXZcTxeZrGCuz+CLElgEBjbcTA4= +golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/service/clickhouse/clickhouse_service.go b/service/clickhouse/clickhouse_service.go index b26e3fac..0d4763dc 100644 --- a/service/clickhouse/clickhouse_service.go +++ b/service/clickhouse/clickhouse_service.go @@ -151,22 +151,35 @@ func UnmarshalClusters(data []byte) (map[string]interface{}, error) { return clustersMap, nil } -func MergeCkClusterConfig(src map[string]interface{}) error { - if src == nil { - return nil +func AddCkClusterConfigVersion() { + version, ok := CkClusters.Load(ClickHouseConfigVersionKey) + if !ok { + CkClusters.Store(ClickHouseConfigVersionKey, 0) } - srcVersion, ok := src[ClickHouseConfigVersionKey] + CkClusters.Store(ClickHouseConfigVersionKey, version.(int)+1) +} + +func UpdateLocalCkClusterConfig(data []byte) (updated bool, err error) { + var ck map[string]interface{} + ck, err = UnmarshalClusters(data) + if err != nil || ck == nil { + return + } + + srcVersion, ok := ck[ClickHouseConfigVersionKey] if !ok { - return errors.Errorf("can't find source version") + err = errors.Errorf("can't find source version") + return } dstVersion, ok := CkClusters.Load(ClickHouseConfigVersionKey) if !ok { - return errors.Errorf("can't find version") + err = errors.Errorf("can't find version") + return } if int(srcVersion.(float64)) <= dstVersion.(int) { - return nil + return } // delete old CkClusters config @@ -176,7 +189,7 @@ func MergeCkClusterConfig(src map[string]interface{}) error { }) // merge new CkClusters config - for key, value := range src { + for key, value := range ck { v, ok := value.(map[string]interface{}) if ok { var conf model.CKManClickHouseConfig @@ -188,25 +201,8 @@ func MergeCkClusterConfig(src map[string]interface{}) error { } } - return nil -} - -func AddCkClusterConfigVersion() { - version, ok := CkClusters.Load(ClickHouseConfigVersionKey) - if !ok { - CkClusters.Store(ClickHouseConfigVersionKey, 0) - } - - CkClusters.Store(ClickHouseConfigVersionKey, version.(int)+1) -} - -func UpdateLocalCkClusterConfig(data []byte) error { - ck, err := UnmarshalClusters(data) - if err != nil { - return err - } - - return MergeCkClusterConfig(ck) + updated = true + return } func ParseCkClusterConfigFile() error { @@ -217,7 +213,7 @@ func ParseCkClusterConfigFile() error { if data != nil { CkClusters.Store(ClickHouseConfigVersionKey, -1) - err := UpdateLocalCkClusterConfig(data) + _, err := UpdateLocalCkClusterConfig(data) if err != nil { return err } diff --git a/service/nacos/nacos.go b/service/nacos/nacos.go index a34cba5e..f7ac48a3 100644 --- a/service/nacos/nacos.go +++ b/service/nacos/nacos.go @@ -240,9 +240,10 @@ func (c *NacosClient) ListenConfig() error { func ListenConfigCallback(namespace, group, dataId, data string) { if data != "" { - clickhouse.UpdateLocalCkClusterConfig([]byte(data)) - buf, _ := clickhouse.MarshalClusters() - clickhouse.WriteClusterConfigFile(buf) + if updated, err := clickhouse.UpdateLocalCkClusterConfig([]byte(data)); err == nil && updated { + buf, _ := clickhouse.MarshalClusters() + clickhouse.WriteClusterConfigFile(buf) + } } }