From 561e0f6f84e3b7011c69d257e204c316354e581e Mon Sep 17 00:00:00 2001 From: zianazhao Date: Wed, 31 Aug 2022 11:21:43 +0800 Subject: [PATCH 01/10] dts --- go.mod | 3 +- go.sum | 5 + pkg/client/client.go | 15 ++- pkg/collector/handler_dts.go | 228 +++++++++++++++++++++++++++++++++ pkg/config/config.go | 1 + pkg/instance/cache.go | 2 + pkg/instance/instance_dts.go | 36 ++++++ pkg/instance/repository_dts.go | 159 +++++++++++++++++++++++ pkg/metric/repository.go | 2 + 9 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 pkg/collector/handler_dts.go create mode 100644 pkg/instance/instance_dts.go create mode 100644 pkg/instance/repository_dts.go diff --git a/go.mod b/go.mod index be60410..75b0c4f 100644 --- a/go.mod +++ b/go.mod @@ -15,11 +15,12 @@ require ( github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ckafka v1.0.334 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb v1.0.334 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cmq v1.0.334 - github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437 + github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.479 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.413 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb v1.0.334 + github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts v1.0.479 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.334 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lighthouse v1.0.334 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mariadb v1.0.334 diff --git a/go.sum b/go.sum index ac0ab7b..d15025d 100644 --- a/go.sum +++ b/go.sum @@ -211,6 +211,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tencentcloud/tencentcloud-sdk-go v1.0.485 h1:l1xeZUO2ddcouOpxO7dhkXcmcYembuCG5EZ6O+Cox/o= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cbs v1.0.334 h1:GijOjoDBcWXtra6hmzpj4IXOahWmsTE3bwpOcp5VBDw= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cbs v1.0.334/go.mod h1:PTp058qpOV//RukBVdYQT962rZg71lIt6eHLK1zdvEc= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdb v1.0.413 h1:6cQPfHc3I1pKUj23csRd9P/xxUPcnGRLRK2e5NGqtzc= @@ -228,6 +229,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.413/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.430/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437 h1:Qa0q6mhOkQ/0RoAqOheJAzPFksAf3jsjOZVt4FGcvyQ= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.479 h1:3kwDb6p1J3LxmwnNgSSEheemPffo+vMewoDzKysYdig= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.479/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334 h1:ulfSODMy8rpKa8MfnTIPbe5HyOArnlB4RJ1qmpj09to= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334/go.mod h1:AqyM/ZZMD7q5mHBqNY9YImbSpEpoEe7E/vrTbUWX+po= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.413 h1:MomwSkFrSLB16s51Yu1h4JO+p3Pzc1yesIj+oNwAVM0= @@ -236,6 +239,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334 h1:inkOmQwx github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334/go.mod h1:5WGSrlIZJOhwIqPjjafb6vzrPEZieSHPhPMjjGPXOSU= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb v1.0.334 h1:Xxe889sr1FVhUPPFdZC4Z5IyJObgnGh0ELe+MJH17Mo= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb v1.0.334/go.mod h1:uvcrduqH3pPyyZoZyEX0WXlXIR554ys3ctSsKoeJp64= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts v1.0.479 h1:Tuo8zLhHaO+AxHlk7WOYGs6j/3bfD3FMNB3fWAgIxWM= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts v1.0.479/go.mod h1:CNPxkXeOC/vOmu9a/yaxp+2immIjDL/WsGC/H2rcWQA= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.334 h1:oXOMQ2EmNjqMfpShQeCrWS5QH2MwrAhJfd0rLJOxccE= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.334/go.mod h1:UegCt4vv9jAlzpgDu31ZJTuRP5T2BTV8w+jZBTsZIzg= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= diff --git a/pkg/client/client.go b/pkg/client/client.go index 4dcd8d9..a3b457f 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -15,6 +15,7 @@ import ( cynosdb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb/v20190107" dc "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" dcdb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb/v20180411" + dts "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts/v20180330" es "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es/v20180416" lh "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lighthouse/v20200324" mariadb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mariadb/v20170312" @@ -257,14 +258,24 @@ func NewCosClient(cred common.CredentialIface, conf *config.TencentConfig) (*cos // 用于Get Service 查询, service域名暂时只支持外网 su, _ := url.Parse("http://cos." + conf.Credential.Region + ".myqcloud.com") b := &cos.BaseURL{BucketURL: nil, ServiceURL: su} - //client := cos.NewClient(b, &http.Client{ + // client := cos.NewClient(b, &http.Client{ // Transport: &cos.AuthorizationTransport{ // SecretID: conf.Credential.AccessKey, // SecretKey: conf.Credential.SecretKey, // }, - //}) + // }) client := cos.NewClient(b, &http.Client{ Transport: common.NewCredentialTransport(cred.GetRole()), }) return client, nil } + +func NewDTSClient(cred common.CredentialIface, conf *config.TencentConfig) (*dts.Client, error) { + cpf := profile.NewClientProfile() + if conf.Credential.IsInternal == true { + cpf.HttpProfile.Endpoint = "dts.internal.tencentcloudapi.com" + } else { + cpf.HttpProfile.Endpoint = "dts.tencentcloudapi.com" + } + return dts.NewClient(cred, conf.Credential.Region, cpf) +} diff --git a/pkg/collector/handler_dts.go b/pkg/collector/handler_dts.go new file mode 100644 index 0000000..44f8e8a --- /dev/null +++ b/pkg/collector/handler_dts.go @@ -0,0 +1,228 @@ +package collector + +import ( + "fmt" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/tencentyun/tencentcloud-exporter/pkg/common" + "github.com/tencentyun/tencentcloud-exporter/pkg/instance" + "github.com/tencentyun/tencentcloud-exporter/pkg/metric" + "github.com/tencentyun/tencentcloud-exporter/pkg/util" +) + +const ( + DTSNamespace = "QCE/DTS" + DTSInstanceidKey = "SubscribeId" +) + +func init() { + registerHandler(DTSNamespace, defaultHandlerEnabled, NewDTSHandler) +} + +type dtsHandler struct { + baseProductHandler + // replicationRepo instance.DtsTcInstanceReplicationsRepository + migrateInfosRepo instance.DtsTcInstanceMigrateInfosRepository +} + +func (h *dtsHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { + return true +} + +func (h *dtsHandler) GetNamespace() string { + return DTSNamespace +} + +func (h *dtsHandler) IsMetricVaild(m *metric.TcmMetric) bool { + _, ok := excludeMetricName[m.Meta.MetricName] + if ok { + return false + } + p, err := m.Meta.GetPeriod(m.Conf.StatPeriodSeconds) + if err != nil { + return false + } + if p != m.Conf.StatPeriodSeconds { + return false + } + return true +} + +func (h *dtsHandler) GetSeries(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + if m.Conf.IsIncludeOnlyInstance() { + return h.GetSeriesByOnly(m) + } + + if m.Conf.IsIncludeAllInstance() { + return h.GetSeriesByAll(m) + } + + if m.Conf.IsCustomQueryDimensions() { + return h.GetSeriesByCustom(m) + } + + return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions") +} + +func (h *dtsHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, insId := range m.Conf.OnlyIncludeInstances { + ins, err := h.collector.InstanceRepo.Get(insId) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "id", insId) + continue + } + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *dtsHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) + if err != nil { + return nil, err + } + for _, ins := range insList { + if len(m.Conf.ExcludeInstances) != 0 && util.IsStrInList(m.Conf.ExcludeInstances, ins.GetInstanceId()) { + continue + } + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *dtsHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, ql := range m.Conf.CustomQueryDimensions { + v, ok := ql[h.monitorQueryKey] + if !ok { + level.Error(h.logger).Log( + "msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey), + "ql", fmt.Sprintf("%v", ql)) + continue + } + ins, err := h.collector.InstanceRepo.Get(v) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v) + continue + } + + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *dtsHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var dimensions []string + for _, v := range m.Meta.SupportDimensions { + dimensions = append(dimensions, v) + fmt.Println(v) + } + if util.IsStrInList(dimensions, "replicationjobid") { + return h.getReplicationSeries(m, ins) + } else if util.IsStrInList(dimensions, "migratejobid") { + return h.getMigrateInfoSeries(m, ins) + } else { + return h.getInstanceSeries(m, ins) + } +} + +func (h *dtsHandler) getInstanceSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + series = append(series, s) + + return series, nil +} + +func (h *dtsHandler) getReplicationSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + // replicationResp, err := h.namespaceRepo.GetRocketMQNamespacesInfo(ins.GetInstanceId()) + // if err != nil { + // return nil, err + // } + // for _, namespace := range namespacesResp.Response.Namespaces { + // topicsResp, err := h.topicRepo.GetRocketMQTopicsInfo(ins.GetInstanceId(), *namespace.NamespaceId) + // if err != nil { + // return nil, err + // } + // for _, topic := range topicsResp.Response.Topics { + // ql := map[string]string{ + // "tenantId": ins.GetMonitorQueryKey(), + // "environmentId": *namespace.NamespaceId, + // "topicName": *topic.Name, + // } + // s, err := metric.NewTcmSeries(m, ql, ins) + // if err != nil { + // return nil, err + // } + // series = append(series, s) + // } + // } + return series, nil +} +func (h *dtsHandler) getMigrateInfoSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + migrateInfos, err := h.migrateInfosRepo.GetMigrateInfosInfo() + if err != nil { + return nil, err + } + for _, migrateInfo := range migrateInfos.Response.JobList { + ql := map[string]string{ + "migratejob_id": *migrateInfo.JobId, + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + series = append(series, s) + + } + return series, nil +} + +func NewDTSHandler(cred common.CredentialIface, c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { + migrateInfosRepoCahe, err := instance.NewDtsTcInstanceMigrateInfosRepository(cred, c.Conf, logger) + if err != nil { + return nil, err + } + // relodInterval := time.Duration(c.ProductConf.RelodIntervalMinutes * int64(time.Minute)) + // migrateInfosRepoCahe := instance.NewDtsTcInstanceMigrateInfosCache(migrateInfoRepo, relodInterval, logger) + + handler = &dtsHandler{ + baseProductHandler: baseProductHandler{ + monitorQueryKey: DTSInstanceidKey, + collector: c, + logger: logger, + }, + migrateInfosRepo: migrateInfosRepoCahe, + } + return + +} diff --git a/pkg/config/config.go b/pkg/config/config.go index a22c8af..c64e816 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -63,6 +63,7 @@ var ( "vpngw": "QCE/VPNGW", // for vpngw "vpnx": "QCE/VPNX", // for vpnx "cynosdb_mysql": "QCE/CYNOSDB_MYSQL", // for cynosdb_mysql + "dts": "QCE/DTS", // for dts } SupportStatisticsTypes = map[string]bool{ diff --git a/pkg/instance/cache.go b/pkg/instance/cache.go index 516fa16..9f5ca81 100644 --- a/pkg/instance/cache.go +++ b/pkg/instance/cache.go @@ -401,3 +401,5 @@ func NewTcNacosInstanceInterfaceCache(repo NacosTcInstanceInterfaceRepository, r } return cache } + + diff --git a/pkg/instance/instance_dts.go b/pkg/instance/instance_dts.go new file mode 100644 index 0000000..bd32651 --- /dev/null +++ b/pkg/instance/instance_dts.go @@ -0,0 +1,36 @@ +package instance + +import ( + "fmt" + "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts/v20180330" +) + +type DtsTcInstance struct { + baseTcInstance + meta *sdk.SubscribeInfo + subscribeMeta *sdk.SubscribeInfo + migrateInfoMeta *sdk.MigrateJobInfo +} + +func (ins *DtsTcInstance) GetMeta() interface{} { + return ins.meta +} + +func NewDtsTcInstance(instanceId string, meta *sdk.SubscribeInfo) (ins *DtsTcInstance, err error) { + if instanceId == "" { + return nil, fmt.Errorf("instanceId is empty ") + } + if meta == nil { + return nil, fmt.Errorf("meta is empty ") + } + ins = &DtsTcInstance{ + baseTcInstance: baseTcInstance{ + instanceId: instanceId, + value: reflect.ValueOf(*meta), + }, + meta: meta, + } + return +} diff --git a/pkg/instance/repository_dts.go b/pkg/instance/repository_dts.go new file mode 100644 index 0000000..22c598a --- /dev/null +++ b/pkg/instance/repository_dts.go @@ -0,0 +1,159 @@ +package instance + +import ( + "fmt" + + "github.com/tencentyun/tencentcloud-exporter/pkg/config" + + selfcommon "github.com/tencentyun/tencentcloud-exporter/pkg/common" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts/v20180330" + + "github.com/tencentyun/tencentcloud-exporter/pkg/client" +) + +func init() { + registerRepository("QCE/DTS", NewDTSTcInstanceRepository) +} + +type DTSTcInstanceRepository struct { + client *sdk.Client + logger log.Logger +} + +func (repo *DTSTcInstanceRepository) GetInstanceKey() string { + return "InstanceId" +} + +func (repo *DTSTcInstanceRepository) Get(id string) (instance TcInstance, err error) { + req := sdk.NewDescribeSubscribesRequest() + req.InstanceId = &id + resp, err := repo.client.DescribeSubscribes(req) + if err != nil { + return + } + if len(resp.Response.Items) != 1 { + return nil, fmt.Errorf("Response instanceDetails size != 1, id=%s ", id) + } + meta := resp.Response.Items[0] + instance, err = NewDtsTcInstance(id, meta) + if err != nil { + return + } + return +} + +func (repo *DTSTcInstanceRepository) ListByIds(id []string) (instances []TcInstance, err error) { + return +} + +func (repo *DTSTcInstanceRepository) ListByFilters(filters map[string]string) (instances []TcInstance, err error) { + req := sdk.NewDescribeSubscribesRequest() + var offset int64 = 0 + var limit int64 = 100 + var total int64 = -1 + + req.Offset = &offset + req.Limit = &limit + +getMoreInstances: + resp, err := repo.client.DescribeSubscribes(req) + if err != nil { + return + } + if total == -1 { + total = *resp.Response.TotalCount + } + for _, meta := range resp.Response.Items { + ins, e := NewDtsTcInstance(*meta.SubscribeId, meta) + if e != nil { + level.Error(repo.logger).Log("msg", "Create Dts instance fail", "id", *meta.SubscribeId) + continue + } + instances = append(instances, ins) + } + offset += limit + if offset < total { + req.Offset = &offset + goto getMoreInstances + } + + return +} + +// Replications +// type DtsTcInstanceReplicationsRepository interface { +// GetReplicationsInfo(instanceId string) (*sdk.DescribeRocketMQNamespacesResponse, error) +// } +// +// type DtsTcInstanceReplicationsRepositoryImpl struct { +// client *sdk.Client +// logger log.Logger +// } +// +// func (repo *DtsTcInstanceReplicationsRepositoryImpl) GetReplicationsInfo(instanceId string) (*sdk.DescribeRocketMQNamespacesResponse, error) { +// req := sdk.NewDescribeRocketMQNamespacesRequest() +// var offset uint64 = 0 +// var limit uint64 = 100 +// req.Limit = &limit +// req.Offset = &offset +// req.ClusterId = common.StringPtr(instanceId) +// return repo.client.DescribeRocketMQNamespaces(req) +// } +// +// func NewDtsTcInstanceReplicationsRepository(cred selfcommon.CredentialIface, c *config.TencentConfig, logger log.Logger) (TdmqTcInstanceRocketMQNameSpacesRepository, error) { +// cli, err := client.NewTDMQClient(cred, c) +// if err != nil { +// return nil, err +// } +// repo := &TdmqTcInstanceRocketMQNameSpacesRepositoryImpl{ +// client: cli, +// logger: logger, +// } +// return repo, nil +// } + +// MigrateInfos +type DtsTcInstanceMigrateInfosRepository interface { + GetMigrateInfosInfo() (*sdk.DescribeMigrateJobsResponse, error) +} + +type DtsTcInstanceMigrateInfosRepositoryImpl struct { + client *sdk.Client + logger log.Logger +} + +func (repo *DtsTcInstanceMigrateInfosRepositoryImpl) GetMigrateInfosInfo() (*sdk.DescribeMigrateJobsResponse, error) { + req := sdk.NewDescribeMigrateJobsRequest() + var offset uint64 = 0 + var limit uint64 = 100 + req.Limit = &limit + req.Offset = &offset + return repo.client.DescribeMigrateJobs(req) +} + +func NewDtsTcInstanceMigrateInfosRepository(cred selfcommon.CredentialIface, c *config.TencentConfig, logger log.Logger) (DtsTcInstanceMigrateInfosRepository, error) { + cli, err := client.NewDTSClient(cred, c) + if err != nil { + return nil, err + } + repo := &DtsTcInstanceMigrateInfosRepositoryImpl{ + client: cli, + logger: logger, + } + return repo, nil +} + +func NewDTSTcInstanceRepository(cred selfcommon.CredentialIface, c *config.TencentConfig, logger log.Logger) (repo TcInstanceRepository, err error) { + cli, err := client.NewDTSClient(cred, c) + if err != nil { + return + } + repo = &DTSTcInstanceRepository{ + client: cli, + logger: logger, + } + return +} diff --git a/pkg/metric/repository.go b/pkg/metric/repository.go index 0a68bcb..1898495 100644 --- a/pkg/metric/repository.go +++ b/pkg/metric/repository.go @@ -129,6 +129,7 @@ func (repo *TcmMetricRepositoryImpl) GetSamples(s *TcmSeries, st int64, et int64 } response, err := repo.monitorClient.GetMonitorData(request) + level.Info(repo.logger).Log("request",response.Response.RequestId) if err != nil { return } @@ -175,6 +176,7 @@ func (repo *TcmMetricRepositoryImpl) listSampleByBatch( request := repo.buildGetMonitorDataRequest(m, seriesList, st, et) response, err := repo.monitorClient.GetMonitorData(request) + level.Info(repo.logger).Log("request",response.Response.RequestId) if err != nil { return nil, err } From a94e5470c8b32fed37ebf12de3575bfced1be759 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:27:17 +0800 Subject: [PATCH 02/10] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 437f78f..9e3fe77 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # 腾讯云监控 Exporter v2 +腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费，需要手动开通 API 付费，详见计费说明与开通指引：https://cloud.tencent.com/document/product/248/77914 + 通过qcloud exporter将云监控支持的产品监控指标自动批量导出 (`兼容v1版本`) From 2a92191cb9dbddd6cd510194619f8300ee5e8674 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:28:37 +0800 Subject: [PATCH 03/10] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e3fe77..970808f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # 腾讯云监控 Exporter v2 -腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费，需要手动开通 API 付费，详见计费说明与开通指引：https://cloud.tencent.com/document/product/248/77914 +腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费，需要手动开通 API 付费，详见计费说明与开通指引。 +开通页面：https://buy.cloud.tencent.com/APIRequestBuy +资源消耗页：https://console.cloud.tencent.com/monitor/consumer/products +计费文档：https://cloud.tencent.com/document/product/248/77914 通过qcloud exporter将云监控支持的产品监控指标自动批量导出 (`兼容v1版本`) From 04568b7a6d437439cd5f2613cf32989d382f44d3 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:30:42 +0800 Subject: [PATCH 04/10] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 970808f..bd5714c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # 腾讯云监控 Exporter v2 -腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费，需要手动开通 API 付费，详见计费说明与开通指引。 -开通页面：https://buy.cloud.tencent.com/APIRequestBuy -资源消耗页：https://console.cloud.tencent.com/monitor/consumer/products -计费文档：https://cloud.tencent.com/document/product/248/77914 +腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费，需要手动开通 API 付费，详见计费说明与开通指引。 +开通页面：https://buy.cloud.tencent.com/APIRequestBuy +资源消耗页：https://console.cloud.tencent.com/monitor/consumer/products +计费文档：https://cloud.tencent.com/document/product/248/77914 通过qcloud exporter将云监控支持的产品监控指标自动批量导出 (`兼容v1版本`) From 8408b962660d6783d77b09c2c3c8dc1bde886a6b Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:34:28 +0800 Subject: [PATCH 05/10] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 437f78f..bd5714c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # 腾讯云监控 Exporter v2 +腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费，需要手动开通 API 付费，详见计费说明与开通指引。 +开通页面：https://buy.cloud.tencent.com/APIRequestBuy +资源消耗页：https://console.cloud.tencent.com/monitor/consumer/products +计费文档：https://cloud.tencent.com/document/product/248/77914 + 通过qcloud exporter将云监控支持的产品监控指标自动批量导出 (`兼容v1版本`) From 8ab33f8f457f3e788e7df5fb75ca006de9b1ab51 Mon Sep 17 00:00:00 2001 From: zianazhao Date: Sun, 4 Sep 2022 20:33:45 +0800 Subject: [PATCH 06/10] for vbc --- .../cachedtransactiongather.go | 6 +- .../cachedtransactiongather_test.go | 196 ++++++++++++++++++ pkg/collector/handler_dcg.go | 55 +++++ pkg/collector/handler_dts.go | 25 +-- pkg/collector/handler_vbc.go | 178 ++++++++++++++++ pkg/config/config.go | 2 + pkg/instance/instance_dcg.go | 34 +++ pkg/instance/instance_vbc.go | 34 +++ pkg/instance/repository_dcg.go | 94 +++++++++ pkg/instance/repository_vbc.go | 94 +++++++++ pkg/metric/metric.go | 1 - 11 files changed, 698 insertions(+), 21 deletions(-) create mode 100644 pkg/cachedtransactiongather/cachedtransactiongather_test.go create mode 100644 pkg/collector/handler_dcg.go create mode 100644 pkg/collector/handler_vbc.go create mode 100644 pkg/instance/instance_dcg.go create mode 100644 pkg/instance/instance_vbc.go create mode 100644 pkg/instance/repository_dcg.go create mode 100644 pkg/instance/repository_vbc.go diff --git a/pkg/cachedtransactiongather/cachedtransactiongather.go b/pkg/cachedtransactiongather/cachedtransactiongather.go index 7938c38..e4d0b62 100644 --- a/pkg/cachedtransactiongather/cachedtransactiongather.go +++ b/pkg/cachedtransactiongather/cachedtransactiongather.go @@ -40,12 +40,10 @@ type cachedTransactionGather struct { } func (c *cachedTransactionGather) Gather() ([]*io_prometheus_client.MetricFamily, func(), error) { - c.lock.RLock() + c.lock.Lock() shouldGather := time.Now().After(c.nextCollectionTime) - c.lock.RUnlock() if shouldGather { begin := time.Now() - c.lock.Lock() c.nextCollectionTime = c.nextCollectionTime.Add(c.cacheInterval) metrics, done, err := c.gather.Gather() if err != nil { @@ -60,6 +58,8 @@ func (c *cachedTransactionGather) Gather() ([]*io_prometheus_client.MetricFamily c.lock.Unlock() duration := time.Since(begin) level.Info(c.logger).Log("msg", "Collect all products done", "duration_seconds", duration.Seconds()) + } else { + c.lock.Unlock() } c.lock.RLock() defer c.lock.RUnlock() diff --git a/pkg/cachedtransactiongather/cachedtransactiongather_test.go b/pkg/cachedtransactiongather/cachedtransactiongather_test.go new file mode 100644 index 0000000..79e5d5b --- /dev/null +++ b/pkg/cachedtransactiongather/cachedtransactiongather_test.go @@ -0,0 +1,196 @@ +package cachedtransactiongather + +import ( + "fmt" + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/prometheus/common/promlog" + "sort" + "sync" + "testing" + "time" +) + +type mockGatherer struct { + sleepUntil time.Duration +} + +func (m mockGatherer) Gather() ([]*io_prometheus_client.MetricFamily, error) { + fmt.Println("start gather: " + m.sleepUntil.String()) + time.Sleep(m.sleepUntil) + fmt.Println("end gather: " + m.sleepUntil.String()) + return []*io_prometheus_client.MetricFamily{}, nil +} + +func newMockGatherer(duration time.Duration) prometheus.Gatherer { + return &mockGatherer{ + sleepUntil: duration, + } +} + +type multiTRegistry struct { + tGatherers []prometheus.TransactionalGatherer +} + +func newMultiConcurrencyRegistry(tGatherers ...prometheus.TransactionalGatherer) *multiTRegistry { + return &multiTRegistry{ + tGatherers: tGatherers, + } +} + +// Gather implements TransactionalGatherer interface. +func (r *multiTRegistry) Gather() (mfs []*io_prometheus_client.MetricFamily, done func(), err error) { + dFns := make([]func(), 0, len(r.tGatherers)) + wait := sync.WaitGroup{} + wait.Add(len(r.tGatherers)) + for i := range r.tGatherers { + go func(i int) { + _, _, _ = r.tGatherers[i].Gather() + wait.Done() + }(i) + } + wait.Wait() + + sort.Slice(mfs, func(i, j int) bool { + return *mfs[i].Name < *mfs[j].Name + }) + return mfs, func() { + for _, d := range dFns { + d() + } + }, nil +} + +func TestCache(t *testing.T) { + promlogConfig := &promlog.Config{} + cacheInterval := 60 * time.Second + logger := promlog.New(promlogConfig) + + t.Run("gather with multiple calls should not error", func(t *testing.T) { + gather := NewCachedTransactionGather( + newMultiConcurrencyRegistry( + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*40)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*23)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*7)), + ), + cacheInterval, logger, + ) + wait := sync.WaitGroup{} + wait.Add(10) + for range [10]int{} { + go func() { + begin := time.Now() + mfs, done, err := gather.Gather() + defer done() + if err != nil { + logger.Log("err", err) + t.Errorf("gather error: %v", err) + } + logger.Log("mfs", mfs, "done", "err", err) + if time.Since(begin) > cacheInterval { + t.Errorf("gather cost more than cacheInterval %v", time.Since(begin).String()) + } + wait.Done() + }() + } + wait.Wait() + }) + + t.Run("gather success", func(t *testing.T) { + gather := NewCachedTransactionGather( + newMultiConcurrencyRegistry( + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*40)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*23)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*7)), + ), + cacheInterval, logger, + ) + wait := sync.WaitGroup{} + wait.Add(3) + go func() { + mfs, done, err := gather.Gather() + defer done() + if err != nil { + logger.Log("err", err) + t.Errorf("gather error: %v", err) + } + logger.Log("mfs", mfs, "done", "err", err) + wait.Done() + }() + go func() { + mfs, done, err := gather.Gather() + defer done() + if err != nil { + logger.Log("err", err) + t.Errorf("gather error: %v", err) + } + logger.Log("mfs", mfs, "done", "err", err) + wait.Done() + }() + go func() { + mfs, done, err := gather.Gather() + defer done() + if err != nil { + logger.Log("err", err) + t.Errorf("gather error: %v", err) + } + logger.Log("mfs", mfs, "done", "err", err) + wait.Done() + }() + wait.Wait() + }) + + t.Run("gather with 5s step", func(t *testing.T) { + gather := NewCachedTransactionGather( + newMultiConcurrencyRegistry( + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*40)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*23)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*7)), + ), + cacheInterval, logger, + ) + wait := sync.WaitGroup{} + wait.Add(10) + for range [10]int{} { + time.Sleep(time.Second * 5) + go func() { + mfs, done, err := gather.Gather() + defer done() + if err != nil { + logger.Log("err", err) + t.Errorf("gather error: %v", err) + } + logger.Log("mfs", mfs, "done", "err", err) + wait.Done() + }() + } + wait.Wait() + }) + + t.Run("gather with 65s step", func(t *testing.T) { + gather := NewCachedTransactionGather( + newMultiConcurrencyRegistry( + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*40)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*23)), + prometheus.ToTransactionalGatherer(newMockGatherer(time.Second*7)), + ), + cacheInterval, logger, + ) + wait := sync.WaitGroup{} + wait.Add(3) + for range [3]int{} { + time.Sleep(time.Second * 65) + go func() { + mfs, done, err := gather.Gather() + defer done() + if err != nil { + logger.Log("err", err) + t.Errorf("gather error: %v", err) + } + logger.Log("mfs", mfs, "done", "err", err) + wait.Done() + }() + } + wait.Wait() + }) +} diff --git a/pkg/collector/handler_dcg.go b/pkg/collector/handler_dcg.go new file mode 100644 index 0000000..c0d645c --- /dev/null +++ b/pkg/collector/handler_dcg.go @@ -0,0 +1,55 @@ +package collector + +import ( + "github.com/go-kit/log" + "github.com/tencentyun/tencentcloud-exporter/pkg/common" + "github.com/tencentyun/tencentcloud-exporter/pkg/metric" +) + +const ( + DcgNamespace = "QCE/DCG" + DcgInstanceidKey = "directConnectGatewayId" +) + +func init() { + registerHandler(DcgNamespace, defaultHandlerEnabled, NewDcgHandler) +} + +type DcgHandler struct { + baseProductHandler +} + +func (h *DcgHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { + return true +} + +func (h *DcgHandler) GetNamespace() string { + return DcgNamespace +} + +func (h *DcgHandler) IsMetricVaild(m *metric.TcmMetric) bool { + _, ok := excludeMetricName[m.Meta.MetricName] + if ok { + return false + } + p, err := m.Meta.GetPeriod(m.Conf.StatPeriodSeconds) + if err != nil { + return false + } + if p != m.Conf.StatPeriodSeconds { + return false + } + return true +} + +func NewDcgHandler(cred common.CredentialIface, c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { + handler = &tdmqHandler{ + baseProductHandler: baseProductHandler{ + monitorQueryKey: DcgInstanceidKey, + collector: c, + logger: logger, + }, + } + return + +} diff --git a/pkg/collector/handler_dts.go b/pkg/collector/handler_dts.go index 44f8e8a..8eab6b0 100644 --- a/pkg/collector/handler_dts.go +++ b/pkg/collector/handler_dts.go @@ -135,7 +135,6 @@ func (h *dtsHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcI var dimensions []string for _, v := range m.Meta.SupportDimensions { dimensions = append(dimensions, v) - fmt.Println(v) } if util.IsStrInList(dimensions, "replicationjobid") { return h.getReplicationSeries(m, ins) @@ -163,27 +162,19 @@ func (h *dtsHandler) getInstanceSeries(m *metric.TcmMetric, ins instance.TcInsta func (h *dtsHandler) getReplicationSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { var series []*metric.TcmSeries - // replicationResp, err := h.namespaceRepo.GetRocketMQNamespacesInfo(ins.GetInstanceId()) + // replications, err := h.replicationsRepo.GetReplicationInfo() // if err != nil { // return nil, err // } - // for _, namespace := range namespacesResp.Response.Namespaces { - // topicsResp, err := h.topicRepo.GetRocketMQTopicsInfo(ins.GetInstanceId(), *namespace.NamespaceId) + // for _, replication := range replications.Response.JobList { + // ql := map[string]string{ + // "replicationjobid": *replication.JobId, + // } + // s, err := metric.NewTcmSeries(m, ql, ins) // if err != nil { // return nil, err // } - // for _, topic := range topicsResp.Response.Topics { - // ql := map[string]string{ - // "tenantId": ins.GetMonitorQueryKey(), - // "environmentId": *namespace.NamespaceId, - // "topicName": *topic.Name, - // } - // s, err := metric.NewTcmSeries(m, ql, ins) - // if err != nil { - // return nil, err - // } - // series = append(series, s) - // } + // series = append(series, s) // } return series, nil } @@ -202,8 +193,8 @@ func (h *dtsHandler) getMigrateInfoSeries(m *metric.TcmMetric, ins instance.TcIn return nil, err } series = append(series, s) - } + return series, nil } diff --git a/pkg/collector/handler_vbc.go b/pkg/collector/handler_vbc.go new file mode 100644 index 0000000..9edf17e --- /dev/null +++ b/pkg/collector/handler_vbc.go @@ -0,0 +1,178 @@ +package collector + +import ( + "fmt" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/tencentyun/tencentcloud-exporter/pkg/common" + "github.com/tencentyun/tencentcloud-exporter/pkg/config" + "github.com/tencentyun/tencentcloud-exporter/pkg/instance" + "github.com/tencentyun/tencentcloud-exporter/pkg/metric" + "github.com/tencentyun/tencentcloud-exporter/pkg/util" +) + +const ( + VbcNamespace = "QCE/VBC" + VbcInstanceidKey = "CcnId" +) + +var conf *config.TencentConfig + +func init() { + registerHandler(VbcNamespace, defaultHandlerEnabled, NewVbcHandler) +} + +type VbcHandler struct { + baseProductHandler +} + +func (h *VbcHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { + return true +} + +func (h *VbcHandler) GetNamespace() string { + return VbcNamespace +} + +func (h *VbcHandler) IsMetricVaild(m *metric.TcmMetric) bool { + + _, ok := excludeMetricName[m.Meta.MetricName] + if ok { + return false + } + p, err := m.Meta.GetPeriod(m.Conf.StatPeriodSeconds) + if err != nil { + return false + } + if p != m.Conf.StatPeriodSeconds { + return false + } + return true +} + +func (h *VbcHandler) GetSeries(m *metric.TcmMetric) (slist []*metric.TcmSeries, err error) { + + if m.Conf.IsIncludeOnlyInstance() { + return h.GetSeriesByOnly(m) + } + + if m.Conf.IsIncludeAllInstance() { + return h.GetSeriesByAll(m) + } + + if m.Conf.IsCustomQueryDimensions() { + return h.GetSeriesByCustom(m) + } + + return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions") +} +func (h *VbcHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, insId := range m.Conf.OnlyIncludeInstances { + ins, err := h.collector.InstanceRepo.Get(insId) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "id", insId) + continue + } + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *VbcHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) + if err != nil { + return nil, err + } + for _, ins := range insList { + if len(m.Conf.ExcludeInstances) != 0 && util.IsStrInList(m.Conf.ExcludeInstances, ins.GetInstanceId()) { + continue + } + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *VbcHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, ql := range m.Conf.CustomQueryDimensions { + if !h.checkMonitorQueryKeys(m, ql) { + continue + } + + s, err := metric.NewTcmSeries(m, ql, nil) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, + "ql", fmt.Sprintf("%v", ql)) + continue + } + slist = append(slist, s) + } + return slist, nil +} +func (h *VbcHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var dimensions []string + for _, v := range m.Meta.SupportDimensions { + dimensions = append(dimensions, v) + } + if util.IsStrInList(dimensions, "DRegion") { + return h.getSingleRegionSeries(m, ins) + } else { + return h.getSingleRegionSeries(m, ins) + } +} +func (h *VbcHandler) getSingleRegionSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), + "SRegion": h.collector.Conf.Credential.Region, + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + series = append(series, s) + + return series, nil +} + +func (h *VbcHandler) getRegionsSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + + return series, nil +} +func (h *VbcHandler) checkMonitorQueryKeys(m *metric.TcmMetric, ql map[string]string) bool { + for k := range ql { + if !util.IsStrInList(m.Meta.SupportDimensions, k) { + level.Error(h.logger).Log("msg", fmt.Sprintf("not found %s in supportQueryDimensions", k), + "ql", fmt.Sprintf("%v", ql)) + return false + } + } + return true +} + +func NewVbcHandler(cred common.CredentialIface, c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { + handler = &VbcHandler{ + baseProductHandler: baseProductHandler{ + monitorQueryKey: VbcInstanceidKey, + collector: c, + logger: logger, + }, + } + return + +} diff --git a/pkg/config/config.go b/pkg/config/config.go index c64e816..0badafb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -64,6 +64,8 @@ var ( "vpnx": "QCE/VPNX", // for vpnx "cynosdb_mysql": "QCE/CYNOSDB_MYSQL", // for cynosdb_mysql "dts": "QCE/DTS", // for dts + "dcg": "QCE/DCG", // for dcg + "vbc": "QCE/VBC", // for vbc } SupportStatisticsTypes = map[string]bool{ diff --git a/pkg/instance/instance_dcg.go b/pkg/instance/instance_dcg.go new file mode 100644 index 0000000..96fe966 --- /dev/null +++ b/pkg/instance/instance_dcg.go @@ -0,0 +1,34 @@ +package instance + +import ( + "fmt" + "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" +) + +type DcgTcInstance struct { + baseTcInstance + meta *sdk.DirectConnectGateway +} + +func (ins *DcgTcInstance) GetMeta() interface{} { + return ins.meta +} + +func NewDcgTcInstance(instanceId string, meta *sdk.DirectConnectGateway) (ins *DcgTcInstance, err error) { + if instanceId == "" { + return nil, fmt.Errorf("instanceId is empty ") + } + if meta == nil { + return nil, fmt.Errorf("meta is empty ") + } + ins = &DcgTcInstance{ + baseTcInstance: baseTcInstance{ + instanceId: instanceId, + value: reflect.ValueOf(*meta), + }, + meta: meta, + } + return +} diff --git a/pkg/instance/instance_vbc.go b/pkg/instance/instance_vbc.go new file mode 100644 index 0000000..cebf09b --- /dev/null +++ b/pkg/instance/instance_vbc.go @@ -0,0 +1,34 @@ +package instance + +import ( + "fmt" + "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" +) + +type VbcTcInstance struct { + baseTcInstance + meta *sdk.CCN +} + +func (ins *VbcTcInstance) GetMeta() interface{} { + return ins.meta +} + +func NewVbcTcInstance(instanceId string, meta *sdk.CCN) (ins *VbcTcInstance, err error) { + if instanceId == "" { + return nil, fmt.Errorf("instanceId is empty ") + } + if meta == nil { + return nil, fmt.Errorf("meta is empty ") + } + ins = &VbcTcInstance{ + baseTcInstance: baseTcInstance{ + instanceId: instanceId, + value: reflect.ValueOf(*meta), + }, + meta: meta, + } + return +} diff --git a/pkg/instance/repository_dcg.go b/pkg/instance/repository_dcg.go new file mode 100644 index 0000000..ddb304d --- /dev/null +++ b/pkg/instance/repository_dcg.go @@ -0,0 +1,94 @@ +package instance + +import ( + "fmt" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" + + "github.com/tencentyun/tencentcloud-exporter/pkg/client" + "github.com/tencentyun/tencentcloud-exporter/pkg/common" + "github.com/tencentyun/tencentcloud-exporter/pkg/config" +) + +func init() { + registerRepository("QCE/DCG", NewDcgTcInstanceRepository) +} + +type DcgTcInstanceRepository struct { + client *sdk.Client + logger log.Logger +} + +func (repo *DcgTcInstanceRepository) GetInstanceKey() string { + return "InstanceId" +} + +func (repo *DcgTcInstanceRepository) Get(id string) (instance TcInstance, err error) { + req := sdk.NewDescribeDirectConnectGatewaysRequest() + // req.Filters.Name = []*string{&id} + resp, err := repo.client.DescribeDirectConnectGateways(req) + if err != nil { + return + } + if len(resp.Response.DirectConnectGatewaySet) != 1 { + return nil, fmt.Errorf("Response instanceDetails size != 1, id=%s ", id) + } + meta := resp.Response.DirectConnectGatewaySet[0] + instance, err = NewDcgTcInstance(id, meta) + if err != nil { + return + } + return +} + +func (repo *DcgTcInstanceRepository) ListByIds(id []string) (instances []TcInstance, err error) { + return +} + +func (repo *DcgTcInstanceRepository) ListByFilters(filters map[string]string) (instances []TcInstance, err error) { + req := sdk.NewDescribeDirectConnectGatewaysRequest() + var offset uint64 = 0 + var limit uint64 = 100 + var total int64 = -1 + + req.Offset = &offset + req.Limit = &limit + +getMoreInstances: + resp, err := repo.client.DescribeDirectConnectGateways(req) + if err != nil { + return + } + if total == -1 { + total = int64(*resp.Response.TotalCount) + } + for _, meta := range resp.Response.DirectConnectGatewaySet { + ins, e := NewDcgTcInstance(*meta.DirectConnectGatewayId, meta) + if e != nil { + level.Error(repo.logger).Log("msg", "Create Dcg instance fail", "id", *meta.DirectConnectGatewayId) + continue + } + instances = append(instances, ins) + } + offset += limit + if offset < uint64(total) { + req.Offset = &offset + goto getMoreInstances + } + + return +} + +func NewDcgTcInstanceRepository(cred common.CredentialIface, c *config.TencentConfig, logger log.Logger) (repo TcInstanceRepository, err error) { + cli, err := client.NewVpvClient(cred, c) + if err != nil { + return + } + repo = &DcgTcInstanceRepository{ + client: cli, + logger: logger, + } + return +} diff --git a/pkg/instance/repository_vbc.go b/pkg/instance/repository_vbc.go new file mode 100644 index 0000000..8cf04bd --- /dev/null +++ b/pkg/instance/repository_vbc.go @@ -0,0 +1,94 @@ +package instance + +import ( + "fmt" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" + + "github.com/tencentyun/tencentcloud-exporter/pkg/client" + "github.com/tencentyun/tencentcloud-exporter/pkg/common" + "github.com/tencentyun/tencentcloud-exporter/pkg/config" +) + +func init() { + registerRepository("QCE/VBC", NewVbcTcInstanceRepository) +} + +type VbcTcInstanceRepository struct { + client *sdk.Client + logger log.Logger +} + +func (repo *VbcTcInstanceRepository) GetInstanceKey() string { + return "InstanceId" +} + +func (repo *VbcTcInstanceRepository) Get(id string) (instance TcInstance, err error) { + req := sdk.NewDescribeCcnsRequest() + // req. + resp, err := repo.client.DescribeCcns(req) + if err != nil { + return + } + if len(resp.Response.CcnSet) != 1 { + return nil, fmt.Errorf("Response instanceDetails size != 1, id=%s ", id) + } + meta := resp.Response.CcnSet[0] + instance, err = NewVbcTcInstance(id, meta) + if err != nil { + return + } + return +} + +func (repo *VbcTcInstanceRepository) ListByIds(id []string) (instances []TcInstance, err error) { + return +} + +func (repo *VbcTcInstanceRepository) ListByFilters(filters map[string]string) (instances []TcInstance, err error) { + req := sdk.NewDescribeCcnsRequest() + var offset uint64 = 0 + var limit uint64 = 100 + var total int64 = -1 + + req.Offset = &offset + req.Limit = &limit + +getMoreInstances: + resp, err := repo.client.DescribeCcns(req) + if err != nil { + return + } + if total == -1 { + total = int64(*resp.Response.TotalCount) + } + for _, meta := range resp.Response.CcnSet { + ins, e := NewVbcTcInstance(*meta.CcnId, meta) + if e != nil { + level.Error(repo.logger).Log("msg", "Create Vbc instance fail", "id", *meta.CcnId) + continue + } + instances = append(instances, ins) + } + offset += limit + if offset < uint64(total) { + req.Offset = &offset + goto getMoreInstances + } + + return +} + +func NewVbcTcInstanceRepository(cred common.CredentialIface, c *config.TencentConfig, logger log.Logger) (repo TcInstanceRepository, err error) { + cli, err := client.NewVpvClient(cred, c) + if err != nil { + return + } + repo = &VbcTcInstanceRepository{ + client: cli, + logger: logger, + } + return +} diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 43b7eae..a120e4f 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -160,7 +160,6 @@ func (m *TcmMetric) GetSeriesSplitByBatch(batch int) (steps [][]*TcmSeries) { // 创建TcmMetric func NewTcmMetric(meta *TcmMeta, conf *TcmMetricConfig) (*TcmMetric, error) { id := fmt.Sprintf("%s-%s", meta.Namespace, meta.MetricName) - labels, err := NewTcmLabels(meta.SupportDimensions, conf.InstanceLabelNames, conf.ConstLabels) if err != nil { return nil, err From 6241285b289bc802665b53f599c3b881b47406d4 Mon Sep 17 00:00:00 2001 From: zianazhao Date: Tue, 6 Sep 2022 19:53:56 +0800 Subject: [PATCH 07/10] dts --- pkg/collector/handler_vbc.go | 28 +++++++++++++++++++++------- pkg/metric/repository.go | 2 -- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pkg/collector/handler_vbc.go b/pkg/collector/handler_vbc.go index 9edf17e..fd9d120 100644 --- a/pkg/collector/handler_vbc.go +++ b/pkg/collector/handler_vbc.go @@ -16,6 +16,14 @@ const ( VbcInstanceidKey = "CcnId" ) +var ( + // BetweenRegionMetricNames = []string{ + // "OutDropBandwidth", "InBandwidthRate", "OutBandwidthRate", "OutDropPkg", "OutDropPkgRate", "InBandwidth", "InPkg", "OutPkg", "OutBandwidth", + // } + SingleRegionMetricNames = []string{ + "Regioninbandwidthbm", "Regionoutbandwidthbm", "Regionoutdropbandwidthbm", "Regioninpkgbm", "Regionoutbandwidthbmrate", "Regionoutdroppkgbmrate", "Regionoutpkgbm", "Regionoutdroppkgbm", + } +) var conf *config.TencentConfig func init() { @@ -124,16 +132,22 @@ func (h *VbcHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries return slist, nil } func (h *VbcHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { - var dimensions []string - for _, v := range m.Meta.SupportDimensions { - dimensions = append(dimensions, v) - } - if util.IsStrInList(dimensions, "DRegion") { + // var dimensions []string + // for _, v := range m.Meta.SupportDimensions { + // dimensions = append(dimensions, v) + // } + // if util.IsStrInList(dimensions, "DRegion") { + // return h.getSingleRegionSeries(m, ins) + // } else { + // return h.getSingleRegionSeries(m, ins) + // } + if util.IsStrInList(SingleRegionMetricNames, m.Meta.MetricName) { return h.getSingleRegionSeries(m, ins) } else { - return h.getSingleRegionSeries(m, ins) + return h.getBetweenRegionSeries(m, ins) } } + func (h *VbcHandler) getSingleRegionSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { var series []*metric.TcmSeries ql := map[string]string{ @@ -149,7 +163,7 @@ func (h *VbcHandler) getSingleRegionSeries(m *metric.TcmMetric, ins instance.TcI return series, nil } -func (h *VbcHandler) getRegionsSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { +func (h *VbcHandler) getBetweenRegionSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { var series []*metric.TcmSeries return series, nil diff --git a/pkg/metric/repository.go b/pkg/metric/repository.go index 1898495..0a68bcb 100644 --- a/pkg/metric/repository.go +++ b/pkg/metric/repository.go @@ -129,7 +129,6 @@ func (repo *TcmMetricRepositoryImpl) GetSamples(s *TcmSeries, st int64, et int64 } response, err := repo.monitorClient.GetMonitorData(request) - level.Info(repo.logger).Log("request",response.Response.RequestId) if err != nil { return } @@ -176,7 +175,6 @@ func (repo *TcmMetricRepositoryImpl) listSampleByBatch( request := repo.buildGetMonitorDataRequest(m, seriesList, st, et) response, err := repo.monitorClient.GetMonitorData(request) - level.Info(repo.logger).Log("request",response.Response.RequestId) if err != nil { return nil, err } From 2a61ed587a3572f969db5d3891b2cfdf160f5550 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:06:25 +0800 Subject: [PATCH 08/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd5714c..d417e96 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ TDMQ RocketMQ 版|QCE/TDMQ|[指标详情](https://cloud.tencent.com/document/pro VPN 网关|QCE/VPNGW|[指标详情](https://cloud.tencent.com/document/product/248/45070) VPN 通道|QCE/VPNX|[指标详情](https://cloud.tencent.com/document/product/248/45071) CYNOSDB_MYSQL|QCE/CYNOSDB_MYSQL|[指标详情](https://cloud.tencent.com/document/product/248/45106) - +云联网|QCE/VBC|[指标详情](https://cloud.tencent.com/document/product/248/75629) `后续会有更多的产品支持` ## 二、快速开始 From 623f39e41abdf884482ae6f44a802cb76dd9c395 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:06:37 +0800 Subject: [PATCH 09/10] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d417e96..b697678 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ VPN 网关|QCE/VPNGW|[指标详情](https://cloud.tencent.com/document/product/2 VPN 通道|QCE/VPNX|[指标详情](https://cloud.tencent.com/document/product/248/45071) CYNOSDB_MYSQL|QCE/CYNOSDB_MYSQL|[指标详情](https://cloud.tencent.com/document/product/248/45106) 云联网|QCE/VBC|[指标详情](https://cloud.tencent.com/document/product/248/75629) + `后续会有更多的产品支持` ## 二、快速开始 From 1208b067cea024e064d5bcaa37ef7c1108619ddf Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:07:46 +0800 Subject: [PATCH 10/10] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b697678..f8fe6d3 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ VPN 网关|QCE/VPNGW|[指标详情](https://cloud.tencent.com/document/product/2 VPN 通道|QCE/VPNX|[指标详情](https://cloud.tencent.com/document/product/248/45071) CYNOSDB_MYSQL|QCE/CYNOSDB_MYSQL|[指标详情](https://cloud.tencent.com/document/product/248/45106) 云联网|QCE/VBC|[指标详情](https://cloud.tencent.com/document/product/248/75629) +数据传输 |QCE/DTS|指标详情说明文档(待上线) +专线网关 |QCE/DCG|指标详情说明文档(待上线) `后续会有更多的产品支持`