Skip to content

Commit

Permalink
Merge branch 'master' into time-format
Browse files Browse the repository at this point in the history
  • Loading branch information
zero3233 authored Sep 7, 2022
2 parents 99a3045 + 6ea2e80 commit 89029a0
Show file tree
Hide file tree
Showing 16 changed files with 949 additions and 2 deletions.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# 腾讯云监控 Exporter v2

腾讯云监控已于2022年09月01日开始对超出免费额度的 API 接口的请求进行计费,需要手动开通 API 付费,详见计费说明与开通指引。
开通页面:https://buy.cloud.tencent.com/APIRequestBuy
资源消耗页:https://console.cloud.tencent.com/monitor/consumer/products
计费文档:https://cloud.tencent.com/document/product/248/77914

通过qcloud exporter将云监控支持的产品监控指标自动批量导出
(`兼容v1版本`)

Expand Down Expand Up @@ -36,6 +41,9 @@ TDMQ RocketMQ 版|QCE/TDMQ|[指标详情](https://cloud.tencent.com/document/pro
VPN 网关|QCE/VPNGW|[指标详情](https://cloud.tencent.com/document/product/248/45070)
VPN 通道|QCE/VPNX|[指标详情](https://cloud.tencent.com/document/product/248/45071)
CYNOSDB_MYSQL|QCE/CYNOSDB_MYSQL|[指标详情](https://cloud.tencent.com/document/product/248/45106)
云联网|QCE/VBC|[指标详情](https://cloud.tencent.com/document/product/248/75629)
数据传输 |QCE/DTS|指标详情说明文档(待上线)
专线网关 |QCE/DCG|指标详情说明文档(待上线)

`后续会有更多的产品支持`

Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@ require (
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ckafka v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cmq v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.479
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.413
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts v1.0.479
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lighthouse v1.0.334
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mariadb v1.0.334
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tencentcloud/tencentcloud-sdk-go v1.0.485 h1:l1xeZUO2ddcouOpxO7dhkXcmcYembuCG5EZ6O+Cox/o=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cbs v1.0.334 h1:GijOjoDBcWXtra6hmzpj4IXOahWmsTE3bwpOcp5VBDw=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cbs v1.0.334/go.mod h1:PTp058qpOV//RukBVdYQT962rZg71lIt6eHLK1zdvEc=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdb v1.0.413 h1:6cQPfHc3I1pKUj23csRd9P/xxUPcnGRLRK2e5NGqtzc=
Expand All @@ -228,6 +229,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.413/go.mod
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.430/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437 h1:Qa0q6mhOkQ/0RoAqOheJAzPFksAf3jsjOZVt4FGcvyQ=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.479 h1:3kwDb6p1J3LxmwnNgSSEheemPffo+vMewoDzKysYdig=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.479/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334 h1:ulfSODMy8rpKa8MfnTIPbe5HyOArnlB4RJ1qmpj09to=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334/go.mod h1:AqyM/ZZMD7q5mHBqNY9YImbSpEpoEe7E/vrTbUWX+po=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.413 h1:MomwSkFrSLB16s51Yu1h4JO+p3Pzc1yesIj+oNwAVM0=
Expand All @@ -236,6 +239,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334 h1:inkOmQwx
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334/go.mod h1:5WGSrlIZJOhwIqPjjafb6vzrPEZieSHPhPMjjGPXOSU=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb v1.0.334 h1:Xxe889sr1FVhUPPFdZC4Z5IyJObgnGh0ELe+MJH17Mo=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb v1.0.334/go.mod h1:uvcrduqH3pPyyZoZyEX0WXlXIR554ys3ctSsKoeJp64=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts v1.0.479 h1:Tuo8zLhHaO+AxHlk7WOYGs6j/3bfD3FMNB3fWAgIxWM=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts v1.0.479/go.mod h1:CNPxkXeOC/vOmu9a/yaxp+2immIjDL/WsGC/H2rcWQA=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.334 h1:oXOMQ2EmNjqMfpShQeCrWS5QH2MwrAhJfd0rLJOxccE=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.334/go.mod h1:UegCt4vv9jAlzpgDu31ZJTuRP5T2BTV8w+jZBTsZIzg=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
Expand Down
12 changes: 12 additions & 0 deletions pkg/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
cynosdb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb/v20190107"
dc "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410"
dcdb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dcdb/v20180411"
dts "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dts/v20180330"
es "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es/v20180416"
lh "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lighthouse/v20200324"
mariadb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mariadb/v20170312"
Expand Down Expand Up @@ -270,5 +271,16 @@ func NewCosClient(cred common.CredentialIface, conf *config.TencentConfig) (*cos
Transport: common.NewCredentialTransport(cred.GetRole()),
})
}

return client, nil
}

func NewDTSClient(cred common.CredentialIface, conf *config.TencentConfig) (*dts.Client, error) {
cpf := profile.NewClientProfile()
if conf.Credential.IsInternal == true {
cpf.HttpProfile.Endpoint = "dts.internal.tencentcloudapi.com"
} else {
cpf.HttpProfile.Endpoint = "dts.tencentcloudapi.com"
}
return dts.NewClient(cred, conf.Credential.Region, cpf)
}
55 changes: 55 additions & 0 deletions pkg/collector/handler_dcg.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package collector

import (
"github.com/go-kit/log"
"github.com/tencentyun/tencentcloud-exporter/pkg/common"
"github.com/tencentyun/tencentcloud-exporter/pkg/metric"
)

const (
DcgNamespace = "QCE/DCG"
DcgInstanceidKey = "directConnectGatewayId"
)

func init() {
registerHandler(DcgNamespace, defaultHandlerEnabled, NewDcgHandler)
}

type DcgHandler struct {
baseProductHandler
}

func (h *DcgHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool {
return true
}

func (h *DcgHandler) GetNamespace() string {
return DcgNamespace
}

func (h *DcgHandler) IsMetricVaild(m *metric.TcmMetric) bool {
_, ok := excludeMetricName[m.Meta.MetricName]
if ok {
return false
}
p, err := m.Meta.GetPeriod(m.Conf.StatPeriodSeconds)
if err != nil {
return false
}
if p != m.Conf.StatPeriodSeconds {
return false
}
return true
}

func NewDcgHandler(cred common.CredentialIface, c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) {
handler = &tdmqHandler{
baseProductHandler: baseProductHandler{
monitorQueryKey: DcgInstanceidKey,
collector: c,
logger: logger,
},
}
return

}
219 changes: 219 additions & 0 deletions pkg/collector/handler_dts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
package collector

import (
"fmt"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/tencentyun/tencentcloud-exporter/pkg/common"
"github.com/tencentyun/tencentcloud-exporter/pkg/instance"
"github.com/tencentyun/tencentcloud-exporter/pkg/metric"
"github.com/tencentyun/tencentcloud-exporter/pkg/util"
)

const (
DTSNamespace = "QCE/DTS"
DTSInstanceidKey = "SubscribeId"
)

func init() {
registerHandler(DTSNamespace, defaultHandlerEnabled, NewDTSHandler)
}

type dtsHandler struct {
baseProductHandler
// replicationRepo instance.DtsTcInstanceReplicationsRepository
migrateInfosRepo instance.DtsTcInstanceMigrateInfosRepository
}

func (h *dtsHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool {
return true
}

func (h *dtsHandler) GetNamespace() string {
return DTSNamespace
}

func (h *dtsHandler) IsMetricVaild(m *metric.TcmMetric) bool {
_, ok := excludeMetricName[m.Meta.MetricName]
if ok {
return false
}
p, err := m.Meta.GetPeriod(m.Conf.StatPeriodSeconds)
if err != nil {
return false
}
if p != m.Conf.StatPeriodSeconds {
return false
}
return true
}

func (h *dtsHandler) GetSeries(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
if m.Conf.IsIncludeOnlyInstance() {
return h.GetSeriesByOnly(m)
}

if m.Conf.IsIncludeAllInstance() {
return h.GetSeriesByAll(m)
}

if m.Conf.IsCustomQueryDimensions() {
return h.GetSeriesByCustom(m)
}

return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions")
}

func (h *dtsHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
var slist []*metric.TcmSeries
for _, insId := range m.Conf.OnlyIncludeInstances {
ins, err := h.collector.InstanceRepo.Get(insId)
if err != nil {
level.Error(h.logger).Log("msg", "Instance not found", "id", insId)
continue
}
sl, err := h.getSeriesByMetricType(m, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail",
"metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, sl...)
}
return slist, nil
}

func (h *dtsHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
var slist []*metric.TcmSeries
insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters)
if err != nil {
return nil, err
}
for _, ins := range insList {
if len(m.Conf.ExcludeInstances) != 0 && util.IsStrInList(m.Conf.ExcludeInstances, ins.GetInstanceId()) {
continue
}
sl, err := h.getSeriesByMetricType(m, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail",
"metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, sl...)
}
return slist, nil
}

func (h *dtsHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
var slist []*metric.TcmSeries
for _, ql := range m.Conf.CustomQueryDimensions {
v, ok := ql[h.monitorQueryKey]
if !ok {
level.Error(h.logger).Log(
"msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey),
"ql", fmt.Sprintf("%v", ql))
continue
}
ins, err := h.collector.InstanceRepo.Get(v)
if err != nil {
level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v)
continue
}

sl, err := h.getSeriesByMetricType(m, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail",
"metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, sl...)
}
return slist, nil
}

func (h *dtsHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) {
var dimensions []string
for _, v := range m.Meta.SupportDimensions {
dimensions = append(dimensions, v)
}
if util.IsStrInList(dimensions, "replicationjobid") {
return h.getReplicationSeries(m, ins)
} else if util.IsStrInList(dimensions, "migratejobid") {
return h.getMigrateInfoSeries(m, ins)
} else {
return h.getInstanceSeries(m, ins)
}
}

func (h *dtsHandler) getInstanceSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) {
var series []*metric.TcmSeries

ql := map[string]string{
h.monitorQueryKey: ins.GetMonitorQueryKey(),
}
s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
return nil, err
}
series = append(series, s)

return series, nil
}

func (h *dtsHandler) getReplicationSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) {
var series []*metric.TcmSeries
// replications, err := h.replicationsRepo.GetReplicationInfo()
// if err != nil {
// return nil, err
// }
// for _, replication := range replications.Response.JobList {
// ql := map[string]string{
// "replicationjobid": *replication.JobId,
// }
// s, err := metric.NewTcmSeries(m, ql, ins)
// if err != nil {
// return nil, err
// }
// series = append(series, s)
// }
return series, nil
}
func (h *dtsHandler) getMigrateInfoSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) {
var series []*metric.TcmSeries
migrateInfos, err := h.migrateInfosRepo.GetMigrateInfosInfo()
if err != nil {
return nil, err
}
for _, migrateInfo := range migrateInfos.Response.JobList {
ql := map[string]string{
"migratejob_id": *migrateInfo.JobId,
}
s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
return nil, err
}
series = append(series, s)
}

return series, nil
}

func NewDTSHandler(cred common.CredentialIface, c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) {
migrateInfosRepoCahe, err := instance.NewDtsTcInstanceMigrateInfosRepository(cred, c.Conf, logger)
if err != nil {
return nil, err
}
// relodInterval := time.Duration(c.ProductConf.RelodIntervalMinutes * int64(time.Minute))
// migrateInfosRepoCahe := instance.NewDtsTcInstanceMigrateInfosCache(migrateInfoRepo, relodInterval, logger)

handler = &dtsHandler{
baseProductHandler: baseProductHandler{
monitorQueryKey: DTSInstanceidKey,
collector: c,
logger: logger,
},
migrateInfosRepo: migrateInfosRepoCahe,
}
return

}
Loading

0 comments on commit 89029a0

Please sign in to comment.