From 681a20aedf9d2f256e4144bc449a374078e9dd72 Mon Sep 17 00:00:00 2001 From: juexiaolin Date: Thu, 10 Dec 2020 22:09:32 +0800 Subject: [PATCH] v2.2.0 (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: product instances support auto reload interval * feat: add redis mem product * docs: add redis mem docs Co-authored-by: juexiaolin(林觉霄) --- cmd/qcloud-exporter/qcloud_exporter.go | 5 +- go.mod | 5 +- go.sum | 13 ++ pkg/collector/collector.go | 27 ++- pkg/collector/handler.go | 164 ++++++++----- pkg/collector/handler_cdb.go | 6 +- pkg/collector/handler_cdn.go | 7 +- pkg/collector/handler_clb.go | 6 +- pkg/collector/handler_clb7.go | 9 +- pkg/collector/handler_cos.go | 7 +- pkg/collector/handler_cvm.go | 9 +- pkg/collector/handler_dc.go | 6 +- pkg/collector/handler_dcx.go | 9 +- pkg/collector/handler_mongo.go | 158 +++++++------ pkg/collector/handler_nat.go | 6 +- pkg/collector/handler_redis.go | 9 +- pkg/collector/handler_redis_mem.go | 222 ++++++++++++++++++ pkg/collector/product.go | 311 ++++++++++++++++--------- pkg/config/config.go | 42 +++- pkg/constant/cache.go | 1 + pkg/constant/instance.go | 7 + pkg/instance/cache.go | 87 +++++-- pkg/instance/cache_test.go | 57 +++++ pkg/instance/instance.go | 6 - pkg/instance/instance_cdb.go | 3 +- pkg/instance/instance_clb.go | 3 +- pkg/instance/instance_cvm.go | 3 +- pkg/instance/instance_dc.go | 3 +- pkg/instance/instance_dcx.go | 3 +- pkg/instance/instance_mongo.go | 3 +- pkg/instance/instance_nat.go | 3 +- pkg/instance/instance_redis.go | 8 +- pkg/instance/repository.go | 1 + pkg/instance/repository_cdb.go | 1 + pkg/instance/repository_clb.go | 3 +- pkg/instance/repository_cvm.go | 1 + pkg/instance/repository_dc.go | 1 + pkg/instance/repository_dcx.go | 1 + pkg/instance/repository_mongo.go | 3 +- pkg/instance/repository_nat.go | 1 + pkg/instance/repository_redis.go | 41 +++- pkg/metric/cache.go | 5 +- pkg/metric/conf.go | 24 +- pkg/metric/meta.go | 5 +- pkg/metric/metric.go | 16 +- pkg/metric/repository.go | 7 +- pkg/metric/sample.go | 1 + pkg/metric/series.go | 1 + readme.md | 2 + 49 files changed, 991 insertions(+), 331 deletions(-) create mode 100644 pkg/collector/handler_redis_mem.go create mode 100644 pkg/constant/cache.go create mode 100644 pkg/constant/instance.go create mode 100644 pkg/instance/cache_test.go diff --git a/cmd/qcloud-exporter/qcloud_exporter.go b/cmd/qcloud-exporter/qcloud_exporter.go index 4756278..d56905a 100644 --- a/cmd/qcloud-exporter/qcloud_exporter.go +++ b/cmd/qcloud-exporter/qcloud_exporter.go @@ -2,6 +2,9 @@ package main import ( "fmt" + "net/http" + "os" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -12,8 +15,6 @@ import ( "github.com/tencentyun/tencentcloud-exporter/pkg/collector" "github.com/tencentyun/tencentcloud-exporter/pkg/config" kingpin "gopkg.in/alecthomas/kingpin.v2" - "net/http" - "os" ) func newHandler(c *config.TencentConfig, includeExporterMetrics bool, maxRequests int, logger log.Logger) (*http.Handler, error) { diff --git a/go.mod b/go.mod index be7c9d9..35450d2 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,11 @@ require ( github.com/prometheus/client_golang v1.5.1 github.com/prometheus/common v0.9.1 github.com/prometheus/tsdb v0.7.1 // indirect - github.com/tencentcloud/tencentcloud-sdk-go v3.0.226+incompatible + github.com/tencentcloud/tencentcloud-sdk-go v1.0.67 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.2.5 + github.com/golang/mock v1.4.4 + github.com/stretchr/testify v1.6.1 + ) diff --git a/go.sum b/go.sum index 5b32254..d515173 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -28,6 +29,8 @@ github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80n github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -56,6 +59,7 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= @@ -91,6 +95,10 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tencentcloud/tencentcloud-sdk-go v1.0.67 h1:fKSwJ7hrvHTxr33EcmrbKcavYJ/U2zNIH8Lvsj2FNTE= +github.com/tencentcloud/tencentcloud-sdk-go v1.0.67/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI= github.com/tencentcloud/tencentcloud-sdk-go v3.0.73-0.20190704135516-e86c9d8b05ee+incompatible h1:bqYHIfgYCNbwEDT+06X2KWi5g7svrzAhZAZCGnK6Mks= github.com/tencentcloud/tencentcloud-sdk-go v3.0.73-0.20190704135516-e86c9d8b05ee+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentcloud/tencentcloud-sdk-go v3.0.226+incompatible h1:e6tY2NFYiRHEhBZiugeZhAjlL6fw+Gq2BtCm1tnAtmg= @@ -101,9 +109,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -116,6 +126,7 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -127,3 +138,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index f35bcad..5451206 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -1,14 +1,16 @@ package collector import ( + "context" "fmt" + "sync" + "time" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/tencentyun/tencentcloud-exporter/pkg/config" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" - "sync" - "time" ) const exporterNamespace = "tcm" @@ -39,6 +41,7 @@ var ( // 总指标采集器, 包含多个产品的采集器 type TcMonitorCollector struct { Collectors map[string]*TcProductCollector + Reloaders map[string]*TcProductCollectorReloader config *config.TencentConfig logger log.Logger lock sync.Mutex @@ -85,6 +88,7 @@ func collect(name string, c *TcProductCollector, ch chan<- prometheus.Metric, lo func NewTcMonitorCollector(conf *config.TencentConfig, logger log.Logger) (*TcMonitorCollector, error) { collectors := make(map[string]*TcProductCollector) + reloaders := make(map[string]*TcProductCollectorReloader) metricRepo, err := metric.NewTcmMetricRepository(conf, logger) if err != nil { @@ -99,18 +103,35 @@ func NewTcMonitorCollector(conf *config.TencentConfig, logger log.Logger) (*TcMo continue } - collector, err := NewTcProductCollector(namespace, metricRepoCache, conf, logger) + pconf, err := conf.GetProductConfig(namespace) + if err != nil { + return nil, err + } + + collector, err := NewTcProductCollector(namespace, metricRepoCache, conf, &pconf, logger) if err != nil { panic(fmt.Sprintf("Create product collecter fail, err=%s, Namespace=%s", err, namespace)) } collectors[namespace] = collector collectorState[namespace] = 1 level.Info(logger).Log("msg", "Create product collecter ok", "Namespace", namespace) + + if pconf.IsReloadEnable() { + relodInterval := time.Duration(pconf.RelodIntervalMinutes * int64(time.Minute)) + reloader := NewTcProductCollectorReloader(context.TODO(), collector, relodInterval, logger) + reloaders[namespace] = reloader + go reloader.Run() + level.Info(logger).Log( + "msg", fmt.Sprintf("reload %s instances every %d minutes", + namespace, pconf.RelodIntervalMinutes), + ) + } } level.Info(logger).Log("msg", "Create all product collecter ok", "num", len(collectors)) return &TcMonitorCollector{ Collectors: collectors, + Reloaders: reloaders, config: conf, logger: logger, }, nil diff --git a/pkg/collector/handler.go b/pkg/collector/handler.go index 99b6737..b656c2d 100644 --- a/pkg/collector/handler.go +++ b/pkg/collector/handler.go @@ -2,29 +2,34 @@ package collector import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" ) var ( - handlerFactoryMap = make(map[string]func(*TcProductCollector, log.Logger) (productHandler, error)) + handlerFactoryMap = make(map[string]func(*TcProductCollector, log.Logger) (ProductHandler, error)) ) // 每个产品的指标处理逻辑 -type productHandler interface { +type ProductHandler interface { // 获取云监控指标namespace GetNamespace() string - // 对指标元数据做检验和补充 - CheckMetricMeta(meta *metric.TcmMeta) bool - // 是否包含该指标, ture=包含, false=不包含 - IsIncludeMetric(m *metric.TcmMetric) bool + // 对指标元数据做检验, true=可用, false=跳过 + IsMetricMetaVaild(meta *metric.TcmMeta) bool + // 修改指标元数据 + ModifyMetricMeta(meta *metric.TcmMeta) error + // 对指标做校验, true=可用, false=跳过 + IsMetricVaild(m *metric.TcmMetric) bool + // 修改指标 + ModifyMetric(m *metric.TcmMetric) error // 获取该指标下符合条件的所有实例, 并生成所有的series GetSeries(tcmMetric *metric.TcmMetric) (series []*metric.TcmSeries, err error) } // 将对应的产品handler注册到Factory中 -func registerHandler(namespace string, isDefaultEnabled bool, factory func(*TcProductCollector, log.Logger) (productHandler, error)) { +func registerHandler(namespace string, isDefaultEnabled bool, factory func(*TcProductCollector, log.Logger) (ProductHandler, error)) { handlerFactoryMap[namespace] = factory } @@ -34,61 +39,104 @@ type baseProductHandler struct { logger log.Logger } -func (h *baseProductHandler) GetSeries(m *metric.TcmMetric) (slist []*metric.TcmSeries, err error) { - if len(m.Conf.OnlyIncludeInstances) != 0 { - for _, insId := range m.Conf.OnlyIncludeInstances { - ins, err := h.collector.InstanceRepo.Get(insId) - if err != nil { - level.Error(h.logger).Log("msg", "Instance not found", "id", insId) - continue - } - ql := map[string]string{ - h.monitorQueryKey: ins.GetMonitorQueryKey(), - } - s, err := metric.NewTcmSeries(m, ql, ins) - if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", insId) - continue - } - slist = append(slist, s) +func (h *baseProductHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { + return true +} + +func (h *baseProductHandler) ModifyMetricMeta(meta *metric.TcmMeta) error { + return nil +} + +func (h *baseProductHandler) IsMetricVaild(m *metric.TcmMetric) bool { + return true +} + +func (h *baseProductHandler) ModifyMetric(m *metric.TcmMetric) error { + return nil +} + +func (h *baseProductHandler) GetSeries(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + if m.Conf.IsIncludeOnlyInstance() { + return h.GetSeriesByOnly(m) + } + + if m.Conf.IsIncludeAllInstance() { + return h.GetSeriesByAll(m) + } + + if m.Conf.IsCustomQueryDimensions() { + return h.GetSeriesByCustom(m) + } + + return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions") +} + +func (h *baseProductHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, insId := range m.Conf.OnlyIncludeInstances { + ins, err := h.collector.InstanceRepo.Get(insId) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "id", insId) + continue + } + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", insId) + continue + } + slist = append(slist, s) + } + return slist, nil +} + +func (h *baseProductHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) + if err != nil { + return nil, err + } + for _, ins := range insList { + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), } - } else if m.Conf.AllInstances { - insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) + s, err := metric.NewTcmSeries(m, ql, ins) if err != nil { - return nil, err + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, s) + } + return slist, nil +} + +func (h *baseProductHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, ql := range m.Conf.CustomQueryDimensions { + v, ok := ql[h.monitorQueryKey] + if !ok { + level.Error(h.logger).Log( + "msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey), + "ql", fmt.Sprintf("%v", ql)) + continue } - for _, ins := range insList { - ql := map[string]string{ - h.monitorQueryKey: ins.GetMonitorQueryKey(), - } - s, err := metric.NewTcmSeries(m, ql, ins) - if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) - continue - } - slist = append(slist, s) + ins, err := h.collector.InstanceRepo.Get(v) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v) + continue } - } else { - for _, ql := range m.Conf.CustomQueryDimensions { - v, ok := ql[h.monitorQueryKey] - if !ok { - level.Error(h.logger).Log("msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey), - "ql", fmt.Sprintf("%v", ql)) - continue - } - ins, err := h.collector.InstanceRepo.Get(v) - if err != nil { - level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v) - continue - } - - s, err := metric.NewTcmSeries(m, ql, ins) - if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "err", err, "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) - continue - } - slist = append(slist, s) + + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "err", err, "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue } + slist = append(slist, s) } - return + return slist, nil } diff --git a/pkg/collector/handler_cdb.go b/pkg/collector/handler_cdb.go index 554a756..b833357 100644 --- a/pkg/collector/handler_cdb.go +++ b/pkg/collector/handler_cdb.go @@ -18,7 +18,7 @@ type cdbHandler struct { baseProductHandler } -func (h *cdbHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *cdbHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { return true } @@ -26,11 +26,11 @@ func (h *cdbHandler) GetNamespace() string { return CdbNamespace } -func (h *cdbHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *cdbHandler) IsMetricVaild(m *metric.TcmMetric) bool { return true } -func NewCdbHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewCdbHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &cdbHandler{ baseProductHandler{ monitorQueryKey: CdbInstanceidKey, diff --git a/pkg/collector/handler_cdn.go b/pkg/collector/handler_cdn.go index a7c074d..0b1b7b7 100644 --- a/pkg/collector/handler_cdn.go +++ b/pkg/collector/handler_cdn.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" @@ -20,7 +21,7 @@ type cdnHandler struct { baseProductHandler } -func (h *cdnHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *cdnHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { return true } @@ -28,7 +29,7 @@ func (h *cdnHandler) GetNamespace() string { return CdnNamespace } -func (h *cdnHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *cdnHandler) IsMetricVaild(m *metric.TcmMetric) bool { return true } @@ -60,7 +61,7 @@ func (h *cdnHandler) checkMonitorQueryKeys(m *metric.TcmMetric, ql map[string]st return true } -func NewCdnHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewCdnHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &cdnHandler{ baseProductHandler{ collector: c, diff --git a/pkg/collector/handler_clb.go b/pkg/collector/handler_clb.go index ae6af99..701e8e2 100644 --- a/pkg/collector/handler_clb.go +++ b/pkg/collector/handler_clb.go @@ -19,7 +19,7 @@ type clbHandler struct { baseProductHandler } -func (h *clbHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *clbHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { if !util.IsStrInList(meta.SupportDimensions, ClbInstanceidKey) { meta.SupportDimensions = append(meta.SupportDimensions, ClbInstanceidKey) } @@ -31,11 +31,11 @@ func (h *clbHandler) GetNamespace() string { return ClbNamespace } -func (h *clbHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *clbHandler) IsMetricVaild(m *metric.TcmMetric) bool { return true } -func NewClbHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewClbHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &clbHandler{ baseProductHandler{ monitorQueryKey: ClbInstanceidKey, diff --git a/pkg/collector/handler_clb7.go b/pkg/collector/handler_clb7.go index 087a252..7ac8843 100644 --- a/pkg/collector/handler_clb7.go +++ b/pkg/collector/handler_clb7.go @@ -1,10 +1,11 @@ package collector import ( + "strings" + "github.com/go-kit/kit/log" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" "github.com/tencentyun/tencentcloud-exporter/pkg/util" - "strings" ) const ( @@ -32,7 +33,7 @@ type clb7Handler struct { baseProductHandler } -func (h *clb7Handler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *clb7Handler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { if !util.IsStrInList(meta.SupportDimensions, Clb7InstanceidKey) { meta.SupportDimensions = append(meta.SupportDimensions, Clb7InstanceidKey) } @@ -44,14 +45,14 @@ func (h *clb7Handler) GetNamespace() string { return Clb7Namespace } -func (h *clb7Handler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *clb7Handler) IsMetricVaild(m *metric.TcmMetric) bool { if util.IsStrInList(Clb7ExcludeMetrics, strings.ToLower(m.Meta.MetricName)) { return false } return true } -func NewClb7Handler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewClb7Handler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &clb7Handler{ baseProductHandler{ monitorQueryKey: Clb7InstanceidKey, diff --git a/pkg/collector/handler_cos.go b/pkg/collector/handler_cos.go index 1833831..b1c3d0e 100644 --- a/pkg/collector/handler_cos.go +++ b/pkg/collector/handler_cos.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" @@ -24,7 +25,7 @@ type cosHandler struct { baseProductHandler } -func (h *cosHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *cosHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { return true } @@ -32,7 +33,7 @@ func (h *cosHandler) GetNamespace() string { return CosNamespace } -func (h *cosHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *cosHandler) IsMetricVaild(m *metric.TcmMetric) bool { // cos大部分指标不支持300以下的统计纬度 if m.Conf.StatPeriodSeconds < 300 { m.Conf.StatPeriodSeconds = 300 @@ -70,7 +71,7 @@ func (h *cosHandler) checkMonitorQueryKeys(m *metric.TcmMetric, ql map[string]st return true } -func NewCosHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewCosHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &cosHandler{ baseProductHandler{ collector: c, diff --git a/pkg/collector/handler_cvm.go b/pkg/collector/handler_cvm.go index 78a2bec..c28b777 100644 --- a/pkg/collector/handler_cvm.go +++ b/pkg/collector/handler_cvm.go @@ -1,10 +1,11 @@ package collector import ( + "strings" + "github.com/go-kit/kit/log" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" "github.com/tencentyun/tencentcloud-exporter/pkg/util" - "strings" ) const ( @@ -24,7 +25,7 @@ type cvmHandler struct { baseProductHandler } -func (h *cvmHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *cvmHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { if !util.IsStrInList(meta.SupportDimensions, CvmInstanceidKey) { meta.SupportDimensions = append(meta.SupportDimensions, CvmInstanceidKey) } @@ -36,7 +37,7 @@ func (h *cvmHandler) GetNamespace() string { return CvmNamespace } -func (h *cvmHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *cvmHandler) IsMetricVaild(m *metric.TcmMetric) bool { if util.IsStrInList(CvmInvalidMetricNames, strings.ToLower(m.Meta.MetricName)) { return false } @@ -50,7 +51,7 @@ func (h *cvmHandler) GetSeries(m *metric.TcmMetric) (slist []*metric.TcmSeries, return h.baseProductHandler.GetSeries(m) } -func NewCvmHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewCvmHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &cvmHandler{ baseProductHandler{ monitorQueryKey: CvmInstanceidKey, diff --git a/pkg/collector/handler_dc.go b/pkg/collector/handler_dc.go index d4feebc..6fac058 100644 --- a/pkg/collector/handler_dc.go +++ b/pkg/collector/handler_dc.go @@ -18,7 +18,7 @@ type dcHandler struct { baseProductHandler } -func (h *dcHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *dcHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { return true } @@ -26,11 +26,11 @@ func (h *dcHandler) GetNamespace() string { return DcNamespace } -func (h *dcHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *dcHandler) IsMetricVaild(m *metric.TcmMetric) bool { return true } -func NewDcHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewDcHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &dcHandler{ baseProductHandler{ monitorQueryKey: DcInstanceidKey, diff --git a/pkg/collector/handler_dcx.go b/pkg/collector/handler_dcx.go index a576089..449827b 100644 --- a/pkg/collector/handler_dcx.go +++ b/pkg/collector/handler_dcx.go @@ -1,10 +1,11 @@ package collector import ( + "strings" + "github.com/go-kit/kit/log" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" "github.com/tencentyun/tencentcloud-exporter/pkg/util" - "strings" ) const ( @@ -24,7 +25,7 @@ type dcxHandler struct { baseProductHandler } -func (h *dcxHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *dcxHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { if util.IsStrInList(DcxInvalidMetricNames, strings.ToLower(meta.MetricName)) { return false } @@ -35,11 +36,11 @@ func (h *dcxHandler) GetNamespace() string { return DcxNamespace } -func (h *dcxHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *dcxHandler) IsMetricVaild(m *metric.TcmMetric) bool { return true } -func NewDcxHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewDcxHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &dcxHandler{ baseProductHandler{ monitorQueryKey: DcxInstanceidKey, diff --git a/pkg/collector/handler_mongo.go b/pkg/collector/handler_mongo.go index 4b7e735..93bb853 100644 --- a/pkg/collector/handler_mongo.go +++ b/pkg/collector/handler_mongo.go @@ -2,13 +2,14 @@ package collector import ( "fmt" + "strings" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" mongodb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mongodb/v20190725" "github.com/tencentyun/tencentcloud-exporter/pkg/instance" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" "github.com/tencentyun/tencentcloud-exporter/pkg/util" - "strings" ) const ( @@ -34,80 +35,97 @@ func init() { } type mongoHandler struct { - collector *TcProductCollector - logger log.Logger -} - -func (h *mongoHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { - return true + baseProductHandler } func (h *mongoHandler) GetNamespace() string { return MongoNamespace } -func (h *mongoHandler) IsIncludeMetric(m *metric.TcmMetric) bool { - return true -} - -func (h *mongoHandler) GetSeries(m *metric.TcmMetric) (slist []*metric.TcmSeries, err error) { +func (h *mongoHandler) ModifyMetric(m *metric.TcmMetric) error { if m.Meta.MetricName == "Commands" { if m.Conf.StatPeriodSeconds == 60 { // 该指标不支持60统计 m.Conf.StatPeriodSeconds = 300 } } + return nil +} - if len(m.Conf.OnlyIncludeInstances) != 0 { - for _, insId := range m.Conf.OnlyIncludeInstances { - ins, err := h.collector.InstanceRepo.Get(insId) - if err != nil { - level.Error(h.logger).Log("msg", "Instance not found", "id", insId) - continue - } - sl, err := h.getSeriesByMetricType(m, ins) - if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", insId) - continue - } - slist = append(slist, sl...) - } +func (h *mongoHandler) GetSeries(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + if m.Conf.IsIncludeOnlyInstance() { + return h.GetSeriesByOnly(m) + } + + if m.Conf.IsIncludeAllInstance() { + return h.GetSeriesByAll(m) + } + + if m.Conf.IsCustomQueryDimensions() { + return h.GetSeriesByCustom(m) + } + + return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions") +} - } else if m.Conf.AllInstances { - insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) +func (h *mongoHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, insId := range m.Conf.OnlyIncludeInstances { + ins, err := h.collector.InstanceRepo.Get(insId) if err != nil { - return nil, err + level.Error(h.logger).Log("msg", "Instance not found", "id", insId) + continue } - for _, ins := range insList { - if len(m.Conf.ExcludeInstances) != 0 && util.IsStrInList(m.Conf.ExcludeInstances, ins.GetInstanceId()) { - continue - } - sl, err := h.getSeriesByMetricType(m, ins) - if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) - continue - } - slist = append(slist, sl...) + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", insId) + continue } - } else { - for _, ql := range m.Conf.CustomQueryDimensions { - v, ok := ql[MongoInstanceidKey] - if !ok { - return nil, fmt.Errorf("not found %s in queryDimensions", MongoInstanceidKey) - } - ins, err := h.collector.InstanceRepo.Get(v) - if err != nil { - return nil, err - } - s, err := metric.NewTcmSeries(m, ql, ins) - if err != nil { - return nil, err - } - slist = append(slist, s) + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *mongoHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) + if err != nil { + return nil, err + } + for _, ins := range insList { + if len(m.Conf.ExcludeInstances) != 0 && util.IsStrInList(m.Conf.ExcludeInstances, ins.GetInstanceId()) { + continue + } + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue } + slist = append(slist, sl...) } + return slist, nil +} - return +func (h *mongoHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, ql := range m.Conf.CustomQueryDimensions { + v, ok := ql[MongoInstanceidKey] + if !ok { + return nil, fmt.Errorf("not found %s in queryDimensions", MongoInstanceidKey) + } + ins, err := h.collector.InstanceRepo.Get(v) + if err != nil { + return nil, err + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + slist = append(slist, s) + } + return slist, nil } func (h *mongoHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) (slist []*metric.TcmSeries, err error) { @@ -119,14 +137,16 @@ func (h *mongoHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.T } s, err := metric.NewTcmSeries(m, ql, ins) if err != nil { - return nil, fmt.Errorf("create metric series fail, metric=%s, instacne=%s", m.Meta.MetricName, ins.GetInstanceId()) + return nil, fmt.Errorf("create metric series fail, metric=%s, instacne=%s", + m.Meta.MetricName, ins.GetInstanceId()) } slist = append(slist, s) } else if util.IsStrInList(MongoReplicaMetrics, strings.ToLower(m.Meta.MetricName)) { // 副本集纬度 meta, ok := ins.GetMeta().(*mongodb.InstanceDetail) if !ok { - return nil, fmt.Errorf("get instacne raw meta fail, metric=%s, instacne=%s", m.Meta.MetricName, ins.GetInstanceId()) + return nil, fmt.Errorf("get instacne raw meta fail, metric=%s, instacne=%s", + m.Meta.MetricName, ins.GetInstanceId()) } for _, rep := range meta.ReplicaSets { // cmgo-6ielucen_0 @@ -135,7 +155,8 @@ func (h *mongoHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.T } s, err := metric.NewTcmSeries(m, ql, ins) if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", *rep.ReplicaSetId) + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", *rep.ReplicaSetId) } else { slist = append(slist, s) } @@ -144,7 +165,8 @@ func (h *mongoHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.T // 节点纬度 meta, ok := ins.GetMeta().(*mongodb.InstanceDetail) if !ok { - return nil, fmt.Errorf("get instacne raw meta fail, metric=%s, instacne=%s", m.Meta.MetricName, ins.GetInstanceId()) + return nil, fmt.Errorf("get instacne raw meta fail, metric=%s, instacne=%s", + m.Meta.MetricName, ins.GetInstanceId()) } for _, rep := range meta.ReplicaSets { // cmgo-6ielucen_0-node-primary @@ -154,7 +176,8 @@ func (h *mongoHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.T } s, err := metric.NewTcmSeries(m, ql, ins) if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", nprimary) + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", nprimary) } else { slist = append(slist, s) } @@ -167,23 +190,24 @@ func (h *mongoHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.T } s, err := metric.NewTcmSeries(m, ql, ins) if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", nslave) + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", nslave) } else { slist = append(slist, s) } } } - } else { - level.Warn(h.logger).Log("msg", "not found metric type", "metric", m.Meta.MetricName) } return } -func NewMongoHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewMongoHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &mongoHandler{ - collector: c, - logger: logger, + baseProductHandler: baseProductHandler{ + monitorQueryKey: MongoInstanceidKey, + collector: c, + logger: logger, + }, } return - } diff --git a/pkg/collector/handler_nat.go b/pkg/collector/handler_nat.go index 629aa64..70a9af3 100644 --- a/pkg/collector/handler_nat.go +++ b/pkg/collector/handler_nat.go @@ -18,7 +18,7 @@ type natHandler struct { baseProductHandler } -func (h *natHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *natHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { return true } @@ -26,11 +26,11 @@ func (h *natHandler) GetNamespace() string { return NatNamespace } -func (h *natHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *natHandler) IsMetricVaild(m *metric.TcmMetric) bool { return true } -func NewNatHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewNatHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &natHandler{ baseProductHandler{ monitorQueryKey: NatMonitorQueryKey, diff --git a/pkg/collector/handler_redis.go b/pkg/collector/handler_redis.go index 8d4941d..74577ca 100644 --- a/pkg/collector/handler_redis.go +++ b/pkg/collector/handler_redis.go @@ -1,10 +1,11 @@ package collector import ( + "strings" + "github.com/go-kit/kit/log" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" "github.com/tencentyun/tencentcloud-exporter/pkg/util" - "strings" ) const ( @@ -35,7 +36,7 @@ type redisHandler struct { baseProductHandler } -func (h *redisHandler) CheckMetricMeta(meta *metric.TcmMeta) bool { +func (h *redisHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool { return true } @@ -43,7 +44,7 @@ func (h *redisHandler) GetNamespace() string { return RedisNamespace } -func (h *redisHandler) IsIncludeMetric(m *metric.TcmMetric) bool { +func (h *redisHandler) IsMetricVaild(m *metric.TcmMetric) bool { if strings.ToLower(m.Conf.CustomProductName) == "cluster_redis" { if util.IsStrInList(RedisClusterMetricNames, strings.ToLower(m.Meta.MetricName)) { return true @@ -57,7 +58,7 @@ func (h *redisHandler) IsIncludeMetric(m *metric.TcmMetric) bool { return false } -func NewRedisHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) { +func NewRedisHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) { handler = &redisHandler{ baseProductHandler{ monitorQueryKey: RedisInstanceidKey, diff --git a/pkg/collector/handler_redis_mem.go b/pkg/collector/handler_redis_mem.go new file mode 100644 index 0000000..6dff7a3 --- /dev/null +++ b/pkg/collector/handler_redis_mem.go @@ -0,0 +1,222 @@ +package collector + +import ( + "fmt" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/tencentyun/tencentcloud-exporter/pkg/instance" + "github.com/tencentyun/tencentcloud-exporter/pkg/metric" + "strings" + "time" +) + +const ( + RedisMemNamespace = "QCE/REDIS_MEM" + RedisMemInstanceidKey = "instanceid" +) + +var ( +//RedisMemInstanceMetricNames = []string{ +// "cpuutil", "cpumaxutil", "memused", "memutil", "memmaxutil", "keys", "expired", "evicted", "connections", +// "connectionsutil", "inflow", "inbandwidthutil", "inflowlimit", "outflow", "outbandwidthutil", +// "outflowlimit", "latencyavg", "latencymax", "latencyread", "latencywrite", "latencyother", +// "commands", "cmdread", "cmdwrite", "cmdother", "cmdbigvalue", "cmdkeycount", "cmdmget", "cmdslow", +// "cmdhits", "cmdmiss", "cmderr", "cmdhitsratio", +//} +//RedisMemProxyMetricNames = []string{ +// "cpuutilproxy", "commandsproxy", "cmdkeycountproxy", "cmdmgetproxy", "cmderrproxy", "cmdbigvalueproxy", +// "connectionsproxy", "connectionsutilproxy", "inflowproxy", "inbandwidthutilproxy", +// "inflowlimitproxy", "outflowproxy", "outbandwidthutilproxy", "outflowlimitproxy", +// "latencyavgproxy", "latencymaxproxy", "latencyreadproxy", "latencywriteproxy", "latencyotherproxy", +//} +//RedisMemNodeMetricNames = []string{ +// "cpuutilnode", "connectionsnode", "connectionsutilnode", "memusednode", "memutilnode", +// "keysnode", "expirednode", "evictednode", "repldelaynode", "commandsnode", "cmdreadnode", +// "cmdwritenode", "cmdothernode", "cmdslownode", "cmdhitsnode", "cmdmissnode", "cmdhitsrationode", +//} +) + +func init() { + registerHandler(RedisMemNamespace, defaultHandlerEnabled, NewRedisMemHandler) +} + +type redisMemHandler struct { + baseProductHandler + + nodeRepo instance.RedisTcInstanceNodeRepository +} + +func (h *redisMemHandler) GetNamespace() string { + return RedisMemNamespace +} + +func (h *redisMemHandler) GetSeries(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + if m.Conf.IsIncludeOnlyInstance() { + return h.GetSeriesByOnly(m) + } + + if m.Conf.IsIncludeAllInstance() { + return h.GetSeriesByAll(m) + } + + if m.Conf.IsCustomQueryDimensions() { + return h.GetSeriesByCustom(m) + } + + return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions") +} + +func (h *redisMemHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, insId := range m.Conf.OnlyIncludeInstances { + ins, err := h.collector.InstanceRepo.Get(insId) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "id", insId) + continue + } + + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *redisMemHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters) + if err != nil { + return nil, err + } + for _, ins := range insList { + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *redisMemHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries + for _, ql := range m.Conf.CustomQueryDimensions { + v, ok := ql[h.monitorQueryKey] + if !ok { + level.Error(h.logger).Log( + "msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey), + "ql", fmt.Sprintf("%v", ql)) + continue + } + ins, err := h.collector.InstanceRepo.Get(v) + if err != nil { + level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v) + continue + } + + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue + } + slist = append(slist, sl...) + } + return slist, nil +} + +func (h *redisMemHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + if strings.HasSuffix(m.Meta.MetricName, "Proxy") { + return h.getProxySeries(m, ins) + } + if strings.HasSuffix(m.Meta.MetricName, "Node") { + return h.getNodeSeries(m, ins) + } + return h.getInstanceSeries(m, ins) +} + +func (h *redisMemHandler) getInstanceSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + series = append(series, s) + + return series, nil +} + +func (h *redisMemHandler) getProxySeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + + resp, err := h.nodeRepo.GetNodeInfo(ins.GetInstanceId()) + if err != nil { + return nil, err + } + + for _, node := range resp.Response.Proxy { + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), + "pnodeid": *node.NodeId, + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + series = append(series, s) + } + + return series, nil +} + +func (h *redisMemHandler) getNodeSeries(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { + var series []*metric.TcmSeries + + resp, err := h.nodeRepo.GetNodeInfo(ins.GetInstanceId()) + if err != nil { + return nil, err + } + + for _, node := range resp.Response.Redis { + ql := map[string]string{ + h.monitorQueryKey: ins.GetMonitorQueryKey(), + "rnodeid": *node.NodeId, + } + s, err := metric.NewTcmSeries(m, ql, ins) + if err != nil { + return nil, err + } + series = append(series, s) + } + + return series, nil +} + +func NewRedisMemHandler(c *TcProductCollector, logger log.Logger) (ProductHandler, error) { + nodeRepo, err := instance.NewRedisTcInstanceNodeRepository(c.Conf, logger) + if err != nil { + return nil, err + } + relodInterval := time.Duration(c.ProductConf.RelodIntervalMinutes * int64(time.Minute)) + nodeRepoCahe := instance.NewTcRedisInstanceNodeCache(nodeRepo, relodInterval, logger) + + handler := &redisMemHandler{ + baseProductHandler: baseProductHandler{ + monitorQueryKey: RedisMemInstanceidKey, + collector: c, + logger: logger, + }, + nodeRepo: nodeRepoCahe, + } + return handler, nil +} diff --git a/pkg/collector/product.go b/pkg/collector/product.go index f38335c..c617493 100644 --- a/pkg/collector/product.go +++ b/pkg/collector/product.go @@ -1,7 +1,13 @@ package collector import ( + "context" "fmt" + "github.com/tencentyun/tencentcloud-exporter/pkg/constant" + "strings" + "sync" + "time" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -9,8 +15,6 @@ import ( "github.com/tencentyun/tencentcloud-exporter/pkg/instance" "github.com/tencentyun/tencentcloud-exporter/pkg/metric" "github.com/tencentyun/tencentcloud-exporter/pkg/util" - "strings" - "sync" ) // 每个产品的指标采集默认实现, 不同的逻辑通过对应的productHandler实现 @@ -22,54 +26,38 @@ type TcProductCollector struct { InstanceMap map[string]instance.TcInstance Querys metric.TcmQuerySet Conf *config.TencentConfig - handler productHandler + ProductConf *config.TencentProduct + handler ProductHandler logger log.Logger lock sync.Mutex } // 指标纬度配置 -func (c *TcProductCollector) loadMetricsByMetricConf() (err error) { +func (c *TcProductCollector) LoadMetricsByMetricConf() error { if len(c.MetricMap) == 0 { c.MetricMap = make(map[string]*metric.TcmMetric) } for _, mconf := range c.Conf.GetMetricConfigs(c.Namespace) { - meta, err := c.MetricRepo.GetMeta(c.Namespace, mconf.MetricName) + nm, err := c.createMetricWithMetricConf(mconf) if err != nil { - level.Error(c.logger).Log("msg", "not found metric meta", "err", err, "Namespace", c.Namespace, "name", mconf.MetricName) + level.Warn(c.logger).Log("msg", "Create metric fail", "err", err, + "Namespace", c.Namespace, "name", mconf.MetricName) continue } + c.MetricMap[nm.Meta.MetricName] = nm - m, ok := c.MetricMap[meta.MetricName] - if !ok { - conf, err := metric.NewTcmMetricConfigWithMetricYaml(mconf, meta) - if err != nil { - level.Error(c.logger).Log("msg", "parse metric config err", "err", err, "Namespace", c.Namespace, "name", mconf.MetricName) - continue - } - nm, err := metric.NewTcmMetric(meta, conf) - if err != nil { - level.Error(c.logger).Log("msg", "create metric err", "err", err, "Namespace", c.Namespace, "name", mconf.MetricName) - continue - } - // 指标过滤 - if !c.handler.IsIncludeMetric(nm) { - level.Error(c.logger).Log("msg", " Metric not support, skip", "Namespace", c.Namespace, "name", nm.Meta.MetricName) - continue - } - c.MetricMap[meta.MetricName] = nm - m = nm - } - - series, err := c.handler.GetSeries(m) + series, err := c.handler.GetSeries(nm) if err != nil { - level.Error(c.logger).Log("msg", "create metric series err", "err", err, "Namespace", c.Namespace, "name", mconf.MetricName) + level.Error(c.logger).Log("msg", "create metric series err", "err", err, + "Namespace", c.Namespace, "name", mconf.MetricName) continue } - err = m.LoadSeries(series) + err = nm.LoadSeries(series) if err != nil { - level.Error(c.logger).Log("msg", "load metric series err", "err", err, "Namespace", c.Namespace, "name", mconf.MetricName) + level.Error(c.logger).Log("msg", "load metric series err", "err", err, + "Namespace", c.Namespace, "name", mconf.MetricName) continue } } @@ -77,95 +65,152 @@ func (c *TcProductCollector) loadMetricsByMetricConf() (err error) { } // 产品纬度配置 -func (c *TcProductCollector) loadMetricsByProductConf() (err error) { +func (c *TcProductCollector) LoadMetricsByProductConf() error { if len(c.MetricMap) == 0 { c.MetricMap = make(map[string]*metric.TcmMetric) } - for _, pconf := range c.Conf.GetProductConfigs(c.Namespace) { - var metricNames []string - - if len(pconf.OnlyIncludeMetrics) != 0 { - // 导出指定指标列表 - for _, mname := range pconf.OnlyIncludeMetrics { - meta, err := c.MetricRepo.GetMeta(c.Namespace, mname) - if err != nil { - level.Warn(c.logger).Log("msg", "not found metric meta", "Namespace", c.Namespace, "name", mname) - } else { - metricNames = append(metricNames, meta.MetricName) - } - } - } else { - // 导出该namespace下的所有指标 - var excludeMetrics []string - if len(pconf.ExcludeMetrics) != 0 { - for _, em := range pconf.ExcludeMetrics { - excludeMetrics = append(excludeMetrics, strings.ToLower(em)) - } - } - allMetas, err := c.MetricRepo.ListMetaByNamespace(c.Namespace) - if err != nil { - return err - } + pconf, err := c.Conf.GetProductConfig(c.Namespace) + if err != nil { + return err + } + metricNames, err := c.getMetricNames(pconf) + if err != nil { + return err + } + for _, mname := range metricNames { + nm, err := c.createMetricWithProductConf(mname, pconf) + if err != nil { + level.Warn(c.logger).Log("msg", "Create metric fail", "err", err, "Namespace", c.Namespace, "name", mname) + continue + } + c.MetricMap[nm.Meta.MetricName] = nm - for _, meta := range allMetas { - if len(excludeMetrics) != 0 && util.IsStrInList(excludeMetrics, strings.ToLower(meta.MetricName)) { - continue - } - metricNames = append(metricNames, meta.MetricName) - } + // 获取该指标下的所有实例纬度查询或自定义纬度查询 + series, err := c.handler.GetSeries(nm) + if err != nil { + level.Error(c.logger).Log("msg", "create metric series err", "err", err, "Namespace", c.Namespace, "name", mname) + continue + } + err = nm.LoadSeries(series) + if err != nil { + level.Error(c.logger).Log("msg", "load metric series err", "err", err, "Namespace", c.Namespace, "name", mname) + continue } + } + return nil +} + +func (c *TcProductCollector) getMetricNames(pconf config.TencentProduct) ([]string, error) { + var metricNames []string - for _, mname := range metricNames { + if len(pconf.OnlyIncludeMetrics) != 0 { + // 导出指定指标列表 + for _, mname := range pconf.OnlyIncludeMetrics { meta, err := c.MetricRepo.GetMeta(c.Namespace, mname) if err != nil { - level.Error(c.logger).Log("msg", "Not found metric meta", "Namespace", c.Namespace, "name", mname) - continue - } - // 指标元数据处理, false=跳过 - if !c.handler.CheckMetricMeta(meta) { - level.Warn(c.logger).Log("msg", " Metric not support, skip", "Namespace", c.Namespace, "name", meta.MetricName) - continue + level.Warn(c.logger).Log("msg", "not found metric meta", "Namespace", c.Namespace, "name", mname) + } else { + metricNames = append(metricNames, meta.MetricName) } - - m, ok := c.MetricMap[mname] - if !ok { - // 创建TcmMetric模型 - conf, err := metric.NewTcmMetricConfigWithProductYaml(pconf, meta) - if err != nil { - level.Error(c.logger).Log("msg", "parse metric config err", "err", err, "Namespace", c.Namespace, "name", mname) - continue - } - nm, err := metric.NewTcmMetric(meta, conf) - if err != nil { - level.Error(c.logger).Log("msg", "create metric err", "err", err, "Namespace", c.Namespace, "name", mname) - continue - } - // 指标过滤 - if !c.handler.IsIncludeMetric(nm) { - level.Warn(c.logger).Log("msg", " Metric not support, skip", "Namespace", c.Namespace, "name", nm.Meta.MetricName) - continue - } - c.MetricMap[meta.MetricName] = nm - m = nm + } + } else { + // 导出该namespace下的所有指标 + var excludeMetrics []string + if len(pconf.ExcludeMetrics) != 0 { + for _, em := range pconf.ExcludeMetrics { + excludeMetrics = append(excludeMetrics, strings.ToLower(em)) } + } + allMetas, err := c.MetricRepo.ListMetaByNamespace(c.Namespace) + if err != nil { + return nil, err + } - // 获取该指标下的所有实例纬度查询或自定义纬度查询 - series, err := c.handler.GetSeries(m) - if err != nil { - level.Error(c.logger).Log("msg", "create metric series err", "err", err, "Namespace", c.Namespace, "name", mname) + for _, meta := range allMetas { + if len(excludeMetrics) != 0 && util.IsStrInList(excludeMetrics, strings.ToLower(meta.MetricName)) { continue } + metricNames = append(metricNames, meta.MetricName) + } + } + return metricNames, nil +} - err = m.LoadSeries(series) - if err != nil { - level.Error(c.logger).Log("msg", "load metric series err", "err", err, "Namespace", c.Namespace, "name", mname) - continue - } +func (c *TcProductCollector) createMetricWithProductConf(mname string, pconf config.TencentProduct) (*metric.TcmMetric, error) { + meta, err := c.MetricRepo.GetMeta(c.Namespace, mname) + if err != nil { + return nil, err + } + // 指标元数据处理, false=跳过 + if !c.handler.IsMetricMetaVaild(meta) { + return nil, fmt.Errorf("metric not support") + } + err = c.handler.ModifyMetricMeta(meta) + if err != nil { + return nil, err + } + + m, exists := c.MetricMap[mname] + if !exists { + // 创建TcmMetric模型 + conf, err := metric.NewTcmMetricConfigWithProductYaml(pconf, meta) + if err != nil { + return nil, err } + nm, err := metric.NewTcmMetric(meta, conf) + if err != nil { + return nil, err + } + // 指标过滤 + if !c.handler.IsMetricVaild(nm) { + return nil, fmt.Errorf("metric not support") + } + err = c.handler.ModifyMetric(nm) + if err != nil { + return nil, err + } + return nm, nil + } + + return m, nil +} +func (c *TcProductCollector) createMetricWithMetricConf(mconf config.TencentMetric) (*metric.TcmMetric, error) { + meta, err := c.MetricRepo.GetMeta(c.Namespace, mconf.MetricName) + if err != nil { + return nil, err } - return nil + // 指标元数据处理, false=跳过 + if !c.handler.IsMetricMetaVaild(meta) { + return nil, fmt.Errorf("metric not support") + } + err = c.handler.ModifyMetricMeta(meta) + if err != nil { + return nil, err + } + + m, ok := c.MetricMap[meta.MetricName] + if !ok { + conf, err := metric.NewTcmMetricConfigWithMetricYaml(mconf, meta) + if err != nil { + return nil, err + } + nm, err := metric.NewTcmMetric(meta, conf) + if err != nil { + return nil, err + } + // 指标过滤 + if !c.handler.IsMetricVaild(nm) { + return nil, fmt.Errorf("metric not support") + } + err = c.handler.ModifyMetric(nm) + if err != nil { + return nil, err + } + return nm, nil + } + return m, nil } // 一个query管理一个metric的采集 @@ -210,22 +255,61 @@ func (c *TcProductCollector) Collect(ch chan<- prometheus.Metric) (err error) { return } +type TcProductCollectorReloader struct { + collector *TcProductCollector + relodInterval time.Duration + ctx context.Context + cancel context.CancelFunc + logger log.Logger +} + +func (r *TcProductCollectorReloader) Run() { + ticker := time.NewTicker(r.relodInterval) + defer ticker.Stop() + + // sleep when first start + time.Sleep(r.relodInterval) + + for { + e := r.reloadMetricsByProductConf() + if e != nil { + level.Error(r.logger).Log("msg", "reload product error", "err", e, + "namespace", r.collector.Namespace) + } + select { + case <-r.ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (r *TcProductCollectorReloader) Stop() { + r.cancel() +} + +func (r *TcProductCollectorReloader) reloadMetricsByProductConf() error { + return r.collector.LoadMetricsByProductConf() +} + // 创建新的TcProductCollector, 每个产品一个 -func NewTcProductCollector(namespace string, metricRepo metric.TcmMetricRepository, conf *config.TencentConfig, logger log.Logger) (*TcProductCollector, error) { +func NewTcProductCollector(namespace string, metricRepo metric.TcmMetricRepository, + conf *config.TencentConfig, pconf *config.TencentProduct, logger log.Logger) (*TcProductCollector, error) { factory, exists := handlerFactoryMap[namespace] if !exists { return nil, fmt.Errorf("product handler not found, Namespace=%s ", namespace) } var instanceRepoCache instance.TcInstanceRepository - if !util.IsStrInList(instance.NotSupportInstances, namespace) { + if !util.IsStrInList(constant.NotSupportInstanceNamespaces, namespace) { // 支持实例自动发现的产品 instanceRepo, err := instance.NewTcInstanceRepository(namespace, conf, logger) if err != nil { return nil, err } // 使用instance缓存 - instanceRepoCache = instance.NewTcInstanceCache(instanceRepo, logger) + reloadInterval := time.Duration(pconf.RelodIntervalMinutes * int64(time.Minute)) + instanceRepoCache = instance.NewTcInstanceCache(instanceRepo, reloadInterval, logger) } c := &TcProductCollector{ @@ -233,6 +317,7 @@ func NewTcProductCollector(namespace string, metricRepo metric.TcmMetricReposito MetricRepo: metricRepo, InstanceRepo: instanceRepoCache, Conf: conf, + ProductConf: pconf, logger: logger, } @@ -242,11 +327,11 @@ func NewTcProductCollector(namespace string, metricRepo metric.TcmMetricReposito } c.handler = handler - err = c.loadMetricsByMetricConf() + err = c.LoadMetricsByMetricConf() if err != nil { return nil, err } - err = c.loadMetricsByProductConf() + err = c.LoadMetricsByProductConf() if err != nil { return nil, err } @@ -255,5 +340,17 @@ func NewTcProductCollector(namespace string, metricRepo metric.TcmMetricReposito return nil, err } return c, nil +} +func NewTcProductCollectorReloader(ctx context.Context, collector *TcProductCollector, + relodInterval time.Duration, logger log.Logger) *TcProductCollectorReloader { + childCtx, cancel := context.WithCancel(ctx) + reloader := &TcProductCollectorReloader{ + collector: collector, + relodInterval: relodInterval, + ctx: childCtx, + cancel: cancel, + logger: logger, + } + return reloader } diff --git a/pkg/config/config.go b/pkg/config/config.go index b0504ee..00668c8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,15 +2,20 @@ package config import ( "fmt" - "gopkg.in/yaml.v2" "io/ioutil" "os" "strings" + + "github.com/tencentyun/tencentcloud-exporter/pkg/constant" + "github.com/tencentyun/tencentcloud-exporter/pkg/util" + "gopkg.in/yaml.v2" ) const ( - DefaultPeriodSeconds = 60 - DefaultDelaySeconds = 300 + DefaultPeriodSeconds = 60 + DefaultDelaySeconds = 300 + DefaultRelodIntervalMinutes = 60 + DefaultRateLimit = 15 EnvAccessKey = "TENCENTCLOUD_SECRET_ID" EnvSecretKey = "TENCENTCLOUD_SECRET_KEY" @@ -25,6 +30,8 @@ var ( "mysql": "QCE/CDB", "cvm": "QCE/CVM", "redis": "QCE/REDIS", + "redis_cluster": "QCE/REDIS", + "redis_mem": "QCE/REDIS_MEM", "cluster_redis": "QCE/REDIS", "dc": "QCE/DC", "dcx": "QCE/DCX", @@ -82,6 +89,17 @@ type TencentProduct struct { RangeSeconds int64 `yaml:"range_seconds"` DelaySeconds int64 `yaml:"delay_seconds"` MetricNameType int32 `yaml:"metric_name_type"` // 1=大写转下划线, 2=全小写 + RelodIntervalMinutes int64 `yaml:"relod_interval_minutes"` +} + +func (p *TencentProduct) IsReloadEnable() bool { + if len(p.OnlyIncludeMetrics) > 0 { + return false + } + if util.IsStrInList(constant.NotSupportInstanceNamespaces, p.Namespace) { + return false + } + return p.AllInstances } type TencentConfig struct { @@ -162,15 +180,17 @@ func (c *TencentConfig) check() (err error) { if _, exists := Product2Namespace[strings.ToLower(pname)]; !exists { return fmt.Errorf("namespace productName not support, %s", pname) } + if len(pconf.OnlyIncludeInstances) == 0 && !pconf.AllInstances && len(pconf.CustomQueryDimensions) == 0 { + return fmt.Errorf("must set all_instances or only_include_instances or custom_query_dimensions") + } } return nil } func (c *TencentConfig) fillDefault() { - if c.RateLimit <= 0 { - c.RateLimit = 15 + c.RateLimit = DefaultRateLimit } for index, metric := range c.Metrics { @@ -189,6 +209,12 @@ func (c *TencentConfig) fillDefault() { c.Metrics[index].MetricReName = c.Metrics[index].MetricName } } + + for index, product := range c.Products { + if product.RelodIntervalMinutes <= 0 { + c.Products[index].RelodIntervalMinutes = DefaultRelodIntervalMinutes + } + } } func (c *TencentConfig) GetNamespaces() (nps []string) { @@ -218,14 +244,14 @@ func (c *TencentConfig) GetMetricConfigs(namespace string) (mconfigs []TencentMe return } -func (c *TencentConfig) GetProductConfigs(namespace string) (pconfigs []TencentProduct) { +func (c *TencentConfig) GetProductConfig(namespace string) (TencentProduct, error) { for _, pconf := range c.Products { ns := GetStandardNamespaceFromCustomNamespace(pconf.Namespace) if ns == namespace { - pconfigs = append(pconfigs, pconf) + return pconf, nil } } - return + return TencentProduct{}, fmt.Errorf("namespace config not found") } func GetStandardNamespaceFromCustomNamespace(cns string) string { diff --git a/pkg/constant/cache.go b/pkg/constant/cache.go new file mode 100644 index 0000000..3f2495e --- /dev/null +++ b/pkg/constant/cache.go @@ -0,0 +1 @@ +package constant diff --git a/pkg/constant/instance.go b/pkg/constant/instance.go new file mode 100644 index 0000000..da6083f --- /dev/null +++ b/pkg/constant/instance.go @@ -0,0 +1,7 @@ +package constant + +// 不支持实例纬度自动查询的namespace +var NotSupportInstanceNamespaces = []string{ + "QCE/COS", + "QCE/CDN", +} diff --git a/pkg/instance/cache.go b/pkg/instance/cache.go index ece48c2..9672984 100644 --- a/pkg/instance/cache.go +++ b/pkg/instance/cache.go @@ -1,19 +1,27 @@ package instance import ( - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "sync" "time" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/redis/v20180412" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" +) + +const ( + DefaultReloadInterval = 60 * time.Minute ) // 可用于产品的实例的缓存, TcInstanceRepository type TcInstanceCache struct { Raw TcInstanceRepository cache map[string]TcInstance - lastReloadTime int64 + lastReloadTime time.Time logger log.Logger mu sync.Mutex + reloadInterval time.Duration } func (c *TcInstanceCache) GetInstanceKey() string { @@ -21,8 +29,8 @@ func (c *TcInstanceCache) GetInstanceKey() string { } func (c *TcInstanceCache) Get(id string) (TcInstance, error) { - ins, ok := c.cache[id] - if ok { + ins, exists := c.cache[id] + if exists { return ins, nil } @@ -74,11 +82,11 @@ func (c *TcInstanceCache) ListByFilters(filters map[string]string) (insList []Tc return } -func (c *TcInstanceCache) checkNeedreload() (err error) { +func (c *TcInstanceCache) checkNeedreload() error { c.mu.Lock() defer c.mu.Unlock() - if c.lastReloadTime != 0 { + if !c.lastReloadTime.IsZero() && time.Now().Sub(c.lastReloadTime) < c.reloadInterval { return nil } @@ -86,20 +94,67 @@ func (c *TcInstanceCache) checkNeedreload() (err error) { if err != nil { return err } - for _, instance := range inss { - c.cache[instance.GetInstanceId()] = instance + numChanged := 0 + if len(inss) > 0 { + newCache := map[string]TcInstance{} + for _, instance := range inss { + newCache[instance.GetInstanceId()] = instance + } + numChanged = len(newCache) - len(c.cache) + c.cache = newCache } - c.lastReloadTime = time.Now().Unix() - level.Info(c.logger).Log("msg", "Reload instance cache", "num", len(c.cache)) - return + c.lastReloadTime = time.Now() + + level.Info(c.logger).Log("msg", "Reload instance cache", "num", len(c.cache), "changed", numChanged) + return nil } -func NewTcInstanceCache(repo TcInstanceRepository, logger log.Logger) TcInstanceRepository { +func NewTcInstanceCache(repo TcInstanceRepository, reloadInterval time.Duration, logger log.Logger) TcInstanceRepository { cache := &TcInstanceCache{ - Raw: repo, - cache: map[string]TcInstance{}, - logger: logger, + Raw: repo, + cache: map[string]TcInstance{}, + reloadInterval: reloadInterval, + logger: logger, } return cache +} + +type TcRedisInstanceNodeCache struct { + Raw RedisTcInstanceNodeRepository + cache map[string]*sdk.DescribeInstanceNodeInfoResponse + lastReloadTime map[string]time.Time + reloadInterval time.Duration + mu sync.Mutex + logger log.Logger +} + +func (c *TcRedisInstanceNodeCache) GetNodeInfo(instanceId string) (*sdk.DescribeInstanceNodeInfoResponse, error) { + lrtime, exists := c.lastReloadTime[instanceId] + if exists && time.Now().Sub(lrtime) < c.reloadInterval { + node, ok := c.cache[instanceId] + if ok { + return node, nil + } + } + + node, err := c.Raw.GetNodeInfo(instanceId) + if err != nil { + return nil, err + } + c.cache[instanceId] = node + c.lastReloadTime[instanceId] = time.Now() + level.Debug(c.logger).Log("msg", "Get redis node info from api", "instanceId", instanceId) + return node, nil +} + +func NewTcRedisInstanceNodeCache(repo RedisTcInstanceNodeRepository, reloadInterval time.Duration, logger log.Logger) RedisTcInstanceNodeRepository { + cache := &TcRedisInstanceNodeCache{ + Raw: repo, + cache: map[string]*sdk.DescribeInstanceNodeInfoResponse{}, + lastReloadTime: map[string]time.Time{}, + reloadInterval: reloadInterval, + logger: logger, + } + return cache } diff --git a/pkg/instance/cache_test.go b/pkg/instance/cache_test.go new file mode 100644 index 0000000..6b0e901 --- /dev/null +++ b/pkg/instance/cache_test.go @@ -0,0 +1,57 @@ +package instance + +import ( + "github.com/prometheus/common/promlog" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/redis/v20180412" +) + +func TestTcRedisInstanceNodeCache_GetNodeInfo(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockNodeRepo := NewMockRedisTcInstanceNodeRepository(ctrl) + + loglevel := &promlog.AllowedLevel{} + loglevel.Set("debug") + logformat := &promlog.AllowedFormat{} + logformat.Set("logfmt") + + cache := &TcRedisInstanceNodeCache{ + Raw: mockNodeRepo, + cache: map[string]*sdk.DescribeInstanceNodeInfoResponse{}, + lastReloadTime: map[string]time.Time{}, + reloadInterval: 3 * time.Minute, + logger: promlog.New(&promlog.Config{ + Level: loglevel, + Format: logformat, + }), + } + + mockInstanceId := "crs-12345678" + + // case 1: get from api, init + mockNodeRepo.EXPECT().GetNodeInfo(gomock.Any()).Return(&sdk.DescribeInstanceNodeInfoResponse{}, nil) + _, err := cache.GetNodeInfo(mockInstanceId) + assert.NoError(t, err) + + // case 2: get from api, expire + mockCacheNode := &sdk.DescribeInstanceNodeInfoResponse{} + cache.cache[mockInstanceId] = mockCacheNode + cache.lastReloadTime[mockInstanceId] = time.Now().Add(-4 * time.Minute) + mockNodeRepo.EXPECT().GetNodeInfo(gomock.Any()).Return(&sdk.DescribeInstanceNodeInfoResponse{}, nil) + node, err := cache.GetNodeInfo(mockInstanceId) + assert.NoError(t, err) + + // case 3: get from cache + cache.cache[mockInstanceId] = &sdk.DescribeInstanceNodeInfoResponse{} + cache.lastReloadTime[mockInstanceId] = time.Now().Add(-1 * time.Minute) + + node, err = cache.GetNodeInfo(mockInstanceId) + assert.NoError(t, err) + assert.Equal(t, node, cache.cache[mockInstanceId]) +} diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index c0f3bef..3b4757b 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -6,12 +6,6 @@ import ( "strconv" ) -// 不支持实例纬度自动查询的namespace -var NotSupportInstances = []string{ - "QCE/COS", - "QCE/CDN", -} - // 每个产品的实例对象, 可用于配置导出指标的额外label填充, 根据字段名获取值 type TcInstance interface { // 获取实例的id diff --git a/pkg/instance/instance_cdb.go b/pkg/instance/instance_cdb.go index 12da1b9..6df5d49 100644 --- a/pkg/instance/instance_cdb.go +++ b/pkg/instance/instance_cdb.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdb/v20170320" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdb/v20170320" ) type CdbTcInstance struct { diff --git a/pkg/instance/instance_clb.go b/pkg/instance/instance_clb.go index 704574b..1ecc108 100644 --- a/pkg/instance/instance_clb.go +++ b/pkg/instance/instance_clb.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb/v20180317" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb/v20180317" ) type ClbInstance struct { diff --git a/pkg/instance/instance_cvm.go b/pkg/instance/instance_cvm.go index 4c012fc..c4a9da8 100644 --- a/pkg/instance/instance_cvm.go +++ b/pkg/instance/instance_cvm.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312" ) type CvmTcInstance struct { diff --git a/pkg/instance/instance_dc.go b/pkg/instance/instance_dc.go index 1151b61..d84c9a1 100644 --- a/pkg/instance/instance_dc.go +++ b/pkg/instance/instance_dc.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" ) type DcTcInstance struct { diff --git a/pkg/instance/instance_dcx.go b/pkg/instance/instance_dcx.go index bdb90ea..33bab76 100644 --- a/pkg/instance/instance_dcx.go +++ b/pkg/instance/instance_dcx.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" ) type DcxTcInstance struct { diff --git a/pkg/instance/instance_mongo.go b/pkg/instance/instance_mongo.go index 0ee290b..94ff493 100644 --- a/pkg/instance/instance_mongo.go +++ b/pkg/instance/instance_mongo.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mongodb/v20190725" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mongodb/v20190725" ) type MongoTcInstance struct { diff --git a/pkg/instance/instance_nat.go b/pkg/instance/instance_nat.go index f593557..46b0f9e 100644 --- a/pkg/instance/instance_nat.go +++ b/pkg/instance/instance_nat.go @@ -2,8 +2,9 @@ package instance import ( "fmt" - sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" "reflect" + + sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" ) type NatTcInstance struct { diff --git a/pkg/instance/instance_redis.go b/pkg/instance/instance_redis.go index 3660af2..e416a0d 100644 --- a/pkg/instance/instance_redis.go +++ b/pkg/instance/instance_redis.go @@ -11,10 +11,6 @@ type RedisTcInstance struct { meta *sdk.InstanceSet } -func (ins *RedisTcInstance) GetMeta() interface{} { - return ins.meta -} - func NewRedisTcInstance(instanceId string, meta *sdk.InstanceSet) (ins *RedisTcInstance, err error) { if instanceId == "" { return nil, fmt.Errorf("instanceId is empty ") @@ -31,3 +27,7 @@ func NewRedisTcInstance(instanceId string, meta *sdk.InstanceSet) (ins *RedisTcI } return } + +func (ins *RedisTcInstance) GetMeta() interface{} { + return ins.meta +} diff --git a/pkg/instance/repository.go b/pkg/instance/repository.go index 9c7ac0c..15017ad 100644 --- a/pkg/instance/repository.go +++ b/pkg/instance/repository.go @@ -2,6 +2,7 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/tencentyun/tencentcloud-exporter/pkg/config" ) diff --git a/pkg/instance/repository_cdb.go b/pkg/instance/repository_cdb.go index 81d2852..b52ee99 100644 --- a/pkg/instance/repository_cdb.go +++ b/pkg/instance/repository_cdb.go @@ -2,6 +2,7 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdb/v20170320" diff --git a/pkg/instance/repository_clb.go b/pkg/instance/repository_clb.go index 0aa0646..2d3700d 100644 --- a/pkg/instance/repository_clb.go +++ b/pkg/instance/repository_clb.go @@ -2,12 +2,13 @@ package instance import ( "fmt" + "net" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb/v20180317" "github.com/tencentyun/tencentcloud-exporter/pkg/client" "github.com/tencentyun/tencentcloud-exporter/pkg/config" - "net" ) func init() { diff --git a/pkg/instance/repository_cvm.go b/pkg/instance/repository_cvm.go index c97646c..4b62811 100644 --- a/pkg/instance/repository_cvm.go +++ b/pkg/instance/repository_cvm.go @@ -2,6 +2,7 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312" diff --git a/pkg/instance/repository_dc.go b/pkg/instance/repository_dc.go index 2d16406..6a2db6b 100644 --- a/pkg/instance/repository_dc.go +++ b/pkg/instance/repository_dc.go @@ -2,6 +2,7 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" diff --git a/pkg/instance/repository_dcx.go b/pkg/instance/repository_dcx.go index 2cafeec..f61b6ce 100644 --- a/pkg/instance/repository_dcx.go +++ b/pkg/instance/repository_dcx.go @@ -2,6 +2,7 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410" diff --git a/pkg/instance/repository_mongo.go b/pkg/instance/repository_mongo.go index 1fde56e..af383af 100644 --- a/pkg/instance/repository_mongo.go +++ b/pkg/instance/repository_mongo.go @@ -2,12 +2,13 @@ package instance import ( "fmt" + "strconv" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/mongodb/v20190725" "github.com/tencentyun/tencentcloud-exporter/pkg/client" "github.com/tencentyun/tencentcloud-exporter/pkg/config" - "strconv" ) func init() { diff --git a/pkg/instance/repository_nat.go b/pkg/instance/repository_nat.go index 70b3578..720b16b 100644 --- a/pkg/instance/repository_nat.go +++ b/pkg/instance/repository_nat.go @@ -2,6 +2,7 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312" diff --git a/pkg/instance/repository_redis.go b/pkg/instance/repository_redis.go index b92a526..43dee03 100644 --- a/pkg/instance/repository_redis.go +++ b/pkg/instance/repository_redis.go @@ -2,15 +2,20 @@ package instance import ( "fmt" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" sdk "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/redis/v20180412" "github.com/tencentyun/tencentcloud-exporter/pkg/client" "github.com/tencentyun/tencentcloud-exporter/pkg/config" ) +//go:generate mockgen -source=./repository_redis.go -destination=./repository_redis_mock.go -package=instance + func init() { registerRepository("QCE/REDIS", NewRedisTcInstanceRepository) + registerRepository("QCE/REDIS_MEM", NewRedisTcInstanceRepository) } type RedisTcInstanceRepository struct { @@ -18,6 +23,18 @@ type RedisTcInstanceRepository struct { logger log.Logger } +func NewRedisTcInstanceRepository(c *config.TencentConfig, logger log.Logger) (repo TcInstanceRepository, err error) { + cli, err := client.NewRedisClient(c) + if err != nil { + return + } + repo = &RedisTcInstanceRepository{ + client: cli, + logger: logger, + } + return +} + func (repo *RedisTcInstanceRepository) GetInstanceKey() string { return "instanceid" } @@ -51,6 +68,7 @@ func (repo *RedisTcInstanceRepository) ListByFilters(filters map[string]string) var total int64 = -1 req.Offset = &offset req.Limit = &limit + req.Status = []*int64{common.Int64Ptr(2)} getMoreInstances: resp, err := repo.client.DescribeInstances(req) @@ -77,14 +95,29 @@ getMoreInstances: return } -func NewRedisTcInstanceRepository(c *config.TencentConfig, logger log.Logger) (repo TcInstanceRepository, err error) { +type RedisTcInstanceNodeRepository interface { + GetNodeInfo(instanceId string) (*sdk.DescribeInstanceNodeInfoResponse, error) +} + +type RedisTcInstanceNodeRepositoryImpl struct { + client *sdk.Client + logger log.Logger +} + +func (repo *RedisTcInstanceNodeRepositoryImpl) GetNodeInfo(instanceId string) (*sdk.DescribeInstanceNodeInfoResponse, error) { + req := sdk.NewDescribeInstanceNodeInfoRequest() + req.InstanceId = common.StringPtr(instanceId) + return repo.client.DescribeInstanceNodeInfo(req) +} + +func NewRedisTcInstanceNodeRepository(c *config.TencentConfig, logger log.Logger) (RedisTcInstanceNodeRepository, error) { cli, err := client.NewRedisClient(c) if err != nil { - return + return nil, err } - repo = &RedisTcInstanceRepository{ + repo := &RedisTcInstanceNodeRepositoryImpl{ client: cli, logger: logger, } - return + return repo, nil } diff --git a/pkg/metric/cache.go b/pkg/metric/cache.go index cbba361..9e35f74 100644 --- a/pkg/metric/cache.go +++ b/pkg/metric/cache.go @@ -2,10 +2,11 @@ package metric import ( "fmt" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "strings" "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" ) // 腾讯云监控指标缓存, 在TcmMetricRepository封装一层, 指标元数据使用缓存, 转发获取数据点请求 diff --git a/pkg/metric/conf.go b/pkg/metric/conf.go index 9d25f05..9bd6c5a 100644 --- a/pkg/metric/conf.go +++ b/pkg/metric/conf.go @@ -1,8 +1,9 @@ package metric import ( - "github.com/tencentyun/tencentcloud-exporter/pkg/config" "strings" + + "github.com/tencentyun/tencentcloud-exporter/pkg/config" ) type TcmMetricConfig struct { @@ -23,6 +24,27 @@ type TcmMetricConfig struct { ExcludeInstances []string } +func (c *TcmMetricConfig) IsIncludeOnlyInstance() bool { + return len(c.OnlyIncludeInstances) > 0 +} + +func (c *TcmMetricConfig) IsIncludeAllInstance() bool { + if c.IsIncludeOnlyInstance() { + return false + } + return c.AllInstances +} + +func (c *TcmMetricConfig) IsCustomQueryDimensions() bool { + if c.IsIncludeOnlyInstance() { + return false + } + if c.IsIncludeAllInstance() { + return false + } + return len(c.CustomQueryDimensions) > 0 +} + func NewTcmMetricConfigWithMetricYaml(c config.TencentMetric, meta *TcmMeta) (*TcmMetricConfig, error) { conf := &TcmMetricConfig{} diff --git a/pkg/metric/meta.go b/pkg/metric/meta.go index c689679..663d56e 100644 --- a/pkg/metric/meta.go +++ b/pkg/metric/meta.go @@ -3,11 +3,12 @@ package metric import ( "errors" "fmt" - monitor "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/monitor/v20180724" - "github.com/tencentyun/tencentcloud-exporter/pkg/config" "sort" "strconv" "strings" + + monitor "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/monitor/v20180724" + "github.com/tencentyun/tencentcloud-exporter/pkg/config" ) // 代表一个云监控指标的元数据 diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index a423a08..0d22c40 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -2,10 +2,12 @@ package metric import ( "fmt" - "github.com/prometheus/client_golang/prometheus" - "github.com/tencentyun/tencentcloud-exporter/pkg/util" "strings" + "sync" "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/tencentyun/tencentcloud-exporter/pkg/util" ) // 代表一个指标, 包含多个时间线 @@ -16,12 +18,18 @@ type TcmMetric struct { Series map[string]*TcmSeries // 包含的多个时间线 StatPromDesc map[string]*prometheus.Desc // 按统计纬度的Desc, max、min、avg、last Conf *TcmMetricConfig + seriesLock sync.Mutex } func (m *TcmMetric) LoadSeries(series []*TcmSeries) error { + m.seriesLock.Lock() + defer m.seriesLock.Unlock() + + newSeries := make(map[string]*TcmSeries) for _, s := range series { - m.Series[s.Id] = s + newSeries[s.Id] = s } + m.Series = newSeries return nil } @@ -74,7 +82,7 @@ func (m *TcmMetric) GetLatestPromMetrics(repo TcmMetricRepository) (pms []promet return } -func (m TcmMetric) GetSeriesSplitByBatch(batch int) (steps [][]*TcmSeries) { +func (m *TcmMetric) GetSeriesSplitByBatch(batch int) (steps [][]*TcmSeries) { var series []*TcmSeries for _, s := range m.Series { series = append(series, s) diff --git a/pkg/metric/repository.go b/pkg/metric/repository.go index e83f376..74c5363 100644 --- a/pkg/metric/repository.go +++ b/pkg/metric/repository.go @@ -3,13 +3,14 @@ package metric import ( "context" "fmt" + "time" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" monitor "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/monitor/v20180724" "github.com/tencentyun/tencentcloud-exporter/pkg/client" "github.com/tencentyun/tencentcloud-exporter/pkg/config" "golang.org/x/time/rate" - "time" ) var ( @@ -199,8 +200,8 @@ func (repo *TcmMetricRepositoryImpl) ListSamples(m *TcmMetric, st int64, et int6 } samples, e := NewTcmSamples(s, points) if e != nil { - level.Warn(repo.logger).Log( - "msg", "Instance has not monitor data", + level.Debug(repo.logger).Log( + "msg", "The instance has no metric data and may not have traffic", "metric", m.Meta.MetricName, "dimension", fmt.Sprintf("%v", ql)) } else { diff --git a/pkg/metric/sample.go b/pkg/metric/sample.go index 3d6685a..3ebc810 100644 --- a/pkg/metric/sample.go +++ b/pkg/metric/sample.go @@ -2,6 +2,7 @@ package metric import ( "fmt" + monitor "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/monitor/v20180724" ) diff --git a/pkg/metric/series.go b/pkg/metric/series.go index b1cc3a8..8e7c45b 100644 --- a/pkg/metric/series.go +++ b/pkg/metric/series.go @@ -2,6 +2,7 @@ package metric import ( "fmt" + "github.com/tencentyun/tencentcloud-exporter/pkg/instance" ) diff --git a/readme.md b/readme.md index f1072ec..227109b 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,7 @@ MongoDB |QCE/CMONGO|[指标详情](https://cloud.tencent.com/document/product/24 CDB|QCE/CDB|[指标详情](https://cloud.tencent.com/document/product/248/45147) Redis标准版|QCE/REDIS|[指标详情](https://cloud.tencent.com/document/product/248/45111) Redis集群版|QCE/REDIS|[指标详情](https://cloud.tencent.com/document/product/248/45111) +Redis内存版监控指标|QCE/REDIS_MEM|[指标详情](https://cloud.tencent.com/document/product/248/49729) CVM|QCE/CVM|[指标详情](https://cloud.tencent.com/document/product/248/6843) COS|QCE/COS|[指标详情](https://cloud.tencent.com/document/product/248/45140) CDN|QCE/CDN|[指标详情](https://cloud.tencent.com/document/product/248/45138) @@ -90,6 +91,7 @@ products: range_seconds: 300 // 可选, 选取时间范围, 开始时间=now-range_seconds, 结束时间=now delay_seconds: 60 // 可选, 时间偏移量, 结束时间=now-delay_seconds metric_name_type: 1 // 可选，导出指标的名字格式化类型, 1=大写转小写加下划线, 2=转小写; 默认2 + relod_interval_minutes: 60 // 可选, 在all_instances=true时, 周期reload实例列表, 建议频率不要太频繁 // 单个指标纬度配置, 每个指标一个item