Skip to content

Commit

Permalink
v2.2.0 (#20)
Browse files Browse the repository at this point in the history
* feat: product instances support auto reload interval

* feat: add redis mem product

* docs: add redis mem docs

Co-authored-by: juexiaolin(林觉霄) <[email protected]>
  • Loading branch information
juexiaolin and juexiaolin(林觉霄) authored Dec 10, 2020
1 parent 22fe15b commit 681a20a
Show file tree
Hide file tree
Showing 49 changed files with 991 additions and 331 deletions.
5 changes: 3 additions & 2 deletions cmd/qcloud-exporter/qcloud_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package main

import (
"fmt"
"net/http"
"os"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/client_golang/prometheus"
Expand All @@ -12,8 +15,6 @@ import (
"github.com/tencentyun/tencentcloud-exporter/pkg/collector"
"github.com/tencentyun/tencentcloud-exporter/pkg/config"
kingpin "gopkg.in/alecthomas/kingpin.v2"
"net/http"
"os"
)

func newHandler(c *config.TencentConfig, includeExporterMetrics bool, maxRequests int, logger log.Logger) (*http.Handler, error) {
Expand Down
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ require (
github.com/prometheus/client_golang v1.5.1
github.com/prometheus/common v0.9.1
github.com/prometheus/tsdb v0.7.1 // indirect
github.com/tencentcloud/tencentcloud-sdk-go v3.0.226+incompatible
github.com/tencentcloud/tencentcloud-sdk-go v1.0.67
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v2 v2.2.5
github.com/golang/mock v1.4.4
github.com/stretchr/testify v1.6.1

)
13 changes: 13 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
Expand All @@ -28,6 +29,8 @@ github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80n
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -56,6 +59,7 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
Expand Down Expand Up @@ -91,6 +95,10 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tencentcloud/tencentcloud-sdk-go v1.0.67 h1:fKSwJ7hrvHTxr33EcmrbKcavYJ/U2zNIH8Lvsj2FNTE=
github.com/tencentcloud/tencentcloud-sdk-go v1.0.67/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.73-0.20190704135516-e86c9d8b05ee+incompatible h1:bqYHIfgYCNbwEDT+06X2KWi5g7svrzAhZAZCGnK6Mks=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.73-0.20190704135516-e86c9d8b05ee+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.226+incompatible h1:e6tY2NFYiRHEhBZiugeZhAjlL6fw+Gq2BtCm1tnAtmg=
Expand All @@ -101,9 +109,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -116,6 +126,7 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
Expand All @@ -127,3 +138,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
27 changes: 24 additions & 3 deletions pkg/collector/collector.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
package collector

import (
"context"
"fmt"
"sync"
"time"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/tencentyun/tencentcloud-exporter/pkg/config"
"github.com/tencentyun/tencentcloud-exporter/pkg/metric"
"sync"
"time"
)

const exporterNamespace = "tcm"
Expand Down Expand Up @@ -39,6 +41,7 @@ var (
// 总指标采集器, 包含多个产品的采集器
type TcMonitorCollector struct {
Collectors map[string]*TcProductCollector
Reloaders map[string]*TcProductCollectorReloader
config *config.TencentConfig
logger log.Logger
lock sync.Mutex
Expand Down Expand Up @@ -85,6 +88,7 @@ func collect(name string, c *TcProductCollector, ch chan<- prometheus.Metric, lo

func NewTcMonitorCollector(conf *config.TencentConfig, logger log.Logger) (*TcMonitorCollector, error) {
collectors := make(map[string]*TcProductCollector)
reloaders := make(map[string]*TcProductCollectorReloader)

metricRepo, err := metric.NewTcmMetricRepository(conf, logger)
if err != nil {
Expand All @@ -99,18 +103,35 @@ func NewTcMonitorCollector(conf *config.TencentConfig, logger log.Logger) (*TcMo
continue
}

collector, err := NewTcProductCollector(namespace, metricRepoCache, conf, logger)
pconf, err := conf.GetProductConfig(namespace)
if err != nil {
return nil, err
}

collector, err := NewTcProductCollector(namespace, metricRepoCache, conf, &pconf, logger)
if err != nil {
panic(fmt.Sprintf("Create product collecter fail, err=%s, Namespace=%s", err, namespace))
}
collectors[namespace] = collector
collectorState[namespace] = 1
level.Info(logger).Log("msg", "Create product collecter ok", "Namespace", namespace)

if pconf.IsReloadEnable() {
relodInterval := time.Duration(pconf.RelodIntervalMinutes * int64(time.Minute))
reloader := NewTcProductCollectorReloader(context.TODO(), collector, relodInterval, logger)
reloaders[namespace] = reloader
go reloader.Run()
level.Info(logger).Log(
"msg", fmt.Sprintf("reload %s instances every %d minutes",
namespace, pconf.RelodIntervalMinutes),
)
}
}

level.Info(logger).Log("msg", "Create all product collecter ok", "num", len(collectors))
return &TcMonitorCollector{
Collectors: collectors,
Reloaders: reloaders,
config: conf,
logger: logger,
}, nil
Expand Down
164 changes: 106 additions & 58 deletions pkg/collector/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,34 @@ package collector

import (
"fmt"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/tencentyun/tencentcloud-exporter/pkg/metric"
)

var (
handlerFactoryMap = make(map[string]func(*TcProductCollector, log.Logger) (productHandler, error))
handlerFactoryMap = make(map[string]func(*TcProductCollector, log.Logger) (ProductHandler, error))
)

// 每个产品的指标处理逻辑
type productHandler interface {
type ProductHandler interface {
// 获取云监控指标namespace
GetNamespace() string
// 对指标元数据做检验和补充
CheckMetricMeta(meta *metric.TcmMeta) bool
// 是否包含该指标, ture=包含, false=不包含
IsIncludeMetric(m *metric.TcmMetric) bool
// 对指标元数据做检验, true=可用, false=跳过
IsMetricMetaVaild(meta *metric.TcmMeta) bool
// 修改指标元数据
ModifyMetricMeta(meta *metric.TcmMeta) error
// 对指标做校验, true=可用, false=跳过
IsMetricVaild(m *metric.TcmMetric) bool
// 修改指标
ModifyMetric(m *metric.TcmMetric) error
// 获取该指标下符合条件的所有实例, 并生成所有的series
GetSeries(tcmMetric *metric.TcmMetric) (series []*metric.TcmSeries, err error)
}

// 将对应的产品handler注册到Factory中
func registerHandler(namespace string, isDefaultEnabled bool, factory func(*TcProductCollector, log.Logger) (productHandler, error)) {
func registerHandler(namespace string, isDefaultEnabled bool, factory func(*TcProductCollector, log.Logger) (ProductHandler, error)) {
handlerFactoryMap[namespace] = factory
}

Expand All @@ -34,61 +39,104 @@ type baseProductHandler struct {
logger log.Logger
}

func (h *baseProductHandler) GetSeries(m *metric.TcmMetric) (slist []*metric.TcmSeries, err error) {
if len(m.Conf.OnlyIncludeInstances) != 0 {
for _, insId := range m.Conf.OnlyIncludeInstances {
ins, err := h.collector.InstanceRepo.Get(insId)
if err != nil {
level.Error(h.logger).Log("msg", "Instance not found", "id", insId)
continue
}
ql := map[string]string{
h.monitorQueryKey: ins.GetMonitorQueryKey(),
}
s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", insId)
continue
}
slist = append(slist, s)
func (h *baseProductHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool {
return true
}

func (h *baseProductHandler) ModifyMetricMeta(meta *metric.TcmMeta) error {
return nil
}

func (h *baseProductHandler) IsMetricVaild(m *metric.TcmMetric) bool {
return true
}

func (h *baseProductHandler) ModifyMetric(m *metric.TcmMetric) error {
return nil
}

func (h *baseProductHandler) GetSeries(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
if m.Conf.IsIncludeOnlyInstance() {
return h.GetSeriesByOnly(m)
}

if m.Conf.IsIncludeAllInstance() {
return h.GetSeriesByAll(m)
}

if m.Conf.IsCustomQueryDimensions() {
return h.GetSeriesByCustom(m)
}

return nil, fmt.Errorf("must config all_instances or only_include_instances or custom_query_dimensions")
}

func (h *baseProductHandler) GetSeriesByOnly(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
var slist []*metric.TcmSeries
for _, insId := range m.Conf.OnlyIncludeInstances {
ins, err := h.collector.InstanceRepo.Get(insId)
if err != nil {
level.Error(h.logger).Log("msg", "Instance not found", "id", insId)
continue
}
ql := map[string]string{
h.monitorQueryKey: ins.GetMonitorQueryKey(),
}
s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail",
"metric", m.Meta.MetricName, "instacne", insId)
continue
}
slist = append(slist, s)
}
return slist, nil
}

func (h *baseProductHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
var slist []*metric.TcmSeries
insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters)
if err != nil {
return nil, err
}
for _, ins := range insList {
ql := map[string]string{
h.monitorQueryKey: ins.GetMonitorQueryKey(),
}
} else if m.Conf.AllInstances {
insList, err := h.collector.InstanceRepo.ListByFilters(m.Conf.InstanceFilters)
s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
return nil, err
level.Error(h.logger).Log("msg", "Create metric series fail",
"metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, s)
}
return slist, nil
}

func (h *baseProductHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) {
var slist []*metric.TcmSeries
for _, ql := range m.Conf.CustomQueryDimensions {
v, ok := ql[h.monitorQueryKey]
if !ok {
level.Error(h.logger).Log(
"msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey),
"ql", fmt.Sprintf("%v", ql))
continue
}
for _, ins := range insList {
ql := map[string]string{
h.monitorQueryKey: ins.GetMonitorQueryKey(),
}
s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, s)
ins, err := h.collector.InstanceRepo.Get(v)
if err != nil {
level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v)
continue
}
} else {
for _, ql := range m.Conf.CustomQueryDimensions {
v, ok := ql[h.monitorQueryKey]
if !ok {
level.Error(h.logger).Log("msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey),
"ql", fmt.Sprintf("%v", ql))
continue
}
ins, err := h.collector.InstanceRepo.Get(v)
if err != nil {
level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v)
continue
}

s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail", "err", err, "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, s)

s, err := metric.NewTcmSeries(m, ql, ins)
if err != nil {
level.Error(h.logger).Log("msg", "Create metric series fail",
"err", err, "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId())
continue
}
slist = append(slist, s)
}
return
return slist, nil
}
6 changes: 3 additions & 3 deletions pkg/collector/handler_cdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,19 @@ type cdbHandler struct {
baseProductHandler
}

func (h *cdbHandler) CheckMetricMeta(meta *metric.TcmMeta) bool {
func (h *cdbHandler) IsMetricMetaVaild(meta *metric.TcmMeta) bool {
return true
}

func (h *cdbHandler) GetNamespace() string {
return CdbNamespace
}

func (h *cdbHandler) IsIncludeMetric(m *metric.TcmMetric) bool {
func (h *cdbHandler) IsMetricVaild(m *metric.TcmMetric) bool {
return true
}

func NewCdbHandler(c *TcProductCollector, logger log.Logger) (handler productHandler, err error) {
func NewCdbHandler(c *TcProductCollector, logger log.Logger) (handler ProductHandler, err error) {
handler = &cdbHandler{
baseProductHandler{
monitorQueryKey: CdbInstanceidKey,
Expand Down
Loading

0 comments on commit 681a20a

Please sign in to comment.