From 9254c9daa24720f8b973b29e377223033959ffe9 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 2 Dec 2023 00:35:56 +0100 Subject: [PATCH] auth: add some more metrics --- auth/backend.go | 16 ++++++++++------ auth/backend_static.go | 24 ++++++++++++++++++++++-- auth/backend_whawty.go | 33 +++++++++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/auth/backend.go b/auth/backend.go index cac6954..d241422 100644 --- a/auth/backend.go +++ b/auth/backend.go @@ -38,18 +38,22 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -type Config struct { - LDAP *LDAPConfig `yaml:"ldap"` - Static *StaticConfig `yaml:"static"` - Whawty *WhawtyAuthConfig `yaml:"whawty"` -} +const ( + metricsSubsystem = "auth" +) var ( - authRequests = prometheus.NewCounterVec(prometheus.CounterOpts{Name: "auth_requests_total"}, []string{"result"}) + authRequests = prometheus.NewCounterVec(prometheus.CounterOpts{Subsystem: metricsSubsystem, Name: "requests_total"}, []string{"result"}) authRequestsSuccess = authRequests.MustCurryWith(prometheus.Labels{"result": "success"}) authRequestsFailed = authRequests.MustCurryWith(prometheus.Labels{"result": "failed"}) ) +type Config struct { + LDAP *LDAPConfig `yaml:"ldap"` + Static *StaticConfig `yaml:"static"` + Whawty *WhawtyAuthConfig `yaml:"whawty"` +} + type Backend interface { Authenticate(username, password string) error } diff --git a/auth/backend_static.go b/auth/backend_static.go index 6fe2b80..cf6ecc3 100644 --- a/auth/backend_static.go +++ b/auth/backend_static.go @@ -39,6 +39,11 @@ import ( "github.com/tg123/go-htpasswd" ) +var ( + staticReloadFailed = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "static_reload_failed"}) + staticReloadLastSuccess = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "static_successful_reload_timestamp_seconds"}) +) + type StaticConfig struct { HTPasswd string `yaml:"htpasswd"` AutoReload bool `yaml:"autoreload"` @@ -61,6 +66,7 @@ func NewStaticBackend(conf *StaticConfig, prom prometheus.Registerer, infoLog, d b := &StaticBackend{htpasswd: file, infoLog: infoLog, dbgLog: dbgLog} if conf.AutoReload { + staticReloadLastSuccess.SetToCurrentTime() runFileWatcher([]string{conf.HTPasswd}, b.watchFileErrorCB, b.watchFileEventCB) } if prom != nil { @@ -78,18 +84,32 @@ func (b *StaticBackend) watchFileErrorCB(err error) { } func (b *StaticBackend) watchFileEventCB(event fsnotify.Event) { + invalidLines := 0 err := b.htpasswd.Reload(func(err error) { - b.dbgLog.Printf("static: found invalid line: %v", err) + invalidLines = invalidLines + 1 }) if err != nil { + staticReloadFailed.Set(1) b.infoLog.Printf("static: reloading htpasswd file failed: %v, keeping current database", err) return } + staticReloadLastSuccess.SetToCurrentTime() + if invalidLines > 0 { + staticReloadFailed.Set(1) + b.infoLog.Printf("static: reloading htpasswd file was successful but %d invalid lines have been ignored", invalidLines) + return + } + staticReloadFailed.Set(0) b.dbgLog.Printf("static: htpasswd file successfully reloaded") } func (b *StaticBackend) initPrometheus(prom prometheus.Registerer) (err error) { - // TODO: add custom metrics + if err = prom.Register(staticReloadFailed); err != nil { + return + } + if err = prom.Register(staticReloadLastSuccess); err != nil { + return + } return metricsCommon(prom) } diff --git a/auth/backend_whawty.go b/auth/backend_whawty.go index 5657ad7..9306bf6 100644 --- a/auth/backend_whawty.go +++ b/auth/backend_whawty.go @@ -50,6 +50,14 @@ const ( MaxConcurrentRemoteUpgrades = 10 ) +var ( + whawtyRemoteUpgrades = prometheus.NewCounterVec(prometheus.CounterOpts{Subsystem: metricsSubsystem, Name: "whawty_remote_upgrades_total"}, []string{"result"}) + whawtyRemoteUpgradesSuccess = whawtyRemoteUpgrades.MustCurryWith(prometheus.Labels{"result": "success"}) + whawtyRemoteUpgradesFailed = whawtyRemoteUpgrades.MustCurryWith(prometheus.Labels{"result": "failed"}) + whawtyReloadFailed = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "whawty_reload_failed"}) + whawtyReloadLastSuccess = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "whawty_successful_reload_timestamp_seconds"}) +) + type WhawtyAuthConfig struct { ConfigFile string `yaml:"store"` AutoReload bool `yaml:"autoreload"` @@ -95,6 +103,7 @@ func NewWhawtyAuthBackend(conf *WhawtyAuthConfig, prom prometheus.Registerer, in } } if conf.AutoReload { + whawtyReloadLastSuccess.SetToCurrentTime() runFileWatcher([]string{conf.ConfigFile}, b.watchFileErrorCB, b.watchFileEventCB) } if prom != nil { @@ -117,6 +126,7 @@ type whawtyUpgradeRequest struct { func remoteHTTPUpgrade(upgrade whawtyUpgradeRequest, remote, httpHost string, client *http.Client, infoLog, dbgLog *log.Logger) { reqdata, err := json.Marshal(upgrade) if err != nil { + whawtyRemoteUpgradesFailed.WithLabelValues().Inc() infoLog.Printf("whawty-auth: error while encoding remote-upgrade request: %v", err) return } @@ -125,12 +135,15 @@ func remoteHTTPUpgrade(upgrade whawtyUpgradeRequest, remote, httpHost string, cl req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { + whawtyRemoteUpgradesFailed.WithLabelValues().Inc() infoLog.Printf("whawty-auth: error sending remote-upgrade request: %v", err) return } if resp.StatusCode != http.StatusOK { + whawtyRemoteUpgradesFailed.WithLabelValues().Inc() infoLog.Printf("whawty-auth: remote-upgrade: failed for '%s' with status: %s", upgrade.Username, resp.Status) } else { + whawtyRemoteUpgradesSuccess.WithLabelValues().Inc() dbgLog.Printf("whawty-auth: successfully upgraded '%s'", upgrade.Username) } } @@ -184,10 +197,12 @@ func (b *WhawtyAuthBackend) watchFileErrorCB(err error) { func (b *WhawtyAuthBackend) watchFileEventCB(event fsnotify.Event) { newdir, err := store.NewDirFromConfig(event.Name) if err != nil { + whawtyReloadFailed.Set(1) b.infoLog.Printf("whawty-auth: reloading store failed: %v, keeping current configuration", err) return } if err := newdir.Check(); err != nil { + whawtyReloadFailed.Set(1) b.infoLog.Printf("whawty-auth: reloading store failed: %v, keeping current configuration", err) return } @@ -195,17 +210,27 @@ func (b *WhawtyAuthBackend) watchFileEventCB(event fsnotify.Event) { b.storeMutex.Lock() defer b.storeMutex.Unlock() b.store = newdir + whawtyReloadFailed.Set(0) + whawtyReloadLastSuccess.SetToCurrentTime() b.infoLog.Printf("whawty-auth: successfully reloaded from: %s (%d parameter-sets loaded)", event.Name, len(b.store.Params)) } -func (b *WhawtyAuthBackend) initPrometheus(prom prometheus.Registerer) error { - // TODO: add custom metrics +func (b *WhawtyAuthBackend) initPrometheus(prom prometheus.Registerer) (err error) { + if err = prom.Register(whawtyRemoteUpgrades); err != nil { + return + } + whawtyRemoteUpgradesSuccess.WithLabelValues() + whawtyRemoteUpgradesFailed.WithLabelValues() + if err = prom.Register(whawtyReloadFailed); err != nil { + return + } + if err = prom.Register(whawtyReloadLastSuccess); err != nil { + return + } return metricsCommon(prom) } func (b *WhawtyAuthBackend) Authenticate(username, password string) error { - //authRequests.Inc() - b.storeMutex.RLock() defer b.storeMutex.RUnlock() ok, _, upgradeable, _, err := b.store.Authenticate(username, password)