auth: add some more metrics

auth/backend.go

@@ -38,18 +38,22 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-type Config struct {
-	LDAP   *LDAPConfig       `yaml:"ldap"`
-	Static *StaticConfig     `yaml:"static"`
-	Whawty *WhawtyAuthConfig `yaml:"whawty"`
-}
+const (
+	metricsSubsystem = "auth"
+)
 
 var (
-	authRequests        = prometheus.NewCounterVec(prometheus.CounterOpts{Name: "auth_requests_total"}, []string{"result"})
+	authRequests        = prometheus.NewCounterVec(prometheus.CounterOpts{Subsystem: metricsSubsystem, Name: "requests_total"}, []string{"result"})
 	authRequestsSuccess = authRequests.MustCurryWith(prometheus.Labels{"result": "success"})
 	authRequestsFailed  = authRequests.MustCurryWith(prometheus.Labels{"result": "failed"})
 )
 
+type Config struct {
+	LDAP   *LDAPConfig       `yaml:"ldap"`
+	Static *StaticConfig     `yaml:"static"`
+	Whawty *WhawtyAuthConfig `yaml:"whawty"`
+}
+
 type Backend interface {
 	Authenticate(username, password string) error
 }

auth/backend_static.go

@@ -39,6 +39,11 @@ import (
 	"github.com/tg123/go-htpasswd"
 )
 
+var (
+	staticReloadFailed      = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "static_reload_failed"})
+	staticReloadLastSuccess = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "static_successful_reload_timestamp_seconds"})
+)
+
 type StaticConfig struct {
 	HTPasswd   string `yaml:"htpasswd"`
 	AutoReload bool   `yaml:"autoreload"`
@@ -61,6 +66,7 @@ func NewStaticBackend(conf *StaticConfig, prom prometheus.Registerer, infoLog, d
 
 	b := &StaticBackend{htpasswd: file, infoLog: infoLog, dbgLog: dbgLog}
 	if conf.AutoReload {
+		staticReloadLastSuccess.SetToCurrentTime()
 		runFileWatcher([]string{conf.HTPasswd}, b.watchFileErrorCB, b.watchFileEventCB)
 	}
 	if prom != nil {
@@ -78,18 +84,32 @@ func (b *StaticBackend) watchFileErrorCB(err error) {
 }
 
 func (b *StaticBackend) watchFileEventCB(event fsnotify.Event) {
+	invalidLines := 0
 	err := b.htpasswd.Reload(func(err error) {
-		b.dbgLog.Printf("static: found invalid line: %v", err)
+		invalidLines = invalidLines + 1
 	})
 	if err != nil {
+		staticReloadFailed.Set(1)
 		b.infoLog.Printf("static: reloading htpasswd file failed: %v, keeping current database", err)
 		return
 	}
+	staticReloadLastSuccess.SetToCurrentTime()
+	if invalidLines > 0 {
+		staticReloadFailed.Set(1)
+		b.infoLog.Printf("static: reloading htpasswd file was successful but %d invalid lines have been ignored", invalidLines)
+		return
+	}
+	staticReloadFailed.Set(0)
 	b.dbgLog.Printf("static: htpasswd file successfully reloaded")
 }
 
 func (b *StaticBackend) initPrometheus(prom prometheus.Registerer) (err error) {
-	// TODO: add custom metrics
+	if err = prom.Register(staticReloadFailed); err != nil {
+		return
+	}
+	if err = prom.Register(staticReloadLastSuccess); err != nil {
+		return
+	}
 	return metricsCommon(prom)
 }
 

auth/backend_whawty.go

@@ -50,6 +50,14 @@ const (
 	MaxConcurrentRemoteUpgrades = 10
 )
 
+var (
+	whawtyRemoteUpgrades        = prometheus.NewCounterVec(prometheus.CounterOpts{Subsystem: metricsSubsystem, Name: "whawty_remote_upgrades_total"}, []string{"result"})
+	whawtyRemoteUpgradesSuccess = whawtyRemoteUpgrades.MustCurryWith(prometheus.Labels{"result": "success"})
+	whawtyRemoteUpgradesFailed  = whawtyRemoteUpgrades.MustCurryWith(prometheus.Labels{"result": "failed"})
+	whawtyReloadFailed          = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "whawty_reload_failed"})
+	whawtyReloadLastSuccess     = prometheus.NewGauge(prometheus.GaugeOpts{Subsystem: metricsSubsystem, Name: "whawty_successful_reload_timestamp_seconds"})
+)
+
 type WhawtyAuthConfig struct {
 	ConfigFile     string `yaml:"store"`
 	AutoReload     bool   `yaml:"autoreload"`
@@ -95,6 +103,7 @@ func NewWhawtyAuthBackend(conf *WhawtyAuthConfig, prom prometheus.Registerer, in
 		}
 	}
 	if conf.AutoReload {
+		whawtyReloadLastSuccess.SetToCurrentTime()
 		runFileWatcher([]string{conf.ConfigFile}, b.watchFileErrorCB, b.watchFileEventCB)
 	}
 	if prom != nil {
@@ -117,6 +126,7 @@ type whawtyUpgradeRequest struct {
 func remoteHTTPUpgrade(upgrade whawtyUpgradeRequest, remote, httpHost string, client *http.Client, infoLog, dbgLog *log.Logger) {
 	reqdata, err := json.Marshal(upgrade)
 	if err != nil {
+		whawtyRemoteUpgradesFailed.WithLabelValues().Inc()
 		infoLog.Printf("whawty-auth: error while encoding remote-upgrade request: %v", err)
 		return
 	}
@@ -125,12 +135,15 @@ func remoteHTTPUpgrade(upgrade whawtyUpgradeRequest, remote, httpHost string, cl
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := client.Do(req)
 	if err != nil {
+		whawtyRemoteUpgradesFailed.WithLabelValues().Inc()
 		infoLog.Printf("whawty-auth: error sending remote-upgrade request: %v", err)
 		return
 	}
 	if resp.StatusCode != http.StatusOK {
+		whawtyRemoteUpgradesFailed.WithLabelValues().Inc()
 		infoLog.Printf("whawty-auth: remote-upgrade: failed for '%s' with status: %s", upgrade.Username, resp.Status)
 	} else {
+		whawtyRemoteUpgradesSuccess.WithLabelValues().Inc()
 		dbgLog.Printf("whawty-auth: successfully upgraded '%s'", upgrade.Username)
 	}
 }
@@ -184,28 +197,40 @@ func (b *WhawtyAuthBackend) watchFileErrorCB(err error) {
 func (b *WhawtyAuthBackend) watchFileEventCB(event fsnotify.Event) {
 	newdir, err := store.NewDirFromConfig(event.Name)
 	if err != nil {
+		whawtyReloadFailed.Set(1)
 		b.infoLog.Printf("whawty-auth: reloading store failed: %v, keeping current configuration", err)
 		return
 	}
 	if err := newdir.Check(); err != nil {
+		whawtyReloadFailed.Set(1)
 		b.infoLog.Printf("whawty-auth: reloading store failed: %v, keeping current configuration", err)
 		return
 	}
 
 	b.storeMutex.Lock()
 	defer b.storeMutex.Unlock()
 	b.store = newdir
+	whawtyReloadFailed.Set(0)
+	whawtyReloadLastSuccess.SetToCurrentTime()
 	b.infoLog.Printf("whawty-auth: successfully reloaded from: %s (%d parameter-sets loaded)", event.Name, len(b.store.Params))
 }
 
-func (b *WhawtyAuthBackend) initPrometheus(prom prometheus.Registerer) error {
-	// TODO: add custom metrics
+func (b *WhawtyAuthBackend) initPrometheus(prom prometheus.Registerer) (err error) {
+	if err = prom.Register(whawtyRemoteUpgrades); err != nil {
+		return
+	}
+	whawtyRemoteUpgradesSuccess.WithLabelValues()
+	whawtyRemoteUpgradesFailed.WithLabelValues()
+	if err = prom.Register(whawtyReloadFailed); err != nil {
+		return
+	}
+	if err = prom.Register(whawtyReloadLastSuccess); err != nil {
+		return
+	}
 	return metricsCommon(prom)
 }
 
 func (b *WhawtyAuthBackend) Authenticate(username, password string) error {
-	//authRequests.Inc()
-
 	b.storeMutex.RLock()
 	defer b.storeMutex.RUnlock()
 	ok, _, upgradeable, _, err := b.store.Authenticate(username, password)