feat: WARN when 1 of a multi-backend proxy is down, CRIT when all down

Previously, check_haproxy would consider it a crit if any server was DOWN, change it so it only considers a full BACKEND or FRONTEND failure a crit ( Fixes #17. ) Have better active/backup role testing, to ignore backup servers that are down. ( Fixes #14 )
benprew · Mar 21, 2021 · 351b2f1 · 351b2f1
1 parent 3fa6ede
commit 351b2f1
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 29 deletions.
diff --git a/check_haproxy.rb b/check_haproxy.rb
@@ -16,10 +16,9 @@
 status = ['OK', 'WARN', 'CRIT', 'UNKN']
 
 @proxies = []
-@errors = []
-@perfdata = []
+@errors = { OK => [], WARNING => [], CRITICAL => [], UNKNOWN => [] }
 options = OpenStruct.new
-options.proxies = []
+options.proxies = nil
 
 op = OptionParser.new do |opts|
   opts.banner = 'Usage: check_haproxy.rb [options]'
@@ -149,12 +148,6 @@ def haproxy_response(options)
   end
 end
 
-def exit_code(code=0)
-  @exit_code ||= OK
-  @exit_code = code if code > @exit_code
-  @exit_code
-end
-
 header = nil
 
 haproxy_response(options).each do |line|
@@ -168,18 +161,18 @@ def exit_code(code=0)
 
   row = header.zip(CSV.parse_line(line)).reduce({}) { |hash, val| hash.merge({val[0] => val[1]}) }
 
-  next unless options.proxies.empty? || options.proxies.include?(row['pxname'])
+  next if options.proxies && !options.proxies.include?(row['pxname'])
   next if ['statistics', 'admin_stats', 'stats'].include? row['pxname']
   next if row['status'] == 'no check'
 
-  role = row['act'].to_i > 0 ? 'active ' : (row['bck'].to_i > 0 ? 'backup ' : '')
+  role = %w[BACKEND FRONTEND].include?(row['svname']) || row['bck'].to_i == 0 ? :act : :bck
   if row['slim'].to_i == 0
     session_percent_usage = 0
   else
     session_percent_usage = row['scur'].to_i * 100 / row['slim'].to_i
   end
 
-  proxy_name = sprintf("%s %s %s %s", row['pxname'], row['svname'], row['status'], role)
+  proxy_name = sprintf("%s %s (%s) %s", row['pxname'], row['svname'], role, row['status'])
   message = sprintf("%s\tsess=%s/%s(%d%%) smax=%s",
                     proxy_name,
                     row['scur'],
@@ -188,33 +181,29 @@ def exit_code(code=0)
                     row['smax'])
   @proxies << message
 
-  if row['status'].include?('DOWN')
-    @errors << message
-    exit_code CRITICAL
+  if role == :act && row['status'] == 'DOWN'
+    err_level = row['svname'] == 'BACKEND' ? CRITICAL : WARNING
+    @errors[err_level] << message
   end
 
   if options.critical && session_percent_usage >= options.critical.to_i
-    @errors << sprintf("%s - too many sessions %s/%s(%d%%)", proxy_name, row['scur'], row['slim'], session_percent_usage)
-    exit_code CRITICAL
+    @errors[CRITICAL] << sprintf("%s - too many sessions %s/%s(%d%%)", proxy_name, row['scur'], row['slim'], session_percent_usage)
   elsif options.warning && session_percent_usage >= options.warning.to_i
-    @errors << sprintf("%s - too many sessions %s/%s(%d%%)", proxy_name, row['scur'], row['slim'], session_percent_usage)
-    exit_code WARNING
+    @errors[WARNING] << sprintf("%s - too many sessions %s/%s(%d%%)", proxy_name, row['scur'], row['slim'], session_percent_usage)
   end
 end
 
-if @errors.length == 0
-  @errors << sprintf("%d proxies found", @proxies.length)
-end
+@errors[OK] << "#{@proxies.length} proxies found"
 
-if @proxies.length == 0
-  @errors << "No proxies listed as up or down"
-  exit_code UNKNOWN
-end
+@errors[UNKNOWN] << "No proxies listed as up or down" if @proxies.empty?
 
-puts "HAPROXY " + status[exit_code] + ": " + @errors.join('; ')
-puts @proxies.sort
+[CRITICAL, WARNING, UNKNOWN, OK].each do |exit_code|
+  next if @errors[exit_code].empty?
 
-exit exit_code
+  puts "HAPROXY #{status[exit_code]}: #{@errors[exit_code].join('; ')}"
+  puts @proxies.sort
+  exit exit_code
+end
 
 =begin
 Copyright (C) 2013 Ben Prew

diff --git a/test/haproxy/partial-backend-failure.csv b/test/haproxy/partial-backend-failure.csv
@@ -0,0 +1,10 @@
+# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
+ubuntuusers-static,she,0,0,0,0,512,0,0,0,,0,,0,0,0,0,UP,1,1,0,19,5,559046,168,,1,18,1,,0,,2,0,,0,L4OK,,6,0,0,0,0,0,0,0,,,,0,0,,,,,-1,,,0,0,0,0,
+ubuntuusers-static,kanu,0,0,0,0,512,0,0,0,,0,,0,0,0,0,DOWN,1,1,0,1,1,4857267,4857267,,1,18,2,,0,,2,0,,0,L4TOUT,,2001,0,0,0,0,0,0,0,,,,0,0,,,,,-1,,,0,0,0,0,
+ubuntuusers-static,BACKEND,0,0,0,1,820,761,183203,108701,0,0,,0,0,0,0,UP,1,1,0,,5,559046,168,,1,18,0,,0,,1,0,,2,,,,0,0,761,0,0,0,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
+ubuntuusers-media,she,0,0,0,0,512,0,0,0,,0,,0,0,0,0,UP,1,1,0,18,4,559046,163,,1,20,1,,0,,2,0,,0,L4OK,,6,0,0,0,0,0,0,0,,,,0,0,,,,,-1,,,0,0,0,0,
+ubuntuusers-media,kanu,0,0,0,0,512,0,0,0,,0,,0,0,0,0,DOWN,1,1,0,1,1,4857266,4857266,,1,20,2,,0,,2,0,,0,L4TOUT,,2001,0,0,0,0,0,0,0,,,,0,0,,,,,-1,,,0,0,0,0,
+ubuntuusers-media,BACKEND,0,0,0,0,820,0,0,0,0,0,,0,0,0,0,UP,1,1,0,,4,559046,163,,1,20,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
+ubuntuusers-legacy,she,0,0,0,0,512,0,0,0,,0,,0,0,0,0,UP,1,1,0,19,5,559046,167,,1,21,1,,0,,2,0,,0,L4OK,,6,0,0,0,0,0,0,0,,,,0,0,,,,,-1,,,0,0,0,0,
+ubuntuusers-legacy,kanu,0,0,0,0,512,0,0,0,,0,,0,0,0,0,DOWN,1,1,0,1,1,4857266,4857266,,1,21,2,,0,,2,0,,0,L4TOUT,,2001,0,0,0,0,0,0,0,,,,0,0,,,,,-1,,,0,0,0,0,
+ubuntuusers-legacy,BACKEND,0,0,0,0,820,0,0,0,0,0,,0,0,0,0,UP,1,1,0,,5,559046,167,,1,21,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
diff --git a/test/haproxy/partial-backend-failure.csv.expected b/test/haproxy/partial-backend-failure.csv.expected
@@ -0,0 +1,10 @@
+HAPROXY WARN: ubuntuusers-static kanu (act) DOWN	sess=0/512(0%) smax=0; ubuntuusers-media kanu (act) DOWN	sess=0/512(0%) smax=0; ubuntuusers-legacy kanu (act) DOWN	sess=0/512(0%) smax=0
+ubuntuusers-legacy BACKEND (act) UP	sess=0/820(0%) smax=0
+ubuntuusers-legacy kanu (act) DOWN	sess=0/512(0%) smax=0
+ubuntuusers-legacy she (act) UP	sess=0/512(0%) smax=0
+ubuntuusers-media BACKEND (act) UP	sess=0/820(0%) smax=0
+ubuntuusers-media kanu (act) DOWN	sess=0/512(0%) smax=0
+ubuntuusers-media she (act) UP	sess=0/512(0%) smax=0
+ubuntuusers-static BACKEND (act) UP	sess=0/820(0%) smax=1
+ubuntuusers-static kanu (act) DOWN	sess=0/512(0%) smax=0
+ubuntuusers-static she (act) UP	sess=0/512(0%) smax=0