Merge pull request #3059 from gs-kamnas/pr_graceful_reload

Improve signal handling and graceful reload capabilities

CHANGELOG.md

@@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Fixed login into Fortigate when post-login-baned ist enabled. Fixes #2021 (@chrisr0880, @sahdan, @dangoscomb and @robertcheramy)
 - Fixed pre_logout for BDCOM switches
 - Fix 'wpa passphrase' hashed secret for SonicOS devices with built-in wireless #3036 (@lazynooblet)
+- Oxidized will now exit on SIGTERM and SIGINT; SIGHUP will trigger a node list reload and reopen logs (@gs-kamnas)
 - Fix potential busy wait when retries and/or next_adds_job is enabled (@gs-kamnas)
 - Reverting PR #2498 as it broke old procurve models (2510G, 2610, 2824). Fixes #2833, #2871 (@robertcheramy)
 - Fixed regexp used to remove secrets in nxos model. Fixes #3080 (@desnoe)

bin/oxidized

@@ -1,8 +1,5 @@
 #!/usr/bin/env ruby
 
-# FIX ME, killing oxidized needs -9
-trap("INT") { exit } # sinatra will otherwise steal this from us
-
 begin
   require_relative '../lib/oxidized/cli'
   Oxidized::CLI.new.run

lib/oxidized.rb

@@ -13,6 +13,7 @@ class OxidizedError < StandardError; end
   require 'oxidized/nodes'
   require 'oxidized/manager'
   require 'oxidized/hook'
+  require 'oxidized/signals'
   require 'oxidized/core'
 
   def self.asetus

lib/oxidized/core.rb

@@ -15,7 +15,15 @@ def initialize(_args)
       raise NoNodesFound, 'source returns no usable nodes' if nodes.empty?
 
       @worker = Worker.new nodes
-      trap('HUP') { nodes.load }
+      @need_reload = false
+
+      # If we receive a SIGHUP, queue a reload of the state
+      reload_proc = proc do
+        @need_reload = true
+      end
+      Signals.register_signal('HUP', reload_proc)
+
+      # Initialize REST API and webUI if requested
       if Oxidized.config.rest?
         begin
           require 'oxidized/web'
@@ -31,9 +39,20 @@ def initialize(_args)
 
     private
 
+    def reload
+      Oxidized.logger.info("Reloading node list and log files")
+      @worker.reload
+      Oxidized.logger.reopen
+      @need_reload = false
+    end
+
     def run
       Oxidized.logger.debug "lib/oxidized/core.rb: Starting the worker..."
-      @worker.work while sleep Config::SLEEP
+      loop do
+        reload if @need_reload
+        @worker.work
+        sleep Config::SLEEP
+      end
     end
   end
 end

lib/oxidized/signals.rb

@@ -0,0 +1,44 @@
+# Monkey patch Signal.trap for Puma to keep it from overriding our handlers
+# Also prevent Puma from registering its own SIGHUP handler
+module Puma
+  class Signal
+    class << self
+      alias os_trap trap
+      def Signal.trap(sig, &block)
+        sigshortname = sig.gsub "SIG", ''
+        Oxidized::Signals.register_signal(sig, block) unless sigshortname.eql? 'HUP'
+      end
+    end
+  end
+end
+
+module Oxidized
+  class Signals
+    @handlers = Hash.new { |h, k| h[k] = [] }
+    class << self
+      attr_accessor :handlers
+
+      def register_signal(sig, procobj)
+        # Compute short name of the signal (without SIG prefix)
+        sigshortname = sig.gsub "SIG", ''
+        signum = Signal.list[sigshortname]
+
+        # Register the handler with OS
+        Signal.trap signum do
+          Oxidized::Signals.handle_signal(signum)
+        end
+
+        # Add the proc to the handler list for the requested signal
+        @handlers[signum].push(procobj)
+      end
+
+      def handle_signal(signum)
+        return unless handlers.has_key?(signum)
+
+        @handlers[signum].each do |handler|
+          handler.call
+        end
+      end
+    end
+  end
+end

lib/oxidized/worker.rb

@@ -55,6 +55,10 @@ def process(job)
       Oxidized.logger.warn "#{node.group}/#{node.name} not found, removed while collecting?"
     end
 
+    def reload
+      @nodes.load
+    end
+
     private
 
     def process_success(node, job)