Add audit alerts

lib/launcher.js

@@ -37,7 +37,10 @@ const HEALTH_POLL_MAX_STARTUP_ERROR_COUNT = 10
 const HEALTH_POLL_MAX_ERROR_COUNT = 3
 
 /** The number of seconds between resource polling */
-const RESOURCE_POLL_INTERVAL = 5
+const RESOURCE_POLL_INTERVAL = 10
+const RESOURCE_ALERT_SAMPLES = 30
+const CPU_LIMIT = process.env['FORGE_CPU_LIMIT']
+const MEMORY_LIMIT = process.env['FORGE_MEMORY_LIMIT']
 
 const States = {
     STOPPED: 'stopped',
@@ -84,6 +87,8 @@ class Launcher {
 
         // defaults to the last 3.5 hours of samples at 5 second intervals
         this.sampleBuffer = new SampleBuffer(this.options.sampleBufferMax || 2520)
+        this.cpuAuditLogged = 0
+        this.memoryAuditLogged = 0
     }
 
     async loadSettings () {
@@ -516,6 +521,38 @@ class Launcher {
 
             const sample = await resourceSample(pollUrl, RESOURCE_POLL_INTERVAL)
             this.sampleBuffer.add(sample)
+
+            //avg over the last minute for alerts
+            if ( CPU_LIMIT || MEMORY_LIMIT) {
+                const avg = this.sampleBuffer.avgLastX(RESOURCE_ALERT_SAMPLES)
+                if (avg.count == RESOURCE_ALERT_SAMPLES) {
+                    console.log(`${JSON.stringify(avg,null,2)} ${avg.count}`)
+                    if (CPU_LIMIT) {
+                        if (avg.cpu > (CPU_LIMIT * 0.75)) {
+                            if (this.cpuAuditLogged === 0) {
+                                await this.logAuditEvent('resources.cpu',{})
+                                this.cpuAuditLogged = RESOURCE_ALERT_SAMPLES
+                            }
+                        } else {
+                            if (this.cpuAuditLogged > 0) {
+                                this.cpuAuditLogged -= 1
+                            }
+                        }
+                    }
+                    if (MEMORY_LIMIT) {
+                        if (avg.ps > (MEMORY_LIMIT * 0.75))  {
+                            if (this.memoryAuditLogged === 0) {
+                                await this.logAuditEvent('resources.memory',{})
+                                this.memoryAuditLogged = RESOURCE_ALERT_SAMPLES
+                            }
+                        } else {
+                            if (this.memoryAuditLogged > 0) {
+                                this.memoryAuditLogged -= 1
+                            }
+                        }
+                    }
+                }
+            }
         }, RESOURCE_POLL_INTERVAL * 1000)
 
         this.proc.on('close', (code, signal) => {

lib/resources/sampleBuffer.js

@@ -39,6 +39,42 @@ class SampleBuffer {
             return result
         }
     }
+
+    lastX (x) {
+        if (this.head > x) {
+            return this.buffer.slice(this.head - x, this.head)
+        } else {
+            if (this.wrapped) {
+                const d = x - this.head
+                const result = this.buffer.slice(this.size - d, this.size)
+                result.push(...this.buffer.slice(0, this.head))
+                return result
+            } else {
+                return this.buffer.slice(0, this.head)
+            }
+        }
+    }
+
+    avgLastX (x) {
+        const samples  = this.lastX(x)
+        const result = {}
+        samples.forEach(sample => {
+            for (const [key, value] of Object.entries(sample)) {
+                if (key !== 'ts' && key !== 'err') {
+                    if (result[key]) {
+                        result[key] += value
+                    } else {
+                        result[key] = value
+                    }
+                }
+            }
+        })
+        for (const [key, value] of Object.entries(result)) {
+            result[key] = value/samples.length
+        }
+        result.count = samples.length
+        return result
+    }
 }
 
 module.exports = SampleBuffer