From 2a7448dab8a50410f211c4a3b38d23d908995cf9 Mon Sep 17 00:00:00 2001 From: nices96 Date: Fri, 11 Nov 2016 15:09:54 +0900 Subject: [PATCH] Add Elaspsed Time, GC Time and Thread Count thresholds for create alert(s). --- README.md | 12 ++ .../server/alert/email/EmailPlugin.java | 143 +++++++++++++++++- 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be6fca1..a2b36cc 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ - 신규 Agent 연결 - Agent의 연결 해제 - Agent의 재접속 + - 응답시간의 임계치 초과 + - GC Time의 임계치 초과 + - Thread 갯수의 임계치 초과 ### Properties (스카우터 서버 설치 경로 하위의 conf/scouter.conf) * **_ext\_plugin\_email\_send_alert_** : Email 발송 여부 (true / false) - 기본 값은 false @@ -22,6 +25,11 @@ * **_ext\_plugin\_email\_from_address_** : Email 발신자 계정 * **_ext\_plugin\_email\_to_address_** : Email 수신 계정(다중 사용자 지정 시 ',' 구분자 사용) * **_ext\_plugin\_email\_cc_address_** : Email 참조 수신 계정(다중 사용자 지정 시 ',' 구분자 사용) +* **_ext\_plugin\_elapsed\_time_threshold_** : 응답시간의 임계치 (ms) - 기본 값은 0으로, 0일때 응답시간의 임계치 초과 여부를 확인하지 않는다. +* **_ext\_plugin\_gc\_time_threshold_** : GC Time의 임계치 (ms) - 기본 값은 0으로, 0일때 GC Time의 임계치 초과 여부를 확인하지 않는다. +* **_ext\_plugin\_thread\_count_threshold_** : Thread Count의 임계치 (ms) - 기본 값은 0으로, 0일때 Thread Count의 임계치 초과 여부를 확인하지 않는다. + + * Example ``` @@ -37,6 +45,10 @@ ext_plugin_email_tls_enabled=true ext_plugin_email_from_address=noreply@scouter.com ext_plugin_email_to_address=receiver1@scouter.com,receiver2@scouter.com ext_plugin_email_cc_address=ccreceiver@yopmail.com + +ext_plugin_elapsed_time_threshold=5000 +ext_plugin_gc_time_threshold=5000 +ext_plugin_thread_count_threshold=300 ``` ### Dependencies diff --git a/src/scouter/plugin/server/alert/email/EmailPlugin.java b/src/scouter/plugin/server/alert/email/EmailPlugin.java index e14e05f..5e31ac5 100644 --- a/src/scouter/plugin/server/alert/email/EmailPlugin.java +++ b/src/scouter/plugin/server/alert/email/EmailPlugin.java @@ -17,18 +17,37 @@ */ package scouter.plugin.server.alert.email; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.commons.mail.DefaultAuthenticator; import org.apache.commons.mail.Email; import org.apache.commons.mail.SimpleEmail; import scouter.lang.AlertLevel; +import scouter.lang.TextTypes; +import scouter.lang.TimeTypeEnum; +import scouter.lang.counters.CounterConstants; import scouter.lang.pack.AlertPack; +import scouter.lang.pack.MapPack; import scouter.lang.pack.ObjectPack; +import scouter.lang.pack.PerfCounterPack; +import scouter.lang.pack.XLogPack; import scouter.lang.plugin.PluginConstants; import scouter.lang.plugin.annotation.ServerPlugin; +import scouter.net.RequestCmd; import scouter.server.Configure; +import scouter.server.CounterManager; import scouter.server.Logger; import scouter.server.core.AgentManager; +import scouter.server.db.TextRD; +import scouter.server.netio.AgentCall; +import scouter.util.DateUtil; +import scouter.util.HashUtil; /** * Scouter server plugin to send alert via email @@ -39,6 +58,48 @@ public class EmailPlugin { // Get singleton Configure instance from server final Configure conf = Configure.getInstance(); + + private static AtomicInteger ai = new AtomicInteger(0); + private static List javaeeObjHashList = new ArrayList(); + + public EmailPlugin() { + if (ai.incrementAndGet() == 1) { + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + // thread count check + executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + for (int objHash : javaeeObjHashList) { + if (AgentManager.isActive(objHash)) { + ObjectPack objectPack = AgentManager.getAgent(objHash); + MapPack mapPack = new MapPack(); + mapPack.put("objHash", objHash); + + mapPack = AgentCall.call(objectPack, RequestCmd.OBJECT_THREAD_LIST, mapPack); + + int threadCountThreshold = conf.getInt("ext_plugin_thread_count_threshold", 0); + int threadCount = mapPack.getList("name").size(); + + if (threadCountThreshold != 0 && threadCount > threadCountThreshold) { + AlertPack ap = new AlertPack(); + + ap.level = AlertLevel.WARN; + ap.objHash = objHash; + ap.title = "Thread count exceed a threahold."; + ap.message = objectPack.objName + "'s Thread count(" + threadCount + ") exceed a threshold."; + ap.time = System.currentTimeMillis(); + ap.objType = objectPack.objType; + + alert(ap); + } + } + } + } + }, + 0, 5, TimeUnit.SECONDS); + } + } @ServerPlugin(PluginConstants.PLUGIN_SERVER_ALERT) public void alert(final AlertPack pack) { @@ -150,7 +211,12 @@ public void object(ObjectPack pack) { ap.title = "An object has been activated."; ap.message = pack.objName + " is connected."; ap.time = System.currentTimeMillis(); - ap.objType = "scouter"; + + if (AgentManager.getAgent(pack.objHash) != null) { + ap.objType = AgentManager.getAgent(pack.objHash).objType; + } else { + ap.objType = "scouter"; + } alert(ap); } else if (op.alive == false) { @@ -161,13 +227,86 @@ public void object(ObjectPack pack) { ap.title = "An object has been activated."; ap.message = pack.objName + " is reconnected."; ap.time = System.currentTimeMillis(); - ap.objType = "scouter"; + ap.objType = AgentManager.getAgent(pack.objHash).objType; alert(ap); } // inactive state can be handled in alert() method. } } + + @ServerPlugin(PluginConstants.PLUGIN_SERVER_XLOG) + public void xlog(XLogPack pack) { + try { + int elapsedThreshold = conf.getInt("ext_plugin_elapsed_time_threshold", 0); + + if (elapsedThreshold != 0 && pack.elapsed > elapsedThreshold) { + String serviceName = TextRD.getString(DateUtil.yyyymmdd(pack.endTime), TextTypes.SERVICE, pack.service); + + AlertPack ap = new AlertPack(); + + ap.level = AlertLevel.WARN; + ap.objHash = pack.objHash; + ap.title = "Elapsed time exceed a threahold."; + ap.message = "[" + AgentManager.getAgentName(pack.objHash) + "] " + + pack.service + "(" + serviceName + ") " + + "elapsed time(" + pack.elapsed + " ms) exceed a threshold."; + ap.time = System.currentTimeMillis(); + ap.objType = AgentManager.getAgent(pack.objHash).objType; + + alert(ap); + } + + } catch (Exception e) { + Logger.printStackTrace(e); + } + } + + @ServerPlugin(PluginConstants.PLUGIN_SERVER_COUNTER) + public void counter(PerfCounterPack pack) { + String objName = pack.objName; + int objHash = HashUtil.hash(objName); + String objType = null; + String objFamily = null; + + if (AgentManager.getAgent(objHash) != null) { + objType = AgentManager.getAgent(objHash).objType; + } + + if (objType != null) { + objFamily = CounterManager.getInstance().getCounterEngine().getObjectType(objType).getFamily().getName(); + } + + try { + // in case of objFamily is javaee + if (CounterConstants.FAMILY_JAVAEE.equals(objFamily)) { + // save javaee type's objHash + if (!javaeeObjHashList.contains(objHash)) { + javaeeObjHashList.add(objHash); + } + + if (pack.timetype == TimeTypeEnum.REALTIME) { + long gcTimeThreshold = conf.getLong("ext_plugin_gc_time_threshold", 0); + long gcTime = pack.data.getLong(CounterConstants.JAVA_GC_TIME); + + if (gcTimeThreshold != 0 && gcTime > gcTimeThreshold) { + AlertPack ap = new AlertPack(); + + ap.level = AlertLevel.WARN; + ap.objHash = objHash; + ap.title = "Elapsed time exceed a threahold."; + ap.message = objName + "'s GC time(" + gcTime + " ms) exceed a threshold."; + ap.time = System.currentTimeMillis(); + ap.objType = objType; + + alert(ap); + } + } + } + } catch (Exception e) { + Logger.printStackTrace(e); + } + } private void println(Object o) { if (conf.getBoolean("ext_plugin_email_debug", false)) {