From 353da0961ccd88cd3851cd50fb14ca8288dd2d03 Mon Sep 17 00:00:00 2001 From: Evgeny Smyshlyaev Date: Wed, 6 Nov 2019 20:21:06 +0300 Subject: [PATCH] Compatibility with Prometheus Alertmanager #482 (#802) --- server/spec/setup_tests.ts | 4 +- server/src/config.ts | 26 +++- server/src/services/alert_service.ts | 15 +- server/src/services/notification_service.ts | 146 +++++++++++++++----- 4 files changed, 149 insertions(+), 42 deletions(-) diff --git a/server/spec/setup_tests.ts b/server/spec/setup_tests.ts index a0f9c4c51..c3c503cd3 100644 --- a/server/spec/setup_tests.ts +++ b/server/spec/setup_tests.ts @@ -9,7 +9,9 @@ jest.mock('../src/config.ts', () => ({ HASTIC_API_KEY: 'fake-key', ZMQ_IPC_PATH: 'fake-zmq-path', HASTIC_DB_CONNECTION_TYPE: 'nedb', - HASTIC_IN_MEMORY_PERSISTANCE: true + HASTIC_IN_MEMORY_PERSISTANCE: true, + HASTIC_ALERT_TYPE: 'webhook', + AlertTypes: jest.requireActual('../src/config').AlertTypes, })); jest.mock('deasync', () => ({ loopWhile: jest.fn() })); diff --git a/server/src/config.ts b/server/src/config.ts index e69a494d2..82140538c 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -2,10 +2,12 @@ import { getJsonDataSync } from './services/json_service'; import { normalizeUrl } from './utils/url'; import { parseTimeZone } from './utils/time'; +import * as _ from 'lodash'; +import * as moment from 'moment'; import * as path from 'path'; import * as fs from 'fs'; import * as os from 'os'; -import * as moment from 'moment'; + let configFile = path.join(__dirname, '../../config.json'); let configExists = fs.existsSync(configFile); @@ -21,7 +23,8 @@ export type DBConfig = { export const ANALYTICS_PATH = path.join(__dirname, '../../analytics'); export const HASTIC_DB_IN_MEMORY = getConfigField('HASTIC_IN_MEMORY_PERSISTANCE', false); -export const HASTIC_DB_CONNECTION_TYPE = getConfigField('HASTIC_DB_CONNECTION_TYPE', 'nedb'); //nedb or mongodb +// TODO: enum for DB types +export const HASTIC_DB_CONNECTION_TYPE = getConfigField('HASTIC_DB_CONNECTION_TYPE', 'nedb', ['nedb', 'mongodb']); //connection string syntax: :@/ export const HASTIC_DB_CONNECTION_STRING = getConfigField( @@ -45,14 +48,24 @@ export const ZMQ_DEV_PORT = getConfigField('ZMQ_DEV_PORT', '8002'); export const ZMQ_HOST = getConfigField('ZMQ_HOST', '127.0.0.1'); export const HASTIC_API_KEY = getConfigField('HASTIC_API_KEY'); export const GRAFANA_URL = normalizeUrl(getConfigField('GRAFANA_URL', null)); + // TODO: save orgId in analytic_units.db export const ORG_ID = getConfigField('ORG_ID', 1); + +export enum AlertTypes { + WEBHOOK = 'webhook', + ALERTMANAGER = 'alertmanager' +}; +export const HASTIC_ALERT_TYPE = getConfigField('HASTIC_ALERT_TYPE', AlertTypes.WEBHOOK, _.values(AlertTypes)); +export const HASTIC_ALERT_IMAGE = getConfigField('HASTIC_ALERT_IMAGE', false); + export const HASTIC_WEBHOOK_URL = getConfigField('HASTIC_WEBHOOK_URL', null); export const HASTIC_WEBHOOK_TYPE = getConfigField('HASTIC_WEBHOOK_TYPE', 'application/json'); export const HASTIC_WEBHOOK_SECRET = getConfigField('HASTIC_WEBHOOK_SECRET', null); -export const HASTIC_WEBHOOK_IMAGE_ENABLED = getConfigField('HASTIC_WEBHOOK_IMAGE', false); export const TIMEZONE_UTC_OFFSET = getTimeZoneOffset(); +export const HASTIC_ALERTMANAGER_URL = getConfigField('HASTIC_ALERTMANAGER_URL', null); + export const ANLYTICS_PING_INTERVAL = 500; // ms export const PACKAGE_VERSION = getPackageVersion(); export const GIT_INFO = getGitInfo(); @@ -63,7 +76,7 @@ export const ZMQ_CONNECTION_STRING = createZMQConnectionString(); export const HASTIC_INSTANCE_NAME = getConfigField('HASTIC_INSTANCE_NAME', os.hostname()); -function getConfigField(field: string, defaultVal?: any) { +function getConfigField(field: string, defaultVal?: any, allowedVals?: any[]) { let val; if(process.env[field] !== undefined) { @@ -82,6 +95,11 @@ function getConfigField(field: string, defaultVal?: any) { } val = defaultVal; } + + if(allowedVals !== undefined && !_.includes(allowedVals, val)) { + throw new Error(`${field} value must be in ${allowedVals}, got ${val}`); + } + console.log(`${field}: ${val}`); return val; } diff --git a/server/src/services/alert_service.ts b/server/src/services/alert_service.ts index 9fc88db1f..0a7f89552 100644 --- a/server/src/services/alert_service.ts +++ b/server/src/services/alert_service.ts @@ -1,14 +1,14 @@ -import { sendNotification, MetaInfo, AnalyticMeta, WebhookType, Notification } from './notification_service'; +import { getNotifier, AnalyticMeta, WebhookType, Notification, MetaInfo } from './notification_service'; import * as AnalyticUnit from '../models/analytic_units'; import { Segment } from '../models/segment_model'; import { availableReporter } from '../utils/reporter'; import { toTimeZone } from '../utils/time'; -import { ORG_ID, HASTIC_API_KEY, HASTIC_WEBHOOK_IMAGE_ENABLED } from '../config'; +import { ORG_ID, HASTIC_API_KEY, HASTIC_ALERT_IMAGE } from '../config'; import axios from 'axios'; import * as _ from 'lodash'; - +const Notifier = getNotifier(); export class Alert { public enabled = true; constructor(protected analyticUnit: AnalyticUnit.AnalyticUnit) {}; @@ -21,7 +21,7 @@ export class Alert { protected async send(segment) { const notification = await this.makeNotification(segment); try { - await sendNotification(notification); + await Notifier.sendNotification(notification); } catch(error) { console.error(`can't send notification ${error}`); }; @@ -31,7 +31,7 @@ export class Alert { const meta = this.makeMeta(segment); const text = this.makeMessage(meta); let result: Notification = { meta, text }; - if(HASTIC_WEBHOOK_IMAGE_ENABLED) { + if(HASTIC_ALERT_IMAGE) { try { const image = await this.loadImage(); result.image = image; @@ -212,7 +212,10 @@ export class AlertService { from: now, to: now } - sendNotification({ text, meta: infoAlert }); + + Notifier.sendNotification({ text, meta: infoAlert }).catch((err) => { + console.error(`can't send message ${err.message}`); + }); } public sendGrafanaAvailableWebhook() { diff --git a/server/src/services/notification_service.ts b/server/src/services/notification_service.ts index b101c46f1..b0cce8071 100644 --- a/server/src/services/notification_service.ts +++ b/server/src/services/notification_service.ts @@ -1,5 +1,5 @@ import * as AnalyticUnit from '../models/analytic_units'; -import { HASTIC_WEBHOOK_URL, HASTIC_WEBHOOK_TYPE, HASTIC_INSTANCE_NAME } from '../config'; +import * as config from '../config'; import axios from 'axios'; import * as querystring from 'querystring'; @@ -17,7 +17,14 @@ export enum WebhookType { MESSAGE = 'MESSAGE' } -export declare type AnalyticMeta = { +export type MetaInfo = { + type: WebhookType, + from: number, + to: number, + params?: any +} + +export type AnalyticMeta = { type: WebhookType, analyticUnitType: string, analyticUnitName: string, @@ -28,47 +35,124 @@ export declare type AnalyticMeta = { message?: any } -export declare type MetaInfo = { - type: WebhookType, - from: number, - to: number, - params?: any -} - export declare type Notification = { text: string, meta: MetaInfo | AnalyticMeta, image?: any } -export async function sendNotification(notification: Notification) { - if(HASTIC_WEBHOOK_URL === null) { - console.log(`HASTIC_WEBHOOK_URL is not set, skip sending notification: ${notification.text}`); - return; +// TODO: split notifiers into 3 files +export interface Notifier { + sendNotification(notification: Notification): Promise; +} + +// TODO: singleton +export function getNotifier(): Notifier { + if(config.HASTIC_ALERT_TYPE === config.AlertTypes.WEBHOOK) { + return new WebhookNotifier(); } - notification.text += `\nInstance: ${HASTIC_INSTANCE_NAME}`; + if(config.HASTIC_ALERT_TYPE === config.AlertTypes.ALERTMANAGER) { + return new AlertManagerNotifier(); + } + + throw new Error(`${config.HASTIC_ALERT_TYPE} alert type not supported`); +} + +class WebhookNotifier implements Notifier { + async sendNotification(notification: Notification) { + if(config.HASTIC_WEBHOOK_URL === null) { + console.log(`HASTIC_WEBHOOK_URL is not set, skip sending notification: ${notification.text}`); + return; + } + + notification.text += `\nInstance: ${config.HASTIC_INSTANCE_NAME}`; + + let data; + if(config.HASTIC_WEBHOOK_TYPE === ContentType.JSON) { + data = JSON.stringify(notification); + } else if(config.HASTIC_WEBHOOK_TYPE === ContentType.URLENCODED) { + data = querystring.stringify(notification); + } else { + throw new Error(`Unknown webhook type: ${config.HASTIC_WEBHOOK_TYPE}`); + } + + // TODO: use HASTIC_WEBHOOK_SECRET + const options = { + method: 'POST', + url: config.HASTIC_WEBHOOK_URL, + data, + headers: { 'Content-Type': config.HASTIC_WEBHOOK_TYPE } + }; - let data; - if(HASTIC_WEBHOOK_TYPE === ContentType.JSON) { - data = JSON.stringify(notification); - } else if(HASTIC_WEBHOOK_TYPE === ContentType.URLENCODED) { - data = querystring.stringify(notification); - } else { - throw new Error(`Unknown webhook type: ${HASTIC_WEBHOOK_TYPE}`); + await axios(options); } +} + +type PostableAlertLabels = { + alertname: string; + [key: string]: string +}; - // TODO: use HASTIC_WEBHOOK_SECRET - const options = { - method: 'POST', - url: HASTIC_WEBHOOK_URL, - data, - headers: { 'Content-Type': HASTIC_WEBHOOK_TYPE } - }; +type PostableAlertAnnotations = { + message?: string; + summary?: string; +}; - try { +type PostableAlert = { + labels: PostableAlertLabels, + annotations: PostableAlertAnnotations + generatorURL?: string, + endsAt?: string +}; + +class AlertManagerNotifier implements Notifier { + + /** + * @throws {Error} from axios if query fails + */ + async sendNotification(notification: Notification) { + if(config.HASTIC_ALERTMANAGER_URL === null) { + console.log(`HASTIC_ALERTMANAGER_URL is not set, skip sending notification: ${notification.text}`); + return; + } + + let generatorURL: string; + let labels: PostableAlertLabels = { + alertname: notification.meta.type, + instance: config.HASTIC_INSTANCE_NAME + }; + let annotations: PostableAlertAnnotations = { + message: notification.text + }; + + if(_.has(notification.meta, 'grafanaUrl')) { + generatorURL = (notification.meta as AnalyticMeta).grafanaUrl; + labels.alertname = (notification.meta as AnalyticMeta).analyticUnitName; + labels.analyticUnitId = (notification.meta as AnalyticMeta).analyticUnitId; + labels.analyticUnitType = (notification.meta as AnalyticMeta).analyticUnitType; + annotations.message = `${(notification.meta as AnalyticMeta).message}\n${generatorURL}`; + } + + let alertData: PostableAlert = { + labels, + annotations, + generatorURL + }; + + let options = { + method: 'POST', + url: `${config.HASTIC_ALERTMANAGER_URL}/api/v2/alerts`, + data: JSON.stringify([alertData]), + headers: { 'Content-Type': ContentType.JSON } + }; + + //first part: send start request + await axios(options); + //TODO: resolve FAILURE alert only after RECOVERY event + //second part: send end request + alertData.endsAt = (new Date()).toISOString(); + options.data = JSON.stringify([alertData]); await axios(options); - } catch(err) { - console.error(`Can't send notification to ${HASTIC_WEBHOOK_URL}. Error: ${err.message}`); } }