Skip to content

Commit

Permalink
Compatibility with Prometheus Alertmanager #482 (#802)
Browse files Browse the repository at this point in the history
  • Loading branch information
@amper43 @rozetko
amper43 authored and rozetko committed Nov 6, 2019
1 parent 95bd1df commit 353da09
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 42 deletions.
4 changes: 3 additions & 1 deletion server/spec/setup_tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ jest.mock('../src/config.ts', () => ({
HASTIC_API_KEY: 'fake-key',
ZMQ_IPC_PATH: 'fake-zmq-path',
HASTIC_DB_CONNECTION_TYPE: 'nedb',
HASTIC_IN_MEMORY_PERSISTANCE: true
HASTIC_IN_MEMORY_PERSISTANCE: true,
HASTIC_ALERT_TYPE: 'webhook',
AlertTypes: jest.requireActual('../src/config').AlertTypes,
}));

jest.mock('deasync', () => ({ loopWhile: jest.fn() }));
Expand Down
26 changes: 22 additions & 4 deletions server/src/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ import { getJsonDataSync } from './services/json_service';
import { normalizeUrl } from './utils/url';
import { parseTimeZone } from './utils/time';

import * as _ from 'lodash';
import * as moment from 'moment';
import * as path from 'path';
import * as fs from 'fs';
import * as os from 'os';
import * as moment from 'moment';


let configFile = path.join(__dirname, '../../config.json');
let configExists = fs.existsSync(configFile);
Expand All @@ -21,7 +23,8 @@ export type DBConfig = {
export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');

export const HASTIC_DB_IN_MEMORY = getConfigField('HASTIC_IN_MEMORY_PERSISTANCE', false);
export const HASTIC_DB_CONNECTION_TYPE = getConfigField('HASTIC_DB_CONNECTION_TYPE', 'nedb'); //nedb or mongodb
// TODO: enum for DB types
export const HASTIC_DB_CONNECTION_TYPE = getConfigField('HASTIC_DB_CONNECTION_TYPE', 'nedb', ['nedb', 'mongodb']);

//connection string syntax: <db_user>:<db_password>@<db_url>/<db_name>
export const HASTIC_DB_CONNECTION_STRING = getConfigField(
Expand All @@ -45,14 +48,24 @@ export const ZMQ_DEV_PORT = getConfigField('ZMQ_DEV_PORT', '8002');
export const ZMQ_HOST = getConfigField('ZMQ_HOST', '127.0.0.1');
export const HASTIC_API_KEY = getConfigField('HASTIC_API_KEY');
export const GRAFANA_URL = normalizeUrl(getConfigField('GRAFANA_URL', null));

// TODO: save orgId in analytic_units.db
export const ORG_ID = getConfigField('ORG_ID', 1);

export enum AlertTypes {
WEBHOOK = 'webhook',
ALERTMANAGER = 'alertmanager'
};
export const HASTIC_ALERT_TYPE = getConfigField('HASTIC_ALERT_TYPE', AlertTypes.WEBHOOK, _.values(AlertTypes));
export const HASTIC_ALERT_IMAGE = getConfigField('HASTIC_ALERT_IMAGE', false);

export const HASTIC_WEBHOOK_URL = getConfigField('HASTIC_WEBHOOK_URL', null);
export const HASTIC_WEBHOOK_TYPE = getConfigField('HASTIC_WEBHOOK_TYPE', 'application/json');
export const HASTIC_WEBHOOK_SECRET = getConfigField('HASTIC_WEBHOOK_SECRET', null);
export const HASTIC_WEBHOOK_IMAGE_ENABLED = getConfigField('HASTIC_WEBHOOK_IMAGE', false);
export const TIMEZONE_UTC_OFFSET = getTimeZoneOffset();

export const HASTIC_ALERTMANAGER_URL = getConfigField('HASTIC_ALERTMANAGER_URL', null);

export const ANLYTICS_PING_INTERVAL = 500; // ms
export const PACKAGE_VERSION = getPackageVersion();
export const GIT_INFO = getGitInfo();
Expand All @@ -63,7 +76,7 @@ export const ZMQ_CONNECTION_STRING = createZMQConnectionString();
export const HASTIC_INSTANCE_NAME = getConfigField('HASTIC_INSTANCE_NAME', os.hostname());


function getConfigField(field: string, defaultVal?: any) {
function getConfigField(field: string, defaultVal?: any, allowedVals?: any[]) {
let val;

if(process.env[field] !== undefined) {
Expand All @@ -82,6 +95,11 @@ function getConfigField(field: string, defaultVal?: any) {
}
val = defaultVal;
}

if(allowedVals !== undefined && !_.includes(allowedVals, val)) {
throw new Error(`${field} value must be in ${allowedVals}, got ${val}`);
}

console.log(`${field}: ${val}`);
return val;
}
Expand Down
15 changes: 9 additions & 6 deletions server/src/services/alert_service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { sendNotification, MetaInfo, AnalyticMeta, WebhookType, Notification } from './notification_service';
import { getNotifier, AnalyticMeta, WebhookType, Notification, MetaInfo } from './notification_service';
import * as AnalyticUnit from '../models/analytic_units';
import { Segment } from '../models/segment_model';
import { availableReporter } from '../utils/reporter';
import { toTimeZone } from '../utils/time';
import { ORG_ID, HASTIC_API_KEY, HASTIC_WEBHOOK_IMAGE_ENABLED } from '../config';
import { ORG_ID, HASTIC_API_KEY, HASTIC_ALERT_IMAGE } from '../config';

import axios from 'axios';
import * as _ from 'lodash';


const Notifier = getNotifier();
export class Alert {
public enabled = true;
constructor(protected analyticUnit: AnalyticUnit.AnalyticUnit) {};
Expand All @@ -21,7 +21,7 @@ export class Alert {
protected async send(segment) {
const notification = await this.makeNotification(segment);
try {
await sendNotification(notification);
await Notifier.sendNotification(notification);
} catch(error) {
console.error(`can't send notification ${error}`);
};
Expand All @@ -31,7 +31,7 @@ export class Alert {
const meta = this.makeMeta(segment);
const text = this.makeMessage(meta);
let result: Notification = { meta, text };
if(HASTIC_WEBHOOK_IMAGE_ENABLED) {
if(HASTIC_ALERT_IMAGE) {
try {
const image = await this.loadImage();
result.image = image;
Expand Down Expand Up @@ -212,7 +212,10 @@ export class AlertService {
from: now,
to: now
}
sendNotification({ text, meta: infoAlert });

Notifier.sendNotification({ text, meta: infoAlert }).catch((err) => {
console.error(`can't send message ${err.message}`);
});
}

public sendGrafanaAvailableWebhook() {
Expand Down
146 changes: 115 additions & 31 deletions server/src/services/notification_service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as AnalyticUnit from '../models/analytic_units';
import { HASTIC_WEBHOOK_URL, HASTIC_WEBHOOK_TYPE, HASTIC_INSTANCE_NAME } from '../config';
import * as config from '../config';

import axios from 'axios';
import * as querystring from 'querystring';
Expand All @@ -17,7 +17,14 @@ export enum WebhookType {
MESSAGE = 'MESSAGE'
}

export declare type AnalyticMeta = {
export type MetaInfo = {
type: WebhookType,
from: number,
to: number,
params?: any
}

export type AnalyticMeta = {
type: WebhookType,
analyticUnitType: string,
analyticUnitName: string,
Expand All @@ -28,47 +35,124 @@ export declare type AnalyticMeta = {
message?: any
}

export declare type MetaInfo = {
type: WebhookType,
from: number,
to: number,
params?: any
}

export declare type Notification = {
text: string,
meta: MetaInfo | AnalyticMeta,
image?: any
}

export async function sendNotification(notification: Notification) {
if(HASTIC_WEBHOOK_URL === null) {
console.log(`HASTIC_WEBHOOK_URL is not set, skip sending notification: ${notification.text}`);
return;
// TODO: split notifiers into 3 files
export interface Notifier {
sendNotification(notification: Notification): Promise<void>;
}

// TODO: singleton
export function getNotifier(): Notifier {
if(config.HASTIC_ALERT_TYPE === config.AlertTypes.WEBHOOK) {
return new WebhookNotifier();
}

notification.text += `\nInstance: ${HASTIC_INSTANCE_NAME}`;
if(config.HASTIC_ALERT_TYPE === config.AlertTypes.ALERTMANAGER) {
return new AlertManagerNotifier();
}

throw new Error(`${config.HASTIC_ALERT_TYPE} alert type not supported`);
}

class WebhookNotifier implements Notifier {
async sendNotification(notification: Notification) {
if(config.HASTIC_WEBHOOK_URL === null) {
console.log(`HASTIC_WEBHOOK_URL is not set, skip sending notification: ${notification.text}`);
return;
}

notification.text += `\nInstance: ${config.HASTIC_INSTANCE_NAME}`;

let data;
if(config.HASTIC_WEBHOOK_TYPE === ContentType.JSON) {
data = JSON.stringify(notification);
} else if(config.HASTIC_WEBHOOK_TYPE === ContentType.URLENCODED) {
data = querystring.stringify(notification);
} else {
throw new Error(`Unknown webhook type: ${config.HASTIC_WEBHOOK_TYPE}`);
}

// TODO: use HASTIC_WEBHOOK_SECRET
const options = {
method: 'POST',
url: config.HASTIC_WEBHOOK_URL,
data,
headers: { 'Content-Type': config.HASTIC_WEBHOOK_TYPE }
};

let data;
if(HASTIC_WEBHOOK_TYPE === ContentType.JSON) {
data = JSON.stringify(notification);
} else if(HASTIC_WEBHOOK_TYPE === ContentType.URLENCODED) {
data = querystring.stringify(notification);
} else {
throw new Error(`Unknown webhook type: ${HASTIC_WEBHOOK_TYPE}`);
await axios(options);
}
}

type PostableAlertLabels = {
alertname: string;
[key: string]: string
};

// TODO: use HASTIC_WEBHOOK_SECRET
const options = {
method: 'POST',
url: HASTIC_WEBHOOK_URL,
data,
headers: { 'Content-Type': HASTIC_WEBHOOK_TYPE }
};
type PostableAlertAnnotations = {
message?: string;
summary?: string;
};

try {
type PostableAlert = {
labels: PostableAlertLabels,
annotations: PostableAlertAnnotations
generatorURL?: string,
endsAt?: string
};

class AlertManagerNotifier implements Notifier {

/**
* @throws {Error} from axios if query fails
*/
async sendNotification(notification: Notification) {
if(config.HASTIC_ALERTMANAGER_URL === null) {
console.log(`HASTIC_ALERTMANAGER_URL is not set, skip sending notification: ${notification.text}`);
return;
}

let generatorURL: string;
let labels: PostableAlertLabels = {
alertname: notification.meta.type,
instance: config.HASTIC_INSTANCE_NAME
};
let annotations: PostableAlertAnnotations = {
message: notification.text
};

if(_.has(notification.meta, 'grafanaUrl')) {
generatorURL = (notification.meta as AnalyticMeta).grafanaUrl;
labels.alertname = (notification.meta as AnalyticMeta).analyticUnitName;
labels.analyticUnitId = (notification.meta as AnalyticMeta).analyticUnitId;
labels.analyticUnitType = (notification.meta as AnalyticMeta).analyticUnitType;
annotations.message = `${(notification.meta as AnalyticMeta).message}\n${generatorURL}`;
}

let alertData: PostableAlert = {
labels,
annotations,
generatorURL
};

let options = {
method: 'POST',
url: `${config.HASTIC_ALERTMANAGER_URL}/api/v2/alerts`,
data: JSON.stringify([alertData]),
headers: { 'Content-Type': ContentType.JSON }
};

//first part: send start request
await axios(options);
//TODO: resolve FAILURE alert only after RECOVERY event
//second part: send end request
alertData.endsAt = (new Date()).toISOString();
options.data = JSON.stringify([alertData]);
await axios(options);
} catch(err) {
console.error(`Can't send notification to ${HASTIC_WEBHOOK_URL}. Error: ${err.message}`);
}
}

0 comments on commit 353da09

Please sign in to comment.