forked from ElasticSearchCLITools/esTail
-
Notifications
You must be signed in to change notification settings - Fork 0
/
esTail.js
executable file
·259 lines (256 loc) · 8.32 KB
/
esTail.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
#!/usr/bin/env node
/**************************************************
**
** Requirements
**
***************************************************/
var elasticsearch = require('elasticsearch');
var markupjs = require('markup-js');
var fs = require('fs');
var colour = require('colour')
var moment = require('moment')
/**************************************************
**
** Varables
**
***************************************************/
// Display all fields default=undefined
var output=[];
var allfields;
// Is regex flag and the REGEX expression
var regex=false;
// default flags for regex g m i
var regexflags="gm";
// Display entire hit in JSON format or just deplay the message
var rawoutput;
// Disable Info messages
console.info = function (){};
/**************************************************
**
** Varables
**
***************************************************/
// Count of documents retrieved which tells us when the scan/scroll is finished
// flag when the search scan/scoll is retrieved
var searchDone=true;
// the Host to connect to
var hostportlist="localhost:9200"
// How often to query the index
var refreshInterval=1000;
// Default search template (json markup)
var searchFilename=__dirname+"/default.search"
// The DSL Query to Elasticsearch - I'll probably set a default so the script has no requirements to just work
var searchTemplate = "";
// set loglevel
var loglevel="error"
// This is used for the JSON Markup - I'll probably add a file option
var context = {
index:"_all",
from:"now-10m",
fetchsize: 100
}
/***************************************************
**
** Setup
**
***************************************************/
/*******************************
**
** Process Command Line
**
********************************/
console.info("Processing Commandline arguments");
process.argv.forEach(function (val, ind, array) {
if(/^(-h|--help|-\?)$/.test(val) ){
console.log(process.argv[0]+":");
console.log("\t[--hostport="+hostportlist+"]");
console.log("\t[--search=<filename> default: "+searchFilename);
console.log("\t[--regex='([\d\.]+)' default: none");
console.log("\t[--regexflags='gm' default: "+regexflags);
console.log("\t[--allfields default: false ");
console.log("\t[--raw default: false ");
console.log("\t[--fetchsize='20' default: 100 ");
console.log("\t[-i|--refreshInterval='1000' default: "+refreshInterval);
console.log("\t\t\tHow often a new search is issued");
console.log("\t[--context='{ 'custom':'json'}' default:"+JSON.stringify(context) );
console.log("\t\t\tContext is what varables pass to the search template for json markup");
console.log("\t\t\tcontext=<key>=<val> is a way to set any varable inside the context array. Make sure this is used after --contextfile or --context=<customejson>");
console.log("\t[--index=<index>|--context=index=<index> default: "+context.index);
console.log("\t[--from=<datestamp>|--context=from='now-5m' default: "+context.from);
console.log("\t\t\tfrom can be of any valid Elasticsearch timevalue or Caclulation ");
process.exit(1)
}
if (val === "--allfields" ){
allfields = true;
console.info("--allfields="+allfields);
}
if (val === "--raw"){
rawoutput=true;
console.info("--raw="+rawoutput);
}
if(val.indexOf('=') >0){
var s = val.split(/=/);
console.info(s[0] + ' : ' + s[1]);
if (s[0] === "--hostport" ){
hostportlist=s[1];
}
if (s[0] === "--regexflags" ){
regexflags = s[1];
}
if (s[0] === "--regex" ){
regex = s[1];
}
if (s[0] === "--loglevel" ){
loglevel = s[1];
}
if (s[0] === "--refreshinterval" || s[0] === "-i" ){
refreshInterval = s[1];
}
if (s[0] === "--contextfile" ){
context = s[1];
if (fs.existsSync(s[1])) {
var searchTemplate = fs.readFileSync(s[1],'utf8');
console.info(searchTemplate);
}else{
console.error("file does not exist:"+s[1]);
process.exit(2);
}
context = JSON.parse(context);
}
if (s[0] === "--context" && s.length == 2){
context = s[1];
context = JSON.parse(context);
}
if (s[0] === "--context" && s.length > 2 ){
console.log(s);
context[s[1]]=s[2]
console.info("context."+s[1]+"="+s[2]);
}
if (s[0] === "--search"){
searchFilename=s[1];
}
if (s[0] === "--index"){
context.index=s[1];
}
}
});
// Convert CLI options to an actual regex expression and set the regex output to be displayed
regex = new RegExp( regex,regexflags);
// Load the defaultSearch
if (fs.existsSync(searchFilename)) {
var searchTemplate = fs.readFileSync(searchFilename,'utf8');
//console.info(searchTemplate);
}else{
console.error("file does not exist:"+searchFilename);
process.exit(2);
}
// Open the Elasticsearch Connection
var client = new elasticsearch.Client({
host: hostportlist,
protocol: 'http',
index: context.index,
keepAlive: true ,
ignore: [404],
log: loglevel,
suggestCompression: true,
sniffOnStart: true,
sniffInterval: 60000
});
/**************************************************
**
** Test Connection make sure it is available
**
***************************************************/
client.ping({
requestTimeout: 1000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster maybe down!');
process.exit(1);
}else{
console.log('Connected to Elasticsearch cluster.')
}
});
/********************************************************************************
**
** Functions
**
*********************************************************************************/
// Main search
function printOutput(){
//s.sort(function ( a, b){
// a1 = moment(a._source["@timestamp"],"YYYY-MM-DDTHH:mm:ss.SSSZ").format("x");
// b1 = moment(b._source["@timestamp"],"YYYY-MM-DDTHH:mm:ss.SSSZ").format("x");
// console.log(a1-b1);
// return a1-b1;
// });
console.info("INFO".yellow+" inPrintOutput length to print="+output.length);
while ( output.length > 0 ) {
hit = output.shift();
console.info("===="+hit+" of "+output.length);
// If allfields cli option is set show all the fields not just one field
if ( allfields ) {
console.log(hit._source["@timestamp"].red+":\n".green+JSON.stringify(hit._source));
}else{
// If not allfields
// If rawoutput is set Pretty Print the json as output
if (rawoutput) {
console.log(JSON.stringify(hit,null,2));
}else{
//If not rawoutput print <indexed time>: <index>:message
console.log(hit._source["@timestamp"].red+": ".green+hit._index.green+":".green+hit._source.message)
}
}
// If set process the message field via the regex and print the results
if ( regex ) {
var result = hit._source.message.match(regex);
if ( result ){
console.log("\tregex: ".red+JSON.stringify(result).yellow);
}
}
// Set the time of the last message timestamp retrieved so we don't requery the same message
context.from = hit._source["@timestamp"];
};
}
function doSearch(){
console.info("Running search".blue);
if (! searchDone ) {
console.log("Search Not Complete");
return;
}
// convert the Template to a valid search
var search = markupjs.up(searchTemplate,context);
// Execute the Search
client.search( JSON.parse(search) , ph = function printHits(error, response) {
// Loop over the events
if (error != undefined) {
console.error("ERR:".red+error);
return;
}
console.info("INFO".yellow+"Count = "+response.hits.hits.length);
response.hits.hits.forEach(function (hit) {
// If allfields cli option is set show all the fields not just one field
//console.info("INFO".yellow+"Push Object");
output.push(hit);
});
// If the retrieved docements equals the count then we are done
printOutput()
if ( output.length >= response.hits.total ){
searchDone=true;
console.info("Search complete".blue)
return;
}
// Else query the scroll again to get more documents
client.scroll({
scrollId: response._scroll_id,
scroll: '30s'
}, ph);
});
}
/********************************************************************************
**
** Application
**
*********************************************************************************/
// set the loop for retrieving files
setInterval ( function () { if(searchDone) { doSearch()}}, refreshInterval );