From ba5f4e81e0d8b3336a744b8314445532fdba662d Mon Sep 17 00:00:00 2001 From: RageLtMan Date: Wed, 16 Oct 2019 05:24:26 -0400 Subject: [PATCH 1/3] Implement multi-line collection for indented logs Multiple logging implementations utilize tabs or spaces to prefix lines following the top, unindented line starting the entry. The existing multi-line approach does not deal well with variable log lengths using these indented sub-entries as it extracts lines by count, not content. This commit implements a modified postgresql log reader as a generic multi-line parser sending the current buffer downstream when it starts a new log entry (line does not start with ' ' or '\t'), or encounters an empty line. --- src/config/localfile-config.c | 1 + src/logcollector/logcollector.c | 5 +- src/logcollector/logcollector.h | 4 +- src/logcollector/read_multiline_indented.c | 120 +++++++++++++++++++++ 4 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 src/logcollector/read_multiline_indented.c diff --git a/src/config/localfile-config.c b/src/config/localfile-config.c index ba3f367ed..fddad3880 100644 --- a/src/config/localfile-config.c +++ b/src/config/localfile-config.c @@ -279,6 +279,7 @@ int Read_Localfile(XML_NODE node, void *d1, __attribute__((unused)) void *d2) } else if (strcmp(logf[pl].logformat, "command") == 0) { } else if (strcmp(logf[pl].logformat, "full_command") == 0) { } else if (strcmp(logf[pl].logformat, "audit") == 0) { + } else if (strcmp(logf[pl].logformat, "multi-line_indented") == 0) { } else if (strncmp(logf[pl].logformat, "multi-line", 10) == 0) { int x = 0; logf[pl].logformat += 10; diff --git a/src/logcollector/logcollector.c b/src/logcollector/logcollector.c index 70a1447d3..e0be8645f 100644 --- a/src/logcollector/logcollector.c +++ b/src/logcollector/logcollector.c @@ -203,6 +203,8 @@ void LogCollectorStart() logff[i].read = read_multiline; } else if (strcmp("audit", logff[i].logformat) == 0) { logff[i].read = read_audit; + } else if (strcmp("multi-line_indented", logff[i].logformat) == 0) { + logff[i].read = read_multiline_indented; } else { logff[i].read = read_syslog; } @@ -395,7 +397,7 @@ void LogCollectorStart() #ifndef WIN32 /* To help detect a file rollover, temporarily open the file a second time. - * Previously the fstat would work on "cached" file data, but this should + * Previously the fstat would work on "cached" file data, but this should * ensure it's fresh when hardlinks are used (like alerts.log). */ FILE *tf; @@ -713,4 +715,3 @@ void win_format_event_string(char *string) } #endif /* WIN32 */ - diff --git a/src/logcollector/logcollector.h b/src/logcollector/logcollector.h index 694bac45d..0c180d8f4 100644 --- a/src/logcollector/logcollector.h +++ b/src/logcollector/logcollector.h @@ -53,6 +53,9 @@ void *read_postgresql_log(int pos, int *rc, int drop_it); /* read multi line logs */ void *read_multiline(int pos, int *rc, int drop_it); +/* read indented multi line logs */ +void *read_multiline_indented(int pos, int *rc, int drop_it); + /* Read DJB multilog format */ /* Initializes multilog */ int init_djbmultilog(int pos); @@ -80,4 +83,3 @@ extern int open_file_attempts; extern logreader *logff; #endif /* __LOGREADER_H */ - diff --git a/src/logcollector/read_multiline_indented.c b/src/logcollector/read_multiline_indented.c new file mode 100644 index 000000000..c29421bea --- /dev/null +++ b/src/logcollector/read_multiline_indented.c @@ -0,0 +1,120 @@ +/* Copyright (C) 2019, Semper Victus LLC + * Copyright (C) 2009 Trend Micro Inc. + * All rights reserved. + * + * This program is free software; you can redistribute it + * and/or modify it under the terms of the GNU General Public + * License (version 2) as published by the FSF - Free Software + * Foundation. + */ + +/* Read indentend multi line logs */ + +#include "shared.h" +#include "logcollector.h" + + +/* Read multi line indented log files */ +void *read_multiline_indented(int pos, int *rc, int drop_it) { + size_t str_len = 0; + char *p; + char str[OS_MAXSTR + 1]; + char buffer[OS_MAXSTR + 1]; + /* Zero buffer and str */ + buffer[0] = '\0'; + buffer[OS_MAXSTR] = '\0'; + str[OS_MAXSTR] = '\0'; + *rc = 0; + + /* Get new entry */ + while (fgets(str, OS_MAXSTR - OS_LOG_HEADER, logff[pos].fp) != NULL) { + + /* Get buffer size */ + str_len = strlen(str); + + /* Check str_len size. Very useless, but just to make sure.. */ + if (str_len >= sizeof(buffer) - 2) { + str_len = sizeof(buffer) - 10; + } + + /* Get the last occurrence of \n */ + if ((p = strrchr(str, '\n')) != NULL) { + *p = '\0'; + } + +#ifdef WIN32 + if ((p = strrchr(str, '\r')) != NULL) { + *p = '\0'; + } +#endif + /* Look for empty string */ + if ((str_len <= 1) || (str[0] == '\r')) { + /* Send existing data if any in buffer */ + if (buffer[0] != '\0') { + if (SendMSG(logr_queue, buffer, logff[pos].file, POSTGRESQL_MQ) < 0) { + merror(QUEUE_SEND, ARGV0); + if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { + ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); + } + } + buffer[0] = '\0'; + } + continue; + } + + /* Look for lines starting with indents */ + if ((str_len > 2) && (buffer[0] != '\0') && + ((str[0] == ' ') || (str[0] == '\t'))) { + /* Size of the buffer */ + size_t buffer_len = strlen(buffer); + + p = str + 1; + + /* Remove extra spaces and tabs */ + while (*p == ' ' || *p == '\t') { + p++; + } + + /* Add additional message to the saved buffer */ + if (sizeof(buffer) - buffer_len > str_len + 256) { + /* Here we make sure that the size of the buffer + * minus what was used (strlen) is greater than + * the length of the received message. + */ + buffer[buffer_len] = ' '; + buffer[buffer_len + 1] = '\0'; + strncat(buffer, str, str_len + 3); + } + /* Look for lines not starting with indents */ + } else if ((str[0] != ' ') || (str[0] != '\t')) { + /* Flush previous messages */ + if (buffer[0] != '\0') { + if (SendMSG(logr_queue, buffer, logff[pos].file, POSTGRESQL_MQ) < 0) { + merror(QUEUE_SEND, ARGV0); + if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { + ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); + } + } + buffer[0] = '\0'; + } + strncpy(buffer, str, str_len + 2); + continue; + /* Error handling for buffer[0] being '\0' when indents are present */ + } else { + // messages or retries + } + + } + + /* Send whatever is stored */ + if (buffer[0] != '\0') { + if (SendMSG(logr_queue, buffer, logff[pos].file, POSTGRESQL_MQ) < 0) { + merror(QUEUE_SEND, ARGV0); + if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { + ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); + } + } + } + + return (NULL); +} From 5eec48fe86d244a398b059cf7120b7863d38915c Mon Sep 17 00:00:00 2001 From: RageLtMan Date: Sun, 20 Oct 2019 20:27:21 -0400 Subject: [PATCH 2/3] Fix target queue --- src/logcollector/read_multiline_indented.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/logcollector/read_multiline_indented.c b/src/logcollector/read_multiline_indented.c index c29421bea..debe80d15 100644 --- a/src/logcollector/read_multiline_indented.c +++ b/src/logcollector/read_multiline_indented.c @@ -51,7 +51,7 @@ void *read_multiline_indented(int pos, int *rc, int drop_it) { if ((str_len <= 1) || (str[0] == '\r')) { /* Send existing data if any in buffer */ if (buffer[0] != '\0') { - if (SendMSG(logr_queue, buffer, logff[pos].file, POSTGRESQL_MQ) < 0) { + if (SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { merror(QUEUE_SEND, ARGV0); if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); @@ -89,7 +89,7 @@ void *read_multiline_indented(int pos, int *rc, int drop_it) { } else if ((str[0] != ' ') || (str[0] != '\t')) { /* Flush previous messages */ if (buffer[0] != '\0') { - if (SendMSG(logr_queue, buffer, logff[pos].file, POSTGRESQL_MQ) < 0) { + if (SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { merror(QUEUE_SEND, ARGV0); if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); @@ -108,7 +108,7 @@ void *read_multiline_indented(int pos, int *rc, int drop_it) { /* Send whatever is stored */ if (buffer[0] != '\0') { - if (SendMSG(logr_queue, buffer, logff[pos].file, POSTGRESQL_MQ) < 0) { + if (SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { merror(QUEUE_SEND, ARGV0); if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); From 067c71df80e62d05aba8982491757be58427b0b8 Mon Sep 17 00:00:00 2001 From: RageLtMan Date: Thu, 31 Oct 2019 19:55:46 -0400 Subject: [PATCH 3/3] Address unused drop_it param warning Thanks for catching this @ddpbsd - the drop_it param, used by Windows readers only at this time, was throwing an unused parameter warning for keeping the same prototype as the other readers. Address by running the pointless test anyway as it has a negligible cost, without diverging from standard reader calling conventions. --- src/logcollector/read_multiline_indented.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/logcollector/read_multiline_indented.c b/src/logcollector/read_multiline_indented.c index debe80d15..2937b4d48 100644 --- a/src/logcollector/read_multiline_indented.c +++ b/src/logcollector/read_multiline_indented.c @@ -51,7 +51,7 @@ void *read_multiline_indented(int pos, int *rc, int drop_it) { if ((str_len <= 1) || (str[0] == '\r')) { /* Send existing data if any in buffer */ if (buffer[0] != '\0') { - if (SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { + if (drop_it == 0 && SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { merror(QUEUE_SEND, ARGV0); if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); @@ -89,7 +89,7 @@ void *read_multiline_indented(int pos, int *rc, int drop_it) { } else if ((str[0] != ' ') || (str[0] != '\t')) { /* Flush previous messages */ if (buffer[0] != '\0') { - if (SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { + if (drop_it == 0 && SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { merror(QUEUE_SEND, ARGV0); if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH); @@ -108,7 +108,7 @@ void *read_multiline_indented(int pos, int *rc, int drop_it) { /* Send whatever is stored */ if (buffer[0] != '\0') { - if (SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { + if (drop_it == 0 && SendMSG(logr_queue, buffer, logff[pos].file, LOCALFILE_MQ) < 0) { merror(QUEUE_SEND, ARGV0); if ((logr_queue = StartMQ(DEFAULTQPATH, WRITE)) < 0) { ErrorExit(QUEUE_FATAL, ARGV0, DEFAULTQPATH);