diff --git a/core/common/JsonUtil.cpp b/core/common/JsonUtil.cpp index 4a416737f3..32d244ae18 100644 --- a/core/common/JsonUtil.cpp +++ b/core/common/JsonUtil.cpp @@ -77,6 +77,8 @@ std::string CompactJson(const std::string& inJson) { case '\\': if (++ch == inJson.end()) { // skip next char after escape char --ch; + } else { + outJson << '\\'; } break; default: diff --git a/core/config/Config.h b/core/config/Config.h index de75620faf..a704129d0d 100644 --- a/core/config/Config.h +++ b/core/config/Config.h @@ -138,26 +138,26 @@ class Config { std::shared_ptr mFilterRule; bool mLocalStorage; int mVersion; - bool mDiscardNoneUtf8; + bool mDiscardNoneUtf8 = false; std::string mAliuid; std::string mRegion; std::string mStreamLogTag; - bool mDiscardUnmatch; + bool mDiscardUnmatch = false; std::vector mColumnKeys; std::string mSeparator; char mQuote; // for delimiter log, accept logs without enough keys or not // eg, keys -> [a, b, c], raw log "xx|yy", log -> [a->xx, b->yy] - bool mAcceptNoEnoughKeys; - bool mAutoExtend; + bool mAcceptNoEnoughKeys = false; + bool mAutoExtend = true; std::string mTimeKey; std::vector mShardHashKey; bool mTailExisted; std::unordered_map> mSensitiveWordCastOptions; - bool mUploadRawLog; // true to update raw log to sls + bool mUploadRawLog = false; // true to update raw log to sls bool mSimpleLogFlag; bool mTimeZoneAdjust; - int mLogTimeZoneOffsetSecond; + int mLogTimeZoneOffsetSecond = 0; int32_t mCreateTime; // create time of this config int32_t mMaxSendBytesPerSecond; // limit for logstore, not just this config. so if we have multi configs with different diff --git a/core/processor/ProcessorParseApsaraNative.cpp b/core/processor/ProcessorParseApsaraNative.cpp index df063d6f19..77776c7de9 100644 --- a/core/processor/ProcessorParseApsaraNative.cpp +++ b/core/processor/ProcessorParseApsaraNative.cpp @@ -70,6 +70,14 @@ void ProcessorParseApsaraNative::Process(PipelineEventGroup& logGroup) { return; } +/* + * 处理单个日志事件。 + * @param logPath - 日志文件的路径。 + * @param e - 指向待处理日志事件的智能指针。 + * @param lastLogTime - 上一条日志的时间戳(秒)。 + * @param timeStrCache - 缓存时间字符串,用于比较和更新。 + * @return 如果事件被处理且保留,则返回true,如果事件被丢弃,则返回false。 + */ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, PipelineEventPtr& e, LogtailTime& lastLogTime, StringView& timeStrCache) { if (!IsSupportedEvent(e)) { return true; @@ -79,6 +87,9 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, Pipelin return true; } StringView buffer = sourceEvent.GetContent(mSourceKey); + if (buffer.size() == 0) { + return true; + } mProcParseInSizeBytes->Add(buffer.size()); int64_t logTime_in_micro = 0; time_t logTime = ApsaraEasyReadLogTimeParser(buffer, timeStrCache, lastLogTime, logTime_in_micro); @@ -148,13 +159,14 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, Pipelin index = ParseApsaraBaseFields(buffer, sourceEvent); bool sourceKeyOverwritten = mSourceKeyOverwritten; bool rawLogTagOverwritten = false; - if (buffer.data()[index] != 0) { - do { - ++index; - if (buffer.data()[index] == '\t' || buffer.data()[index] == '\0') { + int32_t length = buffer.size(); + if (index < length) { + for (index = index + 1; index <= length; ++index) { + if (index == length || buffer.data()[index] == '\t') { if (colon_index >= 0) { StringView key(buffer.data() + beg_index, colon_index - beg_index); - AddLog(key, StringView(buffer.data() + colon_index + 1, index - colon_index - 1), sourceEvent); + StringView data(buffer.data() + colon_index + 1, index - colon_index - 1); + AddLog(key, data, sourceEvent); if (key == mSourceKey) { sourceKeyOverwritten = true; } @@ -167,7 +179,7 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, Pipelin } else if (buffer.data()[index] == ':' && colon_index == -1) { colon_index = index; } - } while (buffer.data()[index]); + } } // TODO: deprecated if (mAdjustApsaraMicroTimezone) { @@ -189,6 +201,14 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, Pipelin return true; } +/* + * 解析Apsara格式日志的时间。 + * @param buffer - 包含日志数据的字符串视图。 + * @param timeStr - 解析后的时间字符串。 + * @param lastLogTime - 上一条日志的时间戳(秒)。 + * @param microTime - 解析出的微秒时间戳。 + * @return 解析出的时间戳(秒),如果解析失败,则返回0。 + */ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffer, StringView& timeStr, LogtailTime& lastLogTime, int64_t& microTime) { if (buffer[0] != '[') { return 0; @@ -196,7 +216,13 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe if (buffer[1] == '1') // for normal time, e.g 1378882630, starts with '1' { int nanosecondLength = 0; - auto strptimeResult = Strptime(buffer.data() + 1, "%s", &lastLogTime, nanosecondLength); + size_t pos = buffer.find(']', 1); + if (pos == std::string::npos) { + LOG_WARNING(sLogger, ("parse apsara log time", "fail")("string", buffer)); + return 0; + } + std::string strTime = buffer.substr(1, pos).to_string(); + auto strptimeResult = Strptime(strTime.c_str(), "%s", &lastLogTime, nanosecondLength); if (NULL == strptimeResult || strptimeResult[0] != ']') { LOG_WARNING(sLogger, ("parse apsara log time", "fail")("string", buffer)("timeformat", "%s")); @@ -207,14 +233,20 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe } // test other date format case { - if (IsPrefixString(buffer.data() + 1, timeStr) == true) { + size_t pos = buffer.find(']', 1); + if (pos == std::string::npos) { + LOG_WARNING(sLogger, ("parse apsara log time", "fail")("string", buffer)); + return 0; + } + std::string strTime = buffer.substr(1, pos).to_string(); + if (IsPrefixString(strTime.c_str(), timeStr) == true) { microTime = (int64_t)lastLogTime.tv_sec * 1000000 + lastLogTime.tv_nsec / 1000; return lastLogTime.tv_sec; } struct tm tm; memset(&tm, 0, sizeof(tm)); int nanosecondLength = 0; - auto strptimeResult = Strptime(buffer.data() + 1, "%Y-%m-%d %H:%M:%S.%f", &lastLogTime, nanosecondLength); + auto strptimeResult = Strptime(strTime.c_str(), "%Y-%m-%d %H:%M:%S.%f", &lastLogTime, nanosecondLength); if (NULL == strptimeResult || strptimeResult[0] != ']') { LOG_WARNING(sLogger, ("parse apsara log time", "fail")("string", buffer)("timeformat", "%Y-%m-%d %H:%M:%S.%f")); @@ -227,6 +259,12 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe } } +/* + * 检查字符串是否包含指定的前缀。 + * @param all - 完整的字符串。 + * @param prefix - 要检查的前缀。 + * @return 如果字符串以指定前缀开头,则返回true;否则返回false。 + */ bool ProcessorParseApsaraNative::IsPrefixString(const char* all, const StringView& prefix) { if (prefix.size() == 0) return false; @@ -239,13 +277,20 @@ bool ProcessorParseApsaraNative::IsPrefixString(const char* all, const StringVie return true; } -static int32_t FindBaseFields(StringView& buffer, int32_t beginIndexArray[], int32_t endIndexArray[]) { +/* + * 查找Apsara格式日志的基础字段。 + * @param buffer - 包含日志数据的字符串视图。 + * @param beginIndexArray - 字段开始索引的数组。 + * @param endIndexArray - 字段结束索引的数组。 + * @return 解析到的基础字段数量。 + */ +static int32_t FindBaseFields(const StringView& buffer, int32_t beginIndexArray[], int32_t endIndexArray[]) { int32_t baseFieldNum = 0; - for (int32_t i = 0; buffer[i] != 0; i++) { + for (size_t i = 0; i < buffer.size(); i++) { if (buffer[i] == '[') { beginIndexArray[baseFieldNum] = i + 1; } else if (buffer[i] == ']') { - if (buffer[i + 1] == '\t' || buffer[i + 1] == '\0' || buffer[i + 1] == '\n') { + if (buffer[i + 1] == '\t' || buffer[i + 1] == '\n') { endIndexArray[baseFieldNum] = i; baseFieldNum++; } @@ -260,7 +305,14 @@ static int32_t FindBaseFields(StringView& buffer, int32_t beginIndexArray[], int return baseFieldNum; } -static bool IsFieldLevel(StringView& buffer, int32_t beginIndex, int32_t endIndex) { +/* + * 检查是否为日志级别字段。 + * @param buffer - 包含日志数据的字符串视图。 + * @param beginIndex - 字段开始的索引。 + * @param endIndex - 字段结束的索引。 + * @return 如果字段是日志级别,则返回true;否则返回false。 + */ +static bool IsFieldLevel(const StringView& buffer, int32_t beginIndex, int32_t endIndex) { for (int32_t i = beginIndex; i < endIndex; i++) { if (buffer[i] > 'Z' || buffer[i] < 'A') { return false; @@ -269,7 +321,14 @@ static bool IsFieldLevel(StringView& buffer, int32_t beginIndex, int32_t endInde return true; } -static bool IsFieldThread(StringView& buffer, int32_t beginIndex, int32_t endIndex) { +/* + * 检查是否为线程ID字段。 + * @param buffer - 包含日志数据的字符串视图。 + * @param beginIndex - 字段开始的索引。 + * @param endIndex - 字段结束的索引。 + * @return 如果字段是线程ID,则返回true;否则返回false。 + */ +static bool IsFieldThread(const StringView& buffer, int32_t beginIndex, int32_t endIndex) { for (int32_t i = beginIndex; i < endIndex; i++) { if (buffer[i] > '9' || buffer[i] < '0') { return false; @@ -278,7 +337,14 @@ static bool IsFieldThread(StringView& buffer, int32_t beginIndex, int32_t endInd return true; } -static bool IsFieldFileLine(StringView& buffer, int32_t beginIndex, int32_t endIndex) { +/* + * 检查是否为文件和行号字段。 + * @param buffer - 包含日志数据的字符串视图。 + * @param beginIndex - 字段开始的索引。 + * @param endIndex - 字段结束的索引。 + * @return 如果字段是文件和行号,则返回true;否则返回false。 + */ +static bool IsFieldFileLine(const StringView& buffer, int32_t beginIndex, int32_t endIndex) { for (int32_t i = beginIndex; i < endIndex; i++) { if (buffer[i] == '/' || buffer[i] == '.') { return true; @@ -287,7 +353,14 @@ static bool IsFieldFileLine(StringView& buffer, int32_t beginIndex, int32_t endI return false; } -static int32_t FindColonIndex(StringView& buffer, int32_t beginIndex, int32_t endIndex) { +/* + * 查找冒号字符的索引。 + * @param buffer - 包含日志数据的字符串视图。 + * @param beginIndex - 字段开始的索引。 + * @param endIndex - 字段结束的索引。 + * @return 冒号字符的索引,如果未找到,则返回endIndex。 + */ +static int32_t FindColonIndex(const StringView& buffer, int32_t beginIndex, int32_t endIndex) { for (int32_t i = beginIndex; i < endIndex; i++) { if (buffer[i] == ':') { return i; @@ -296,7 +369,13 @@ static int32_t FindColonIndex(StringView& buffer, int32_t beginIndex, int32_t en return endIndex; } -int32_t ProcessorParseApsaraNative::ParseApsaraBaseFields(StringView& buffer, LogEvent& sourceEvent) { +/* + * 解析Apsara日志的基础字段并添加到日志事件中。 + * @param buffer - 包含日志数据的字符串视图。 + * @param sourceEvent - 引用到日志事件对象,用于添加解析出的字段。 + * @return 返回处理完基础字段后的索引位置。 + */ +int32_t ProcessorParseApsaraNative::ParseApsaraBaseFields(const StringView& buffer, LogEvent& sourceEvent) { int32_t beginIndexArray[LogParser::MAX_BASE_FIELD_NUM] = {0}; int32_t endIndexArray[LogParser::MAX_BASE_FIELD_NUM] = {0}; int32_t baseFieldNum = FindBaseFields(buffer, beginIndexArray, endIndexArray); diff --git a/core/processor/ProcessorParseApsaraNative.h b/core/processor/ProcessorParseApsaraNative.h index 1da6ecc2db..43ee3c951f 100644 --- a/core/processor/ProcessorParseApsaraNative.h +++ b/core/processor/ProcessorParseApsaraNative.h @@ -37,7 +37,7 @@ class ProcessorParseApsaraNative : public Processor { time_t ApsaraEasyReadLogTimeParser(StringView& buffer, StringView& timeStr, LogtailTime& lastLogTime, int64_t& microTime); int32_t GetApsaraLogMicroTime(StringView& buffer); bool IsPrefixString(const char* all, const StringView& prefix); - int32_t ParseApsaraBaseFields(StringView& buffer, LogEvent& sourceEvent); + int32_t ParseApsaraBaseFields(const StringView& buffer, LogEvent& sourceEvent); std::string mSourceKey; std::string mRawLogTag; diff --git a/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp b/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp index 1c69a0eae2..9cdea30e11 100644 --- a/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp @@ -12,13 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. #include -#include "unittest/Unittest.h" + #include "common/JsonUtil.h" #include "config/Config.h" -#include "processor/ProcessorDesensitizeNative.h" +#include "config_manager/ConfigManager.h" #include "models/LogEvent.h" #include "plugin/instance/ProcessorInstance.h" -#include "config_manager/ConfigManager.h" +#include "processor/ProcessorDesensitizeNative.h" +#include "processor/ProcessorSplitLogStringNative.h" +#include "processor/ProcessorSplitRegexNative.h" +#include "unittest/Unittest.h" using namespace std; @@ -32,7 +35,7 @@ class ProcessorDesensitizeNativeUnittest : public ::testing::Test { mContext.SetProjectName("project"); mContext.SetRegion("cn-shanghai"); } - Config* GetCastSensWordConfig(string, string, int, bool, string); + Config GetCastSensWordConfig(string, string, int, bool, string); void TestInit(); void TestParseCastSensWordConfig(); void TestCastSensWordConst(); @@ -41,6 +44,8 @@ class ProcessorDesensitizeNativeUnittest : public ::testing::Test { void TestCastSensWordLoggroup(); void TestCastSensWordMulti(); void TestCastWholeKey(); + void TestMultipleLines(); + PipelineContext mContext; }; @@ -60,13 +65,15 @@ UNIT_TEST_CASE(ProcessorDesensitizeNativeUnittest, TestCastSensWordMulti); UNIT_TEST_CASE(ProcessorDesensitizeNativeUnittest, TestCastWholeKey); -Config* ProcessorDesensitizeNativeUnittest::GetCastSensWordConfig(string key = string("cast1"), - string regex = string("(pwd=)[^,]+"), - int type = SensitiveWordCastOption::CONST_OPTION, - bool replaceAll = false, - string constVal = string("\\1********")) { - Config* oneConfig = new Config; - vector& optVec = oneConfig->mSensitiveWordCastOptions[key]; +UNIT_TEST_CASE(ProcessorDesensitizeNativeUnittest, TestMultipleLines); + +Config ProcessorDesensitizeNativeUnittest::GetCastSensWordConfig(string key = string("cast1"), + string regex = string("(pwd=)[^,]+"), + int type = SensitiveWordCastOption::CONST_OPTION, + bool replaceAll = false, + string constVal = string("\\1********")) { + Config oneConfig; + vector& optVec = oneConfig.mSensitiveWordCastOptions[key]; optVec.resize(1); optVec[0].option = SensitiveWordCastOption::CONST_OPTION; optVec[0].key = key; @@ -76,13 +83,116 @@ Config* ProcessorDesensitizeNativeUnittest::GetCastSensWordConfig(string key = s return oneConfig; } +void ProcessorDesensitizeNativeUnittest::TestMultipleLines() { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "asf@@@324 FS2$%pwd,pwd=saf543#$@,, +dbf@@@324 FS2$%pwd,pwd=saf543#$@,,", + "log.file.offset": "0" + }, + "timestampNanosecond" : 0, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + + std::string expectJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "asf@@@324 FS2$%pwd,pwd=********,,", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + }, + { + "contents" : + { + "content" : "dbf@@@324 FS2$%pwd,pwd=********,,", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config = GetCastSensWordConfig("content"); + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + + // run function ProcessorDesensitizeNative + ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config = GetCastSensWordConfig("content"); + config.mLogBeginReg = ".*"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorDesensitizeNative + ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } +} + void ProcessorDesensitizeNativeUnittest::TestInit() { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); // run function ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); } @@ -124,11 +234,11 @@ void ProcessorDesensitizeNativeUnittest::TestParseCastSensWordConfig() { void ProcessorDesensitizeNativeUnittest::TestCastSensWordConst() { // case1 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -169,16 +279,14 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordConst() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case2 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -219,16 +327,14 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordConst() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case3 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -269,17 +375,15 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordConst() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case4 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -320,17 +424,15 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordConst() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case5 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -371,21 +473,19 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordConst() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } } void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { // case1 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -426,18 +526,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case2 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -478,18 +576,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 3 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -530,18 +626,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 4 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -582,18 +676,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 5 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -634,18 +727,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 6 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -686,18 +778,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 7 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -738,18 +829,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 8 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -790,18 +880,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 9 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].option = SensitiveWordCastOption::MD5_OPTION; + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -842,19 +931,18 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMD5() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } } void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { // case 1 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -895,16 +983,15 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 2 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -945,17 +1032,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - delete config; } // case 3 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].constValue = "********"; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].constValue = "********"; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -996,18 +1082,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 4 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].constValue = "\\2********"; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].constValue = "\\2********"; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1048,19 +1132,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 5 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].constValue = "\\2********"; - config->mSensitiveWordCastOptions.begin()->second[0].mRegex.reset(new re2::RE2("pwd=[^,]+")); + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].constValue = "\\2********"; + config.mSensitiveWordCastOptions.begin()->second[0].mRegex.reset(new re2::RE2("pwd=[^,]+")); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1101,20 +1183,18 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 6 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].constValue = "\\2********"; - config->mSensitiveWordCastOptions.begin()->second[0].mRegex.reset(new re2::RE2("pwd=[^,]+")); - config->mSensitiveWordCastOptions.begin()->second[0].mRegex.reset(new re2::RE2("")); + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].constValue = "\\2********"; + config.mSensitiveWordCastOptions.begin()->second[0].mRegex.reset(new re2::RE2("pwd=[^,]+")); + config.mSensitiveWordCastOptions.begin()->second[0].mRegex.reset(new re2::RE2("")); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1155,15 +1235,13 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordFail() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } } void ProcessorDesensitizeNativeUnittest::TestCastSensWordLoggroup() { - Config* config = GetCastSensWordConfig(); - vector& optVec = config->mSensitiveWordCastOptions["id"]; - vector& cntVec = config->mSensitiveWordCastOptions["content"]; + Config config = GetCastSensWordConfig(); + vector& optVec = config.mSensitiveWordCastOptions["id"]; + vector& cntVec = config.mSensitiveWordCastOptions["content"]; optVec.resize(1); optVec[0].option = SensitiveWordCastOption::CONST_OPTION; optVec[0].key = "id"; @@ -1180,7 +1258,7 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordLoggroup() { ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1251,19 +1329,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordLoggroup() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } void ProcessorDesensitizeNativeUnittest::TestCastSensWordMulti() { // case 1 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1304,17 +1380,15 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMulti() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 2 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1355,17 +1429,15 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMulti() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 3 { - Config* config = GetCastSensWordConfig(); + Config config = GetCastSensWordConfig(); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1406,18 +1478,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMulti() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 4 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1458,19 +1528,17 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMulti() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } // case 5 { - Config* config = GetCastSensWordConfig(); - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; - config->mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + Config config = GetCastSensWordConfig(); + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; + config.mSensitiveWordCastOptions.begin()->second[0].replaceAll = true; // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1511,18 +1579,16 @@ void ProcessorDesensitizeNativeUnittest::TestCastSensWordMulti() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } } void ProcessorDesensitizeNativeUnittest::TestCastWholeKey() { - Config* config = GetCastSensWordConfig("pwd", "().*", 1, false, "\\1********"); + Config config = GetCastSensWordConfig("pwd", "().*", 1, false, "\\1********"); // init ProcessorDesensitizeNative& processor = *(new ProcessorDesensitizeNative); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); - ComponentConfig componentConfig(pluginId, *config); + ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); // make events auto sourceBuffer = std::make_shared(); @@ -1563,8 +1629,6 @@ void ProcessorDesensitizeNativeUnittest::TestCastWholeKey() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - - delete config; } } // namespace logtail diff --git a/core/unittest/processor/ProcessorFilterNativeUnittest.cpp b/core/unittest/processor/ProcessorFilterNativeUnittest.cpp index 7aae5657e8..c32e0db2f8 100644 --- a/core/unittest/processor/ProcessorFilterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorFilterNativeUnittest.cpp @@ -191,7 +191,6 @@ void ProcessorFilterNativeUnittest::TestFilter() { ] })"; eventGroup1.FromJsonString(inJson); - std::cout< -#include "unittest/Unittest.h" #include "common/JsonUtil.h" #include "config/Config.h" -#include "processor/ProcessorParseApsaraNative.h" #include "models/LogEvent.h" #include "plugin/instance/ProcessorInstance.h" +#include "processor/ProcessorParseApsaraNative.h" +#include "processor/ProcessorSplitLogStringNative.h" +#include "processor/ProcessorSplitRegexNative.h" +#include "unittest/Unittest.h" namespace logtail { @@ -41,6 +43,7 @@ class ProcessorParseApsaraNativeUnittest : public ::testing::Test { void TestAddLog(); void TestProcessEventKeepUnmatch(); void TestProcessEventDiscardUnmatch(); + void TestMultipleLines(); PipelineContext mContext; }; @@ -53,6 +56,183 @@ UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestUploadRawLog); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestAddLog); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestProcessEventKeepUnmatch); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestProcessEventDiscardUnmatch); +UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestMultipleLines); + +void ProcessorParseApsaraNativeUnittest::TestMultipleLines() { + // 第一个contents 测试多行下的解析,第二个contents测试多行下time的解析 + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "[2023-09-04 13:15:50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1 +[2023-09-04 13:15:33.2]\t[INFO]\t[2]\t/ilogtail/AppConfigBase.cpp:2\t\tAppConfigBase AppConfigBase:2 +[2023-09-04 13:15:22.3]\t[WARNING]\t[3]\t/ilogtail/AppConfigBase.cpp:3\t\tAppConfigBase AppConfigBase:3", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + }, + { + "contents" : + { + "content" : "[2023-09-04 13:15 +:50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1 +[2023-09-04 13:15:22.3]\t[WARNING]\t[3]\t/ilogtail/AppConfigBase.cpp:3\t\tAppConfigBase AppConfigBase:3", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "/ilogtail/AppConfigBase.cpp": "1", + "AppConfigBase AppConfigBase": "1", + "__LEVEL__": "ERROR", + "__THREAD__": "1", + "log.file.offset": "0", + "microtime": "1693833350100000" + }, + "timestamp": 1693833350, + "timestampNanosecond": 100000000, + "type": 1 + }, + { + "contents": { + "/ilogtail/AppConfigBase.cpp": "2", + "AppConfigBase AppConfigBase": "2", + "__LEVEL__": "INFO", + "__THREAD__": "2", + "log.file.offset": "0", + "microtime": "1693833333200000" + }, + "timestamp": 1693833333, + "timestampNanosecond": 200000000, + "type": 1 + }, + { + "contents": { + "/ilogtail/AppConfigBase.cpp": "3", + "AppConfigBase AppConfigBase": "3", + "__LEVEL__": "WARNING", + "__THREAD__": "3", + "log.file.offset": "0", + "microtime": "1693833322300000" + }, + "timestamp": 1693833322, + "timestampNanosecond": 300000000, + "type": 1 + }, + { + "contents": { + "__raw_log__": "[2023-09-04 13:15", + "content": "[2023-09-04 13:15", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "__raw_log__": ":50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1", + "content": ":50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "/ilogtail/AppConfigBase.cpp": "3", + "AppConfigBase AppConfigBase": "3", + "__LEVEL__": "WARNING", + "__THREAD__": "3", + "log.file.offset": "0", + "microtime": "1693833322300000" + }, + "timestamp": 1693833322, + "timestampNanosecond": 300000000, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mLogTimeZoneOffsetSecond = -GetLocalTimeZoneOffsetSecond(); + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + std::string outJson = eventGroup.ToJsonString(); + // run function ProcessorParseApsaraNative + ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + + // judge result + outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mLogTimeZoneOffsetSecond = -GetLocalTimeZoneOffsetSecond(); + config.mLogBeginReg = ".*"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseApsaraNative + ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + std::string outJson2 = CompactJson(outJson); + + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } +} void ProcessorParseApsaraNativeUnittest::TestInit() { Config config; @@ -62,7 +242,7 @@ void ProcessorParseApsaraNativeUnittest::TestInit() { config.mLogTimeZoneOffsetSecond = 0; ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); - processor.SetContext(mContext); + processor.SetContext(mContext); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); ComponentConfig componentConfig(pluginId, config); @@ -85,7 +265,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLine() { { "contents" : { - "content" : "[2023-09-04 13:15:04.862181] [info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[2023-09-04 13:15:04.862181]\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -94,7 +274,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLine() { { "contents" : { - "content" : "[2023-09-04 13:16:04.862181] [info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[2023-09-04 13:16:04.862181]\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -103,7 +283,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLine() { { "contents" : { - "content" : "[1693833364862181] [info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[1693833364862181]\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -114,53 +294,52 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLine() { eventGroup.FromJsonString(inJson); // run function ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); - processor.SetContext(mContext); + processor.SetContext(mContext); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); processorInstance.Process(eventGroup); std::string expectJson = R"({ - "events" : - [ + "events": [ { - "contents" : - { + "contents": { "/ilogtail/AppConfigBase.cpp": "100", "AppConfigBase AppConfigBase": "success", + "__LEVEL__": "INFO", "__THREAD__": "385658", "log.file.offset": "0", "microtime": "1693833304862181" }, - "timestamp" : 1693833304, + "timestamp": 1693833304, "timestampNanosecond": 862181000, - "type" : 1 + "type": 1 }, { - "contents" : - { + "contents": { "/ilogtail/AppConfigBase.cpp": "100", "AppConfigBase AppConfigBase": "success", + "__LEVEL__": "INFO", "__THREAD__": "385658", "log.file.offset": "0", "microtime": "1693833364862181" }, - "timestamp" : 1693833364, + "timestamp": 1693833364, "timestampNanosecond": 862181000, - "type" : 1 + "type": 1 }, { - "contents" : - { + "contents": { "/ilogtail/AppConfigBase.cpp": "100", "AppConfigBase AppConfigBase": "success", + "__LEVEL__": "INFO", "__THREAD__": "385658", "log.file.offset": "0", "microtime": "1693833364862181" }, - "timestamp" : 1693833364, + "timestamp": 1693833364, "timestampNanosecond": 862181000, - "type" : 1 + "type": 1 } ] })"; @@ -185,7 +364,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLinePart() { { "contents" : { - "content" : "[2023-09-04 13:15:0] [info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[2023-09-04 13:15:0]\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -194,7 +373,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLinePart() { { "contents" : { - "content" : "[2023-09-04 13:16:0[info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[2023-09-04 13:16:0[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -203,7 +382,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLinePart() { { "contents" : { - "content" : "[1234560 [info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[1234560\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -214,7 +393,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLinePart() { eventGroup.FromJsonString(inJson); // run function ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); - processor.SetContext(mContext); + processor.SetContext(mContext); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); ComponentConfig componentConfig(pluginId, config); @@ -226,12 +405,12 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLinePart() { // check observablity int count = 3; APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); // discard unmatch, so output is 0 - APSARA_TEST_EQUAL_FATAL(0, processorInstance.mProcOutRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); } void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() { @@ -250,7 +429,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() { { "contents" : { - "content" : "[2023-09-04 13:15:04.862181] [info] [385658] content:100 __raw__:success __raw_log__:success", + "content" : "[2023-09-04 13:15:04.862181]\t[INFO]\t[385658]\tcontent:100\t\t__raw__:success\t\t__raw_log__:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -270,18 +449,17 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() { eventGroup.FromJsonString(inJson); // run function ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); - processor.SetContext(mContext); + processor.SetContext(mContext); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); processorInstance.Process(eventGroup); std::string expectJson = R"({ - "events" : - [ + "events": [ { - "contents" : - { + "contents": { + "__LEVEL__": "INFO", "__THREAD__": "385658", "__raw__": "success", "__raw_log__": "success", @@ -289,21 +467,20 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() { "log.file.offset": "0", "microtime": "1693833304862181" }, - "timestamp" : 1693833304, + "timestamp": 1693833304, "timestampNanosecond": 862181000, - "type" : 1 + "type": 1 }, { - "contents" : - { + "contents": { "__raw__": "value1", "__raw_log__": "value1", "content": "value1", "log.file.offset": "0" }, - "timestamp" : 12345678901, + "timestamp": 12345678901, "timestampNanosecond": 0, - "type" : 1 + "type": 1 } ] })"; @@ -328,7 +505,7 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() { { "contents" : { - "content" : "[2023-09-04 13:15:04.862181] [info] [385658] /ilogtail/AppConfigBase.cpp:100 AppConfigBase AppConfigBase:success", + "content" : "[2023-09-04 13:15:04.862181]\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0" }, "timestamp" : 12345678901, @@ -348,40 +525,38 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() { eventGroup.FromJsonString(inJson); // run function ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); - processor.SetContext(mContext); + processor.SetContext(mContext); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); ComponentConfig componentConfig(pluginId, config); APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); processorInstance.Process(eventGroup); std::string expectJson = R"({ - "events" : - [ + "events": [ { - "contents" : - { + "contents": { "/ilogtail/AppConfigBase.cpp": "100", "AppConfigBase AppConfigBase": "success", + "__LEVEL__": "INFO", "__THREAD__": "385658", - "__raw__": "[2023-09-04 13:15:04.862181]\t[info]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", + "__raw__": "[2023-09-04 13:15:04.862181]\t[INFO]\t[385658]\t/ilogtail/AppConfigBase.cpp:100\t\tAppConfigBase AppConfigBase:success", "log.file.offset": "0", "microtime": "1693833304862181" }, - "timestamp" : 1693833304, + "timestamp": 1693833304, "timestampNanosecond": 862181000, - "type" : 1 + "type": 1 }, { - "contents" : - { + "contents": { "__raw__": "value1", "__raw_log__": "value1", "content": "value1", "log.file.offset": "0" }, - "timestamp" : 12345678901, + "timestamp": 12345678901, "timestampNanosecond": 0, - "type" : 1 + "type": 1 } ] })"; @@ -393,7 +568,7 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() { void ProcessorParseApsaraNativeUnittest::TestAddLog() { Config config; ProcessorParseApsaraNative& processor = *(new ProcessorParseApsaraNative); - processor.SetContext(mContext); + processor.SetContext(mContext); std::string pluginId = "testID"; ProcessorInstance processorInstance(&processor, pluginId); ComponentConfig componentConfig(pluginId, config); @@ -405,7 +580,7 @@ void ProcessorParseApsaraNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(strlen(key) + strlen(value) + 5, processor.GetContext().GetProcessProfile().logGroupSize); + APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), processor.GetContext().GetProcessProfile().logGroupSize); } void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { @@ -544,16 +719,16 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { // check observablity int count = 5; APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length())*count, processor.mProcParseInSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcOutRecordsTotal->GetValue()); expectValue = "__raw_log__value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length())*count, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); } void ProcessorParseApsaraNativeUnittest::TestProcessEventDiscardUnmatch() { @@ -632,14 +807,14 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventDiscardUnmatch() { // check observablity int count = 5; APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length())*count, processor.mProcParseInSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); // discard unmatch, so output is 0 - APSARA_TEST_EQUAL_FATAL(0, processorInstance.mProcOutRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); } } // namespace logtail diff --git a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp index e1e36fe01a..b94f00c931 100644 --- a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp @@ -20,6 +20,7 @@ #include "plugin/instance/ProcessorInstance.h" #include "processor/ProcessorParseDelimiterNative.h" #include "processor/ProcessorSplitLogStringNative.h" +#include "processor/ProcessorSplitRegexNative.h" #include "unittest/Unittest.h" namespace logtail { @@ -43,6 +44,8 @@ class ProcessorParseDelimiterNativeUnittest : public ::testing::Test { void TestAddLog(); void TestProcessEventKeepUnmatch(); void TestProcessEventDiscardUnmatch(); + void TestAutoExtend(); + void TestAcceptNoEnoughKeys(); PipelineContext mContext; }; @@ -56,6 +59,8 @@ UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestUploadRawLog); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestAddLog); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestProcessEventKeepUnmatch); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestProcessEventDiscardUnmatch); +UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestAutoExtend); +UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestAcceptNoEnoughKeys); void ProcessorParseDelimiterNativeUnittest::TestInit() { Config config; @@ -73,82 +78,789 @@ void ProcessorParseDelimiterNativeUnittest::TestInit() { APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); } +void ProcessorParseDelimiterNativeUnittest::TestAcceptNoEnoughKeys() { + // mAcceptNoEnoughKeys false + { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "123@@45 +012@@34", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "__raw_log__": "123@@45", + "content": "123@@45", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "__raw_log__": "012@@34", + "content": "012@@34", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAcceptNoEnoughKeys = false; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAcceptNoEnoughKeys = false; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } + // mAcceptNoEnoughKeys true + { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "123@@45 +012@@34", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "a": "123", + "b": "45", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "a": "012", + "b": "34", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAcceptNoEnoughKeys = true; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAcceptNoEnoughKeys = true; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } +} + +void ProcessorParseDelimiterNativeUnittest::TestAutoExtend() { + // mAutoExtend false + { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "123@@456@@1@@2@@3 +012@@345@@1@@2@@3", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "__column3__": "@@2@@3", + "a": "123", + "b": "456", + "c": "1", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "__column3__": "@@2@@3", + "a": "012", + "b": "345", + "c": "1", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAutoExtend = false; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAutoExtend = false; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } + // mAutoExtend true + { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "123@@456@@1@@2@@3 +012@@345@@1@@2@@3", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "__column3__": "2", + "__column4__": "3", + "a": "123", + "b": "456", + "c": "1", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "__column3__": "2", + "__column4__": "3", + "a": "012", + "b": "345", + "c": "1", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAutoExtend = true; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mAutoExtend = true; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } +} + void ProcessorParseDelimiterNativeUnittest::TestMultipleLines() { - // make config - Config config; - config.mLogType = DELIMITER_LOG; - config.mLogBeginReg = ""; - config.mAdvancedConfig.mEnableLogPositionMeta = false; - config.mSeparator = "@@"; - config.mQuote = '\000'; - config.mColumnKeys = {"a", "b", "c"}; - config.mDiscardUnmatch = true; - config.mUploadRawLog = false; - config.mAdvancedConfig.mRawLogTag = "__raw__"; - // make events - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson = R"({ + //case < field + { + std::string inJson = R"({ "events" : [ { "contents" : { - "content" : "123@@456@@789 -012@@345@@678 -", + "content" : "123@@456 +012@@345", "log.file.offset": "0" }, "timestamp" : 12345678901, "type" : 1 } ] - })"; - eventGroup.FromJsonString(inJson); - // run function ProcessorSplitLogStringNative - ProcessorSplitLogStringNative processorSplitLogStringNative; - processorSplitLogStringNative.SetContext(mContext); - std::string pluginId = "testID"; - ComponentConfig componentConfig(pluginId, config); - APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); - processorSplitLogStringNative.Process(eventGroup); - // run function ProcessorParseDelimiterNative - ProcessorParseDelimiterNative& processorDelimiterNative = *(new ProcessorParseDelimiterNative); - ProcessorInstance processorInstance(&processorDelimiterNative, pluginId); - APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); - processorDelimiterNative.Process(eventGroup); - std::string expectJson = R"({ + })"; + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "__raw_log__": "123@@456", + "content": "123@@456", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "__raw_log__": "012@@345", + "content": "012@@345", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } + + // case > field + { + std::string inJson = R"({ "events" : [ { "contents" : { - "a": "123", - "b": "456", - "c": "789", + "content" : "123@@456@@789 +012@@345@@678", "log.file.offset": "0" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 - }, + } + ] + })"; + + std::string expectJson = R"({ + "events": [ + { + "contents": { + "__column2__": "789", + "a": "123", + "b": "456", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + }, + { + "contents": { + "__column2__": "678", + "a": "012", + "b": "345", + "log.file.offset": "0" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } + + // case = field + { + std::string inJson = R"({ + "events" : + [ { "contents" : { - "a": "012", - "b": "345", - "c": "678", + "content" : "123@@456@@789 +012@@345@@678", "log.file.offset": "0" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 } ] - })"; - // judge result - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + })"; + + std::string expectJson = R"({ + "events" : + [ + { + "contents" : + { + "a": "123", + "b": "456", + "c": "789", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "timestampNanosecond": 0, + "type" : 1 + }, + { + "contents" : + { + "a": "012", + "b": "345", + "c": "678", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "timestampNanosecond": 0, + "type" : 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + // ProcessorSplitRegexNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mLogBeginReg = ".*"; + config.mSeparator = "@@"; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + // run function processorSplitRegexNative + ProcessorSplitRegexNative processorSplitRegexNative; + processorSplitRegexNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitRegexNative.Init(componentConfig)); + processorSplitRegexNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } } void ProcessorParseDelimiterNativeUnittest::TestProcessWholeLine() { @@ -497,10 +1209,9 @@ void ProcessorParseDelimiterNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(strlen(key) + strlen(value) + 5, processor.GetContext().GetProcessProfile().logGroupSize); + APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), processor.GetContext().GetProcessProfile().logGroupSize); } - void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { // make config Config config; @@ -637,16 +1348,16 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { // check observablity int count = 5; APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(expectValue.length() * count), processor.mProcParseInSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcOutRecordsTotal->GetValue()); expectValue = "__raw_log__value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(expectValue.length() * count), processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); } void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { @@ -725,14 +1436,14 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { // check observablity int count = 5; APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "value1"; APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); // discard unmatch, so output is 0 - APSARA_TEST_EQUAL_FATAL(0, processorInstance.mProcOutRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); } } // namespace logtail diff --git a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp index bf62bbefc9..b58f35268e 100644 --- a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. #include -#include "unittest/Unittest.h" #include "common/JsonUtil.h" #include "config/Config.h" -#include "processor/ProcessorParseJsonNative.h" #include "models/LogEvent.h" #include "plugin/instance/ProcessorInstance.h" +#include "processor/ProcessorParseJsonNative.h" +#include "processor/ProcessorSplitLogStringNative.h" +#include "processor/ProcessorSplitRegexNative.h" +#include "unittest/Unittest.h" namespace logtail { @@ -38,6 +40,7 @@ class ProcessorParseJsonNativeUnittest : public ::testing::Test { void TestProcessEventDiscardUnmatch(); void TestProcessJsonContent(); void TestProcessJsonRaw(); + void TestMultipleLines(); PipelineContext mContext; }; @@ -56,6 +59,178 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonContent); UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonRaw); +UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestMultipleLines); + +void ProcessorParseJsonNativeUnittest::TestMultipleLines() { + // error json + { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": \"07/Jul/2022:10:30:28\"}\u0000{\"name\":\"Mike\",\"age\":25,\"is_student\":asdfsadf,\"address\":{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"},\"courses\":[\"Math\",\"English\",\"Science\"],\"scores\":{\"Math\":90,\"English\":85,\"Science\":95}}", + "log.file.offset": "0" + }, + "timestampNanosecond" : 0, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + std::string expectJson = R"({ + "events" : + [ + { + "contents" : + { + "__raw__" : "{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": \"07/Jul/2022:10:30:28\"}", + "log.file.offset": "0", + "time" : "07/Jul/2022:10:30:28", + "url" : "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + }, + { + "contents" : + { + "__raw__" : "{\"name\":\"Mike\",\"age\":25,\"is_student\":asdfsadf,\"address\":{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"},\"courses\":[\"Math\",\"English\",\"Science\"],\"scores\":{\"Math\":90,\"English\":85,\"Science\":95}}", + "__raw_log__" : "{\"name\":\"Mike\",\"age\":25,\"is_student\":asdfsadf,\"address\":{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"},\"courses\":[\"Math\",\"English\",\"Science\"],\"scores\":{\"Math\":90,\"English\":85,\"Science\":95}}", + "content" : "{\"name\":\"Mike\",\"age\":25,\"is_student\":asdfsadf,\"address\":{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"},\"courses\":[\"Math\",\"English\",\"Science\"],\"scores\":{\"Math\":90,\"English\":85,\"Science\":95}}", + "log.file.offset":"0" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mDiscardUnmatch = false; + config.mUploadRawLog = true; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mLogType = JSON_LOG; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + std::string outJson = eventGroup.ToJsonString(); + // run function ProcessorParseJsonNative + ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + + // judge result + outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } + { + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": \"07/Jul/2022:10:30:28\"}\u0000{\"name\":\"Mike\",\"age\":25,\"is_student\":false,\"address\":{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"},\"courses\":[\"Math\",\"English\",\"Science\"],\"scores\":{\"Math\":90,\"English\":85,\"Science\":95}}", + "log.file.offset": "0" + }, + "timestampNanosecond" : 0, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + + std::string expectJson = R"({ + "events" : + [ + { + "contents" : + { + "__raw__" : "{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": \"07/Jul/2022:10:30:28\"}", + "log.file.offset": "0", + "time" : "07/Jul/2022:10:30:28", + "url" : "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + }, + { + "contents" : + { + "__raw__" : "{\"name\":\"Mike\",\"age\":25,\"is_student\":false,\"address\":{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"},\"courses\":[\"Math\",\"English\",\"Science\"],\"scores\":{\"Math\":90,\"English\":85,\"Science\":95}}", + "address" : "{\"city\":\"Hangzhou\",\"postal_code\":\"100000\"}", + "age":"25", + "courses":"[\"Math\",\"English\",\"Science\"]", + "is_student":"false", + "log.file.offset":"0", + "name":"Mike", + "scores":"{\"Math\":90,\"English\":85,\"Science\":95}" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ] + })"; + + // ProcessorSplitLogStringNative + { + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + eventGroup.FromJsonString(inJson); + + // make config + Config config; + config.mDiscardUnmatch = false; + config.mUploadRawLog = true; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + config.mLogType = JSON_LOG; + + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + std::string outJson = eventGroup.ToJsonString(); + // run function ProcessorParseJsonNative + ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative); + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processor.Process(eventGroup); + + // judge result + outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + } + } +} + void ProcessorParseJsonNativeUnittest::TestInit() { Config config; config.mDiscardUnmatch = false; @@ -84,7 +259,7 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(strlen(key) + strlen(value) + 5, processor.GetContext().GetProcessProfile().logGroupSize); + APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), processor.GetContext().GetProcessProfile().logGroupSize); } void ProcessorParseJsonNativeUnittest::TestProcessJson() { @@ -166,7 +341,7 @@ void ProcessorParseJsonNativeUnittest::TestProcessJson() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), 0); + APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0)); } void ProcessorParseJsonNativeUnittest::TestProcessJsonContent() { @@ -227,7 +402,7 @@ void ProcessorParseJsonNativeUnittest::TestProcessJsonContent() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), 0); + APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0)); } void ProcessorParseJsonNativeUnittest::TestProcessJsonRaw() { @@ -287,7 +462,7 @@ void ProcessorParseJsonNativeUnittest::TestProcessJsonRaw() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), 0); + APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0)); } void ProcessorParseJsonNativeUnittest::TestProcessEventKeepUnmatch() { @@ -329,18 +504,18 @@ void ProcessorParseJsonNativeUnittest::TestProcessEventKeepUnmatch() { // check observablity APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": \"07/Jul/2022:10:30:28\""; APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcOutRecordsTotal->GetValue()); expectValue = "__raw_log__{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": " "\"07/Jul/2022:10:30:28\""; - APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(expectValue.length() * count), processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); // judge result std::string expectJson = R"({ @@ -361,7 +536,7 @@ void ProcessorParseJsonNativeUnittest::TestProcessEventKeepUnmatch() { })"; std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), 0); + APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0)); } void ProcessorParseJsonNativeUnittest::TestProcessEventDiscardUnmatch() { @@ -403,20 +578,20 @@ void ProcessorParseJsonNativeUnittest::TestProcessEventDiscardUnmatch() { // check observablity APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "{\"url\": \"POST /PutData?Category=YunOsAccountOpLog HTTP/1.1\",\"time\": \"07/Jul/2022:10:30:28\""; APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); // discard unmatch, so output is 0 - APSARA_TEST_EQUAL_FATAL(0, processorInstance.mProcOutRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(count, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mProcParseErrorTotal->GetValue()); std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL("null", CompactJson(outJson).c_str()); - APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), 0); + APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0)); } } // namespace logtail