diff --git a/config_server/service/Dockerfile b/config_server/service/Dockerfile index 9a969db07f..0173b6f42c 100644 --- a/config_server/service/Dockerfile +++ b/config_server/service/Dockerfile @@ -25,6 +25,8 @@ MAINTAINER TomYu yyuuttaaoo@gmail.com ENV container docker +RUN curl -L -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo + RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.* WORKDIR /config_server diff --git a/core/processor/ProcessorParseJsonNative.cpp b/core/processor/ProcessorParseJsonNative.cpp index f02aaa0fbf..2d19310536 100644 --- a/core/processor/ProcessorParseJsonNative.cpp +++ b/core/processor/ProcessorParseJsonNative.cpp @@ -173,7 +173,7 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent, std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Value& value) { if (value.IsString()) - return value.GetString(); + return std::string(value.GetString(), value.GetStringLength()); else if (value.IsBool()) return ToString(value.GetBool()); else if (value.IsInt()) @@ -193,7 +193,7 @@ std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Va rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); value.Accept(writer); - return buffer.GetString(); + return std::string(buffer.GetString(), buffer.GetLength()); } } diff --git a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp index 5041326639..d210da2ba9 100644 --- a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp @@ -30,6 +30,7 @@ class ProcessorParseJsonNativeUnittest : public ::testing::Test { void TestInit(); void TestProcessJson(); + void TestProcessJsonEscapedNullByte(); void TestAddLog(); void TestProcessEventKeepUnmatch(); void TestProcessEventDiscardUnmatch(); @@ -46,6 +47,8 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestAddLog); UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJson); +UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonEscapedNullByte); + UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventKeepUnmatch); UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventDiscardUnmatch); @@ -258,6 +261,63 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() { processor.GetContext().GetProcessProfile().logGroupSize); } +void ProcessorParseJsonNativeUnittest::TestProcessJsonEscapedNullByte() { + // make config + Json::Value config; + config["SourceKey"] = "content"; + config["KeepingSourceWhenParseFail"] = true; + config["KeepingSourceWhenParseSucceed"] = false; + config["CopingRawLog"] = false; + config["RenamedSourceKey"] = "rawLog"; + + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "{\"level\":\"ERROR\",\"time\":\"2024-07-04T06:59:23.078Z\",\"msg\":\"expect { or n, but found \\u0000, error found in #0 byte of ...\"}" + }, + "timestampNanosecond" : 0, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + eventGroup.FromJsonString(inJson); + // run function + ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative); + std::string pluginId = "testID"; + ProcessorInstance processorInstance(&processor, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); + std::vector eventGroupList; + eventGroupList.emplace_back(std::move(eventGroup)); + processorInstance.Process(eventGroupList); + // judge result + std::string expectJson = R"({ + "events" : + [ + { + "contents" : + { + "level": "ERROR", + "msg": "expect { or n, but found \u0000, error found in #0 byte of ...", + "time": "2024-07-04T06:59:23.078Z" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ] + })"; + std::string outJson = eventGroupList[0].ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); + APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0)); +} + void ProcessorParseJsonNativeUnittest::TestProcessJson() { // make config Json::Value config; diff --git a/docker/Dockerfile.centos7-cve-fix b/docker/Dockerfile.centos7-cve-fix index e4b410ac52..dabc0f6444 100644 --- a/docker/Dockerfile.centos7-cve-fix +++ b/docker/Dockerfile.centos7-cve-fix @@ -1,6 +1,8 @@ FROM centos:centos7.9.2009 ENV container docker +RUN curl -L -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo + RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.* CMD ["/usr/sbin/init"] diff --git a/docker/Dockerfile_production b/docker/Dockerfile_production index 52cca96a87..b6c972a894 100644 --- a/docker/Dockerfile_production +++ b/docker/Dockerfile_production @@ -25,6 +25,8 @@ MAINTAINER TomYu yyuuttaaoo@gmail.com ENV container docker +RUN curl -L -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo + RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.* ARG HOST_OS=Linux