Skip to content

Commit

Permalink
[backport] fix: JSON truncation caused by escaped zero byte (#1594) (#…
Browse files Browse the repository at this point in the history
…1596)

* fix: JSON truncation caused by escaped zero byte (#1594)

Signed-off-by: Tao Yu <[email protected]>

* fix centos 7 deprecated yum repo

---------

Signed-off-by: Tao Yu <[email protected]>
Co-authored-by: abingcbc <[email protected]>
  • Loading branch information
yyuuttaaoo and Abingcbc authored Jul 11, 2024
1 parent 218dc71 commit eee5de4
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 2 deletions.
2 changes: 2 additions & 0 deletions config_server/service/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ MAINTAINER TomYu [email protected]

ENV container docker

RUN curl -L -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.*

WORKDIR /config_server
Expand Down
4 changes: 2 additions & 2 deletions core/processor/ProcessorParseJsonNative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent,

std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Value& value) {
if (value.IsString())
return value.GetString();
return std::string(value.GetString(), value.GetStringLength());
else if (value.IsBool())
return ToString(value.GetBool());
else if (value.IsInt())
Expand All @@ -193,7 +193,7 @@ std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Va
rapidjson::StringBuffer buffer;
rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
value.Accept(writer);
return buffer.GetString();
return std::string(buffer.GetString(), buffer.GetLength());
}
}

Expand Down
60 changes: 60 additions & 0 deletions core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class ProcessorParseJsonNativeUnittest : public ::testing::Test {

void TestInit();
void TestProcessJson();
void TestProcessJsonEscapedNullByte();
void TestAddLog();
void TestProcessEventKeepUnmatch();
void TestProcessEventDiscardUnmatch();
Expand All @@ -46,6 +47,8 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestAddLog);

UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJson);

UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonEscapedNullByte);

UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventKeepUnmatch);

UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventDiscardUnmatch);
Expand Down Expand Up @@ -258,6 +261,63 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() {
processor.GetContext().GetProcessProfile().logGroupSize);
}

void ProcessorParseJsonNativeUnittest::TestProcessJsonEscapedNullByte() {
// make config
Json::Value config;
config["SourceKey"] = "content";
config["KeepingSourceWhenParseFail"] = true;
config["KeepingSourceWhenParseSucceed"] = false;
config["CopingRawLog"] = false;
config["RenamedSourceKey"] = "rawLog";

// make events
auto sourceBuffer = std::make_shared<SourceBuffer>();
PipelineEventGroup eventGroup(sourceBuffer);
std::string inJson = R"({
"events" :
[
{
"contents" :
{
"content" : "{\"level\":\"ERROR\",\"time\":\"2024-07-04T06:59:23.078Z\",\"msg\":\"expect { or n, but found \\u0000, error found in #0 byte of ...\"}"
},
"timestampNanosecond" : 0,
"timestamp" : 12345678901,
"type" : 1
}
]
})";
eventGroup.FromJsonString(inJson);
// run function
ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative);
std::string pluginId = "testID";
ProcessorInstance processorInstance(&processor, pluginId);
APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
std::vector<PipelineEventGroup> eventGroupList;
eventGroupList.emplace_back(std::move(eventGroup));
processorInstance.Process(eventGroupList);
// judge result
std::string expectJson = R"({
"events" :
[
{
"contents" :
{
"level": "ERROR",
"msg": "expect { or n, but found \u0000, error found in #0 byte of ...",
"time": "2024-07-04T06:59:23.078Z"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 1
}
]
})";
std::string outJson = eventGroupList[0].ToJsonString();
APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0));
}

void ProcessorParseJsonNativeUnittest::TestProcessJson() {
// make config
Json::Value config;
Expand Down
2 changes: 2 additions & 0 deletions docker/Dockerfile.centos7-cve-fix
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
FROM centos:centos7.9.2009
ENV container docker

RUN curl -L -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.*

CMD ["/usr/sbin/init"]
2 changes: 2 additions & 0 deletions docker/Dockerfile_production
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ MAINTAINER TomYu [email protected]

ENV container docker

RUN curl -L -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.*

ARG HOST_OS=Linux
Expand Down

0 comments on commit eee5de4

Please sign in to comment.