Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[BugFix] Remove json value from error message (backport #49526) #49534

Closed
wants to merge 1 commit into from
Closed

[BugFix] Remove json value from error message (backport #49526) #49534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions be/src/exec/json_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,19 @@ Status JsonDocumentStreamParser::get_current(simdjson::ondemand::object* row) no
try {
if (_doc_stream_itr != _doc_stream.end()) {
simdjson::ondemand::document_reference doc = *_doc_stream_itr;
<<<<<<< HEAD
=======
// simdjson version 3.9.4 and JsonFunctions::to_json_string may crash when json is invalid.
// https://github.com/StarRocks/StarRocksTest/issues/8327
// TODO: add value in error message
if (doc.type() == simdjson::ondemand::json_type::array) {
return Status::DataQualityError(
"The value is array type in json document stream, you can set strip_outer_array=true to parse "
"each element of the array as individual rows");
} else if (doc.type() != simdjson::ondemand::json_type::object) {
return Status::DataQualityError("The value should be object type in json document stream");
}
>>>>>>> e7d9dc06da ([BugFix] Remove json value from error message (#49526))

_curr = doc.get_object();
*row = _curr;
Expand Down
89 changes: 89 additions & 0 deletions be/test/exec/json_parser_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -563,4 +563,93 @@ PARALLEL_TEST(JsonParserTest, test_big_json) {
ASSERT_TRUE(st.is_end_of_file());
}

<<<<<<< HEAD
=======
PARALLEL_TEST(JsonParserTest, test_document_stream_parser_invalid_type_array) {
std::string input = R"( [{"key":1},{"key":2}] )";
// Reserved for simdjson padding.
auto size = input.size();
input.resize(input.size() + simdjson::SIMDJSON_PADDING);
auto padded_size = input.size();

simdjson::ondemand::parser simdjson_parser;
std::unique_ptr<JsonParser> parser(new JsonDocumentStreamParser(&simdjson_parser));
auto st = parser->parse(input.data(), size, padded_size);
ASSERT_TRUE(st.ok());

simdjson::ondemand::object row;
st = parser->get_current(&row);
ASSERT_TRUE(st.is_data_quality_error());
ASSERT_TRUE(st.message().find("The value is array type in json document stream, you can set strip_outer_array=true "
"to parse each element of the array as individual rows") != std::string::npos);
}

PARALLEL_TEST(JsonParserTest, test_document_stream_parser_invalid_type_not_object) {
std::string input = R"( 1 )";
// Reserved for simdjson padding.
auto size = input.size();
input.resize(input.size() + simdjson::SIMDJSON_PADDING);
auto padded_size = input.size();

simdjson::ondemand::parser simdjson_parser;
std::unique_ptr<JsonParser> parser(new JsonDocumentStreamParser(&simdjson_parser));
auto st = parser->parse(input.data(), size, padded_size);
ASSERT_TRUE(st.ok());

simdjson::ondemand::object row;
st = parser->get_current(&row);
ASSERT_TRUE(st.is_data_quality_error());
ASSERT_TRUE(st.message().find("The value should be object type in json document stream") != std::string::npos);
}

PARALLEL_TEST(JsonParserTest, test_document_stream_parser_with_jsonroot_invalid_type_array) {
// ndjson with ' ', '/t', '\n'
std::string input = R"( {"key0": [{"key1":1},{"key1":2}]} {"key0":[{"key1":3},{"key1":4}]} )";
// Reserved for simdjson padding.
auto size = input.size();
input.resize(input.size() + simdjson::SIMDJSON_PADDING);
auto padded_size = input.size();

std::vector<SimpleJsonPath> jsonroot;
ASSERT_OK(JsonFunctions::parse_json_paths("$.key0", &jsonroot));

simdjson::ondemand::parser simdjson_parser;
std::unique_ptr<JsonParser> parser(new JsonDocumentStreamParserWithRoot(&simdjson_parser, jsonroot));
auto st = parser->parse(input.data(), size, padded_size);
ASSERT_TRUE(st.ok());

simdjson::ondemand::object row;
st = parser->get_current(&row);
ASSERT_TRUE(
st.message().find("The value is array type in json document stream with json root, you can set "
"strip_outer_array=true to parse each element of the array as individual rows, value: "
"[{\"key1\":1},{\"key1\":2}]") != std::string::npos);
}

PARALLEL_TEST(JsonParserTest, test_array_parser_with_jsonroot_invalid_type_array) {
// json array with ' ', '/t', '\n'
std::string input = R"([ {"key0": [{"key1":1},{"key1":2}]}, {"key0": [{"key1":3},{"key1":4}]} ])";
// Reserved for simdjson padding.
auto size = input.size();
input.resize(input.size() + simdjson::SIMDJSON_PADDING);
auto padded_size = input.size();

std::vector<SimpleJsonPath> jsonroot;
ASSERT_OK(JsonFunctions::parse_json_paths("$.key0", &jsonroot));

simdjson::ondemand::parser simdjson_parser;
std::unique_ptr<JsonParser> parser(new JsonArrayParserWithRoot(&simdjson_parser, jsonroot));
auto st = parser->parse(input.data(), size, padded_size);
ASSERT_TRUE(st.ok());

simdjson::ondemand::object row;
st = parser->get_current(&row);
ASSERT_TRUE(st.is_data_quality_error());
ASSERT_TRUE(st.message().find(
"The value is array type in json array with json root, you can set strip_outer_array=true to "
"parse each element of the array as individual rows, value: [{\"key1\":1},{\"key1\":2}]") !=
std::string::npos);
}

>>>>>>> e7d9dc06da ([BugFix] Remove json value from error message (#49526))
} // namespace starrocks
Loading