Skip to content

Commit

Permalink
Review fixes + json to proto map<string, string> conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
@siarheivesialou
siarheivesialou committed Apr 4, 2024
1 parent cfabdcd commit aedc4fd
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 112 deletions.
136 changes: 103 additions & 33 deletions ydb/core/http_proxy/json_proto_conversion.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#include <ydb/library/naming_conventions/naming_conventions.h>
#include <ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h>
#include <ydb/library/http_proxy/error/error.h>
#include <contrib/libs/protobuf/src/google/protobuf/message.h>
#include <contrib/libs/protobuf/src/google/protobuf/reflection.h>

#include <nlohmann/json.hpp>

Expand Down Expand Up @@ -42,9 +44,9 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {

void PrintField(const NProtoBuf::Message& proto, const NProtoBuf::FieldDescriptor& field,
NProtobufJson::IJsonOutput& json, TStringBuf key = {}) override {
if (field.options().HasExtension(Ydb::DataStreams::V1::FieldTransformer)) {
if (field.options().GetExtension(Ydb::DataStreams::V1::FieldTransformer) ==
Ydb::DataStreams::V1::TRANSFORM_BASE64) {
if (field.options().HasExtension(Ydb::FieldTransformation::FieldTransformer)) {
if (field.options().GetExtension(Ydb::FieldTransformation::FieldTransformer) ==
Ydb::FieldTransformation::TRANSFORM_BASE64) {
Y_ENSURE(field.cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_STRING,
"Base64 is only supported for strings");
if (!key) {
Expand All @@ -71,8 +73,8 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
return;
}

if (field.options().GetExtension(Ydb::DataStreams::V1::FieldTransformer) ==
Ydb::DataStreams::V1::TRANSFORM_DOUBLE_S_TO_INT_MS) {
if (field.options().GetExtension(Ydb::FieldTransformation::FieldTransformer) ==
Ydb::FieldTransformation::TRANSFORM_DOUBLE_S_TO_INT_MS) {
Y_ENSURE(field.cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_INT64,
"Double S to Int MS is only supported for int64 timestamps");

Expand All @@ -92,8 +94,8 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
return;
}

if (field.options().GetExtension(Ydb::DataStreams::V1::FieldTransformer) ==
Ydb::DataStreams::V1::TRANSFORM_EMPTY_TO_NOTHING) {
if (field.options().GetExtension(Ydb::FieldTransformation::FieldTransformer) ==
Ydb::FieldTransformation::TRANSFORM_EMPTY_TO_NOTHING) {
Y_ENSURE(field.cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_STRING,
"Empty to nothing is only supported for strings");

Expand Down Expand Up @@ -133,11 +135,70 @@ inline void ProtoToJson(const NProtoBuf::Message& resp, NJson::TJsonValue& value
.SetFormatOutput(false)
.SetMissingSingleKeyMode(NProtobufJson::TProto2JsonConfig::MissingKeyDefault)
.SetNameGenerator(ProxyFieldNameConverter)
.SetMapAsObject(true)
.SetEnumMode(NProtobufJson::TProto2JsonConfig::EnumName);
TYdsProtoToJsonPrinter printer(resp.GetReflection(), config, skipBase64Encode);
printer.Print(resp, *NProtobufJson::CreateJsonMapOutput(value));
}

template<typename JSON, typename MAP>
inline void AddJsonObjectToProtoAsMap(
const google::protobuf::FieldDescriptor* fieldDescriptor,
const google::protobuf::Reflection* reflection,
grpc::protobuf::Message* message,
const JSON& jsonObject,
std::function<const MAP(const JSON&)> extractMap,
std::function<const TString(const JSON&)> valueToString
) {
const auto& protoMap = reflection->GetMutableRepeatedFieldRef<google::protobuf::Message>(message, fieldDescriptor);
for (const auto& [key, value] : extractMap(jsonObject)) {
std::unique_ptr<google::protobuf::Message> stringStringEntry(
google::protobuf::MessageFactory::generated_factory()
->GetPrototype(fieldDescriptor->message_type())
->New(message->GetArena())
);
stringStringEntry
->GetReflection()
->SetString(stringStringEntry.get(), fieldDescriptor->message_type()->field(0), key);
stringStringEntry
->GetReflection()
->SetString(stringStringEntry.get(), fieldDescriptor->message_type()->field(1), valueToString(value));
protoMap.Add(*stringStringEntry);
}
}

inline void AddJsonObjectToProtoAsMap(
const google::protobuf::FieldDescriptor* fieldDescriptor,
const google::protobuf::Reflection* reflection,
grpc::protobuf::Message* message,
const NJson::TJsonValue& jsonObject
) {
AddJsonObjectToProtoAsMap<NJson::TJsonValue, NJson::TJsonValue::TMapType>(
fieldDescriptor,
reflection,
message,
jsonObject,
[](auto& json) { return json.GetMap(); },
[](auto& value) -> const TString { return value.GetString(); }
);
}

inline void AddJsonObjectToProtoAsMap(
const google::protobuf::FieldDescriptor* fieldDescriptor,
const google::protobuf::Reflection* reflection,
grpc::protobuf::Message* message,
const nlohmann::basic_json<>& jsonObject
) {
AddJsonObjectToProtoAsMap<nlohmann::basic_json<>, std::map<TString, nlohmann::basic_json<>>>(
fieldDescriptor,
reflection,
message,
jsonObject,
[](auto&json) { return json.template get<std::map<TString, nlohmann::basic_json<>>>(); },
[](auto& value) -> const TString { return value.template get<TString>(); }
);
}

inline void JsonToProto(const NJson::TJsonValue& jsonValue, NProtoBuf::Message* message, ui32 depth = 0) {
Y_ENSURE(depth < 101, "Json depth is > 100");
Y_ENSURE_EX(
Expand All @@ -155,28 +216,28 @@ inline void JsonToProto(const NJson::TJsonValue& jsonValue, NProtoBuf::Message*
"Unexpected json key: " << key
);
Y_ENSURE(fieldDescriptor, "Unexpected json key: " + key);
auto transformer = Ydb::DataStreams::V1::TRANSFORM_NONE;
if (fieldDescriptor->options().HasExtension(Ydb::DataStreams::V1::FieldTransformer)) {
transformer = fieldDescriptor->options().GetExtension(Ydb::DataStreams::V1::FieldTransformer);
auto transformer = Ydb::FieldTransformation::TRANSFORM_NONE;
if (fieldDescriptor->options().HasExtension(Ydb::FieldTransformation::FieldTransformer)) {
transformer = fieldDescriptor->options().GetExtension(Ydb::FieldTransformation::FieldTransformer);
}

if (value.IsArray()) {
Y_ENSURE(fieldDescriptor->is_repeated());
for (auto& elem : value.GetArray()) {
switch (transformer) {
case Ydb::DataStreams::V1::TRANSFORM_BASE64: {
case Ydb::FieldTransformation::TRANSFORM_BASE64: {
Y_ENSURE(fieldDescriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_STRING,
"Base64 transformer is only applicable to strings");
reflection->AddString(message, fieldDescriptor, Base64Decode(elem.GetString()));
break;
}
case Ydb::DataStreams::V1::TRANSFORM_DOUBLE_S_TO_INT_MS: {
case Ydb::FieldTransformation::TRANSFORM_DOUBLE_S_TO_INT_MS: {
reflection->AddInt64(message, fieldDescriptor, elem.GetDouble() * 1000);
break;
}
case Ydb::DataStreams::V1::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::DataStreams::V1::TRANSFORM_NONE: {
case Ydb::FieldTransformation::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::FieldTransformation::TRANSFORM_NONE: {
switch (fieldDescriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
reflection->AddInt32(message, fieldDescriptor, elem.GetInteger());
Expand Down Expand Up @@ -233,19 +294,19 @@ inline void JsonToProto(const NJson::TJsonValue& jsonValue, NProtoBuf::Message*
}
} else {
switch (transformer) {
case Ydb::DataStreams::V1::TRANSFORM_BASE64: {
case Ydb::FieldTransformation::TRANSFORM_BASE64: {
Y_ENSURE(fieldDescriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_STRING,
"Base64 transformer is applicable only to strings");
reflection->SetString(message, fieldDescriptor, Base64Decode(value.GetString()));
break;
}
case Ydb::DataStreams::V1::TRANSFORM_DOUBLE_S_TO_INT_MS: {
case Ydb::FieldTransformation::TRANSFORM_DOUBLE_S_TO_INT_MS: {
reflection->SetInt64(message, fieldDescriptor, value.GetDouble() * 1000);
break;
}
case Ydb::DataStreams::V1::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::DataStreams::V1::TRANSFORM_NONE: {
case Ydb::FieldTransformation::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::FieldTransformation::TRANSFORM_NONE: {
switch (fieldDescriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
reflection->SetInt32(message, fieldDescriptor, value.GetInteger());
Expand Down Expand Up @@ -286,8 +347,12 @@ inline void JsonToProto(const NJson::TJsonValue& jsonValue, NProtoBuf::Message*
reflection->SetString(message, fieldDescriptor, value.GetString());
break;
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: {
auto *msg = reflection->MutableMessage(message, fieldDescriptor);
JsonToProto(value, msg, depth + 1);
if (fieldDescriptor->is_map()) {
AddJsonObjectToProtoAsMap(fieldDescriptor, reflection, message, value);
} else {
auto *msg = reflection->MutableMessage(message, fieldDescriptor);
JsonToProto(value, msg, depth + 1);
}
break;
}
default:
Expand All @@ -313,16 +378,16 @@ inline void NlohmannJsonToProto(const nlohmann::json& jsonValue, NProtoBuf::Mess
for (const auto& [key, value] : jsonValue.get<std::unordered_map<std::string, nlohmann::json>>()) {
auto* fieldDescriptor = desc->FindFieldByName(NNaming::CamelToSnakeCase(key.c_str()));
Y_ENSURE(fieldDescriptor, "Unexpected json key: " + key);
auto transformer = Ydb::DataStreams::V1::TRANSFORM_NONE;
if (fieldDescriptor->options().HasExtension(Ydb::DataStreams::V1::FieldTransformer)) {
transformer = fieldDescriptor->options().GetExtension(Ydb::DataStreams::V1::FieldTransformer);
auto transformer = Ydb::FieldTransformation::TRANSFORM_NONE;
if (fieldDescriptor->options().HasExtension(Ydb::FieldTransformation::FieldTransformer)) {
transformer = fieldDescriptor->options().GetExtension(Ydb::FieldTransformation::FieldTransformer);
}

if (value.is_array()) {
Y_ENSURE(fieldDescriptor->is_repeated());
for (auto& elem : value) {
switch (transformer) {
case Ydb::DataStreams::V1::TRANSFORM_BASE64: {
case Ydb::FieldTransformation::TRANSFORM_BASE64: {
Y_ENSURE(fieldDescriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_STRING,
"Base64 transformer is only applicable to strings");
Expand All @@ -333,12 +398,12 @@ inline void NlohmannJsonToProto(const nlohmann::json& jsonValue, NProtoBuf::Mess
}
break;
}
case Ydb::DataStreams::V1::TRANSFORM_DOUBLE_S_TO_INT_MS: {
case Ydb::FieldTransformation::TRANSFORM_DOUBLE_S_TO_INT_MS: {
reflection->AddInt64(message, fieldDescriptor, elem.get<double>() * 1000);
break;
}
case Ydb::DataStreams::V1::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::DataStreams::V1::TRANSFORM_NONE: {
case Ydb::FieldTransformation::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::FieldTransformation::TRANSFORM_NONE: {
switch (fieldDescriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
reflection->AddInt32(message, fieldDescriptor, elem.get<i32>());
Expand Down Expand Up @@ -395,7 +460,7 @@ inline void NlohmannJsonToProto(const nlohmann::json& jsonValue, NProtoBuf::Mess
}
} else {
switch (transformer) {
case Ydb::DataStreams::V1::TRANSFORM_BASE64: {
case Ydb::FieldTransformation::TRANSFORM_BASE64: {
Y_ENSURE(fieldDescriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_STRING,
"Base64 transformer is applicable only to strings");
Expand All @@ -406,12 +471,12 @@ inline void NlohmannJsonToProto(const nlohmann::json& jsonValue, NProtoBuf::Mess
}
break;
}
case Ydb::DataStreams::V1::TRANSFORM_DOUBLE_S_TO_INT_MS: {
case Ydb::FieldTransformation::TRANSFORM_DOUBLE_S_TO_INT_MS: {
reflection->SetInt64(message, fieldDescriptor, value.get<double>() * 1000);
break;
}
case Ydb::DataStreams::V1::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::DataStreams::V1::TRANSFORM_NONE: {
case Ydb::FieldTransformation::TRANSFORM_EMPTY_TO_NOTHING:
case Ydb::FieldTransformation::TRANSFORM_NONE: {
switch (fieldDescriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
reflection->SetInt32(message, fieldDescriptor, value.get<i32>());
Expand Down Expand Up @@ -452,11 +517,16 @@ inline void NlohmannJsonToProto(const nlohmann::json& jsonValue, NProtoBuf::Mess
reflection->SetString(message, fieldDescriptor, value.get<std::string>());
break;
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: {
auto *msg = reflection->MutableMessage(message, fieldDescriptor);
NlohmannJsonToProto(value, msg, depth + 1);
if (fieldDescriptor->is_map()) {
AddJsonObjectToProtoAsMap(fieldDescriptor, reflection, message, value);
} else {
auto *msg = reflection->MutableMessage(message, fieldDescriptor);
NlohmannJsonToProto(value, msg, depth + 1);
}
break;
}
default:
Cerr << "KLACK: type is " << std::to_string(fieldDescriptor->cpp_type()) << "\n";
Y_ENSURE(false, "Unexpected type");
}
break;
Expand Down
57 changes: 57 additions & 0 deletions ydb/core/http_proxy/ut/json_proto_conversion_ut.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include <library/cpp/testing/unittest/registar.h>
#include "json_proto_conversion.h"
#include <ydb/public/api/protos/draft/ymq.pb.h>

Y_UNIT_TEST_SUITE(JsonProtoConversion) {

Expand Down Expand Up @@ -164,4 +165,60 @@ Y_UNIT_TEST(NlohmannJsonToProtoArray) {
}

}

Y_UNIT_TEST(JsonToProtoMap) {
{
Ydb::YMQ::CreateQueueRequest message;

NJson::TJsonValue jsonObject;
jsonObject["QueueName"] = "SampleQueueName";

NJson::TJsonMap attributes;
attributes["DelaySeconds"] = "900";
attributes["MaximumMessageSize"] = "1024";

jsonObject["Attributes"] = attributes;

NKikimr::NHttpProxy::JsonToProto(jsonObject, &message);

UNIT_ASSERT_VALUES_EQUAL(message.queue_name(), "SampleQueueName");
UNIT_ASSERT_VALUES_EQUAL(message.attributes().find("DelaySeconds")->second, "900");
UNIT_ASSERT_VALUES_EQUAL(message.attributes().find("MaximumMessageSize")->second, "1024");
}
}

Y_UNIT_TEST(ProtoMapToJson) {
{
Ydb::YMQ::GetQueueAttributesResult message;
message.mutable_attributes()->insert({google::protobuf::MapPair<TString, TString>("DelaySeconds", "900")});
message.mutable_attributes()->insert({google::protobuf::MapPair<TString, TString>("MaximumMessageSize", "1024")});

NJson::TJsonValue jsonObject;
NKikimr::NHttpProxy::ProtoToJson(message, jsonObject, false);

UNIT_ASSERT_VALUES_EQUAL(jsonObject.GetMap().find("Attributes")->second.GetMap().size(), 2);
UNIT_ASSERT_VALUES_EQUAL(jsonObject.GetMap().find("Attributes")->second.GetMap().find("DelaySeconds")->second.GetString(), "900");
UNIT_ASSERT_VALUES_EQUAL(jsonObject.GetMap().find("Attributes")->second.GetMap().find("MaximumMessageSize")->second.GetString(), "1024");
}
}

Y_UNIT_TEST(NlohmannJsonToProtoMap) {
{
nlohmann::json jsonObject;
jsonObject["QueueName"] = "SampleQueueName";

nlohmann::json attributes;
attributes["DelaySeconds"] = "900";
attributes["MaximumMessageSize"] = "1024";
jsonObject["Attributes"] = attributes;
nlohmann::json record;

Ydb::YMQ::CreateQueueRequest message;
NKikimr::NHttpProxy::NlohmannJsonToProto(jsonObject, &message);

UNIT_ASSERT_VALUES_EQUAL(message.queue_name(), "SampleQueueName");
UNIT_ASSERT_VALUES_EQUAL(message.attributes().find("DelaySeconds")->second, "900");
UNIT_ASSERT_VALUES_EQUAL(message.attributes().find("MaximumMessageSize")->second, "1024");
}
}
} // Y_UNIT_TEST_SUITE(JsonProtoConversion)
1 change: 1 addition & 0 deletions ydb/public/api/protos/api-protos-sources.jar
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/home/sergeyveselov/.ya/build/symres/fa4b671eae9645bd0d6b6cf3638dd6ff/api-protos-sources.jar
1 change: 1 addition & 0 deletions ydb/public/api/protos/api-protos.jar
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/home/sergeyveselov/.ya/build/symres/6b252e1a5a7551ee0dba31dffdae2b0c/api-protos.jar
Loading

0 comments on commit aedc4fd

Please sign in to comment.