Skip to content

Commit

Permalink
Replace the fake ParseMessage/ParseGroup function calls in with ones …
Browse files Browse the repository at this point in the history 
…that accept a lambda. It allows the callers to drop the mock "message" types that only exist to have an _InternalParse function in them. It also gives more freedom to the caller to do things like force inlining when it matters.

Do more inlining in certain callers of ParseLoop to avoid the extra stack frame.
This reduces the overall cost of maintaining the stack frames in functions like FastMtS1.

PiperOrigin-RevId: 590943926
  • Loading branch information
@protobuf-github-bot @copybara-github
protobuf-github-bot authored and copybara-github committed Dec 14, 2023
1 parent 34908e2 commit f0a8c47
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 71 deletions.
3 changes: 3 additions & 0 deletions src/google/protobuf/generated_message_tctable_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,9 @@ class PROTOBUF_EXPORT TcParser final {
static const char* ParseLoop(MessageLite* msg, const char* ptr,
ParseContext* ctx,
const TcParseTableBase* table);
static const char* ParseLoopInlined(MessageLite* msg, const char* ptr,
ParseContext* ctx,
const TcParseTableBase* table);

// Functions referenced by generated fast tables (numeric types):
// F: fixed V: varint Z: zigzag
Expand Down
48 changes: 29 additions & 19 deletions src/google/protobuf/generated_message_tctable_lite.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ const char* TcParser::GenericFallbackLite(PROTOBUF_TC_PARAM_DECL) {
// Core fast parsing implementation:
//////////////////////////////////////////////////////////////////////////////

PROTOBUF_NOINLINE const char* TcParser::ParseLoop(
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ParseLoopInlined(
MessageLite* msg, const char* ptr, ParseContext* ctx,
const TcParseTableBase* table) {
// Note: TagDispatch uses a dispatch table at "&table->fast_entries".
Expand All @@ -98,6 +98,12 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop(
return ptr;
}

PROTOBUF_NOINLINE const char* TcParser::ParseLoop(
MessageLite* msg, const char* ptr, ParseContext* ctx,
const TcParseTableBase* table) {
return ParseLoopInlined(msg, ptr, ctx, table);
}

// On the fast path, a (matching) 1-byte tag already has the decoded value.
static uint32_t FastDecodeTag(uint8_t coded_tag) {
return coded_tag;
Expand Down Expand Up @@ -387,11 +393,12 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl(
if (field == nullptr) {
field = inner_table->default_instance->New(msg->GetArena());
}
if (group_coding) {
return ctx->ParseGroup<TcParser>(field, ptr, FastDecodeTag(saved_tag),
inner_table);
}
return ctx->ParseMessage<TcParser>(field, ptr, inner_table);
const auto inner_loop = [&](const char* ptr) {
return ParseLoopInlined(field, ptr, ctx, inner_table);
};
return group_coding ? ctx->ParseGroupInlined(ptr, FastDecodeTag(saved_tag),
inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr, inner_loop);
} else {
if (field == nullptr) {
const MessageLite* default_instance =
Expand Down Expand Up @@ -474,12 +481,12 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
ptr += sizeof(TagType);
MessageLite* submsg = field.AddMessage(default_instance);
if (aux_is_table) {
if (group_coding) {
ptr = ctx->ParseGroup<TcParser>(submsg, ptr,
FastDecodeTag(expected_tag), aux.table);
} else {
ptr = ctx->ParseMessage<TcParser>(submsg, ptr, aux.table);
}
const auto inner_loop = [&](const char* ptr) {
return ParseLoopInlined(submsg, ptr, ctx, aux.table);
};
ptr = group_coding ? ctx->ParseGroupInlined(
ptr, FastDecodeTag(expected_tag), inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr, inner_loop);
} else {
if (group_coding) {
ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag));
Expand Down Expand Up @@ -2369,10 +2376,11 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
if (need_init || field == nullptr) {
field = inner_table->default_instance->New(msg->GetArena());
}
if (is_group) {
return ctx->ParseGroup<TcParser>(field, ptr, decoded_tag, inner_table);
}
return ctx->ParseMessage<TcParser>(field, ptr, inner_table);
const auto inner_loop = [&](const char* ptr) {
return ParseLoop(field, ptr, ctx, inner_table);
};
return is_group ? ctx->ParseGroupInlined(ptr, decoded_tag, inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr, inner_loop);
} else {
if (need_init || field == nullptr) {
const MessageLite* def;
Expand Down Expand Up @@ -2428,9 +2436,11 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) {
uint32_t next_tag;
do {
MessageLite* value = field.AddMessage(default_instance);
ptr = is_group ? ctx->ParseGroup<TcParser>(value, ptr2, decoded_tag,
inner_table)
: ctx->ParseMessage<TcParser>(value, ptr2, inner_table);
const auto inner_loop = [&](const char* ptr) {
return ParseLoop(value, ptr, ctx, inner_table);
};
ptr = is_group ? ctx->ParseGroupInlined(ptr2, decoded_tag, inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr2, inner_loop);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
Expand Down
76 changes: 24 additions & 52 deletions src/google/protobuf/parse_context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,34 +494,18 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {

const char* ParseMessage(MessageLite* msg, const char* ptr);

// This overload supports those few cases where ParseMessage is called
// on a class that is not actually a proto message.
// TODO: Eliminate this use case.
template <typename T,
typename std::enable_if<!std::is_base_of<MessageLite, T>::value,
bool>::type = true>
PROTOBUF_NODISCARD const char* ParseMessage(T* msg, const char* ptr);

// Read the length prefix, push the new limit, call the func(ptr), and then
// pop the limit. Useful for situations that don't value an actual message,
// like map entries.
// pop the limit. Useful for situations that don't have an actual message.
template <typename Func>
PROTOBUF_NODISCARD const char* ParseLengthDelimitedInlined(const char*,
const Func& func);

template <typename TcParser, typename Table>
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseMessage(
MessageLite* msg, const char* ptr, const Table* table) {
LimitToken old;
ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old);
if (ptr == nullptr) return ptr;
auto old_depth = depth_;
ptr = TcParser::ParseLoop(msg, ptr, this, table);
if (ptr != nullptr) ABSL_DCHECK_EQ(old_depth, depth_);
depth_++;
if (!PopLimit(std::move(old))) return nullptr;
return ptr;
}
// Push the recursion depth, call the func(ptr), and then pop depth. Useful
// for situations that don't have an actual message.
template <typename Func>
PROTOBUF_NODISCARD const char* ParseGroupInlined(const char* ptr,
uint32_t start_tag,
const Func& func);

template <typename T>
PROTOBUF_NODISCARD PROTOBUF_NDEBUG_INLINE const char* ParseGroup(
Expand All @@ -541,24 +525,6 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
return ptr;
}

template <typename TcParser, typename Table>
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseGroup(
MessageLite* msg, const char* ptr, uint32_t tag, const Table* table) {
if (--depth_ < 0) return nullptr;
group_depth_++;
auto old_depth = depth_;
auto old_group_depth = group_depth_;
ptr = TcParser::ParseLoop(msg, ptr, this, table);
if (ptr != nullptr) {
ABSL_DCHECK_EQ(old_depth, depth_);
ABSL_DCHECK_EQ(old_group_depth, group_depth_);
}
group_depth_--;
depth_++;
if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr;
return ptr;
}

private:
// Out-of-line routine to save space in ParseContext::ParseMessage<T>
// LimitToken old;
Expand Down Expand Up @@ -1105,15 +1071,14 @@ inline int32_t ReadVarintZigZag32(const char** p) {
return WireFormatLite::ZigZagDecode32(static_cast<uint32_t>(tmp));
}

template <typename T, typename std::enable_if<
!std::is_base_of<MessageLite, T>::value, bool>::type>
PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg,
const char* ptr) {
template <typename Func>
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char*
ParseContext::ParseLengthDelimitedInlined(const char* ptr, const Func& func) {
LimitToken old;
ptr = ReadSizeAndPushLimitAndDepth(ptr, &old);
ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old);
if (ptr == nullptr) return ptr;
auto old_depth = depth_;
ptr = msg->_InternalParse(ptr, this);
PROTOBUF_ALWAYS_INLINE_CALL ptr = func(ptr);
if (ptr != nullptr) ABSL_DCHECK_EQ(old_depth, depth_);
depth_++;
if (!PopLimit(std::move(old))) return nullptr;
Expand All @@ -1122,13 +1087,20 @@ PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg,

template <typename Func>
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char*
ParseContext::ParseLengthDelimitedInlined(const char* ptr, const Func& func) {
LimitToken old;
ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old);
if (ptr == nullptr) return ptr;
ParseContext::ParseGroupInlined(const char* ptr, uint32_t start_tag,
const Func& func) {
if (--depth_ < 0) return nullptr;
group_depth_++;
auto old_depth = depth_;
auto old_group_depth = group_depth_;
PROTOBUF_ALWAYS_INLINE_CALL ptr = func(ptr);
if (ptr != nullptr) {
ABSL_DCHECK_EQ(old_depth, depth_);
ABSL_DCHECK_EQ(old_group_depth, group_depth_);
}
group_depth_--;
depth_++;
if (!PopLimit(std::move(old))) return nullptr;
if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(start_tag))) return nullptr;
return ptr;
}

Expand Down

0 comments on commit f0a8c47

Please sign in to comment.