[feat](spill) spill and reserve #44469
Conversation
|
Thank you for your contribution to Apache Doris. Please clearly describe your PR:
|
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
| @@ -108,6 +112,87 @@ Status HashJoinBuildSinkLocalState::open(RuntimeState* state) { | |||
| return Status::OK(); | |||
| } | |||
|
|
|||
| size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) { | |||
There was a problem hiding this comment.
warning: method 'get_reserve_mem_size' can be made const [readability-make-member-function-const]
be/src/pipeline/exec/hashjoin_build_sink.h:56:
- [[nodiscard]] size_t get_reserve_mem_size(RuntimeState* state, bool eos);
+ [[nodiscard]] size_t get_reserve_mem_size(RuntimeState* state, bool eos) const;| size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) { | |
| size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) const { |
| @@ -29,56 +46,281 @@ MultiCastBlock::MultiCastBlock(vectorized::Block* block, int un_finish_copy, siz | |||
| block->clear(); | |||
| } | |||
|
|
|||
| Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* block, bool* eos) { | |||
| int* un_finish_copy = nullptr; | |||
| Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block, | |||
There was a problem hiding this comment.
warning: function 'pull' has cognitive complexity of 54 (threshold 50) [readability-function-cognitive-complexity]
Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block,
^Additional context
be/src/pipeline/exec/multi_cast_data_streamer.cpp:54: +1, including nesting penalty of 0, nesting level increased to 1
if (!_cached_blocks[sender_idx].empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:69: +1, including nesting penalty of 0, nesting level increased to 1
if (!_spill_readers[sender_idx].empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:71: +2, including nesting penalty of 1, nesting level increased to 2
if (!reader_item->stream->ready_for_reading()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:76: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(reader->open());
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:76: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(reader->open());
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:77: +2, including nesting penalty of 1, nesting level increased to 2
if (reader_item->block_offset != 0) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:82: nesting level increased to 2
auto spill_func = [this, reader_item, sender_idx]() {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:86: +3, including nesting penalty of 2, nesting level increased to 3
while (!spill_eos) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:87: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(reader_item->reader->read(&block, &spill_eos));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:87: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(reader_item->reader->read(&block, &spill_eos));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:88: +4, including nesting penalty of 3, nesting level increased to 4
if (!block.empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:92: +5, including nesting penalty of 4, nesting level increased to 5
if (_cached_blocks[sender_idx].size() >= 32 ||
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:99: +3, including nesting penalty of 2, nesting level increased to 3
if (spill_eos || !_cached_blocks[sender_idx].empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:106: nesting level increased to 2
auto catch_exception_func = [spill_func = std::move(spill_func)]() {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:107: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_CATCH_EXCEPTION(return spill_func(););
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:107: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_CATCH_EXCEPTION(return spill_func(););
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:107: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_CATCH_EXCEPTION(return spill_func(););
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:116: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(thread_pool->submit(std::move(spill_runnable)));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:116: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(thread_pool->submit(std::move(spill_runnable)));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:122: +1, including nesting penalty of 0, nesting level increased to 1
if (pos_to_pull == end) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:137: +1, including nesting penalty of 0, nesting level increased to 1
if (pos_to_pull == end) {
^| @@ -29,56 +46,281 @@ | |||
| block->clear(); | |||
| } | |||
|
|
|||
| Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* block, bool* eos) { | |||
| int* un_finish_copy = nullptr; | |||
| Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block, | |||
There was a problem hiding this comment.
warning: function 'pull' exceeds recommended size/complexity thresholds [readability-function-size]
Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block,
^Additional context
be/src/pipeline/exec/multi_cast_data_streamer.cpp:48: 95 lines including whitespace and comments (threshold 80)
Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block,
^| @@ -117,4 +360,38 @@ | |||
| dep->block(); | |||
| } | |||
|
|
|||
| std::string MultiCastDataStreamer::debug_string() { | |||
There was a problem hiding this comment.
warning: method 'debug_string' can be made const [readability-make-member-function-const]
| std::string MultiCastDataStreamer::debug_string() { | |
| std::string MultiCastDataStreamer::debug_string() const { |
be/src/pipeline/exec/multi_cast_data_streamer.h:115:
- std::string debug_string();
+ std::string debug_string() const;| _peak_mem_usage = ADD_COUNTER(profile(), "PeakMemUsage", TUnit::BYTES); | ||
| _process_rows = ADD_COUNTER(profile(), "ProcessRows", TUnit::UNIT); | ||
| }; | ||
|
|
||
| ~MultiCastDataStreamer() = default; | ||
| ~MultiCastDataStreamer() { |
There was a problem hiding this comment.
warning: use '= default' to define a trivial destructor [modernize-use-equals-default]
~MultiCastDataStreamer() {
^| @@ -75,94 +73,85 @@ int SpillSortLocalState::_calc_spill_blocks_to_merge() const { | |||
| } | |||
| Status SpillSortLocalState::initiate_merge_sort_spill_streams(RuntimeState* state) { | |||
There was a problem hiding this comment.
warning: function 'initiate_merge_sort_spill_streams' has cognitive complexity of 84 (threshold 50) [readability-function-cognitive-complexity]
Status SpillSortLocalState::initiate_merge_sort_spill_streams(RuntimeState* state) {
^Additional context
be/src/pipeline/exec/spill_sort_source_operator.cpp:81: nesting level increased to 1
auto spill_func = [this, state, query_id, &parent] {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:102: +2, including nesting penalty of 1, nesting level increased to 2
while (!state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:109: nesting level increased to 3
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:67: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:117: +3, including nesting penalty of 2, nesting level increased to 3
if (_shared_state->sorted_streams.empty()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +3, including nesting penalty of 2, nesting level increased to 3
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:134: nesting level increased to 4
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:67: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:140: +4, including nesting penalty of 3, nesting level increased to 4
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:146: +4, including nesting penalty of 3, nesting level increased to 4
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +6, including nesting penalty of 5, nesting level increased to 6
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:164: nesting level increased to 1
auto exception_catch_func = [spill_func]() {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:169: +1, including nesting penalty of 0, nesting level increased to 1
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::merge_sort_spill_data_submit_func", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:169: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::merge_sort_spill_data_submit_func", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^|
|
||
| auto spill_func = [this, state, query_id, &parent, submit_timer] { | ||
| _spill_wait_in_queue_timer->update(submit_timer.elapsed_time()); | ||
| auto spill_func = [this, state, query_id, &parent] { |
There was a problem hiding this comment.
warning: lambda has cognitive complexity of 52 (threshold 50) [readability-function-cognitive-complexity]
auto spill_func = [this, state, query_id, &parent] {
^Additional context
be/src/pipeline/exec/spill_sort_source_operator.cpp:102: +1, including nesting penalty of 0, nesting level increased to 1
while (!state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:109: nesting level increased to 2
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:67: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:117: +2, including nesting penalty of 1, nesting level increased to 2
if (_shared_state->sorted_streams.empty()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +2, including nesting penalty of 1, nesting level increased to 2
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:134: nesting level increased to 3
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:67: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:140: +3, including nesting penalty of 2, nesting level increased to 3
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:146: +3, including nesting penalty of 2, nesting level increased to 3
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^| @@ -17,39 +17,131 @@ | |||
|
|
|||
| #pragma once | |||
|
|
|||
| #include <gen_cpp/Types_types.h> | |||
There was a problem hiding this comment.
warning: 'gen_cpp/Types_types.h' file not found [clang-diagnostic-error]
#include <gen_cpp/Types_types.h>
^| query_id(std::move(query_id_)), | ||
| all_tasks_finished_callback(std::move(all_tasks_finished_callback_)) {} | ||
|
|
||
| ~SpillContext() { |
There was a problem hiding this comment.
warning: use '= default' to define a trivial destructor [modernize-use-equals-default]
~SpillContext() {
^| _remote_scan_bytes_per_second); | ||
| } | ||
|
|
||
| bool WorkloadGroup::add_wg_refresh_interval_memory_growth(int64_t size) { |
There was a problem hiding this comment.
warning: method 'add_wg_refresh_interval_memory_growth' can be made const [readability-make-member-function-const]
be/src/runtime/workload_group/workload_group.h:118:
- bool add_wg_refresh_interval_memory_growth(int64_t size);
+ bool add_wg_refresh_interval_memory_growth(int64_t size) const;| bool WorkloadGroup::add_wg_refresh_interval_memory_growth(int64_t size) { | |
| bool WorkloadGroup::add_wg_refresh_interval_memory_growth(int64_t size) const { |
mrhhsg
force-pushed
the
spill_rebased
branch
from
598192c to
1f4d2c7
Compare
|
run buildall |
| @@ -479,11 +582,11 @@ Status PartitionedHashJoinSinkOperatorX::_setup_internal_operator(RuntimeState* | |||
| Status PartitionedHashJoinSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block, | |||
There was a problem hiding this comment.
warning: function 'sink' has cognitive complexity of 76 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block,
^Additional context
be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:587: +1, including nesting penalty of 0, nesting level increased to 1
if (!local_state._shared_state->_spill_status.ok()) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:596: +1, including nesting penalty of 0, nesting level increased to 1
if (rows == 0) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:597: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:602: +3, including nesting penalty of 2, nesting level increased to 3
if (need_to_spill) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:604: +1, nesting level increased to 3
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:605: +4, including nesting penalty of 3, nesting level increased to 4
if (UNLIKELY(!local_state._shared_state->inner_runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:606: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:606: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:608: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink_eos", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:608: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink_eos", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:614: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:614: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:625: nesting level increased to 3
[&](auto& block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:626: +4, including nesting penalty of 3, nesting level increased to 4
if (block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:636: +1, including nesting penalty of 0, nesting level increased to 1
if (need_to_spill) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:637: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(local_state._partition_block(state, in_block, 0, rows));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:637: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(local_state._partition_block(state, in_block, 0, rows));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:638: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:640: +1, nesting level increased to 2
} else if (revocable_mem_size(state) > vectorized::SpillStream::MAX_SPILL_WRITE_BATCH_MEM) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:643: +1, nesting level increased to 1
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:644: +2, including nesting penalty of 1, nesting level increased to 2
if (UNLIKELY(!local_state._shared_state->inner_runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:645: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:645: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:647: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:647: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:653: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:653: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:656: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^| @@ -215,6 +251,91 @@ void WorkloadGroup::add_mem_tracker_limiter(std::shared_ptr<MemTrackerLimiter> m | |||
| _mem_tracker_limiter_pool[group_num].trackers.end(), mem_tracker_ptr); | |||
| } | |||
|
|
|||
| int64_t WorkloadGroup::free_overcommited_memory(int64_t need_free_mem, RuntimeProfile* profile) { | |||
There was a problem hiding this comment.
warning: function 'free_overcommited_memory' exceeds recommended size/complexity thresholds [readability-function-size]
int64_t WorkloadGroup::free_overcommited_memory(int64_t need_free_mem, RuntimeProfile* profile) {
^Additional context
be/src/runtime/workload_group/workload_group.cpp:253: 83 lines including whitespace and comments (threshold 80)
int64_t WorkloadGroup::free_overcommited_memory(int64_t need_free_mem, RuntimeProfile* profile) {
^| * strategy 4: If any query exceed query's memlimit, then do spill disk or cancel it. | ||
| * strategy 5: If any query exceed process's memlimit and cache is zero, then do following: | ||
| */ | ||
| void WorkloadGroupMgr::handle_paused_queries() { |
There was a problem hiding this comment.
warning: function 'handle_paused_queries' exceeds recommended size/complexity thresholds [readability-function-size]
void WorkloadGroupMgr::handle_paused_queries() {
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:298: 219 lines including whitespace and comments (threshold 80)
void WorkloadGroupMgr::handle_paused_queries() {
^| // If the query could release some memory, for example, spill disk, then the return value is true. | ||
| // If the query could not release memory, then cancel the query, the return value is true. | ||
| // If the query is not ready to do these tasks, it means just wait, then return value is false. | ||
| bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx, |
There was a problem hiding this comment.
warning: function 'handle_single_query_' exceeds recommended size/complexity thresholds [readability-function-size]
bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx,
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:647: 100 lines including whitespace and comments (threshold 80)
bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx,
^| return true; | ||
| } | ||
|
|
||
| void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit) { |
There was a problem hiding this comment.
warning: function 'update_queries_limit_' exceeds recommended size/complexity thresholds [readability-function-size]
void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit) {
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:751: 98 lines including whitespace and comments (threshold 80)
void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit) {
^| bool is_low_watermark = false; | ||
| bool is_high_watermark = false; | ||
| wg->check_mem_used(&is_low_watermark, &is_high_watermark); | ||
| int64_t wg_high_water_mark_limit = |
There was a problem hiding this comment.
warning: use auto when initializing with a cast to avoid duplicating the type name [modernize-use-auto]
| int64_t wg_high_water_mark_limit = | |
| auto wg_high_water_mark_limit = |
| @@ -19,6 +19,8 @@ | |||
|
|
|||
| #include <glog/logging.h> | |||
There was a problem hiding this comment.
warning: 'glog/logging.h' file not found [clang-diagnostic-error]
#include <glog/logging.h>
^
TPC-H: Total hot run time: 41016 ms |
|
run buildall |
mrhhsg
force-pushed
the
spill_rebased
branch
from
1f4d2c7 to
d70e6dd
Compare
|
run buildall |
|
run buildall |
TPC-H: Total hot run time: 40956 ms |
| @@ -16,6 +16,7 @@ | |||
| // under the License. | |||
|
|
|||
| #include <fmt/core.h> | |||
There was a problem hiding this comment.
warning: 'fmt/core.h' file not found [clang-diagnostic-error]
#include <fmt/core.h>
^| @@ -29,65 +46,306 @@ MultiCastBlock::MultiCastBlock(vectorized::Block* block, int un_finish_copy, siz | |||
| block->clear(); | |||
| } | |||
|
|
|||
| Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* block, bool* eos) { | |||
| int* un_finish_copy = nullptr; | |||
| Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block, | |||
There was a problem hiding this comment.
warning: function 'pull' has cognitive complexity of 54 (threshold 50) [readability-function-cognitive-complexity]
Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block,
^Additional context
be/src/pipeline/exec/multi_cast_data_streamer.cpp:54: +1, including nesting penalty of 0, nesting level increased to 1
if (!_cached_blocks[sender_idx].empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:69: +1, including nesting penalty of 0, nesting level increased to 1
if (!_spill_readers[sender_idx].empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:71: +2, including nesting penalty of 1, nesting level increased to 2
if (!reader_item->stream->ready_for_reading()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:76: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(reader->open());
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:76: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(reader->open());
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:77: +2, including nesting penalty of 1, nesting level increased to 2
if (reader_item->block_offset != 0) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:82: nesting level increased to 2
auto spill_func = [this, reader_item, sender_idx]() {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:86: +3, including nesting penalty of 2, nesting level increased to 3
while (!spill_eos) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:87: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(reader_item->reader->read(&block, &spill_eos));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:87: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(reader_item->reader->read(&block, &spill_eos));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:88: +4, including nesting penalty of 3, nesting level increased to 4
if (!block.empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:92: +5, including nesting penalty of 4, nesting level increased to 5
if (_cached_blocks[sender_idx].size() >= 32 ||
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:99: +3, including nesting penalty of 2, nesting level increased to 3
if (spill_eos || !_cached_blocks[sender_idx].empty()) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:106: nesting level increased to 2
auto catch_exception_func = [spill_func = std::move(spill_func)]() {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:107: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_CATCH_EXCEPTION(return spill_func(););
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:107: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_CATCH_EXCEPTION(return spill_func(););
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:107: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_CATCH_EXCEPTION(return spill_func(););
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:116: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(thread_pool->submit(std::move(spill_runnable)));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:116: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(thread_pool->submit(std::move(spill_runnable)));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:122: +1, including nesting penalty of 0, nesting level increased to 1
if (pos_to_pull == end) {
^be/src/pipeline/exec/multi_cast_data_streamer.cpp:139: +1, including nesting penalty of 0, nesting level increased to 1
if (pos_to_pull == end) {
^| @@ -29,65 +46,306 @@ | |||
| block->clear(); | |||
| } | |||
|
|
|||
| Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* block, bool* eos) { | |||
| int* un_finish_copy = nullptr; | |||
| Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block, | |||
There was a problem hiding this comment.
warning: function 'pull' exceeds recommended size/complexity thresholds [readability-function-size]
Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block,
^Additional context
be/src/pipeline/exec/multi_cast_data_streamer.cpp:48: 97 lines including whitespace and comments (threshold 80)
Status MultiCastDataStreamer::pull(RuntimeState* state, int sender_idx, vectorized::Block* block,
^| return size; | ||
| } | ||
|
|
||
| Status PartitionedAggSinkLocalState::revoke_memory( |
There was a problem hiding this comment.
warning: function 'revoke_memory' exceeds recommended size/complexity thresholds [readability-function-size]
Status PartitionedAggSinkLocalState::revoke_memory(
^Additional context
be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp:261: 88 lines including whitespace and comments (threshold 80)
Status PartitionedAggSinkLocalState::revoke_memory(
^| if (Base::_shared_state->in_mem_shared_state->aggregate_data_container->iterator != | ||
| Base::_shared_state->in_mem_shared_state->aggregate_data_container->end() || | ||
| _shared_state->spill_partitions.empty()) { | ||
| Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) { |
There was a problem hiding this comment.
warning: function 'recover_blocks_from_disk' has cognitive complexity of 76 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) {
^Additional context
be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:242: +1, including nesting penalty of 0, nesting level increased to 1
if (_shared_state->spill_partitions.empty()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:249: nesting level increased to 1
auto spill_func = [this, state, query_id] {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:262: +2, including nesting penalty of 1, nesting level increased to 2
while (!state->is_cancelled() && !has_agg_data &&
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:264: +3, including nesting penalty of 2, nesting level increased to 3
while (!_shared_state->spill_partitions[0]->spill_streams_.empty() &&
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:270: +4, including nesting penalty of 3, nesting level increased to 4
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:270: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:272: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::recover_spill_data",
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:272: +6, including nesting penalty of 5, nesting level increased to 6
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::recover_spill_data",
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:278: +5, including nesting penalty of 4, nesting level increased to 5
if (status.ok()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:282: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:282: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:284: +5, including nesting penalty of 4, nesting level increased to 5
if (!block.empty()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:289: +6, including nesting penalty of 5, nesting level increased to 6
if (accumulated_blocks_size >=
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:298: +4, including nesting penalty of 3, nesting level increased to 4
if (_current_partition_eos) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:304: +3, including nesting penalty of 2, nesting level increased to 3
if (_shared_state->spill_partitions[0]->spill_streams_.empty()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:317: nesting level increased to 1
auto exception_catch_func = [spill_func, query_id]() {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:318: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::merge_spill_data_cancel", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:318: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::merge_spill_data_cancel", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:326: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:326: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:326: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:326: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:332: +1, including nesting penalty of 0, nesting level increased to 1
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::submit_func", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:332: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::submit_func", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^| @@ -75,94 +73,85 @@ int SpillSortLocalState::_calc_spill_blocks_to_merge() const { | |||
| } | |||
| Status SpillSortLocalState::initiate_merge_sort_spill_streams(RuntimeState* state) { | |||
There was a problem hiding this comment.
warning: function 'initiate_merge_sort_spill_streams' has cognitive complexity of 84 (threshold 50) [readability-function-cognitive-complexity]
Status SpillSortLocalState::initiate_merge_sort_spill_streams(RuntimeState* state) {
^Additional context
be/src/pipeline/exec/spill_sort_source_operator.cpp:81: nesting level increased to 1
auto spill_func = [this, state, query_id, &parent] {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:102: +2, including nesting penalty of 1, nesting level increased to 2
while (!state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:109: nesting level increased to 3
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:68: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:117: +3, including nesting penalty of 2, nesting level increased to 3
if (_shared_state->sorted_streams.empty()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +3, including nesting penalty of 2, nesting level increased to 3
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:134: nesting level increased to 4
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:68: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:140: +4, including nesting penalty of 3, nesting level increased to 4
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:146: +4, including nesting penalty of 3, nesting level increased to 4
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +6, including nesting penalty of 5, nesting level increased to 6
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:164: nesting level increased to 1
auto exception_catch_func = [spill_func]() {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:165: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:169: +1, including nesting penalty of 0, nesting level increased to 1
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::merge_sort_spill_data_submit_func", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:169: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::merge_sort_spill_data_submit_func", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^|
|
||
| auto spill_func = [this, state, query_id, &parent, submit_timer] { | ||
| _spill_wait_in_queue_timer->update(submit_timer.elapsed_time()); | ||
| auto spill_func = [this, state, query_id, &parent] { |
There was a problem hiding this comment.
warning: lambda has cognitive complexity of 52 (threshold 50) [readability-function-cognitive-complexity]
auto spill_func = [this, state, query_id, &parent] {
^Additional context
be/src/pipeline/exec/spill_sort_source_operator.cpp:102: +1, including nesting penalty of 0, nesting level increased to 1
while (!state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:109: nesting level increased to 2
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:68: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:114: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:117: +2, including nesting penalty of 1, nesting level increased to 2
if (_shared_state->sorted_streams.empty()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:126: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +2, including nesting penalty of 1, nesting level increased to 2
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:131: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:134: nesting level increased to 3
SCOPED_TIMER(Base::_spill_recover_time);
^be/src/util/runtime_profile.h:68: expanded from macro 'SCOPED_TIMER'
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:135: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::recover_spill_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:140: +3, including nesting penalty of 2, nesting level increased to 3
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:144: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:146: +3, including nesting penalty of 2, nesting level increased to 3
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:147: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::spill_sort_source::spill_merged_data", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_source_operator.cpp:153: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^| _remote_scan_bytes_per_second); | ||
| } | ||
|
|
||
| bool WorkloadGroup::add_wg_refresh_interval_memory_growth(int64_t size) { |
There was a problem hiding this comment.
warning: method 'add_wg_refresh_interval_memory_growth' can be made const [readability-make-member-function-const]
be/src/runtime/workload_group/workload_group.h:115:
- bool add_wg_refresh_interval_memory_growth(int64_t size);
+ bool add_wg_refresh_interval_memory_growth(int64_t size) const;| bool WorkloadGroup::add_wg_refresh_interval_memory_growth(int64_t size) { | |
| bool WorkloadGroup::add_wg_refresh_interval_memory_growth(int64_t size) const { |
| * strategy 4: If any query exceed query's memlimit, then do spill disk or cancel it. | ||
| * strategy 5: If any query exceed process's memlimit and cache is zero, then do following: | ||
| */ | ||
| void WorkloadGroupMgr::handle_paused_queries() { |
There was a problem hiding this comment.
warning: function 'handle_paused_queries' exceeds recommended size/complexity thresholds [readability-function-size]
void WorkloadGroupMgr::handle_paused_queries() {
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:296: 219 lines including whitespace and comments (threshold 80)
void WorkloadGroupMgr::handle_paused_queries() {
^| // If the query could release some memory, for example, spill disk, then the return value is true. | ||
| // If the query could not release memory, then cancel the query, the return value is true. | ||
| // If the query is not ready to do these tasks, it means just wait, then return value is false. | ||
| bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx, |
There was a problem hiding this comment.
warning: function 'handle_single_query_' exceeds recommended size/complexity thresholds [readability-function-size]
bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx,
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:645: 102 lines including whitespace and comments (threshold 80)
bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx,
^
mrhhsg
force-pushed
the
spill_rebased
branch
from
811264d to
145e019
Compare
| @@ -108,6 +113,93 @@ Status HashJoinBuildSinkLocalState::open(RuntimeState* state) { | |||
| return Status::OK(); | |||
| } | |||
|
|
|||
| size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) { | |||
There was a problem hiding this comment.
warning: function 'get_reserve_mem_size' exceeds recommended size/complexity thresholds [readability-function-size]
size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) {
^Additional context
be/src/pipeline/exec/hashjoin_build_sink.cpp:115: 85 lines including whitespace and comments (threshold 80)
size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) {
^
mrhhsg
force-pushed
the
spill_rebased
branch
from
145e019 to
1692348
Compare
| return size; | ||
| } | ||
|
|
||
| Status PartitionedAggSinkLocalState::revoke_memory( |
There was a problem hiding this comment.
warning: function 'revoke_memory' exceeds recommended size/complexity thresholds [readability-function-size]
Status PartitionedAggSinkLocalState::revoke_memory(
^Additional context
be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp:255: 85 lines including whitespace and comments (threshold 80)
Status PartitionedAggSinkLocalState::revoke_memory(
^| if (Base::_shared_state->in_mem_shared_state->aggregate_data_container->iterator != | ||
| Base::_shared_state->in_mem_shared_state->aggregate_data_container->end() || | ||
| _shared_state->spill_partitions.empty()) { | ||
| Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) { |
There was a problem hiding this comment.
warning: function 'recover_blocks_from_disk' has cognitive complexity of 76 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) {
^Additional context
be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:240: +1, including nesting penalty of 0, nesting level increased to 1
if (_shared_state->spill_partitions.empty()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:247: nesting level increased to 1
auto spill_func = [this, state, query_id] {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:260: +2, including nesting penalty of 1, nesting level increased to 2
while (!state->is_cancelled() && !has_agg_data &&
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:262: +3, including nesting penalty of 2, nesting level increased to 3
while (!_shared_state->spill_partitions[0]->spill_streams_.empty() &&
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:268: +4, including nesting penalty of 3, nesting level increased to 4
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:268: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:270: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::recover_spill_data",
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:270: +6, including nesting penalty of 5, nesting level increased to 6
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::recover_spill_data",
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:276: +5, including nesting penalty of 4, nesting level increased to 5
if (status.ok()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:280: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:280: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:282: +5, including nesting penalty of 4, nesting level increased to 5
if (!block.empty()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:287: +6, including nesting penalty of 5, nesting level increased to 6
if (accumulated_blocks_size >=
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:296: +4, including nesting penalty of 3, nesting level increased to 4
if (_current_partition_eos) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:302: +3, including nesting penalty of 2, nesting level increased to 3
if (_shared_state->spill_partitions[0]->spill_streams_.empty()) {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:315: nesting level increased to 1
auto exception_catch_func = [spill_func, query_id]() {
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:316: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::merge_spill_data_cancel", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:316: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::merge_spill_data_cancel", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:324: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:324: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:324: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:324: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:330: +1, including nesting penalty of 0, nesting level increased to 1
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::submit_func", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:330: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_agg_source::submit_func", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^| if (Base::_shared_state->in_mem_shared_state->aggregate_data_container->iterator != | ||
| Base::_shared_state->in_mem_shared_state->aggregate_data_container->end() || | ||
| _shared_state->spill_partitions.empty()) { | ||
| Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) { |
There was a problem hiding this comment.
warning: function 'recover_blocks_from_disk' exceeds recommended size/complexity thresholds [readability-function-size]
Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) {
^Additional context
be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp:237: 107 lines including whitespace and comments (threshold 80)
Status PartitionedAggLocalState::recover_blocks_from_disk(RuntimeState* state, bool& has_data) {
^| local_state._shared_state->inner_runtime_state.get()); | ||
|
|
||
| COUNTER_SET(local_state._hash_table_memory_usage, | ||
| sink_local_state->profile()->get_counter("MemoryUsageHashTable")->value()); | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| Status PartitionedHashJoinProbeOperatorX::pull(doris::RuntimeState* state, |
There was a problem hiding this comment.
warning: function 'pull' has cognitive complexity of 72 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinProbeOperatorX::pull(doris::RuntimeState* state,
^Additional context
be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:707: +1, including nesting penalty of 0, nesting level increased to 1
if (local_state._recovered_build_block && !local_state._recovered_build_block->empty()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:707: +1
if (local_state._recovered_build_block && !local_state._recovered_build_block->empty()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:710: +2, including nesting penalty of 1, nesting level increased to 2
if (!mutable_block) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:712: +1, nesting level increased to 2
} else {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:713: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(mutable_block->merge(local_state._recovered_build_block->to_block()));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:713: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(mutable_block->merge(local_state._recovered_build_block->to_block()));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:718: +1, including nesting penalty of 0, nesting level increased to 1
if (local_state._need_to_setup_internal_operators) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:720: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(local_state.recover_build_blocks_from_disk(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:720: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(local_state.recover_build_blocks_from_disk(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:722: +2, including nesting penalty of 1, nesting level increased to 2
if (has_data) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:727: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(local_state.finish_spilling(partition_index));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:727: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(local_state.finish_spilling(partition_index));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:728: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_setup_internal_operators(local_state, state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:728: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_setup_internal_operators(local_state, state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:731: +2, including nesting penalty of 1, nesting level increased to 2
if (mutable_block && !mutable_block->empty()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:731: +1
if (mutable_block && !mutable_block->empty()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:737: +1, including nesting penalty of 0, nesting level increased to 1
while (_inner_probe_operator->need_more_input_data(runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:738: +2, including nesting penalty of 1, nesting level increased to 2
if (probe_blocks.empty()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:741: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:741: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:743: +3, including nesting penalty of 2, nesting level increased to 3
if (!has_data) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:745: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_inner_probe_operator->push(runtime_state, &block, true));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:745: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_inner_probe_operator->push(runtime_state, &block, true));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:750: +1, nesting level increased to 3
} else {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:757: +2, including nesting penalty of 1, nesting level increased to 2
if (!block.empty()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:758: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_inner_probe_operator->push(runtime_state, &block, false));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:758: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_inner_probe_operator->push(runtime_state, &block, false));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:762: +1, including nesting penalty of 0, nesting level increased to 1
RETURN_IF_ERROR(_inner_probe_operator->pull(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:762: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_inner_probe_operator->pull(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:766: +1, including nesting penalty of 0, nesting level increased to 1
if (in_mem_eos) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:771: +2, including nesting penalty of 1, nesting level increased to 2
if (local_state._partition_cursor == _partition_count) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:773: +1, nesting level increased to 2
} else {
^| @@ -479,12 +520,9 @@ Status PartitionedHashJoinSinkOperatorX::_setup_internal_operator(RuntimeState* | |||
| Status PartitionedHashJoinSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block, | |||
There was a problem hiding this comment.
warning: function 'sink' has cognitive complexity of 75 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block,
^Additional context
be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:531: +1, including nesting penalty of 0, nesting level increased to 1
if (rows == 0) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:532: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:537: +3, including nesting penalty of 2, nesting level increased to 3
if (need_to_spill) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:539: +1, nesting level increased to 3
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:540: +4, including nesting penalty of 3, nesting level increased to 4
if (UNLIKELY(!local_state._shared_state->inner_runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:541: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:541: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:543: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink_eos", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:543: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink_eos", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:549: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:549: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:560: nesting level increased to 3
[&](auto& block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:561: +4, including nesting penalty of 3, nesting level increased to 4
if (block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:571: +1, including nesting penalty of 0, nesting level increased to 1
if (need_to_spill) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:572: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(local_state._partition_block(state, in_block, 0, rows));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:572: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(local_state._partition_block(state, in_block, 0, rows));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:573: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:575: +1, nesting level increased to 2
} else if (revocable_mem_size(state) > vectorized::SpillStream::MAX_SPILL_WRITE_BATCH_MEM) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:578: +1, nesting level increased to 1
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:579: +2, including nesting penalty of 1, nesting level increased to 2
if (UNLIKELY(!local_state._shared_state->inner_runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:580: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:580: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:582: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:582: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:587: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:587: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:590: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^| eos); | ||
| } | ||
|
|
||
| Status SpillSortSinkLocalState::revoke_memory(RuntimeState* state, |
There was a problem hiding this comment.
warning: function 'revoke_memory' has cognitive complexity of 51 (threshold 50) [readability-function-cognitive-complexity]
Status SpillSortSinkLocalState::revoke_memory(RuntimeState* state,
^Additional context
be/src/pipeline/exec/spill_sort_sink_operator.cpp:191: +1, including nesting penalty of 0, nesting level increased to 1
if (!_shared_state->is_spilled) {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:204: +1, including nesting penalty of 0, nesting level increased to 1
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:204: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:211: +1, including nesting penalty of 0, nesting level increased to 1
if (!_eos) {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:216: nesting level increased to 1
auto spill_func = [this, state, query_id, &parent] {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:242: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:242: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:249: +2, including nesting penalty of 1, nesting level increased to 2
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:249: +1
while (!eos && !state->is_cancelled()) {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:256: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:256: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:258: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(status);
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:258: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(status);
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:266: nesting level increased to 1
auto exception_catch_func = [query_id, spill_func]() {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:267: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::spill_sort_sink::revoke_memory_cancel", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:267: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::spill_sort_sink::revoke_memory_cancel", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:275: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:275: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:275: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:275: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:280: +1, including nesting penalty of 0, nesting level increased to 1
DBUG_EXECUTE_IF("fault_inject::spill_sort_sink::revoke_memory_submit_func", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:280: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::spill_sort_sink::revoke_memory_submit_func", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:285: +1, including nesting penalty of 0, nesting level increased to 1
if (status.ok()) {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:293: +1, including nesting penalty of 0, nesting level increased to 1
if (!status.ok()) {
^be/src/pipeline/exec/spill_sort_sink_operator.cpp:294: +2, including nesting penalty of 1, nesting level increased to 2
if (!_eos) {
^| eos); | ||
| } | ||
|
|
||
| Status SpillSortSinkLocalState::revoke_memory(RuntimeState* state, |
There was a problem hiding this comment.
warning: function 'revoke_memory' exceeds recommended size/complexity thresholds [readability-function-size]
Status SpillSortSinkLocalState::revoke_memory(RuntimeState* state,
^Additional context
be/src/pipeline/exec/spill_sort_sink_operator.cpp:189: 109 lines including whitespace and comments (threshold 80)
Status SpillSortSinkLocalState::revoke_memory(RuntimeState* state,
^| // If the query could release some memory, for example, spill disk, then the return value is true. | ||
| // If the query could not release memory, then cancel the query, the return value is true. | ||
| // If the query is not ready to do these tasks, it means just wait, then return value is false. | ||
| bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx, |
There was a problem hiding this comment.
warning: function 'handle_single_query_' exceeds recommended size/complexity thresholds [readability-function-size]
bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx,
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:645: 105 lines including whitespace and comments (threshold 80)
bool WorkloadGroupMgr::handle_single_query_(const std::shared_ptr<QueryContext>& query_ctx,
^| return true; | ||
| } | ||
|
|
||
| void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit) { |
There was a problem hiding this comment.
warning: function 'update_queries_limit_' exceeds recommended size/complexity thresholds [readability-function-size]
void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit) {
^Additional context
be/src/runtime/workload_group/workload_group_manager.cpp:754: 98 lines including whitespace and comments (threshold 80)
void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit) {
^…eed to be restricted to constants
mrhhsg
force-pushed
the
spill_rebased
branch
from
c02fd5d to
267d580
Compare
| return size_to_reserve; | ||
| } | ||
|
|
||
| Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block( |
There was a problem hiding this comment.
warning: function '_revoke_unpartitioned_block' has cognitive complexity of 63 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(
^Additional context
be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:161: +1, including nesting penalty of 0, nesting level increased to 1
if (auto* tmp_sink_state = _shared_state->inner_runtime_state->get_sink_local_state()) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:165: +1, including nesting penalty of 0, nesting level increased to 1
if (inner_sink_state) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:174: +1, including nesting penalty of 0, nesting level increased to 1
if (inner_sink_state) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:179: +1, including nesting penalty of 0, nesting level increased to 1
if (build_block.rows() <= 1) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:182: +2, including nesting penalty of 1, nesting level increased to 2
if (spill_context) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:188: +1, including nesting penalty of 0, nesting level increased to 1
if (build_block.columns() > num_slots) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:193: nesting level increased to 1
auto spill_func = [build_block = std::move(build_block), state, this]() mutable {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:201: nesting level increased to 2
[](std::vector<uint32_t>& indices) { indices.reserve(reserved_size); });
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:205: +2, including nesting penalty of 1, nesting level increased to 2
while (offset < total_rows) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:209: +3, including nesting penalty of 2, nesting level increased to 3
for (size_t i = 0; i != build_block.columns(); ++i) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:227: +3, including nesting penalty of 2, nesting level increased to 3
for (size_t i = 0; i != sub_block.rows(); ++i) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:231: +3, including nesting penalty of 2, nesting level increased to 3
for (uint32_t partition_idx = 0; partition_idx != p._partition_count; ++partition_idx) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:237: +4, including nesting penalty of 3, nesting level increased to 4
if (UNLIKELY(!partition_block)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:245: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(partition_block->add_rows(&sub_block, begin, end));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:245: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(partition_block->add_rows(&sub_block, begin, end));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:250: +4, including nesting penalty of 3, nesting level increased to 4
if (partition_block->rows() >= reserved_size || is_last_block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:250: +1
if (partition_block->rows() >= reserved_size || is_last_block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:252: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(spilling_stream->spill_block(state, block, false));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:252: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(spilling_stream->spill_block(state, block, false));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:256: +1, nesting level increased to 4
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:265: nesting level increased to 1
auto exception_catch_func = [spill_func]() mutable {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:266: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION(return spill_func()); }();
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:266: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION(return spill_func()); }();
^be/src/common/exception.h:79: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:266: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION(return spill_func()); }();
^be/src/common/exception.h:84: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:266: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION(return spill_func()); }();
^be/src/common/exception.h:85: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:277: +1, including nesting penalty of 0, nesting level increased to 1
DBUG_EXECUTE_IF(
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:277: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF(
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^| return size_to_reserve; | ||
| } | ||
|
|
||
| Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block( |
There was a problem hiding this comment.
warning: function '_revoke_unpartitioned_block' exceeds recommended size/complexity thresholds [readability-function-size]
Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(
^Additional context
be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:157: 126 lines including whitespace and comments (threshold 80)
Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(
^| @@ -479,12 +521,9 @@ | |||
| Status PartitionedHashJoinSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block, | |||
There was a problem hiding this comment.
warning: function 'sink' has cognitive complexity of 75 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block,
^Additional context
be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:532: +1, including nesting penalty of 0, nesting level increased to 1
if (rows == 0) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:533: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:538: +3, including nesting penalty of 2, nesting level increased to 3
if (need_to_spill) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:540: +1, nesting level increased to 3
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:541: +4, including nesting penalty of 3, nesting level increased to 4
if (UNLIKELY(!local_state._shared_state->inner_runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:542: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:542: +6, including nesting penalty of 5, nesting level increased to 6
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:544: +4, including nesting penalty of 3, nesting level increased to 4
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink_eos", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:544: +5, including nesting penalty of 4, nesting level increased to 5
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink_eos", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:550: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:550: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:561: nesting level increased to 3
[&](auto& block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:562: +4, including nesting penalty of 3, nesting level increased to 4
if (block) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:572: +1, including nesting penalty of 0, nesting level increased to 1
if (need_to_spill) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:573: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(local_state._partition_block(state, in_block, 0, rows));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:573: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(local_state._partition_block(state, in_block, 0, rows));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:574: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:576: +1, nesting level increased to 2
} else if (revocable_mem_size(state) > vectorized::SpillStream::MAX_SPILL_WRITE_BATCH_MEM) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:579: +1, nesting level increased to 1
} else {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:580: +2, including nesting penalty of 1, nesting level increased to 2
if (UNLIKELY(!local_state._shared_state->inner_runtime_state)) {
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:581: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:581: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(_setup_internal_operator(state));
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:583: +2, including nesting penalty of 1, nesting level increased to 2
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink", {
^be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'
if (UNLIKELY(config::enable_debug_points)) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:583: +3, including nesting penalty of 2, nesting level increased to 3
DBUG_EXECUTE_IF("fault_inject::partitioned_hash_join_sink::sink", {
^be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'
if (dp) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:588: +2, including nesting penalty of 1, nesting level increased to 2
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:642: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:588: +3, including nesting penalty of 2, nesting level increased to 3
RETURN_IF_ERROR(_inner_sink_operator->sink(
^be/src/common/status.h:644: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp:591: +2, including nesting penalty of 1, nesting level increased to 2
if (eos) {
^
What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
Release note
None
Check List (For Author)
Test
Behavior changed:
Does this need documentation?
Check List (For Reviewer who merge this PR)