From 04d11c5e2a51f07dcde5a2ee213384bd5cca503a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 28 Nov 2024 11:30:47 +0800 Subject: [PATCH 1/2] [fix](runtime filter) Fix wrong synced size --- be/src/exprs/runtime_filter.cpp | 8 +++----- be/src/exprs/runtime_filter_slots.h | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 6eed49e8567e1c..efe96ae3395432 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -1114,9 +1114,6 @@ Status IRuntimeFilter::send_filter_size(RuntimeState* state, uint64_t local_filt std::lock_guard l(*local_merge_filters->lock); local_merge_filters->merge_size_times--; local_merge_filters->local_merged_size += local_filter_size; - if (_has_local_target) { - set_synced_size(local_filter_size); - } if (local_merge_filters->merge_size_times) { return Status::OK(); } else { @@ -1539,9 +1536,10 @@ void IRuntimeFilter::update_runtime_filter_type_to_profile(uint64_t local_merge_ std::string IRuntimeFilter::debug_string() const { return fmt::format( "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, " - "build_bf_cardinality: {}, error_msg: {}", + "build_bf_cardinality: {}, ignored: {}, error_msg: {}", _filter_id, to_string(_runtime_filter_type), _is_broadcast_join, - _wrapper->get_build_bf_cardinality(), _wrapper->_context->err_msg); + _wrapper->get_build_bf_cardinality(), _wrapper->is_ignored(), + _wrapper->_context->err_msg); } Status IRuntimeFilter::merge_from(const RuntimePredicateWrapper* wrapper) { diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h index 3c18735e4e82ce..cf6669c637fad2 100644 --- a/be/src/exprs/runtime_filter_slots.h +++ b/be/src/exprs/runtime_filter_slots.h @@ -71,6 +71,9 @@ class VRuntimeFilterSlots { // process ignore duplicate IN_FILTER std::unordered_set has_in_filter; for (auto filter : _runtime_filters) { + if (filter->filter_id() == 20) { + LOG(WARNING) << "=======1 " << filter->debug_string(); + } if (filter->get_ignored()) { continue; } @@ -98,6 +101,9 @@ class VRuntimeFilterSlots { continue; } filter->set_ignored(); + if (filter->filter_id() == 20) { + LOG(WARNING) << "=======2 " << filter->debug_string(); + } } return Status::OK(); } @@ -112,6 +118,8 @@ class VRuntimeFilterSlots { Status init_filters(RuntimeState* state, uint64_t local_hash_table_size) { // process IN_OR_BLOOM_FILTER's real type for (auto filter : _runtime_filters) { + LOG(WARNING) << "=======3 " << filter->debug_string() << " " << local_hash_table_size + << " " << filter->get_synced_size(); if (filter->type() == RuntimeFilterType::IN_OR_BLOOM_FILTER && get_real_size(filter.get(), local_hash_table_size) > state->runtime_filter_max_in_num()) { From 1762a40dffd616afa848ddb94956d1dd3e948166 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 28 Nov 2024 11:34:12 +0800 Subject: [PATCH 2/2] update --- be/src/exprs/runtime_filter_slots.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h index cf6669c637fad2..3c18735e4e82ce 100644 --- a/be/src/exprs/runtime_filter_slots.h +++ b/be/src/exprs/runtime_filter_slots.h @@ -71,9 +71,6 @@ class VRuntimeFilterSlots { // process ignore duplicate IN_FILTER std::unordered_set has_in_filter; for (auto filter : _runtime_filters) { - if (filter->filter_id() == 20) { - LOG(WARNING) << "=======1 " << filter->debug_string(); - } if (filter->get_ignored()) { continue; } @@ -101,9 +98,6 @@ class VRuntimeFilterSlots { continue; } filter->set_ignored(); - if (filter->filter_id() == 20) { - LOG(WARNING) << "=======2 " << filter->debug_string(); - } } return Status::OK(); } @@ -118,8 +112,6 @@ class VRuntimeFilterSlots { Status init_filters(RuntimeState* state, uint64_t local_hash_table_size) { // process IN_OR_BLOOM_FILTER's real type for (auto filter : _runtime_filters) { - LOG(WARNING) << "=======3 " << filter->debug_string() << " " << local_hash_table_size - << " " << filter->get_synced_size(); if (filter->type() == RuntimeFilterType::IN_OR_BLOOM_FILTER && get_real_size(filter.get(), local_hash_table_size) > state->runtime_filter_max_in_num()) {