Merge branch 'branch-3.0' into branch-3.0_20241119_fix_flight_sql

be/src/exprs/bloom_filter_func.h

@@ -100,14 +100,14 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
     virtual ~BloomFilterFuncBase() = default;
 
     void init_params(const RuntimeFilterParams* params) {
-        _bloom_filter_length =
-                params->runtime_bloom_filter_min_size > 0
-                        ? std::max(params->bloom_filter_size, params->runtime_bloom_filter_min_size)
-                        : params->bloom_filter_size;
+        _bloom_filter_length = params->bloom_filter_size;
+
         _build_bf_exactly = params->build_bf_exactly;
         _runtime_bloom_filter_min_size = params->runtime_bloom_filter_min_size;
+        _runtime_bloom_filter_max_size = params->runtime_bloom_filter_max_size;
         _null_aware = params->null_aware;
         _bloom_filter_size_calculated_by_ndv = params->bloom_filter_size_calculated_by_ndv;
+        _limit_length();
     }
 
     Status init_with_fixed_length() { return init_with_fixed_length(_bloom_filter_length); }
@@ -128,17 +128,11 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
             // if FE do use ndv stat to predict the bf size, BE only use the row count. FE have more
             // exactly row count stat. which one is min is more correctly.
             if (_bloom_filter_size_calculated_by_ndv) {
-                _bloom_filter_length =
-                        _runtime_bloom_filter_min_size > 0
-                                ? std::max(_runtime_bloom_filter_min_size,
-                                           std::min(be_calculate_size, _bloom_filter_length))
-                                : std::min(be_calculate_size, _bloom_filter_length);
+                _bloom_filter_length = std::min(be_calculate_size, _bloom_filter_length);
             } else {
-                _bloom_filter_length =
-                        _runtime_bloom_filter_min_size > 0
-                                ? std::max(_runtime_bloom_filter_min_size, be_calculate_size)
-                                : be_calculate_size;
+                _bloom_filter_length = be_calculate_size;
             }
+            _limit_length();
         }
         return init_with_fixed_length(_bloom_filter_length);
     }
@@ -229,13 +223,24 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
                                                 uint16_t* offsets, int number,
                                                 bool is_parse_column) = 0;
 
+private:
+    void _limit_length() {
+        if (_runtime_bloom_filter_min_size > 0) {
+            _bloom_filter_length = std::max(_bloom_filter_length, _runtime_bloom_filter_min_size);
+        }
+        if (_runtime_bloom_filter_max_size > 0) {
+            _bloom_filter_length = std::min(_bloom_filter_length, _runtime_bloom_filter_max_size);
+        }
+    }
+
 protected:
     // bloom filter size
     int32_t _bloom_filter_alloced;
     std::shared_ptr<BloomFilterAdaptor> _bloom_filter;
     bool _inited = false;
     int64_t _bloom_filter_length;
     int64_t _runtime_bloom_filter_min_size;
+    int64_t _runtime_bloom_filter_max_size;
     bool _build_bf_exactly = false;
     bool _bloom_filter_size_calculated_by_ndv = false;
 };

be/src/exprs/runtime_filter.cpp

@@ -1337,6 +1337,9 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
     params.runtime_bloom_filter_min_size = options->__isset.runtime_bloom_filter_min_size
                                                    ? options->runtime_bloom_filter_min_size
                                                    : 0;
+    params.runtime_bloom_filter_max_size = options->__isset.runtime_bloom_filter_max_size
+                                                   ? options->runtime_bloom_filter_max_size
+                                                   : 0;
     // We build runtime filter by exact distinct count iff three conditions are met:
     // 1. Only 1 join key
     // 2. Do not have remote target (e.g. do not need to merge), or broadcast join

be/src/exprs/runtime_filter.h

@@ -128,6 +128,7 @@ struct RuntimeFilterParams {
     int64_t bloom_filter_size;
     int32_t max_in_num;
     int64_t runtime_bloom_filter_min_size;
+    int64_t runtime_bloom_filter_max_size;
     int32_t filter_id;
     bool bitmap_filter_not_in;
     bool build_bf_exactly;

build-for-release.sh

@@ -128,6 +128,7 @@ FE="fe"
 BE="be"
 CLOUD="ms"
 EXT="extensions"
+TOOLS="tools"
 PACKAGE="apache-doris-${VERSION}-bin-${ARCH}"
 
 if [[ "${_USE_AVX2}" == "0" ]]; then
@@ -139,6 +140,7 @@ OUTPUT_FE="${OUTPUT}/${FE}"
 OUTPUT_EXT="${OUTPUT}/${EXT}"
 OUTPUT_BE="${OUTPUT}/${BE}"
 OUTPUT_CLOUD="${OUTPUT}/${CLOUD}"
+OUTPUT_TOOLS="${OUTPUT}/${TOOLS}"
 
 echo "Package Name:"
 echo "FE:    ${OUTPUT_FE}"
@@ -152,7 +154,7 @@ sh build.sh --clean &&
 
 echo "Begin to pack"
 rm -rf "${OUTPUT}"
-mkdir -p "${OUTPUT_FE}" "${OUTPUT_BE}" "${OUTPUT_EXT}" "${OUTPUT_CLOUD}"
+mkdir -p "${OUTPUT_FE}" "${OUTPUT_BE}" "${OUTPUT_EXT}" "${OUTPUT_CLOUD}" "${OUTPUT_TOOLS}"
 
 # FE
 cp -R "${ORI_OUTPUT}"/fe/* "${OUTPUT_FE}"/
@@ -177,5 +179,8 @@ if [[ "${TAR}" -eq 1 ]]; then
     cd -
 fi
 
+# TOOL
+cp -R "${ORI_OUTPUT}"/tools/* "${OUTPUT_TOOLS}"/
+
 echo "Output dir: ${OUTPUT}"
 exit 0

cloud/script/run_all_tests.sh

@@ -143,11 +143,13 @@ for i in *_test; do
             patchelf --set-rpath "$(pwd)" "${i}"
         fi
 
+        set -euo pipefail
         if [[ "${filter}" == "" ]]; then
             LLVM_PROFILE_FILE="./report/${i}.profraw" "./${i}" --gtest_print_time=true --gtest_output="xml:${i}.xml"
         else
             LLVM_PROFILE_FILE="./report/${i}.profraw" "./${i}" --gtest_print_time=true --gtest_output="xml:${i}.xml" --gtest_filter="${filter}"
         fi
+        set +euo pipefail
         unittest_files[${#unittest_files[*]}]="${i}"
         echo "--------------------------"
     fi

cloud/src/meta-service/meta_service_http.cpp

@@ -409,8 +409,8 @@ static HttpResponse process_adjust_rate_limit(MetaServiceImpl* service, brpc::Co
     processors[0b101] = std::move(set_instance_qps_limit);
     processors[0b111] = std::move(set_instance_rpc_qps_limit);
 
-    uint8_t level = (0x01 & qps_limit_str.empty()) | ((0x01 & rpc_name.empty()) << 1) |
-                    ((0x01 & instance_id.empty()) << 2);
+    uint8_t level = (0x01 & !qps_limit_str.empty()) | ((0x01 & !rpc_name.empty()) << 1) |
+                    ((0x01 & !instance_id.empty()) << 2);
 
     DCHECK_LT(level, 8);
 
@@ -420,21 +420,21 @@ static HttpResponse process_adjust_rate_limit(MetaServiceImpl* service, brpc::Co
 static HttpResponse process_query_rate_limit(MetaServiceImpl* service, brpc::Controller* cntl) {
     auto rate_limiter = service->rate_limiter();
     rapidjson::Document d;
+    d.SetObject();
     auto get_qps_limit = [&d](std::string_view rpc_name,
                               std::shared_ptr<RpcRateLimiter> rpc_limiter) {
         rapidjson::Document node;
+        node.SetObject();
         rapidjson::Document sub;
+        sub.SetObject();
         auto get_qps_token_limit = [&](std::string_view instance_id,
                                        std::shared_ptr<RpcRateLimiter::QpsToken> qps_token) {
             sub.AddMember(rapidjson::StringRef(instance_id.data(), instance_id.size()),
                           qps_token->max_qps_limit(), d.GetAllocator());
         };
         rpc_limiter->for_each_qps_token(std::move(get_qps_token_limit));
 
-        auto max_qps_limit = std::to_string(rpc_limiter->max_qps_limit());
-        node.AddMember("RPC qps limit",
-                       rapidjson::StringRef(max_qps_limit.data(), max_qps_limit.size()),
-                       d.GetAllocator());
+        node.AddMember("RPC qps limit", rpc_limiter->max_qps_limit(), d.GetAllocator());
         node.AddMember("instance specific qps limit", sub, d.GetAllocator());
         d.AddMember(rapidjson::StringRef(rpc_name.data(), rpc_name.size()), node, d.GetAllocator());
     };
@@ -443,7 +443,7 @@ static HttpResponse process_query_rate_limit(MetaServiceImpl* service, brpc::Con
     rapidjson::StringBuffer sb;
     rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(sb);
     d.Accept(writer);
-    return http_json_reply(MetaServiceCode::OK, sb.GetString());
+    return http_json_reply(MetaServiceCode::OK, "", sb.GetString());
 }
 
 static HttpResponse process_decode_key(MetaServiceImpl*, brpc::Controller* ctrl) {

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyConditionalFunction.java

@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.NullIf;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 
@@ -98,7 +99,7 @@ private static Expression rewriteNvl(Nvl nvl) {
      */
     private static Expression rewriteNullIf(NullIf nullIf) {
         if (nullIf.child(0) instanceof NullLiteral || nullIf.child(1) instanceof NullLiteral) {
-            return nullIf.child(0);
+            return new Nullable(nullIf.child(0));
         } else {
             return nullIf;
         }

fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java

@@ -35,6 +35,7 @@ public final class GlobalVariable {
 
     public static final int VARIABLE_VERSION_0 = 0;
     public static final int VARIABLE_VERSION_100 = 100;
+    public static final int VARIABLE_VERSION_101 = 101;
     public static final int VARIABLE_VERSION_200 = 200;
     public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_200;
     public static final String VARIABLE_VERSION = "variable_version";

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

@@ -1912,7 +1912,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
                 "Maximum table width to enable auto analyze, "
                     + "table with more columns than this value will not be auto analyzed."},
             flag = VariableMgr.GLOBAL)
-    public int autoAnalyzeTableWidthThreshold = 100;
+    public int autoAnalyzeTableWidthThreshold = 300;
 
     @VariableMgr.VarAttr(name = AUTO_ANALYZE_START_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
             description = {"该参数定义自动ANALYZE例程的开始时间",
@@ -1977,7 +1977,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
                             + "exceeds (100 - table_stats_health_threshold)% since the last "
                             + "statistics collection operation, the statistics for this table are"
                             + "considered outdated."})
-    public int tableStatsHealthThreshold = 60;
+    public int tableStatsHealthThreshold = 90;
 
     @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward = true,
             description = {"是否开启基于结构信息的物化视图透明改写",
@@ -3842,6 +3842,7 @@ public TQueryOptions toThrift() {
         tResult.setRuntimeFilterWaitTimeMs(runtimeFilterWaitTimeMs);
         tResult.setRuntimeFilterMaxInNum(runtimeFilterMaxInNum);
         tResult.setRuntimeBloomFilterMinSize(runtimeBloomFilterMinSize);
+        tResult.setRuntimeBloomFilterMaxSize(runtimeBloomFilterMaxSize);
         tResult.setRuntimeFilterWaitInfinitely(runtimeFilterWaitInfinitely);
 
         if (cpuResourceLimit > 0) {

fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java

@@ -34,6 +34,8 @@
 import org.apache.doris.common.util.SerializationUtils;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.persist.GlobalVarPersistInfo;
+import org.apache.doris.statistics.StatisticConstants;
+import org.apache.doris.statistics.util.StatisticsUtil;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -984,6 +986,20 @@ public static void forceUpdateVariables() {
                     SessionVariable.ENABLE_PIPELINE_X_ENGINE,
                     String.valueOf(true));
         }
+        if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_101) {
+            if (StatisticsUtil.getAutoAnalyzeTableWidthThreshold()
+                    < StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD) {
+                VariableMgr.refreshDefaultSessionVariables("update variable version",
+                        SessionVariable.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD,
+                        String.valueOf(StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD));
+            }
+            if (StatisticsUtil.getTableStatsHealthThreshold()
+                    < StatisticConstants.TABLE_STATS_HEALTH_THRESHOLD) {
+                VariableMgr.refreshDefaultSessionVariables("update variable version",
+                        SessionVariable.TABLE_STATS_HEALTH_THRESHOLD,
+                        String.valueOf(StatisticConstants.TABLE_STATS_HEALTH_THRESHOLD));
+            }
+        }
         if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) {
             // update from 3.0.2 or below to 3.0.3 or higher
             VariableMgr.refreshDefaultSessionVariables("update variable version",

fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java

@@ -93,13 +93,13 @@ public class StatisticConstants {
 
     public static final long EXTERNAL_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS = TimeUnit.HOURS.toMillis(24);
 
-    public static final int TABLE_STATS_HEALTH_THRESHOLD = 60;
+    public static final int TABLE_STATS_HEALTH_THRESHOLD = 90;
 
     public static final int ANALYZE_TIMEOUT_IN_SEC = 43200;
 
     public static final int TASK_QUEUE_CAP = 1;
 
-    public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 100;
+    public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 300;
 
     public static final int MSG_LEN_UPPER_BOUND = 1024;
 

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyConditionalFunctionTest.java

@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.NullIf;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.types.BooleanType;
@@ -99,13 +100,14 @@ public void testNullIf() {
         SlotReference slot = new SlotReference("a", StringType.INSTANCE, true);
         SlotReference nonNullableSlot = new SlotReference("b", StringType.INSTANCE, false);
         // nullif(null, slot) -> null
-        assertRewrite(new NullIf(NullLiteral.INSTANCE, slot), new NullLiteral(VarcharType.SYSTEM_DEFAULT));
+        assertRewrite(new NullIf(NullLiteral.INSTANCE, slot),
+                new Nullable(new NullLiteral(VarcharType.SYSTEM_DEFAULT)));
 
         // nullif(nullable_slot, null) -> slot
-        assertRewrite(new NullIf(slot, NullLiteral.INSTANCE), slot);
+        assertRewrite(new NullIf(slot, NullLiteral.INSTANCE), new Nullable(slot));
 
         // nullif(non-nullable_slot, null) -> non-nullable_slot
-        assertRewrite(new NullIf(nonNullableSlot, NullLiteral.INSTANCE), nonNullableSlot);
+        assertRewrite(new NullIf(nonNullableSlot, NullLiteral.INSTANCE), new Nullable(nonNullableSlot));
     }
 
 }

fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java

@@ -344,11 +344,11 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
         tableMeta.partitionChanged.set(false);
         Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
-        // Test update rows changed more than threshold.
+        // Test row count changed more than threshold.
         new MockUp<OlapTable>() {
             @Mock
             public long getRowCount() {
-                return 120;
+                return 111;
             }
         };
         new MockUp<TableStatsMeta>() {
@@ -358,12 +358,29 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
             }
         };
         tableMeta.partitionChanged.set(false);
-        tableMeta.updatedRows.set(200);
+        tableMeta.updatedRows.set(80);
         Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
-        // Test update rows changed less than threshold
+        // Test update rows changed more than threshold
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getRowCount() {
+                return 101;
+            }
+        };
+        tableMeta.partitionChanged.set(false);
+        tableMeta.updatedRows.set(91);
+        Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
+
+        // Test row count and update rows changed less than threshold
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getRowCount() {
+                return 100;
+            }
+        };
         tableMeta.partitionChanged.set(false);
-        tableMeta.updatedRows.set(100);
+        tableMeta.updatedRows.set(85);
         Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
     }

gensrc/script/gen_build_version.sh

@@ -31,7 +31,7 @@ build_version_prefix="doris"
 build_version_major=3
 build_version_minor=0
 build_version_patch=3
-build_version_rc_version="rc01"
+build_version_rc_version="rc02"
 
 build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}"