diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index fcbe8519ef17bc..e1779895bba65a 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -183,6 +183,11 @@ class RuntimeState { return _query_options.__isset.enable_decimal256 && _query_options.enable_decimal256; } + bool new_is_ip_address_in_range() const { + return _query_options.__isset.new_is_ip_address_in_range && + _query_options.new_is_ip_address_in_range; + } + bool enable_common_expr_pushdown() const { return _query_options.__isset.enable_common_expr_pushdown && _query_options.enable_common_expr_pushdown; diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp index 3192653a816f2c..427059bba41a43 100644 --- a/be/src/vec/exprs/vectorized_fn_call.cpp +++ b/be/src/vec/exprs/vectorized_fn_call.cpp @@ -107,7 +107,9 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, // get the function. won't prepare function. _function = SimpleFunctionFactory::instance().get_function( _fn.name.function_name, argument_template, _data_type, - {.enable_decimal256 = state->enable_decimal256()}, state->be_exec_version()); + {.enable_decimal256 = state->enable_decimal256(), + .new_is_ip_address_in_range = state->new_is_ip_address_in_range()}, + state->be_exec_version()); } if (_function == nullptr) { return Status::InternalError("Could not find function {}, arg {} return {} ", diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index 4702a4b7af0bbf..ff987d130cda58 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -49,6 +49,7 @@ namespace doris::vectorized { struct FunctionAttr { bool enable_decimal256 {false}; + bool new_is_ip_address_in_range {false}; }; #define RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(TYPE) \ diff --git a/be/src/vec/functions/function_ip.cpp b/be/src/vec/functions/function_ip.cpp index ae5a2399981bf2..30b31901624ee5 100644 --- a/be/src/vec/functions/function_ip.cpp +++ b/be/src/vec/functions/function_ip.cpp @@ -47,6 +47,8 @@ void register_function_ip(SimpleFunctionFactory& factory) { factory.register_function>(); factory.register_function(); + factory.register_function(); + /// CIDR part factory.register_function(); factory.register_function(); diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h index b60e1f393f80b7..864d0cbc220e2c 100644 --- a/be/src/vec/functions/function_ip.h +++ b/be/src/vec/functions/function_ip.h @@ -789,6 +789,81 @@ class FunctionIsIPAddressInRange : public IFunction { } }; +// old version throw exception when meet null value +class FunctionIsIPAddressInRangeOld : public IFunction { +public: + static constexpr auto name = "is_ip_address_in_range"; + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 2; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + bool use_default_implementation_for_nulls() const override { return false; } + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) const override { + const auto& addr_column_with_type_and_name = block.get_by_position(arguments[0]); + const auto& cidr_column_with_type_and_name = block.get_by_position(arguments[1]); + WhichDataType addr_type(addr_column_with_type_and_name.type); + WhichDataType cidr_type(cidr_column_with_type_and_name.type); + const auto& [addr_column, addr_const] = + unpack_if_const(addr_column_with_type_and_name.column); + const auto& [cidr_column, cidr_const] = + unpack_if_const(cidr_column_with_type_and_name.column); + const ColumnString* str_addr_column = nullptr; + const ColumnString* str_cidr_column = nullptr; + const NullMap* null_map_addr = nullptr; + const NullMap* null_map_cidr = nullptr; + if (addr_type.is_nullable()) { + const auto* addr_column_nullable = + assert_cast(addr_column.get()); + str_addr_column = assert_cast( + addr_column_nullable->get_nested_column_ptr().get()); + null_map_addr = &addr_column_nullable->get_null_map_data(); + } else { + str_addr_column = assert_cast(addr_column.get()); + } + + if (cidr_type.is_nullable()) { + const auto* cidr_column_nullable = + assert_cast(cidr_column.get()); + str_cidr_column = assert_cast( + cidr_column_nullable->get_nested_column_ptr().get()); + null_map_cidr = &cidr_column_nullable->get_null_map_data(); + } else { + str_cidr_column = assert_cast(cidr_column.get()); + } + + auto col_res = ColumnUInt8::create(input_rows_count, 0); + auto& col_res_data = col_res->get_data(); + for (size_t i = 0; i < input_rows_count; ++i) { + auto addr_idx = index_check_const(i, addr_const); + auto cidr_idx = index_check_const(i, cidr_const); + if (null_map_addr && (*null_map_addr)[addr_idx]) [[unlikely]] { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "The arguments of function {} must be String, not NULL", + get_name()); + } + if (null_map_cidr && (*null_map_cidr)[cidr_idx]) [[unlikely]] { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "The arguments of function {} must be String, not NULL", + get_name()); + } + const auto addr = + IPAddressVariant(str_addr_column->get_data_at(addr_idx).to_string_view()); + const auto cidr = + parse_ip_with_cidr(str_cidr_column->get_data_at(cidr_idx).to_string_view()); + col_res_data[i] = is_address_in_range(addr, cidr) ? 1 : 0; + } + block.replace_by_position(result, std::move(col_res)); + return Status::OK(); + } +}; + class FunctionIPv4CIDRToRange : public IFunction { public: static constexpr auto name = "ipv4_cidr_to_range"; diff --git a/be/src/vec/functions/simple_function_factory.h b/be/src/vec/functions/simple_function_factory.h index dfcb756bba5236..5a1718a028351d 100644 --- a/be/src/vec/functions/simple_function_factory.h +++ b/be/src/vec/functions/simple_function_factory.h @@ -157,6 +157,11 @@ class SimpleFunctionFactory { int be_version = BeExecVersionManager::get_newest_version()) { std::string key_str = name; + // special function replacement + if (key_str == "is_ip_address_in_range" && !attr.new_is_ip_address_in_range) [[unlikely]] { + key_str = "__is_ip_address_in_range_OLD__"; + } + if (function_alias.contains(name)) { key_str = function_alias[name]; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 675bcbaf7988d5..0b9ac4c89e47a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -681,6 +681,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_COOLDOWN_REPLICA_AFFINITY = "enable_cooldown_replica_affinity"; + + public static final String NEW_IS_IP_ADDRESS_IN_RANGE = "new_is_ip_address_in_range"; /** * Inserting overwrite for auto partition table allows creating partition for * datas which cannot find partition to overwrite. @@ -2309,6 +2311,11 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { }) public int adaptivePipelineTaskSerialReadOnLimit = 10000; + // only to control some function behaviour. not visible or mutable. + @VariableMgr.VarAttr(name = NEW_IS_IP_ADDRESS_IN_RANGE, needForward = true, flag = VariableMgr.INVISIBLE + | VariableMgr.READ_ONLY) + public boolean newIsIpAddressInRange = true; + public void setEnableEsParallelScroll(boolean enableESParallelScroll) { this.enableESParallelScroll = enableESParallelScroll; } @@ -3957,6 +3964,8 @@ public TQueryOptions toThrift() { tResult.setOrcOnceMaxReadBytes(orcOnceMaxReadBytes); tResult.setIgnoreRuntimeFilterError(ignoreRuntimeFilterError); + tResult.setNewIsIpAddressInRange(newIsIpAddressInRange); + return tResult; } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 354ab4d3c840cb..e29ad1c1a04d6f 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -355,6 +355,10 @@ struct TQueryOptions { 140: optional i64 orc_max_merge_distance_bytes = 1048576; 141: optional bool ignore_runtime_filter_error = false; + + // upgrade options. keep them same in every branch. + 200: optional bool new_is_ip_address_in_range = false; + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query.