From 8ed83f87a4288c03cb20c9b93e9af39b27639cc4 Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Sun, 11 Oct 2026 20:41:09 +0800 Subject: [PATCH 01/10] feat(function): add bitmap binary conversion functions --- be/src/core/value/bitmap_value.h | 257 ++++++++++++++++++ be/src/exprs/function/function_bitmap.cpp | 95 +++++++ be/test/core/value/bitmap_value_test.cpp | 127 +++++++++ .../exprs/function/function_bitmap_test.cpp | 189 +++++++++++++ .../doris/catalog/BuiltinScalarFunctions.java | 4 + .../functions/scalar/BitmapFromBinary.java | 80 ++++++ .../functions/scalar/BitmapToBinary.java | 78 ++++++ .../visitor/ScalarFunctionVisitor.java | 10 + .../bitmap_functions/test_bitmap_function.out | 45 +++ .../test_bitmap_function.groovy | 20 ++ 10 files changed, 905 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBinary.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapToBinary.java diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h index ae4249c5b19cb5..cea224d61a7814 100644 --- a/be/src/core/value/bitmap_value.h +++ b/be/src/core/value/bitmap_value.h @@ -832,6 +832,128 @@ class Roaring64Map { return result; } + /** + * Read a bitmap from a serialized version, bounded by max_bytes. + * + * Unlike read(), this never reads past buf + max_bytes. It is meant for user + * controlled inputs (e.g. bitmap_from_binary over external catalog bytes), where a + * truncated or corrupted value must be rejected instead of reading out of bounds. + * + * @param is_valid_ptr set to false when the buffer does not hold a complete bitmap, + * true when the whole payload parsed cleanly. + */ + static Roaring64Map read_safe(const char* buf, size_t max_bytes, bool* is_valid_ptr) { + if (max_bytes < 1) { + *is_valid_ptr = false; + return {}; + } + bool is_v1 = BitmapTypeCode::BITMAP32 == *buf || BitmapTypeCode::BITMAP64 == *buf; + bool is_bitmap32 = BitmapTypeCode::BITMAP32 == *buf || BitmapTypeCode::BITMAP32_V2 == *buf; + if (is_bitmap32) { + return read_bitmap32_safe(buf, max_bytes, is_v1, is_valid_ptr); + } + DCHECK(BitmapTypeCode::BITMAP64 == *buf || BitmapTypeCode::BITMAP64_V2 == *buf); + return read_bitmap64_safe(buf, max_bytes, is_v1, is_valid_ptr); + } + +private: + // BITMAP32/BITMAP32_V2 payload := one Roaring bitmap in portable(V1)/non-portable(V2) form. + static Roaring64Map read_bitmap32_safe(const char* buf, size_t max_bytes, bool is_v1, + bool* is_valid_ptr) { + Roaring64Map result; + bool valid = false; + roaring::Roaring read = read_single_roaring_safe(buf + 1, max_bytes - 1, is_v1, &valid); + if (!valid) { + *is_valid_ptr = false; + return result; + } + result.emplaceOrInsert(0, std::move(read)); + *is_valid_ptr = true; + return result; + } + + // BITMAP64/BITMAP64_V2 payload := NumRoaring(varint64), { MapKey, Roaring }^NumRoaring. + static Roaring64Map read_bitmap64_safe(const char* buf, size_t max_bytes, bool is_v1, + bool* is_valid_ptr) { + Roaring64Map result; + // consume the type code first; the remaining checks track the payload size only + buf++; + max_bytes--; + + // get map size (varint64 took 1~10 bytes) + uint64_t map_size; + const char* next = reinterpret_cast(decode_varint64_ptr( + reinterpret_cast(buf), + reinterpret_cast(buf) + std::min(10, max_bytes), + &map_size)); + if (next == nullptr) { + *is_valid_ptr = false; + return result; + } + max_bytes -= next - buf; + buf = next; + + // each entry needs at least a 4-byte key; reject an impossible count up front so a + // corrupted varint cannot drive an (almost) unbounded loop below. + if (map_size > max_bytes / sizeof(uint32_t)) { + *is_valid_ptr = false; + return result; + } + + for (uint64_t lcv = 0; lcv < map_size; lcv++) { + // get map key + if (max_bytes < sizeof(uint32_t)) { + *is_valid_ptr = false; + return result; + } + uint32_t key = decode_fixed32_le(reinterpret_cast(buf)); + buf += sizeof(uint32_t); + max_bytes -= sizeof(uint32_t); + + // read map value Roaring + bool valid = false; + roaring::Roaring read_var = read_single_roaring_safe(buf, max_bytes, is_v1, &valid); + if (!valid) { + *is_valid_ptr = false; + return result; + } + // forward buffer past the last Roaring Bitmap + size_t consumed = read_var.getSizeInBytes(is_v1); + if (consumed > max_bytes) { + *is_valid_ptr = false; + return result; + } + buf += consumed; + max_bytes -= consumed; + // A map key must appear at most once. emplaceOrInsert would silently keep the + // first segment and drop the rest, i.e. return a wrong bitmap for corrupted + // input instead of rejecting it. + if (!result.tryEmplaceOrInsert(key, std::move(read_var))) { + *is_valid_ptr = false; + return result; + } + } + // the whole payload parsed cleanly + *is_valid_ptr = true; + return result; + } + + // Bounded counterpart of roaring::Roaring::read(buf, portable). portable format uses + // roaring_bitmap_portable_deserialize_safe; the V1 non-portable format uses + // roaring_bitmap_deserialize_safe. Both return NULL on a truncated/invalid buffer. + static roaring::Roaring read_single_roaring_safe(const char* buf, size_t max_bytes, + bool portable, bool* is_valid_ptr) { + auto* r = portable ? roaring::api::roaring_bitmap_portable_deserialize_safe(buf, max_bytes) + : roaring::api::roaring_bitmap_deserialize_safe(buf, max_bytes); + if (r == nullptr) { + *is_valid_ptr = false; + return {}; + } + *is_valid_ptr = true; + return roaring::Roaring(r); + } + +public: /** * How many bytes are required to serialize this bitmap */ @@ -993,6 +1115,14 @@ class Roaring64Map { roarings.emplace(key, value); } + // Same as emplaceOrInsert, but reports whether the key was actually inserted. Used by the + // bounded reader to reject a payload that repeats a map key instead of silently dropping + // the later segment. + bool tryEmplaceOrInsert(const uint32_t key, roaring::Roaring&& value) { + // Pass `value` as an lvalue for the same relocation reason as emplaceOrInsert above. + return roarings.emplace(key, value).second; + } + roaring::Roaring& lookupOrCreateInner(uint32_t key) { auto& bitmap = roarings[key]; bitmap.setCopyOnWrite(copyOnWrite); @@ -2468,6 +2598,133 @@ class BitmapValue { } } + // Deserialize a bitmap value from `src`, bounded by max_bytes. + // Unlike deserialize(), this validates the buffer length before every access, so it is + // safe for user-controlled inputs (external catalog binary columns, UDF output). + // Returns false when the buffer is too short, holds an unknown type code, or fails to + // decode a complete bitmap payload; on failure the value is left as an empty bitmap. + bool valid_and_deserialize(const char* src, size_t max_bytes) { + reset(); + if (max_bytes < 1) { + return false; + } + switch (*src) { + case BitmapTypeCode::EMPTY: + // An EMPTY payload is exactly one byte. A longer buffer is a corrupted value and + // must not be silently accepted as an empty bitmap. + if (max_bytes != 1) { + return false; + } + _type = EMPTY; + return true; + case BitmapTypeCode::SINGLE32: + if (max_bytes != 1 + sizeof(uint32_t)) { + return false; + } + _type = SINGLE; + _sv = decode_fixed32_le(reinterpret_cast(src + 1)); + if (config::enable_set_in_bitmap_value) { + _type = SET; + _set.clear(); + _set.insert(_sv); + } + return true; + case BitmapTypeCode::SINGLE64: + if (max_bytes != 1 + sizeof(uint64_t)) { + return false; + } + _type = SINGLE; + _sv = decode_fixed64_le(reinterpret_cast(src + 1)); + if (config::enable_set_in_bitmap_value) { + _type = SET; + _set.clear(); + _set.insert(_sv); + } + return true; + case BitmapTypeCode::BITMAP32: + case BitmapTypeCode::BITMAP64: + case BitmapTypeCode::BITMAP32_V2: + case BitmapTypeCode::BITMAP64_V2: { + bool valid = false; + detail::Roaring64Map bitmap = detail::Roaring64Map::read_safe(src, max_bytes, &valid); + if (!valid) { + // This path is reachable from user supplied bytes (bitmap_from_binary), so it + // must not log at a level that is on by default: one log line per input row + // lets any user with SELECT privilege flood be.log. Keep it at VLOG_DEBUG. + VLOG_DEBUG << "Decode roaring bitmap failed: truncated or corrupted payload"; + return false; + } + _type = BITMAP; + _is_shared = false; + _bitmap = std::make_shared(std::move(bitmap)); + return true; + } + case BitmapTypeCode::SET: + return valid_and_deserialize_set(src, max_bytes); + case BitmapTypeCode::SET_V2: + return valid_and_deserialize_set_v2(src, max_bytes); + default: + // Same reasoning as above: user supplied bytes, so no default-visible logging. + VLOG_DEBUG << "BitmapTypeCode invalid, should be one of EMPTY(0)/SINGLE32(1)/" + "BITMAP32(2)/SINGLE64(3)/BITMAP64(4)/SET(5)/SET_V2(10)/" + "BITMAP32_V2(12)/BITMAP64_V2(13), actual is " + << static_cast(*src); + return false; + } + } + +private: + // SET payload := TypeCode::SET(1 byte) | count(1 byte) | uint64 values in little-endian order. + bool valid_and_deserialize_set(const char* src, size_t max_bytes) { + if (max_bytes < 2) { + return false; + } + uint8_t count = *(src + 1); + if (count > SET_TYPE_THRESHOLD) { + return false; + } + if (max_bytes != 2 + sizeof(uint64_t) * count) { + return false; + } + _type = SET; + if (!_set.read(src + 2, count)) { + // Duplicated values can only come from a corrupted payload; the trusted + // deserialize() path throws here, but for user bytes a NULL result is enough. + reset(); + return false; + } + if (!config::enable_set_in_bitmap_value) { + _convert_set_to_bitmap(); + } + return true; + } + + // SET_V2 payload := TypeCode::SET_V2(1 byte) | count(uint32 little-endian) | uint64 values. + bool valid_and_deserialize_set_v2(const char* src, size_t max_bytes) { + if (max_bytes < 1 + sizeof(uint32_t)) { + return false; + } + uint32_t count = decode_fixed32_le(reinterpret_cast(src + 1)); + if (max_bytes != 1 + sizeof(uint32_t) + sizeof(uint64_t) * count) { + return false; + } + const char* values = src + 1 + sizeof(uint32_t); + if (!config::enable_set_in_bitmap_value || count > SET_TYPE_THRESHOLD) { + _type = BITMAP; + _prepare_bitmap_for_write(); + if (count > 0) { + std::vector vals(count); + decode_fixed64_le_array(vals.data(), values, count); + _bitmap->addMany(count, vals.data()); + } + } else { + _type = SET; + _set.read(values, count, true); + } + return true; + } + +public: // Deserialize a bitmap value from `src`. // Return false if `src` begins with unknown type code, true otherwise. bool deserialize(const char* src) { diff --git a/be/src/exprs/function/function_bitmap.cpp b/be/src/exprs/function/function_bitmap.cpp index 6ba5974cd4191e..b47c6e000f5e15 100644 --- a/be/src/exprs/function/function_bitmap.cpp +++ b/be/src/exprs/function/function_bitmap.cpp @@ -45,6 +45,7 @@ #include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_array.h" @@ -52,6 +53,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" #include "core/field.h" #include "core/types.h" #include "core/value/bitmap_value.h" @@ -312,6 +314,40 @@ struct BitmapFromBase64 { return Status::OK(); } }; +struct NameBitmapFromBinary { + static constexpr auto name = "bitmap_from_binary"; +}; +struct BitmapFromBinary { + using ArgumentType = DataTypeVarbinary; + + static constexpr auto name = "bitmap_from_binary"; + + // Deserialize Doris bitmap serialized bytes (see BitmapTypeCode) into a BitmapValue. + // The input column is ColumnVarbinary (e.g. a hive BINARY column read through a catalog + // with enable.mapping.varbinary=true, or the output of bitmap_to_binary), whose bytes + // are user controlled, so the bounded valid_and_deserialize is used and an invalid or + // truncated payload yields NULL instead of reading out of bounds. + static Status vector(const ColumnVarbinary::Container& data, std::vector& res, + NullMap& null_map, size_t input_rows_count) { + res.reserve(input_rows_count); + for (size_t i = 0; i < input_rows_count; ++i) { + const auto& src_value = data[i]; + if (src_value.size() == 0) { + res.emplace_back(); + null_map[i] = 1; + continue; + } + BitmapValue bitmap_val; + if (!bitmap_val.valid_and_deserialize(src_value.data(), src_value.size())) { + res.emplace_back(); + null_map[i] = 1; + } else { + res.emplace_back(std::move(bitmap_val)); + } + } + return Status::OK(); + } +}; struct BitmapFromArray { using ArgumentType = DataTypeArray; static constexpr auto name = "bitmap_from_array"; @@ -381,6 +417,10 @@ class FunctionBitmapAlwaysNull : public IFunction { const ColumnString::Chars& data = str_column.get_chars(); const ColumnString::Offsets& offsets = str_column.get_offsets(); RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count)); + } else if constexpr (std::is_same_v) { + const auto& binary_column = static_cast(*argument_column); + RETURN_IF_ERROR( + Impl::vector(binary_column.get_data(), res, null_map, input_rows_count)); } else if constexpr (std::is_same_v) { auto argument_type = remove_nullable( assert_cast(*block.get_by_position(arguments[0]).type) @@ -979,6 +1019,58 @@ struct BitmapToBase64 { } }; +struct NameBitmapToBinary { + static constexpr auto name = "bitmap_to_binary"; +}; + +class FunctionBitmapToBinary : public IFunction { +public: + static constexpr auto name = NameBitmapToBinary::name; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + // Serialize each BitmapValue with BitmapValue::write_to and return the raw bytes as + // VARBINARY. This is the byte-level counterpart of bitmap_to_base64, meant for exporting + // bitmaps into binary file formats (parquet/orc) or hive BINARY columns. The output is + // not nullable: bitmap_to_binary(NULL) is handled by the default NULL propagation + // (use_default_implementation_for_nulls), same as bitmap_to_base64. + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + auto& col_ptr = block.get_by_position(arguments[0]).column; + if (const auto* col = check_and_get_column(col_ptr.get())) { + auto col_res = ColumnVarbinary::create(); + const auto& data = col->get_data(); + // Serialize into a reusable buffer; each value is bounded by its own getSizeInBytes() + // and the destination StringView is filled by insert_data (inline vs arena). + std::string ser_buff; + size_t last_ser_size = 0; + for (size_t i = 0; i < data.size(); ++i) { + size_t cur_ser_size = data[i].getSizeInBytes(); + if (cur_ser_size > last_ser_size) { + ser_buff.resize(cur_ser_size); + last_ser_size = cur_ser_size; + } + data[i].write_to(ser_buff.data()); + col_res->insert_data(ser_buff.data(), cur_ser_size); + } + block.replace_by_position(result, std::move(col_res)); + } else { + return Status::RuntimeError("Illegal column {} of argument of function {}", + block.get_by_position(arguments[0]).column->get_name(), + get_name()); + } + return Status::OK(); + } +}; + struct SubBitmap { static constexpr auto name = "sub_bitmap"; using TData1 = std::vector; @@ -1200,6 +1292,7 @@ using FunctionBitmapMax = FunctionBitmapSingle; using FunctionBitmapToString = FunctionUnaryToType; using FunctionBitmapToBase64 = FunctionUnaryToType; using FunctionBitmapFromBase64 = FunctionBitmapAlwaysNull; +using FunctionBitmapFromBinary = FunctionBitmapAlwaysNull; using FunctionBitmapNot = FunctionBinaryToType; using FunctionBitmapContains = @@ -1222,6 +1315,8 @@ void register_function_bitmap(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 97cbdef56cb0db..51f285d5db8d50 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -749,6 +749,133 @@ TEST(BitmapValueTest, write_read) { config::enable_set_in_bitmap_value = false; } +// Serialize `src` and hand back the raw bytes; the caller owns the returned buffer's +// lifetime via the out-param std::string. +static std::string serialize_for_valid_and_deserialize(const BitmapValue& src) { + std::string buf(src.getSizeInBytes(), '\0'); + src.write_to(buf.data()); + return buf; +} + +TEST(BitmapValueTest, valid_and_deserialize_rejects_duplicate_bitmap64_map_key) { + // A bitmap64 payload that repeats a map key is corrupted: keeping only the first segment + // would silently return a wrong bitmap, so the bounded reader must reject it. + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP64); + bytes += static_cast(2); // varint map size = 2 + for (uint16_t value : {42, 43}) { + uint32_t map_key = 7; // the same key in both segments + bytes.append(reinterpret_cast(&map_key), 4); + uint32_t cookie = 12347; // SERIAL_COOKIE (hasrun), size-1 = 0 + bytes.append(reinterpret_cast(&cookie), 4); + bytes += '\0'; // run bitmap: container 0 is not a run container + uint16_t key = 0, card_minus_one = 0; // one array container holding {value} + bytes.append(reinterpret_cast(&key), 2); + bytes.append(reinterpret_cast(&card_minus_one), 2); + bytes.append(reinterpret_cast(&value), 2); + } + + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(deserialized.cardinality(), 0); +} + +TEST(BitmapValueTest, valid_and_deserialize) { + config::enable_set_in_bitmap_value = false; + + // round-trip: every type code must survive the bounded deserialize + BitmapValue bitmap_empty; + BitmapValue bitmap_single(1); + BitmapValue bitmap_single64((uint64_t)4294967296); + BitmapValue bitmap32({1, 9999999}); + BitmapValue bitmap64({1, (uint64_t)4294967296}); + std::vector many(128); + std::iota(many.begin(), many.begin() + 32, 0); + std::iota(many.begin() + 32, many.begin() + 64, 4294967297); + std::iota(many.begin() + 64, many.end(), 8589934594); + BitmapValue bitmap_many(many); + + for (const auto* src : + {&bitmap_empty, &bitmap_single, &bitmap_single64, &bitmap32, &bitmap64, &bitmap_many}) { + std::string bytes = serialize_for_valid_and_deserialize(*src); + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())) + << "type_code=" << static_cast(src->get_type_code()); + check_bitmap_equal(deserialized, *src); + } + + // an empty buffer carries no type code at all + { + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize("", 0)); + EXPECT_EQ(deserialized.cardinality(), 0); + } + + // truncated payloads of every type must be rejected, never crash: cut the last byte off + // each serialized form (a 1-byte EMPTY payload can only be truncated to zero bytes, + // already covered above) + for (const auto* src : {&bitmap_single, &bitmap_single64, &bitmap32, &bitmap64, &bitmap_many}) { + std::string bytes = serialize_for_valid_and_deserialize(*src); + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size() - 1)) + << "type_code=" << static_cast(src->get_type_code()); + EXPECT_EQ(deserialized.cardinality(), 0); + } + + // unknown type code + { + std::string bytes("\x7f", 1); + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + } + + // trailing bytes must be rejected for every fixed-size encoding: a longer buffer is a + // corrupted value, not a valid bitmap with padding. (The single-roaring BITMAP32 form is + // covered by valid_and_deserialize_bitmap32_trailing_bytes once read_safe reports the + // consumed byte count.) + for (const auto* src : {&bitmap_empty, &bitmap_single, &bitmap_single64}) { + std::string bytes = serialize_for_valid_and_deserialize(*src); + bytes += '\x7f'; + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())) + << "type_code=" << static_cast(src->get_type_code()); + EXPECT_EQ(deserialized.cardinality(), 0); + } + + // SET payload round-trips and, with set disabled, converts to BITMAP with equal contents + { + config::enable_set_in_bitmap_value = true; + BitmapValue set_bitmap; + set_bitmap.add(1); + set_bitmap.add(2); + EXPECT_EQ(set_bitmap.get_type_code(), BitmapTypeCode::SET); + std::string bytes = serialize_for_valid_and_deserialize(set_bitmap); + + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + check_bitmap_equal(deserialized, set_bitmap); + + // reader with set disabled: contents equal, representation becomes BITMAP + config::enable_set_in_bitmap_value = false; + BitmapValue deserialized2; + EXPECT_TRUE(deserialized2.valid_and_deserialize(bytes.data(), bytes.size())); + check_bitmap_equal(deserialized2, set_bitmap); + + // truncated SET (count byte present, values cut off) + EXPECT_FALSE(deserialized2.valid_and_deserialize(bytes.data(), 2)); + EXPECT_EQ(deserialized2.cardinality(), 0); + + // trailing byte after the SET payload + std::string padded = bytes; + padded += '\x7f'; + BitmapValue deserialized3; + EXPECT_FALSE(deserialized3.valid_and_deserialize(padded.data(), padded.size())); + EXPECT_EQ(deserialized3.cardinality(), 0); + } + + config::enable_set_in_bitmap_value = false; +} + TEST(BitmapValueTest, set_to_string) { config::enable_set_in_bitmap_value = true; BitmapValue bitmap; diff --git a/be/test/exprs/function/function_bitmap_test.cpp b/be/test/exprs/function/function_bitmap_test.cpp index aba66d453ba3cf..1b1ce798ad0ee9 100644 --- a/be/test/exprs/function/function_bitmap_test.cpp +++ b/be/test/exprs/function/function_bitmap_test.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,7 @@ #include "core/data_type/data_type_bitmap.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" #include "core/types.h" #include "core/value/bitmap_value.h" #include "exprs/function/function_test_util.h" @@ -33,6 +35,8 @@ namespace doris { +using namespace ut_type; + TEST(function_bitmap_test, function_bitmap_min_test) { std::string func_name = "bitmap_min"; InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP}; @@ -587,4 +591,189 @@ TEST(function_bitmap_test, function_bitmap_from_string_multiple_rows) { static_cast(check_function(func_name, input_types, data_set)); } +namespace { +// Keep a byte buffer alive and hand back a non-owning VARBINARY view over it. +// ut_type::VARBINARY is a StringView that does not own its bytes and StringView rejects +// std::string&&, so dynamic payloads must be parked somewhere stable for the lifetime of +// the test process; check_function copies the bytes into the input column immediately. +// deque never relocates its elements (unlike vector), so a view over an SSO string inside it +// stays valid across later insertions. +ut_type::VARBINARY keep_alive_binary(std::string bytes) { + static std::deque keep_alive; + keep_alive.push_back(std::move(bytes)); + return ut_type::VARBINARY(keep_alive.back().data(), keep_alive.back().size()); +} + +// Serialize a bitmap to the Doris wire format, used to build bitmap_from_binary inputs and +// bitmap_to_binary expectations without hard-coding byte strings (which depend on +// enable_set_in_bitmap_value / bitmap_serialize_version, see function_bitmap_to_base64). +ut_type::VARBINARY serialize_bitmap_to_binary(const BitmapValue& bitmap) { + std::string serialized(bitmap.getSizeInBytes(), '\0'); + bitmap.write_to(serialized.data()); + return keep_alive_binary(std::move(serialized)); +} +} // namespace + +TEST(function_bitmap_test, function_bitmap_to_binary) { + config::Register::Field field("bool", "enable_set_in_bitmap_value", + &config::enable_set_in_bitmap_value, "false", false); + config::Register::_s_field_map->insert( + std::make_pair(std::string("enable_set_in_bitmap_value"), field)); + config::Register::Field field_ser_ver("int16_t", "bitmap_serialize_version", + &config::bitmap_serialize_version, "1", false); + config::Register::_s_field_map->insert( + std::make_pair(std::string("bitmap_serialize_version"), field_ser_ver)); + + std::string func_name = "bitmap_to_binary"; + InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP}; + + EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "false", false, true).ok()); + BitmapValue bitmap32_1(1); // single + BitmapValue bitmap32_2({1, 9999999}); // bitmap + BitmapValue bitmap64_1((uint64_t)4294967296); // single + BitmapValue bitmap64_2({1, (uint64_t)4294967296}); // bitmap + BitmapValue empty_bitmap; + + { + DataSet data_set = {{{&bitmap32_1}, VARBINARY(serialize_bitmap_to_binary(bitmap32_1))}, + {{&bitmap32_2}, VARBINARY(serialize_bitmap_to_binary(bitmap32_2))}, + {{&bitmap64_1}, VARBINARY(serialize_bitmap_to_binary(bitmap64_1))}, + {{&bitmap64_2}, VARBINARY(serialize_bitmap_to_binary(bitmap64_2))}, + {{&empty_bitmap}, VARBINARY(serialize_bitmap_to_binary(empty_bitmap))}, + {{Null()}, Null()}}; + + static_cast( + check_function(func_name, input_types, data_set)); + } + + // SET serialization order is not canonical (see function_bitmap_to_base64), so the + // expectation is built from the same serialize helper, keeping the round-trip contract. + EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "true", false, true).ok()); + BitmapValue set_bitmap({1, 9999999}); + { + EXPECT_EQ(set_bitmap.get_type_code(), BitmapTypeCode::SET); + DataSet data_set = {{{&set_bitmap}, VARBINARY(serialize_bitmap_to_binary(set_bitmap))}, + {{Null()}, Null()}}; + static_cast( + check_function(func_name, input_types, data_set)); + } + + // serialize version 2 emits BITMAP32_V2 / BITMAP64_V2 type codes; the round trip must hold. + // enable_set_in_bitmap_value was flipped to true above, so reset it first: with it on, a + // 6-element bitmap serializes as SET and the V2 bitmap-code assertion below would not hold. + EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "false", false, true).ok()); + EXPECT_TRUE(config::set_config("bitmap_serialize_version", "2", false, true).ok()); + BitmapValue bitmap32_3({0, 1, 2, 3, 4, 5}); + { + EXPECT_EQ(bitmap32_3.get_type_code(), BitmapTypeCode::BITMAP32_V2); + DataSet data_set = {{{&bitmap32_3}, VARBINARY(serialize_bitmap_to_binary(bitmap32_3))}}; + static_cast( + check_function(func_name, input_types, data_set)); + } + EXPECT_TRUE(config::set_config("bitmap_serialize_version", "1", false, true).ok()); +} + +TEST(function_bitmap_test, function_bitmap_from_binary) { + config::Register::Field field("bool", "enable_set_in_bitmap_value", + &config::enable_set_in_bitmap_value, "false", false); + config::Register::_s_field_map->insert( + std::make_pair(std::string("enable_set_in_bitmap_value"), field)); + config::Register::Field field_ser_ver("int16_t", "bitmap_serialize_version", + &config::bitmap_serialize_version, "1", false); + config::Register::_s_field_map->insert( + std::make_pair(std::string("bitmap_serialize_version"), field_ser_ver)); + + std::string func_name = "bitmap_from_binary"; + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY}; + + EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "false", false, true).ok()); + BitmapValue bitmap32_1(1); // single + BitmapValue bitmap32_2({1, 9999999}); // bitmap + BitmapValue bitmap64_1((uint64_t)4294967296); // single + BitmapValue bitmap64_2({1, (uint64_t)4294967296}); // bitmap + BitmapValue empty_bitmap; + + { + // valid payloads round-trip back to the original bitmaps + DataSet data_set = {{{VARBINARY(serialize_bitmap_to_binary(bitmap32_1))}, &bitmap32_1}, + {{VARBINARY(serialize_bitmap_to_binary(bitmap32_2))}, &bitmap32_2}, + {{VARBINARY(serialize_bitmap_to_binary(bitmap64_1))}, &bitmap64_1}, + {{VARBINARY(serialize_bitmap_to_binary(bitmap64_2))}, &bitmap64_2}, + {{VARBINARY(serialize_bitmap_to_binary(empty_bitmap))}, &empty_bitmap}, + {{Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + } + + { + // invalid / truncated user-controlled bytes must yield NULL, never crash: + // - empty payload (no type code at all) + // - unknown type code + // - SINGLE32 / SINGLE64 / SET payloads cut short + // - BITMAP32 roaring payload truncated + std::string single32 = serialize_bitmap_to_binary(bitmap32_1); + std::string single64 = serialize_bitmap_to_binary(bitmap64_1); + std::string set_bytes = [&] { + config::enable_set_in_bitmap_value = true; + BitmapValue set_bitmap; + set_bitmap.add(1); + set_bitmap.add(9999999); + std::string serialized(set_bitmap.getSizeInBytes(), '\0'); + set_bitmap.write_to(serialized.data()); + config::enable_set_in_bitmap_value = false; + return serialized; + }(); + std::string bitmap32 = serialize_bitmap_to_binary(bitmap32_2); + + auto truncated = [](std::string s, size_t n) { return keep_alive_binary(s.substr(0, n)); }; + + DataSet data_set = {{{VARBINARY("")}, Null()}, + {{keep_alive_binary(std::string("\x7f", 1))}, Null()}, + {{truncated(single32, 3)}, Null()}, + {{truncated(single64, 5)}, Null()}, + {{truncated(set_bytes, 3)}, Null()}, + {{truncated(bitmap32, 4)}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + } + + // SET payloads (from enable_set_in_bitmap_value=true writers) must decode to the same + // bitmap regardless of the reader's enable_set_in_bitmap_value setting. + EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "true", false, true).ok()); + ut_type::VARBINARY set_payload; + { + // build via repeated add(): the vector ctor goes through add_many which always + // produces BITMAP, while SET is the <=32-element type formed by add() + BitmapValue set_bitmap; + set_bitmap.add(1); + set_bitmap.add(9999999); + EXPECT_EQ(set_bitmap.get_type_code(), BitmapTypeCode::SET); + set_payload = serialize_bitmap_to_binary(set_bitmap); + BitmapValue set_result({1, 9999999}); + DataSet data_set = {{{set_payload}, &set_result}}; + static_cast(check_function(func_name, input_types, data_set)); + } + EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "false", false, true).ok()); + { + // the same SET payload, read with set disabled: converts to BITMAP internally, + // contents must stay equal + BitmapValue set_result({1, 9999999}); + DataSet data_set = {{{set_payload}, &set_result}}; + static_cast(check_function(func_name, input_types, data_set)); + } + + // serialize version 2 payloads (BITMAP32_V2 / BITMAP64_V2) round-trip too + EXPECT_TRUE(config::set_config("bitmap_serialize_version", "2", false, true).ok()); + { + BitmapValue bitmap32_3({0, 1, 2, 3, 4, 5}); + BitmapValue bitmap64_3({1, (uint64_t)4294967296}); + EXPECT_EQ(bitmap32_3.get_type_code(), BitmapTypeCode::BITMAP32_V2); + EXPECT_EQ(bitmap64_3.get_type_code(), BitmapTypeCode::BITMAP64_V2); + DataSet data_set = {{{VARBINARY(serialize_bitmap_to_binary(bitmap32_3))}, &bitmap32_3}, + {{VARBINARY(serialize_bitmap_to_binary(bitmap64_3))}, &bitmap64_3}}; + static_cast(check_function(func_name, input_types, data_set)); + } + EXPECT_TRUE(config::set_config("bitmap_serialize_version", "1", false, true).ok()); +} + } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 2d7aaecd38d4d3..6f60c32cf5a7eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -111,6 +111,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromString; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHasAll; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHasAny; @@ -126,6 +127,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetLimit; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToString; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXor; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXorCount; @@ -721,6 +723,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(BitmapEmpty.class, "bitmap_empty"), scalar(BitmapFromArray.class, "bitmap_from_array"), scalar(BitmapFromBase64.class, "bitmap_from_base64"), + scalar(BitmapFromBinary.class, "bitmap_from_binary"), scalar(BitmapFromString.class, "bitmap_from_string"), scalar(BitmapHasAll.class, "bitmap_has_all"), scalar(BitmapHasAny.class, "bitmap_has_any"), @@ -736,6 +739,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(BitmapSubsetLimit.class, "bitmap_subset_limit"), scalar(BitmapToArray.class, "bitmap_to_array"), scalar(BitmapToBase64.class, "bitmap_to_base64"), + scalar(BitmapToBinary.class, "bitmap_to_binary"), scalar(BitmapToString.class, "bitmap_to_string"), scalar(BitmapXor.class, "bitmap_xor"), scalar(BitmapXorCount.class, "bitmap_xor_count"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBinary.java new file mode 100644 index 00000000000000..85ad50edbbc5fa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBinary.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BitmapType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'bitmap_from_binary'. + * + *

Converts Doris serialized bitmap bytes (the format produced by {@code bitmap_to_binary}, + * by the hive-udf to_bitmap/bitmap_union UDAFs, or by Spark load) into a BITMAP. The input + * is a VARBINARY value, typically a hive BINARY column read through a catalog with + * {@code enable.mapping.varbinary=true}. Invalid or truncated bytes yield NULL. + */ +public class BitmapFromBinary extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BitmapType.INSTANCE).args(VarBinaryType.MAX_VARBINARY_TYPE) + ); + + /** + * constructor with 1 argument. + */ + public BitmapFromBinary(Expression arg) { + super("bitmap_from_binary", arg); + } + + /** constructor for withChildren and reuse signature */ + private BitmapFromBinary(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public BitmapFromBinary withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new BitmapFromBinary(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitBitmapFromBinary(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapToBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapToBinary.java new file mode 100644 index 00000000000000..bd443f9d3ecc7e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapToBinary.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BitmapType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'bitmap_to_binary'. + * + *

Serializes a BITMAP into its raw Doris serialized bytes and returns them as VARBINARY. + * This is the byte-level counterpart of {@code bitmap_to_base64} and is meant for exporting + * bitmaps into binary file formats (parquet/orc) or hive BINARY columns. + */ +public class BitmapToBinary extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(VarBinaryType.MAX_VARBINARY_TYPE).args(BitmapType.INSTANCE) + ); + + /** + * constructor with 1 argument. + */ + public BitmapToBinary(Expression arg) { + super("bitmap_to_binary", arg); + } + + /** constructor for withChildren and reuse signature */ + private BitmapToBinary(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public BitmapToBinary withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new BitmapToBinary(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitBitmapToBinary(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index b5120939365edf..6853b85b3698a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -121,6 +121,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapFromString; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHasAll; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHasAny; @@ -136,6 +137,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetLimit; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToString; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXor; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXorCount; @@ -989,10 +991,18 @@ default R visitBitmapFromBase64(BitmapFromBase64 bitmapFromBase64, C context) { return visitScalarFunction(bitmapFromBase64, context); } + default R visitBitmapFromBinary(BitmapFromBinary bitmapFromBinary, C context) { + return visitScalarFunction(bitmapFromBinary, context); + } + default R visitBitmapToBase64(BitmapToBase64 bitmapToBase64, C context) { return visitScalarFunction(bitmapToBase64, context); } + default R visitBitmapToBinary(BitmapToBinary bitmapToBinary, C context) { + return visitScalarFunction(bitmapToBinary, context); + } + default R visitBitmapHasAll(BitmapHasAll bitmapHasAll, C context) { return visitScalarFunction(bitmapHasAll, context); } diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index 2df96803a0ff1c..79b01b5ab185d6 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -603,6 +603,51 @@ true 0,1,2,3,4294967296 1,9999999 +-- !sql_bitmap_binary_nereids0 -- +\N + +-- !sql_bitmap_binary_nereids1 -- + + +-- !sql_bitmap_binary_nereids2 -- +1 + +-- !sql_bitmap_binary_nereids3 -- +0,1,2,3 + +-- !sql_bitmap_binary_nereids4 -- +0,1,2,3,4294967296 + +-- !sql_bitmap_binary_nereids5 -- +33 + +-- !sql_bitmap_binary_nereids6 -- +1 + +-- !sql_bitmap_binary_invalid_empty -- +\N + +-- !sql_bitmap_binary_invalid_code -- +\N + +-- !sql_bitmap_binary_truncated -- +\N + +-- !sql_bitmap_binary_table0 -- + +0 +0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 +0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296 +0,1,2,3,4294967296 +1,9999999 + +-- !sql_bitmap_binary_table1 -- +0 +0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 +0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296 +0,1,2,3,4294967296 +1,9999999 + -- !sql_bitmap_remove_nereids0 -- 1,2 diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index 540b396f37fc2d..6a15288558ecd5 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -936,6 +936,26 @@ suite("test_bitmap_function") { """ qt_sql_bitmap_base64_not_null0 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(id))) s from test_bitmap_base64_not_null order by s; """ + ///////////////////////////// + // test bitmap binary: raw serialized bytes carried through a VARBINARY value, + // the byte-level counterpart of the base64 cases above. + qt_sql_bitmap_binary_nereids0 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(null))); """ + qt_sql_bitmap_binary_nereids1 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(bitmap_from_string("")))); """ + qt_sql_bitmap_binary_nereids2 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(bitmap_from_string("1")))); """ + qt_sql_bitmap_binary_nereids3 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(bitmap_from_string("0, 1, 2, 3")))); """ + qt_sql_bitmap_binary_nereids4 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(bitmap_from_string("0, 1, 2, 3, 4294967296")))); """ + qt_sql_bitmap_binary_nereids5 """ select bitmap_count(bitmap_from_binary(bitmap_to_binary(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32")))); """ + qt_sql_bitmap_binary_nereids6 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(to_bitmap(1)))); """ + // invalid user-controlled bytes decode to NULL instead of crashing + qt_sql_bitmap_binary_invalid_empty """ select bitmap_from_binary(to_binary("")); """ + qt_sql_bitmap_binary_invalid_code """ select bitmap_from_binary(to_binary("zz")); """ + // a SINGLE32 payload (5 bytes) cut to its first 3 bytes: truncated user data must be NULL + qt_sql_bitmap_binary_truncated """ select bitmap_to_string(bitmap_from_binary(sub_binary(bitmap_to_binary(to_bitmap(1)), 1, 3))); """ + + // binary round-trip over a bitmap table column + qt_sql_bitmap_binary_table0 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(id))) s from test_bitmap_base64 order by s; """ + qt_sql_bitmap_binary_table1 """ select bitmap_to_string(bitmap_from_binary(bitmap_to_binary(id))) s from test_bitmap_base64_not_null order by s; """ + /////////////////// // test bitmap_remove qt_sql_bitmap_remove_nereids0 """ From db631104907ca03aaffa3b3b3b2a6feef554421b Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Sun, 11 Oct 2026 23:56:26 +0800 Subject: [PATCH 02/10] feat(function): add HLL binary conversion functions --- be/src/exprs/function/function_hll.cpp | 117 +++++++++++++++++ be/test/exprs/function/function_hll_test.cpp | 124 ++++++++++++++++++ .../doris/catalog/BuiltinScalarFunctions.java | 4 + .../functions/scalar/HllFromBinary.java | 81 ++++++++++++ .../functions/scalar/HllToBinary.java | 78 +++++++++++ .../visitor/ScalarFunctionVisitor.java | 10 ++ .../hll_functions/test_hll_functions.out | 49 +++++++ .../hll_functions/test_hll_functions.groovy | 26 ++++ 8 files changed, 489 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllFromBinary.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllToBinary.java diff --git a/be/src/exprs/function/function_hll.cpp b/be/src/exprs/function/function_hll.cpp index a3e10bdc4331fb..bbcf0c5c88e776 100644 --- a/be/src/exprs/function/function_hll.cpp +++ b/be/src/exprs/function/function_hll.cpp @@ -31,11 +31,13 @@ #include "core/column/column_complex.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_hll.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" #include "core/types.h" #include "core/value/hll.h" #include "exprs/function/function.h" @@ -254,6 +256,14 @@ struct NameHllToBase64 { static constexpr auto name = "hll_to_base64"; }; +struct NameHllFromBinary { + static constexpr auto name = "hll_from_binary"; +}; + +struct NameHllToBinary { + static constexpr auto name = "hll_to_binary"; +}; + struct HllToBase64 { using ReturnType = DataTypeString; static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_HLL; @@ -308,12 +318,119 @@ using FunctionHLLEmpty = FunctionConst; using FunctionHLLHash = FunctionAlwaysNotNullable; using FunctionHllToBase64 = FunctionUnaryToType; +// hll_from_binary: the input VARBINARY holds the serialized bytes of a Doris HLL (see +// HyperLogLog::serialize), e.g. a hive BINARY column read through a catalog with +// enable.mapping.varbinary=true, or the output of hll_to_binary. Unlike the bitmap +// counterpart, HyperLogLog::deserialize(Slice) validates the input length fully +// (HyperLogLog::is_valid, including the exact total length), so deserialization is +// bounded by the column value length. Invalid or truncated bytes yield row-level +// NULL, consistent with the sibling hll_from_base64 tolerance for bad input. +class FunctionHllFromBinary : public IFunction { +public: + static constexpr auto name = NameHllFromBinary::name; + + String get_name() const override { return name; } + + static FunctionPtr create() { return std::make_shared(); } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + + size_t get_number_of_arguments() const override { return 1; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + auto res_null_map = ColumnUInt8::create(input_rows_count, 0); + auto res_data_column = ColumnHLL::create(); + auto& null_map = res_null_map->get_data(); + auto& res = res_data_column->get_data(); + + auto& argument_column = block.get_by_position(arguments[0]).column; + const auto& binary_column = static_cast(*argument_column); + const auto& data = binary_column.get_data(); + + res.reserve(input_rows_count); + for (size_t i = 0; i < input_rows_count; ++i) { + const auto& src_value = data[i]; + if (src_value.size() == 0) { + res.emplace_back(); + null_map[i] = 1; + continue; + } + doris::HyperLogLog hll_val; + doris::Slice decoded_slice(src_value.data(), src_value.size()); + if (!hll_val.deserialize(decoded_slice)) { + res.emplace_back(); + null_map[i] = 1; + } else { + res.emplace_back(std::move(hll_val)); + } + } + + block.get_by_position(result).column = + ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); + return Status::OK(); + } +}; + +// hll_to_binary: serialize each HLL with HyperLogLog::serialize and return the raw bytes +// as VARBINARY. This is the byte-level counterpart of hll_to_base64, meant for exporting +// HLLs into binary file formats (parquet/orc) or hive BINARY columns. NULL input is +// handled by the default NULL propagation (use_default_implementation_for_nulls), same +// as hll_to_base64. +class FunctionHllToBinary : public IFunction { +public: + static constexpr auto name = NameHllToBinary::name; + + String get_name() const override { return name; } + + static FunctionPtr create() { return std::make_shared(); } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + auto& col_ptr = block.get_by_position(arguments[0]).column; + if (const auto* col = check_and_get_column(col_ptr.get())) { + auto col_res = ColumnVarbinary::create(); + const auto& data = col->get_data(); + // Serialize into a reusable buffer; each value is bounded by its own + // max_serialized_size() and the destination StringView is filled by insert_data + // (inline vs arena). + std::string ser_buff; + size_t last_ser_size = 0; + for (size_t i = 0; i < data.size(); ++i) { + size_t cur_ser_size = data[i].max_serialized_size(); + if (cur_ser_size > last_ser_size) { + ser_buff.resize(cur_ser_size); + last_ser_size = cur_ser_size; + } + size_t real_size = data[i].serialize(reinterpret_cast(ser_buff.data())); + col_res->insert_data(ser_buff.data(), real_size); + } + block.replace_by_position(result, std::move(col_res)); + } else { + return Status::RuntimeError("Illegal column {} of argument of function {}", + block.get_by_position(arguments[0]).column->get_name(), + get_name()); + } + return Status::OK(); + } +}; + void register_function_hll(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); } } // namespace doris diff --git a/be/test/exprs/function/function_hll_test.cpp b/be/test/exprs/function/function_hll_test.cpp index ec94eef3088049..635f1ecb109dc3 100644 --- a/be/test/exprs/function/function_hll_test.cpp +++ b/be/test/exprs/function/function_hll_test.cpp @@ -18,14 +18,18 @@ #include #include +#include #include +#include #include "core/data_type/data_type_hll.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_varbinary.h" #include "core/types.h" #include "exprs/function/function_test_util.h" namespace doris { +using namespace ut_type; TEST(function_hll_test, function_hll_cardinality_test) { std::string func_name = "hll_cardinality"; @@ -133,4 +137,124 @@ TEST(function_hll_test, function_hll_from_base64_test) { static_cast(check_function(func_name, input_types, data_set)); } + +namespace { +// Keep a byte buffer alive and hand back a non-owning VARBINARY view over it. +// ut_type::VARBINARY is a StringView that does not own its bytes and StringView rejects +// std::string&&, so dynamic payloads must be parked somewhere stable for the lifetime of +// the test process; check_function copies the bytes into the input column immediately. +// deque never relocates its elements (unlike vector), so a view over an SSO string inside it +// stays valid across later insertions. +ut_type::VARBINARY keep_alive_binary(std::string bytes) { + static std::deque keep_alive; + keep_alive.push_back(std::move(bytes)); + return ut_type::VARBINARY(keep_alive.back().data(), keep_alive.back().size()); +} + +// Serialize an HLL to the Doris wire format (HyperLogLog::serialize), used to build +// hll_from_binary inputs and hll_to_binary expectations. +ut_type::VARBINARY serialize_hll_to_binary(const HyperLogLog& hll) { + std::string serialized(hll.max_serialized_size(), '\0'); + size_t real_size = hll.serialize(reinterpret_cast(serialized.data())); + serialized.resize(real_size); + return keep_alive_binary(std::move(serialized)); +} +} // namespace + +TEST(function_hll_test, function_hll_to_binary_test) { + std::string func_name = "hll_to_binary"; + InputTypeSet input_types = {PrimitiveType::TYPE_HLL}; + + const std::string input1 = "test"; + const uint64_t hash_value1 = + HashUtil::murmur_hash64A(input1.data(), input1.size(), HashUtil::MURMUR_SEED); + HyperLogLog hll1(hash_value1); + + const std::string input2 = " "; + const uint64_t hash_value2 = + HashUtil::murmur_hash64A(input2.data(), input2.size(), HashUtil::MURMUR_SEED); + HyperLogLog hll2(hash_value2); + + HyperLogLog hll3; + hll3.update(hash_value1); + hll3.update(hash_value2); + + HyperLogLog empty_hll; + + DataSet data_set = {{{&hll1}, VARBINARY(serialize_hll_to_binary(hll1))}, + {{&hll2}, VARBINARY(serialize_hll_to_binary(hll2))}, + {{&hll3}, VARBINARY(serialize_hll_to_binary(hll3))}, + {{&empty_hll}, VARBINARY(serialize_hll_to_binary(empty_hll))}, + {{Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); +} + +TEST(function_hll_test, function_hll_from_binary_test) { + std::string func_name = "hll_from_binary"; + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY}; + + const std::string input1 = "test"; + const uint64_t hash_value1 = + HashUtil::murmur_hash64A(input1.data(), input1.size(), HashUtil::MURMUR_SEED); + HyperLogLog hll1(hash_value1); + + const std::string input2 = " "; + const uint64_t hash_value2 = + HashUtil::murmur_hash64A(input2.data(), input2.size(), HashUtil::MURMUR_SEED); + HyperLogLog hll2(hash_value2); + + HyperLogLog hll3; + hll3.update(hash_value1); + hll3.update(hash_value2); + + HyperLogLog empty_hll; + + { + // valid payloads round-trip back to the original HLLs + DataSet data_set = {{{VARBINARY(serialize_hll_to_binary(hll1))}, &hll1}, + {{VARBINARY(serialize_hll_to_binary(hll2))}, &hll2}, + {{VARBINARY(serialize_hll_to_binary(hll3))}, &hll3}, + {{VARBINARY(serialize_hll_to_binary(empty_hll))}, &empty_hll}, + {{Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + } + + { + // invalid / truncated user-controlled bytes must yield NULL, never crash: + // - empty payload (no type code at all) + // - unknown type code + // - EXPLICIT payload cut short + // - EXPLICIT with a count larger than the actual bytes + // - SPARSE with a register count larger than the actual bytes + // - SPARSE with a well-formed length but a register index past the register array + // - FULL payload shorter than the full register array + std::string explicit_bytes = serialize_hll_to_binary(hll3); + explicit_bytes.resize(1); // type byte only + std::string explicit_bad_count = explicit_bytes; + explicit_bad_count += static_cast(8); + explicit_bad_count.resize(2); // num_explicits = 8 but no hash bytes follow + std::string sparse_bad = std::string("\x02", 1); + sparse_bad += std::string("\xff\xff\xff\xff", 4); // huge num_registers + // type=SPARSE, num_registers=1, entry: index=16384 (== HLL_REGISTERS_COUNT), value=1 + std::string sparse_oob_index = std::string("\x02\x01\x00\x00\x00\x00\x40\x01", 8); + // entry: index=65535, value=1 + std::string sparse_oob_index_max = std::string("\x02\x01\x00\x00\x00\xff\xff\x01", 8); + std::string full_short = std::string(4, '\0'); + full_short[0] = HLL_DATA_FULL; + + DataSet data_set = { + {{VARBINARY(keep_alive_binary(std::string()))}, Null()}, + {{VARBINARY(keep_alive_binary(std::string("\x7f", 1)))}, Null()}, + {{VARBINARY(keep_alive_binary(std::move(explicit_bytes)))}, Null()}, + {{VARBINARY(keep_alive_binary(std::move(explicit_bad_count)))}, Null()}, + {{VARBINARY(keep_alive_binary(std::move(sparse_bad)))}, Null()}, + {{VARBINARY(keep_alive_binary(std::move(sparse_oob_index)))}, Null()}, + {{VARBINARY(keep_alive_binary(std::move(sparse_oob_index_max)))}, Null()}, + {{VARBINARY(keep_alive_binary(std::move(full_short)))}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + } +} } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 6f60c32cf5a7eb..6d24d088d49599 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -250,8 +250,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.HllCardinality; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllEmpty; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllFromBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HllFromBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllToBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HllToBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Hour; import org.apache.doris.nereids.trees.expressions.functions.scalar.HourCeil; import org.apache.doris.nereids.trees.expressions.functions.scalar.HourFloor; @@ -857,8 +859,10 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(HllCardinality.class, "hll_cardinality"), scalar(HllEmpty.class, "hll_empty"), scalar(HllFromBase64.class, "hll_from_base64"), + scalar(HllFromBinary.class, "hll_from_binary"), scalar(HllHash.class, "hll_hash"), scalar(HllToBase64.class, "hll_to_base64"), + scalar(HllToBinary.class, "hll_to_binary"), scalar(Hour.class, "hour"), scalar(HourCeil.class, "hour_ceil"), scalar(HourFloor.class, "hour_floor"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllFromBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllFromBinary.java new file mode 100644 index 00000000000000..856b99f8e17d8f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllFromBinary.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.HllType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'hll_from_binary'. + * + * Build an HLL from the serialized bytes of a Doris HLL. This is the byte-level + * counterpart of hll_from_base64: the input VARBINARY holds the raw serialized + * bytes (e.g. a hive BINARY column read through a catalog with + * 'enable.mapping.varbinary' = 'true', or the output of hll_to_binary), not the + * Base64 text form that a TEXTFILE binary column stores. + */ +public class HllFromBinary extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(HllType.INSTANCE).args(VarBinaryType.MAX_VARBINARY_TYPE) + ); + + /** + * constructor with 1 argument. + */ + public HllFromBinary(Expression arg) { + super("hll_from_binary", arg); + } + + /** constructor for withChildren and reuse signature */ + private HllFromBinary(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public HllFromBinary withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new HllFromBinary(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitHllFromBinary(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllToBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllToBinary.java new file mode 100644 index 00000000000000..66f8a0ff51fed7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllToBinary.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.HllType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'hll_to_binary'. + * + * Convert an HLL into its serialized bytes returned as VARBINARY. This is the + * byte-level counterpart of hll_to_base64, meant for exporting HLLs into binary + * file formats (parquet/orc) or hive BINARY columns without the Base64 layer. + */ +public class HllToBinary extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(VarBinaryType.MAX_VARBINARY_TYPE).args(HllType.INSTANCE) + ); + + /** + * constructor with 1 argument. + */ + public HllToBinary(Expression arg) { + super("hll_to_binary", arg); + } + + /** constructor for withChildren and reuse signature */ + private HllToBinary(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public HllToBinary withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new HllToBinary(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitHllToBinary(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 6853b85b3698a6..d3ceb5662e1c32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -263,8 +263,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.HllCardinality; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllEmpty; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllFromBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HllFromBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllToBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HllToBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Hour; import org.apache.doris.nereids.trees.expressions.functions.scalar.HourCeil; import org.apache.doris.nereids.trees.expressions.functions.scalar.HourFloor; @@ -1629,6 +1631,10 @@ default R visitHllFromBase64(HllFromBase64 hllFromBase64, C context) { return visitScalarFunction(hllFromBase64, context); } + default R visitHllFromBinary(HllFromBinary hllFromBinary, C context) { + return visitScalarFunction(hllFromBinary, context); + } + default R visitHllHash(HllHash hllHash, C context) { return visitScalarFunction(hllHash, context); } @@ -1637,6 +1643,10 @@ default R visitHllToBase64(HllToBase64 hllToBase64, C context) { return visitScalarFunction(hllToBase64, context); } + default R visitHllToBinary(HllToBinary hllToBinary, C context) { + return visitScalarFunction(hllToBinary, context); + } + default R visitHour(Hour hour, C context) { return visitScalarFunction(hour, context); } diff --git a/regression-test/data/query_p0/sql_functions/hll_functions/test_hll_functions.out b/regression-test/data/query_p0/sql_functions/hll_functions/test_hll_functions.out index b34c741473e5cc..3db2487c2a9cc4 100644 --- a/regression-test/data/query_p0/sql_functions/hll_functions/test_hll_functions.out +++ b/regression-test/data/query_p0/sql_functions/hll_functions/test_hll_functions.out @@ -38,3 +38,52 @@ true -- !sparse_len_positive -- true + +-- !hll_binary_table_roundtrip -- +3 + +-- !hll_binary_table_roundtrip2 -- +beijing 3 +hebei 1 +jiangsu 1 +shanghai 2 +shanxi 1 + +-- !hll_binary_table_card -- +0 + +-- !hll_binary_const_roundtrip -- +1 + +-- !hll_binary_const_empty -- +0 + +-- !hll_binary_const_null -- +0 + +-- !hll_binary_to_null -- +\N + +-- !hll_binary_invalid_empty -- +true + +-- !hll_binary_invalid_type -- +true + +-- !hll_binary_invalid_truncated -- +true + +-- !hll_binary_valid_not_null -- +false + +-- !hll_binary_equals_base64 -- +true +true +true +true +true +true +true +true +true + diff --git a/regression-test/suites/query_p0/sql_functions/hll_functions/test_hll_functions.groovy b/regression-test/suites/query_p0/sql_functions/hll_functions/test_hll_functions.groovy index 90226543d55c58..1d20e435788916 100644 --- a/regression-test/suites/query_p0/sql_functions/hll_functions/test_hll_functions.groovy +++ b/regression-test/suites/query_p0/sql_functions/hll_functions/test_hll_functions.groovy @@ -80,4 +80,30 @@ suite("test_hll_functions") { // oversized vs offsets and tripped sanity_check. qt_sparse_roundtrip "select hll_cardinality(hll_from_base64(hll_to_base64(hll_union(hll_hash(number))))) > 400 from numbers(\"number\"=\"500\");" qt_sparse_len_positive "select length(hll_to_base64(hll_union(hll_hash(number)))) > 0 from numbers(\"number\"=\"500\");" + + // hll_to_binary / hll_from_binary: byte-level counterparts of hll_to_base64 / + // hll_from_base64. The binary form is what an ORC/Parquet hive BINARY column + // (mapped to VARBINARY via 'enable.mapping.varbinary') holds. + qt_hll_binary_table_roundtrip "select hll_union_agg(hll_from_binary(hll_to_binary(pv))) from test_hll_func;" + qt_hll_binary_table_roundtrip2 "select province, hll_union_agg(hll_from_binary(hll_to_binary(pv))) from test_hll_func group by province order by province;" + qt_hll_binary_table_card "select hll_cardinality(hll_from_binary(hll_to_binary(pv))) as res from test_hll_func order by res limit 1;" + + qt_hll_binary_const_roundtrip "select hll_cardinality(hll_from_binary(hll_to_binary(hll_hash('abc'))));" + qt_hll_binary_const_empty "select hll_cardinality(hll_from_binary(hll_to_binary(hll_empty())));" + qt_hll_binary_const_null "select hll_cardinality(hll_from_binary(hll_to_binary(hll_hash(NULL))));" + qt_hll_binary_to_null "select hll_to_binary(NULL);" + + // invalid / truncated bytes yield row-level NULL instead of an error, unlike + // hll_from_base64 which fails the query. Assert `is null` directly: hll_cardinality + // maps a NULL argument to 0, so wrapping the call would also pass if the function + // wrongly returned a valid empty HLL instead of NULL. + qt_hll_binary_invalid_empty "select hll_from_binary(to_binary('')) is null;" + qt_hll_binary_invalid_type "select hll_from_binary(to_binary('7f')) is null;" + qt_hll_binary_invalid_truncated "select hll_from_binary(sub_binary(hll_to_binary(hll_hash('abc')), 1, 2)) is null;" + // positive control: a valid payload must not be NULL, otherwise the three cases + // above would pass for the wrong reason + qt_hll_binary_valid_not_null "select hll_from_binary(hll_to_binary(hll_hash('abc'))) is null;" + + // mixed base64/binary byte level consistency + qt_hll_binary_equals_base64 "select hll_cardinality(hll_from_binary(hll_to_binary(pv))) = hll_cardinality(hll_from_base64(hll_to_base64(pv))) from test_hll_func order by 1;" } From 5ab5d43d39ad5c7861898b032a0f7492b3ad6a6c Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Mon, 12 Oct 2026 16:24:32 +0800 Subject: [PATCH 03/10] test(hive): cover bitmap and HLL binary round trips --- .../test_hive_bitmap_hll_binary_roundtrip.out | 89 +++++ ...st_hive_bitmap_hll_binary_roundtrip.groovy | 354 ++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 regression-test/data/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.groovy diff --git a/regression-test/data/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.out b/regression-test/data/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.out new file mode 100644 index 00000000000000..ee5b6d75048e93 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.out @@ -0,0 +1,89 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bitmap_text_hex -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !bitmap_orc_hex -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !bitmap_parquet_hex -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !bitmap_text_doris -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !bitmap_orc_doris -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !bitmap_parquet_doris -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !hll_text_hex -- +1 1 +2 10 +3 503 + +-- !hll_doris_all_formats -- +orc 1 1 +orc 2 10 +orc 3 503 +orc_hex 1 1 +orc_hex 2 10 +orc_hex 3 503 +parquet 1 1 +parquet 2 10 +parquet 3 503 +text 1 1 +text 2 10 +text 3 503 + +-- !hll_approx_band -- +1 true +2 true +3 true + +-- !bitmap_inner -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !hll_inner -- +1 1 +2 10 +3 503 + +-- !bitmap_roundtrip_again -- +1 1 1 +2 1 4294967296 +3 7 1,2,3,100,1000,65535,4294967295 +4 0 + +-- !hll_roundtrip_again -- +1 1 +2 10 +3 503 + +-- !bitmap_union_agree -- +true + +-- !hll_union_agree -- +true + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.groovy b/regression-test/suites/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.groovy new file mode 100644 index 00000000000000..268947c455d273 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_bitmap_hll_binary_roundtrip.groovy @@ -0,0 +1,354 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under +// the License. + +// End-to-end round-trip test for storing Doris BITMAP / HLL values in Hive +// `binary` columns, exercised through bitmap_to_binary / bitmap_from_binary +// and hll_to_binary / hll_from_binary: +// +// 1. write the serialized values into Hive binary columns, from two sources: +// a. Hive-side unhex() - equivalent to any external tool writing the +// raw serialized bytes (TEXTFILE stores them Base64-encoded per the +// Hive text serde contract, ORC / Parquet store them raw); +// b. Doris INSERT INTO the external table with bitmap_to_binary / +// hll_to_binary expressions - the same serialization the Doris Hive +// UDF jar's to_bitmap / to_hll produce, going through the real +// external-table write path; +// 2. query the external tables directly with bitmap_from_binary / +// hll_from_binary; +// 3. load them into Doris internal tables and query again; +// 4. convert the loaded values back with bitmap_to_binary / hll_to_binary +// and load once more (round trip through VARBINARY); +// 5. all results must agree across the three storage formats, both write +// sources, and every load stage. +// +// Note on the Hive UDF jar: the to_bitmap / to_hll UDFs emit exactly the +// bytes covered by source (b) (verified against the jar output), so they +// are not exercised separately here; deploying the jar on the CI hive +// docker images is additionally blocked because the master jar targets +// Java 17 while those images run Java 8. +suite("test_hive_bitmap_hll_binary_roundtrip", "p0,external") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return + } + + // Bitmap test data, keyed by k: single uint32, single uint64 (crossing + // the uint32 boundary), a multi-value bitmap, and the empty bitmap. + def bitmapData = [ + 1 : "to_bitmap(1)", + 2 : "to_bitmap(4294967296)", + 3 : "bitmap_from_string('1,2,3,100,1000,65535,4294967295')", + 4 : "bitmap_empty()", + ] + // HLL test data, keyed by k: distinct-element counts 1, 10, 500. + def hllData = [ + 1 : 1, + 2 : 10, + 3 : 500, + ] + // upper bound of the HLL relative error band used by the approx asserts + def hllBand = { n -> [n * 95 / 100, n * 105 / 100] } + + // staging table for producing multi-element HLLs: insert one hll_hash per + // input value and let the HLL_UNION aggregate do the merging + sql """ drop table if exists hll_binary_roundtrip_staging """ + sql """ + create table hll_binary_roundtrip_staging ( + k int not null, + v hll hll_union not null + ) aggregate key(k) distributed by hash(k) buckets 1 + properties("replication_num" = "1") + """ + + // Single Hive version, like the majority of the hive suites: every qt_* below runs + // once per iteration, so looping over two versions would need two copies of each + // result block in the .out file. + for (String hivePrefix : ["hive3"]) { + setHivePrefix(hivePrefix) + // skip hive versions that are not deployed in this environment + try { + hive_docker "select 1" + } catch (Throwable t) { + logger.info("hive docker ${hivePrefix} unreachable, skip it: ${t.getMessage()}") + continue + } + try { + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog_name = "test_${hivePrefix}_binary_roundtrip" + String test_db = "test_binary_roundtrip_db" + + sql """drop catalog if exists ${catalog_name}""" + hive_docker """drop database if exists ${test_db} cascade""" + hive_docker """create database ${test_db}""" + + sql """create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + )""" + + // ------------------------------------------------------------------ + // Serialize the test values to their hex form on the Doris side + // ------------------------------------------------------------------ + def bitmapHex = [:] + bitmapData.each { k, expr -> + def res = sql "select hex(bitmap_to_binary(${expr})) as h" + bitmapHex[k] = res[0][0] + } + def hllHex = [:] + hllData.each { k, n -> + sql """insert into hll_binary_roundtrip_staging + select ${k}, hll_hash(number) from numbers("number" = "${n}") + """ + def res = sql """select hex(hll_to_binary(v)) from hll_binary_roundtrip_staging where k = ${k}""" + hllHex[k] = res[0][0] + } + + // ------------------------------------------------------------------ + // Create the source tables, one per storage format and kind. + // The _hex tables receive bytes written from the Hive side + // (unhex); the _doris tables receive bytes written by Doris + // INSERT INTO ... *_to_binary(...) - the external write path. + // ------------------------------------------------------------------ + for (kind in ["bitmap", "hll"]) { + for (format in ["text", "orc", "parquet"]) { + def textRowFormat = format == "text" + ? "row format delimited fields terminated by '\\t'" : "" + hive_docker """ + create table ${test_db}.src_${kind}_${format}_hex (k int, v binary) + ${textRowFormat} + stored as ${format == 'text' ? 'textfile' : format} + """ + hive_docker """ + create table ${test_db}.src_${kind}_${format}_doris (k int, v binary) + ${textRowFormat} + stored as ${format == 'text' ? 'textfile' : format} + """ + } + } + // ---- write source a: Hive-side unhex() of the Doris hex ---- + bitmapData.each { k, expr -> + def h = bitmapHex[k] + for (format in ["text", "orc", "parquet"]) { + hive_docker """insert into ${test_db}.src_bitmap_${format}_hex + values (${k}, unhex('${h}'))""" + } + } + hllData.each { k, n -> + def h = hllHex[k] + for (format in ["text", "orc", "parquet"]) { + hive_docker """insert into ${test_db}.src_hll_${format}_hex + values (${k}, unhex('${h}'))""" + } + } + // ---- write source b: Doris INSERT INTO with *_to_binary ---- + for (format in ["text", "orc", "parquet"]) { + bitmapData.each { k, expr -> + sql """insert into `${catalog_name}`.`${test_db}`.`src_bitmap_${format}_doris` + select ${k}, bitmap_to_binary(${expr})""" + } + hllData.each { k, n -> + sql """insert into `${catalog_name}`.`${test_db}`.`src_hll_${format}_doris` + select ${k}, v_binary from ( + select hll_to_binary(v) as v_binary from hll_binary_roundtrip_staging + where k = ${k} + ) t""" + } + } + + // ------------------------------------------------------------------ + // 1. direct read from the external tables, per format and source + // ------------------------------------------------------------------ + qt_bitmap_text_hex """ + select k, bitmap_count(bitmap_from_binary(v)) as cnt, + bitmap_to_string(bitmap_from_binary(v)) as vals + from ${catalog_name}.${test_db}.src_bitmap_text_hex order by k + """ + qt_bitmap_orc_hex """ + select k, bitmap_count(bitmap_from_binary(v)) as cnt, + bitmap_to_string(bitmap_from_binary(v)) as vals + from ${catalog_name}.${test_db}.src_bitmap_orc_hex order by k + """ + qt_bitmap_parquet_hex """ + select k, bitmap_count(bitmap_from_binary(v)) as cnt, + bitmap_to_string(bitmap_from_binary(v)) as vals + from ${catalog_name}.${test_db}.src_bitmap_parquet_hex order by k + """ + qt_bitmap_text_doris """ + select k, bitmap_count(bitmap_from_binary(v)) as cnt, + bitmap_to_string(bitmap_from_binary(v)) as vals + from ${catalog_name}.${test_db}.src_bitmap_text_doris order by k + """ + qt_bitmap_orc_doris """ + select k, bitmap_count(bitmap_from_binary(v)) as cnt, + bitmap_to_string(bitmap_from_binary(v)) as vals + from ${catalog_name}.${test_db}.src_bitmap_orc_doris order by k + """ + qt_bitmap_parquet_doris """ + select k, bitmap_count(bitmap_from_binary(v)) as cnt, + bitmap_to_string(bitmap_from_binary(v)) as vals + from ${catalog_name}.${test_db}.src_bitmap_parquet_doris order by k + """ + // HLL is approximate: exact values from one format, error band + // from the others, and per-source agreement on the band + qt_hll_text_hex """ + select k, hll_cardinality(hll_from_binary(v)) as card + from ${catalog_name}.${test_db}.src_hll_text_hex order by k + """ + qt_hll_doris_all_formats """ + select format, k, hll_cardinality(hll_from_binary(v)) as card + from ( + select 'text' as format, k, v from ${catalog_name}.${test_db}.src_hll_text_doris + union all + select 'orc' as format, k, v from ${catalog_name}.${test_db}.src_hll_orc_doris + union all + select 'parquet' as format, k, v from ${catalog_name}.${test_db}.src_hll_parquet_doris + union all + select 'orc_hex' as format, k, v from ${catalog_name}.${test_db}.src_hll_orc_hex + ) t order by format, k + """ + qt_hll_approx_band """ + select k, hll_cardinality(hll_from_binary(v)) between ${hllBand(1)[0]} and ${hllBand(1)[1]} as in_band + from ${catalog_name}.${test_db}.src_hll_orc_doris where k = 1 + union all + select k, hll_cardinality(hll_from_binary(v)) between ${hllBand(10)[0]} and ${hllBand(10)[1]} as in_band + from ${catalog_name}.${test_db}.src_hll_orc_doris where k = 2 + union all + select k, hll_cardinality(hll_from_binary(v)) between ${hllBand(500)[0]} and ${hllBand(500)[1]} as in_band + from ${catalog_name}.${test_db}.src_hll_orc_doris where k = 3 + order by k + """ + + // ------------------------------------------------------------------ + // 2. load into Doris internal tables and query back + // ------------------------------------------------------------------ + sql """ drop table if exists bitmap_binary_roundtrip_inner """ + sql """ + create table bitmap_binary_roundtrip_inner ( + k int not null, + v bitmap bitmap_union not null + ) aggregate key(k) distributed by hash(k) buckets 1 + properties("replication_num" = "1") + """ + // load from both sources of every format: equal keys merge under + // BITMAP_UNION and must produce the same values again + for (source in ["hex", "doris"]) { + for (format in ["text", "orc", "parquet"]) { + sql """ + insert into bitmap_binary_roundtrip_inner + select k, bitmap_from_binary(v) + from ${catalog_name}.${test_db}.src_bitmap_${format}_${source} + """ + } + } + qt_bitmap_inner """ + select k, bitmap_count(v) as cnt, bitmap_to_string(v) as vals + from bitmap_binary_roundtrip_inner order by k + """ + + sql """ drop table if exists hll_binary_roundtrip_inner """ + sql """ + create table hll_binary_roundtrip_inner ( + k int not null, + v hll hll_union not null + ) aggregate key(k) distributed by hash(k) buckets 1 + properties("replication_num" = "1") + """ + for (source in ["hex", "doris"]) { + for (format in ["text", "orc", "parquet"]) { + sql """ + insert into hll_binary_roundtrip_inner + select k, hll_from_binary(v) + from ${catalog_name}.${test_db}.src_hll_${format}_${source} + """ + } + } + qt_hll_inner """ + select k, hll_cardinality(v) as card + from hll_binary_roundtrip_inner order by k + """ + + // ------------------------------------------------------------------ + // 3. round trip the loaded values through VARBINARY again: + // *_to_binary on the internal-table column, then *_from_binary + // ------------------------------------------------------------------ + qt_bitmap_roundtrip_again """ + select k, bitmap_count(bitmap_from_binary(bitmap_to_binary(v))) as cnt, + bitmap_to_string(bitmap_from_binary(bitmap_to_binary(v))) as vals + from bitmap_binary_roundtrip_inner order by k + """ + qt_hll_roundtrip_again """ + select k, hll_cardinality(hll_from_binary(hll_to_binary(v))) as card + from hll_binary_roundtrip_inner order by k + """ + + // ------------------------------------------------------------------ + // 4. all formats, both sources and every load stage must agree + // ------------------------------------------------------------------ + qt_bitmap_union_agree """ + select + (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_text_doris) + = (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_orc_doris) + and + (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_orc_doris) + = (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_parquet_doris) + and + (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_orc_hex) + = (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_orc_doris) + and + (select bitmap_union_count(bitmap_from_binary(v)) from ${catalog_name}.${test_db}.src_bitmap_orc_doris) + = (select bitmap_union_count(v) from bitmap_binary_roundtrip_inner) + and + (select bitmap_union_count(bitmap_from_binary(bitmap_to_binary(v))) from bitmap_binary_roundtrip_inner) + = (select bitmap_union_count(v) from bitmap_binary_roundtrip_inner) + as agree + """ + qt_hll_union_agree """ + select + (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_text_doris) + = (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_orc_doris) + and + (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_orc_doris) + = (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_parquet_doris) + and + (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_orc_hex) + = (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_orc_doris) + and + (select hll_union_agg(hll_from_binary(v)) from ${catalog_name}.${test_db}.src_hll_orc_doris) + = (select hll_union_agg(v) from hll_binary_roundtrip_inner) + and + (select hll_union_agg(hll_from_binary(hll_to_binary(v))) from hll_binary_roundtrip_inner) + = (select hll_union_agg(v) from hll_binary_roundtrip_inner) + as agree + """ + } finally { + try { + if (hivePrefix != null) { + setHivePrefix(hivePrefix) + hive_docker """drop database if exists test_binary_roundtrip_db cascade""" + } + } catch (Throwable t) { + logger.warn("cleanup failed: ${t}") + } + } + } + + sql """ drop table if exists bitmap_binary_roundtrip_inner """ + sql """ drop table if exists hll_binary_roundtrip_inner """ + sql """ drop table if exists hll_binary_roundtrip_staging """ +} From 4b5a778601509c1844b590dd35a62daa43921095 Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Mon, 12 Oct 2026 23:12:17 +0800 Subject: [PATCH 04/10] fix(bitmap): validate deserialized roaring containers --- be/src/core/value/bitmap_value.h | 18 ++++++++++++++++++ be/test/core/value/bitmap_value_test.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h index cea224d61a7814..8fc6065541bebc 100644 --- a/be/src/core/value/bitmap_value.h +++ b/be/src/core/value/bitmap_value.h @@ -941,6 +941,14 @@ class Roaring64Map { // Bounded counterpart of roaring::Roaring::read(buf, portable). portable format uses // roaring_bitmap_portable_deserialize_safe; the V1 non-portable format uses // roaring_bitmap_deserialize_safe. Both return NULL on a truncated/invalid buffer. + // + // The *_deserialize_safe functions only bound the reads: a bitset container keeps + // whatever cardinality the payload declared (bitset_container_read never recomputes + // it), so a corrupt bitmap can claim cardinality 4097 while its words hold 65536 + // bits. bitmap_to_array resizes by the declared cardinality but the element writer + // iterates the actual bits, i.e. a heap out-of-bounds write. roaring_bitmap_internal_validate + // cross-checks the declared and actual cardinalities (plus key order and container + // consistency) and rejects such payloads. static roaring::Roaring read_single_roaring_safe(const char* buf, size_t max_bytes, bool portable, bool* is_valid_ptr) { auto* r = portable ? roaring::api::roaring_bitmap_portable_deserialize_safe(buf, max_bytes) @@ -949,6 +957,16 @@ class Roaring64Map { *is_valid_ptr = false; return {}; } + const char* reason = nullptr; + if (!roaring::api::roaring_bitmap_internal_validate(r, &reason)) { + // Reachable from user supplied bytes (bitmap_from_binary), so it must not log at a + // level that is on by default: one line per input row lets any user with SELECT + // privilege flood be.log. Keep the diagnostic at VLOG_DEBUG. + VLOG_DEBUG << "Decode roaring bitmap failed: " << (reason ? reason : "unknown"); + roaring::api::roaring_bitmap_free(r); + *is_valid_ptr = false; + return {}; + } *is_valid_ptr = true; return roaring::Roaring(r); } diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 51f285d5db8d50..6db0fcad441dd5 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -780,6 +780,30 @@ TEST(BitmapValueTest, valid_and_deserialize_rejects_duplicate_bitmap64_map_key) EXPECT_EQ(deserialized.cardinality(), 0); } +// A BITMAP32 portable payload declaring a bitset cardinality that does not +// match the actual bits set must be rejected by valid_and_deserialize: the +// declared cardinality flows into bitmap_to_array's resize while the element +// writer iterates the actual bits, so accepting such a payload is a heap +// out-of-bounds write (covered by ASan in the ASAN UT build). +TEST(BitmapValueTest, FakeBitsetCardinality) { + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP32); + uint32_t cookie = 12347; // SERIAL_COOKIE (hasrun), size-1 = 0 + bytes.append(reinterpret_cast(&cookie), 4); + bytes += '\0'; // run bitmap: container 0 is not a run container + uint16_t key = 0, card_minus_one = 4096; // thiscard = 4097 > 4096 -> bitset container + bytes.append(reinterpret_cast(&key), 2); + bytes.append(reinterpret_cast(&card_minus_one), 2); + bytes.append(8192, '\xff'); // every one of the 65536 bits set + + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())) + << "declared cardinality must be validated"; + PaddedPODArray out; + deserialized.to_array(out); + EXPECT_EQ(0u, out.size()); +} + TEST(BitmapValueTest, valid_and_deserialize) { config::enable_set_in_bitmap_value = false; From e63f93fc7b1cbe0f003b4062b2d5667319bc34e2 Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Mon, 12 Oct 2026 23:53:44 +0800 Subject: [PATCH 05/10] fix(bitmap): advance bitmap64 parsing by consumed bytes --- be/src/core/value/bitmap_value.h | 59 +++++++++++++++++++--- be/test/core/value/bitmap_value_test.cpp | 62 ++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 6 deletions(-) diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h index 8fc6065541bebc..259ade5291afe2 100644 --- a/be/src/core/value/bitmap_value.h +++ b/be/src/core/value/bitmap_value.h @@ -862,11 +862,20 @@ class Roaring64Map { bool* is_valid_ptr) { Roaring64Map result; bool valid = false; - roaring::Roaring read = read_single_roaring_safe(buf + 1, max_bytes - 1, is_v1, &valid); + size_t consumed = 0; + roaring::Roaring read = + read_single_roaring_safe(buf + 1, max_bytes - 1, is_v1, &valid, &consumed); if (!valid) { *is_valid_ptr = false; return result; } + // bitmap32 holds exactly one Roaring, so the payload must be consumed exactly. + // Trailing bytes mean the value is corrupted (or a fixed-width column was padded); + // accepting them would silently turn damaged input into a valid bitmap. + if (consumed + 1 != max_bytes) { + *is_valid_ptr = false; + return result; + } result.emplaceOrInsert(0, std::move(read)); *is_valid_ptr = true; return result; @@ -912,13 +921,18 @@ class Roaring64Map { // read map value Roaring bool valid = false; - roaring::Roaring read_var = read_single_roaring_safe(buf, max_bytes, is_v1, &valid); + size_t consumed = 0; + roaring::Roaring read_var = + read_single_roaring_safe(buf, max_bytes, is_v1, &valid, &consumed); if (!valid) { *is_valid_ptr = false; return result; } - // forward buffer past the last Roaring Bitmap - size_t consumed = read_var.getSizeInBytes(is_v1); + // forward buffer past the last Roaring Bitmap, by the bytes the ORIGINAL + // encoding actually occupies (see read_single_roaring_safe: the reserialized + // size can differ from the input length, e.g. a portable hasrun-cookie + // bitmap without run containers re-serializes with the no-run cookie and + // 4 extra offset bytes). if (consumed > max_bytes) { *is_valid_ptr = false; return result; @@ -933,7 +947,12 @@ class Roaring64Map { return result; } } - // the whole payload parsed cleanly + // the whole payload parsed cleanly: a bitmap64 map must consume the buffer + // exactly, otherwise the segments were misaligned + if (max_bytes != 0) { + *is_valid_ptr = false; + return result; + } *is_valid_ptr = true; return result; } @@ -949,8 +968,18 @@ class Roaring64Map { // iterates the actual bits, i.e. a heap out-of-bounds write. roaring_bitmap_internal_validate // cross-checks the declared and actual cardinalities (plus key order and container // consistency) and rejects such payloads. + // + // consumed_bytes reports how many bytes the ORIGINAL encoding occupies, which can + // differ from the reserialized getSizeInBytes() (a portable hasrun-cookie bitmap + // without run containers re-serializes with the no-run cookie plus a 4-byte offset + // table; a V2 ARRAY_UINT32 payload with duplicate values shrinks on reserialize). + // The caller advances its buffer by consumed_bytes; using the reserialized size + // there would mis-parse a bitmap64 map (rejecting valid payloads or reading a + // segment at the wrong offset). static roaring::Roaring read_single_roaring_safe(const char* buf, size_t max_bytes, - bool portable, bool* is_valid_ptr) { + bool portable, bool* is_valid_ptr, + size_t* consumed_bytes) { + *consumed_bytes = 0; auto* r = portable ? roaring::api::roaring_bitmap_portable_deserialize_safe(buf, max_bytes) : roaring::api::roaring_bitmap_deserialize_safe(buf, max_bytes); if (r == nullptr) { @@ -968,6 +997,24 @@ class Roaring64Map { return {}; } *is_valid_ptr = true; + // the bytes the deserializer actually consumed, computed from the ORIGINAL + // encoding (never from the reserialized size) + if (portable) { + *consumed_bytes = + roaring::api::roaring_bitmap_portable_deserialize_size(buf, max_bytes); + } else { + // V2 non-portable: 1-byte tag then either uint32 count + count uint32 + // elements, or a portable bitmap - mirror roaring_bitmap_deserialize_safe. + if (static_cast(buf[0]) == 1) { // CROARING_SERIALIZATION_ARRAY_UINT32 + uint32_t card = decode_fixed32_le(reinterpret_cast(buf + 1)); + *consumed_bytes = + 1 + sizeof(uint32_t) + static_cast(card) * sizeof(uint32_t); + } else { // CROARING_SERIALIZATION_CONTAINER + *consumed_bytes = 1 + roaring::api::roaring_bitmap_portable_deserialize_size( + buf + 1, max_bytes - 1); + } + } + DCHECK(*consumed_bytes > 0 && *consumed_bytes <= max_bytes); return roaring::Roaring(r); } diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 6db0fcad441dd5..6be764624607dc 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -804,6 +804,68 @@ TEST(BitmapValueTest, FakeBitsetCardinality) { EXPECT_EQ(0u, out.size()); } +// A BITMAP64 map payload advances per-segment by the bytes the ORIGINAL encoding +// occupies, not by the reserialized getSizeInBytes: a portable hasrun-cookie bitmap +// without run containers re-serializes with the no-run cookie (4 more bytes), so the +// old advance logic rejected a perfectly valid 17-byte map (bitmap_from_binary NULL) +// and mis-parsed multi-segment maps. The input below is a hand-crafted hasrun +// encoding equivalent to bitmap {42} under map key 0. +TEST(BitmapValueTest, valid_and_deserialize_hasrun_bitmap64) { + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP64); + bytes += static_cast(1); // varint map size = 1 + uint32_t map_key = 0; + bytes.append(reinterpret_cast(&map_key), 4); + uint32_t cookie = 12347; // SERIAL_COOKIE (hasrun), size-1 = 0 + bytes.append(reinterpret_cast(&cookie), 4); + bytes += '\0'; // run bitmap: container 0 is not a run container + uint16_t key = 0, card_minus_one = 0; // one array container holding {42} + bytes.append(reinterpret_cast(&key), 2); + bytes.append(reinterpret_cast(&card_minus_one), 2); + uint16_t value = 42; + bytes.append(reinterpret_cast(&value), 2); + + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(1u, deserialized.cardinality()); + EXPECT_TRUE(deserialized.contains(42)); + + // trailing bytes after the last map segment must be rejected (misaligned payload) + std::string trailing = bytes; + trailing += '\0'; + BitmapValue rejected; + EXPECT_FALSE(rejected.valid_and_deserialize(trailing.data(), trailing.size())); +} + +TEST(BitmapValueTest, valid_and_deserialize_bitmap32_trailing_bytes) { + // read_safe now reports the consumed byte count, so a BITMAP32 payload with trailing + // bytes is a corrupted value rather than a valid bitmap with padding. + // + // Two ambient globals decide which encoding serialize_for_valid_and_deserialize produces: + // a 2-element bitmap is serialized as SET while enable_set_in_bitmap_value is on (the + // process default), and bitmap_serialize_version selects the V1 vs V2 type code. Pin both + // and restore them afterwards, so the case passes whatever order gtest runs it in (other + // tests in this binary leave both globals modified). + const bool old_enable_set = config::enable_set_in_bitmap_value; + const int16_t old_serialize_version = config::bitmap_serialize_version; + config::enable_set_in_bitmap_value = false; + config::bitmap_serialize_version = 1; + BitmapValue bitmap32({1, 9999999}); + std::string bytes = serialize_for_valid_and_deserialize(bitmap32); + EXPECT_EQ(BitmapTypeCode::BITMAP32, static_cast(bytes[0])); + + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + check_bitmap_equal(deserialized, bitmap32); + + bytes += '\x7f'; + BitmapValue rejected; + EXPECT_FALSE(rejected.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(rejected.cardinality(), 0); + config::enable_set_in_bitmap_value = old_enable_set; + config::bitmap_serialize_version = old_serialize_version; +} + TEST(BitmapValueTest, valid_and_deserialize) { config::enable_set_in_bitmap_value = false; From 45c0fc374e26166eefe04d9d109665651a0994a6 Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Tue, 13 Oct 2026 12:25:18 +0800 Subject: [PATCH 06/10] fix(bitmap): reject empty roaring containers --- be/src/core/value/bitmap_value.h | 29 +++++++++++++++++++ be/test/core/value/bitmap_value_test.cpp | 27 +++++++++++++++++ .../exprs/function/function_bitmap_test.cpp | 22 +++++++++++++- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h index 259ade5291afe2..30892870ad8af7 100644 --- a/be/src/core/value/bitmap_value.h +++ b/be/src/core/value/bitmap_value.h @@ -34,7 +34,12 @@ #include #include #include +// clang-format off +// CRoaring's public header defines ROARING_CONTAINER_T; its internal headers undefine it, +// so the public header must come first (same ordering as storage/index/snii/format/null_bitmap.cpp). #include +#include +// clang-format on #include #include #include @@ -996,6 +1001,30 @@ class Roaring64Map { *is_valid_ptr = false; return {}; } + // The CRoaring 2.1.2 bundled with Doris predates upstream PR #560 (2024-01-15, + // eleven days after the v2.1.2 tag), so its internal_validate still accepts + // empty containers (run container with n_runs == 0 - the only encoding able to + // declare one, since a non-run container's cardinality is tmp+1 >= 1). An empty + // container breaks the "no empty containers" invariant every container + // operation (AND/OR/ANDNOT) relies on: array_run_container_union dereferences + // the NULL runs pointer of an empty run container and segfaults the process. + // The bundled version cannot tell us this, so check it here. + { + const auto* ra = &r->high_low_container; + for (int32_t i = 0; i < ra->size; ++i) { + if (roaring::internal::container_get_cardinality(ra->containers[i], + ra->typecodes[i]) == 0) { + // Reachable from user supplied bytes (bitmap_from_binary), so keep the + // diagnostic off the default log level: one line per input row would let + // any user with SELECT privilege flood be.log. + VLOG_DEBUG << "Decode roaring bitmap failed: empty container at key " + << ra->keys[i]; + roaring::api::roaring_bitmap_free(r); + *is_valid_ptr = false; + return {}; + } + } + } *is_valid_ptr = true; // the bytes the deserializer actually consumed, computed from the ORIGINAL // encoding (never from the reserialized size) diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 6be764624607dc..88226c5a7e36cb 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -780,6 +780,33 @@ TEST(BitmapValueTest, valid_and_deserialize_rejects_duplicate_bitmap64_map_key) EXPECT_EQ(deserialized.cardinality(), 0); } +// The CRoaring 2.1.2 bundled with Doris predates upstream PR #560, so its +// internal_validate accepts an empty container: a portable hasrun-cookie bitmap +// with one container marked as a run container and n_runs == 0. Every container +// operation (AND/OR/ANDNOT) dereferences the NULL runs pointer of an empty run +// container (array_run_container_union -> run_container_is_full), so accepting +// such a payload crashes the process the moment the bitmap meets any other one. +// read_single_roaring_safe must reject it. +TEST(BitmapValueTest, EmptyRunContainer) { + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP32); + uint32_t cookie = 12347; // SERIAL_COOKIE (hasrun), size-1 = 0 + bytes.append(reinterpret_cast(&cookie), 4); + bytes += '\x01'; // run bitmap: container 0 IS a run container + uint16_t key = 0, card_minus_one = 0; // thiscard = 1 -> not bitset; isrun -> run + bytes.append(reinterpret_cast(&key), 2); + bytes.append(reinterpret_cast(&card_minus_one), 2); + uint16_t n_runs = 0; // the empty container: zero runs + bytes.append(reinterpret_cast(&n_runs), 2); + + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())) + << "empty run container must be rejected"; + PaddedPODArray out; + deserialized.to_array(out); + EXPECT_EQ(0u, out.size()); +} + // A BITMAP32 portable payload declaring a bitset cardinality that does not // match the actual bits set must be rejected by valid_and_deserialize: the // declared cardinality flows into bitmap_to_array's resize while the element diff --git a/be/test/exprs/function/function_bitmap_test.cpp b/be/test/exprs/function/function_bitmap_test.cpp index 1b1ce798ad0ee9..9d1b6a94c39dfa 100644 --- a/be/test/exprs/function/function_bitmap_test.cpp +++ b/be/test/exprs/function/function_bitmap_test.cpp @@ -727,12 +727,32 @@ TEST(function_bitmap_test, function_bitmap_from_binary) { auto truncated = [](std::string s, size_t n) { return keep_alive_binary(s.substr(0, n)); }; + // A run container with zero runs: the CRoaring 2.1.2 bundled with Doris + // predates upstream PR #560, so its internal_validate accepts the empty + // container while every container operation (AND/OR/ANDNOT) dereferences + // its NULL runs pointer and crashes. valid_and_deserialize must reject it + // up front (NULL), never load it. + auto empty_run = [&] { + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP32); + uint32_t cookie = 12347; // SERIAL_COOKIE (hasrun), size-1 = 0 + bytes.append(reinterpret_cast(&cookie), 4); + bytes += '\x01'; // run bitmap: container 0 IS a run container + uint16_t key = 0, card_minus_one = 0; // thiscard = 1 -> run container + bytes.append(reinterpret_cast(&key), 2); + bytes.append(reinterpret_cast(&card_minus_one), 2); + uint16_t n_runs = 0; // the empty container + bytes.append(reinterpret_cast(&n_runs), 2); + return keep_alive_binary(std::move(bytes)); + }(); + DataSet data_set = {{{VARBINARY("")}, Null()}, {{keep_alive_binary(std::string("\x7f", 1))}, Null()}, {{truncated(single32, 3)}, Null()}, {{truncated(single64, 5)}, Null()}, {{truncated(set_bytes, 3)}, Null()}, - {{truncated(bitmap32, 4)}, Null()}}; + {{truncated(bitmap32, 4)}, Null()}, + {{empty_run}, Null()}}; static_cast(check_function(func_name, input_types, data_set)); } From 75f5b1246d23ae2f9c78271424c71d6404c63b3c Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Mon, 19 Oct 2026 17:13:46 +0800 Subject: [PATCH 07/10] fix(fe): allow hll_from_binary in load mappings --- .../doris/nereids/load/NereidsLoadScanProvider.java | 5 ++++- .../load_p0/stream_load/test_stream_load_new.groovy | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java index bc7fb92ca670d8..4604910d760ec4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java @@ -71,6 +71,7 @@ public class NereidsLoadScanProvider { private static final Logger LOG = LogManager.getLogger(NereidsLoadScanProvider.class); private static final String HLL_HASH = "hll_hash"; private static final String HLL_FROM_BASE64 = "hll_from_base64"; + private static final String HLL_FROM_BINARY = "hll_from_binary"; private NereidsFileGroupInfo fileGroupInfo; private Set partialUpdateInputColumns; @@ -327,10 +328,12 @@ private void fillContextExprMap(List columnDescList, Ne String functionName = function.getName(); if (!functionName.equalsIgnoreCase(HLL_HASH) && !functionName.equalsIgnoreCase("hll_empty") - && !functionName.equalsIgnoreCase(HLL_FROM_BASE64)) { + && !functionName.equalsIgnoreCase(HLL_FROM_BASE64) + && !functionName.equalsIgnoreCase(HLL_FROM_BINARY)) { throw new AnalysisException("HLL column must use " + HLL_HASH + " function, like " + columnName + "=" + HLL_HASH + "(xxx) or " + columnName + "=" + HLL_FROM_BASE64 + "(xxx) or " + + columnName + "=" + HLL_FROM_BINARY + "(xxx) or " + columnName + "=hll_empty()"); } } diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_new.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_new.groovy index f2eae1b5488e81..b1e7ce53779c37 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_new.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_new.groovy @@ -550,7 +550,8 @@ suite("test_stream_load_new", "p0") { type_id int, type_name varchar(10), pv_hash hll hll_union not null, - pv_base64 hll hll_union not null + pv_base64 hll hll_union not null, + pv_binary hll hll_union not null ) AGGREGATE KEY(type_id,type_name) DISTRIBUTED BY HASH(type_id) BUCKETS 1 @@ -561,7 +562,7 @@ suite("test_stream_load_new", "p0") { streamLoad { set 'column_separator', ',' - set 'columns', 'type_id,type_name,type_id_base64,pv_hash=hll_hash(type_id),pv_base64=hll_from_base64(type_id_base64)' + set 'columns', 'type_id,type_name,type_id_base64,pv_hash=hll_hash(type_id),pv_base64=hll_from_base64(type_id_base64),pv_binary=hll_from_binary(from_base64(type_id_base64))' table "${tableName13}" time 10000 file 'test_stream_load_hll_type.csv' @@ -578,6 +579,9 @@ suite("test_stream_load_new", "p0") { } sql """ sync; """ qt_sql13 "select type_name, hll_union_agg(pv_hash), hll_union_agg(pv_base64) from ${tableName13} group by type_name order by type_name" + check_sqls_result_equal( + "select type_name, hll_union_agg(pv_base64) from ${tableName13} group by type_name order by type_name", + "select type_name, hll_union_agg(pv_binary) from ${tableName13} group by type_name order by type_name") } finally { try_sql "DROP TABLE IF EXISTS ${tableName13}" } From 9cdca1d91a6faf0d938366e43fb636db721cbd6b Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Sun, 1 Nov 2026 03:21:59 +0800 Subject: [PATCH 08/10] fix(hll): read the HLL type byte as uint8_t in is_valid() --- be/src/core/value/hll.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/be/src/core/value/hll.cpp b/be/src/core/value/hll.cpp index b72147806aeb48..8e6b539e62a265 100644 --- a/be/src/core/value/hll.cpp +++ b/be/src/core/value/hll.cpp @@ -216,7 +216,11 @@ bool HyperLogLog::is_valid(const Slice& slice) { } const auto* ptr = reinterpret_cast(slice.data); const auto* end = ptr + slice.size; - auto type = (HllDataType)*ptr++; + // Keep the type byte as a plain uint8_t until it is known to be one of the HllDataType + // values. The payload is user controlled (hll_from_binary / hll_from_base64), so casting + // an arbitrary byte to HllDataType first would materialize an out-of-range enum value, + // which is undefined behavior (and is reported by -fsanitize=enum). + const uint8_t type = *ptr++; switch (type) { case HLL_DATA_EMPTY: break; @@ -290,8 +294,9 @@ bool HyperLogLog::deserialize(const Slice& slice) { } const uint8_t* ptr = (uint8_t*)slice.data; - // first byte : type - _type = (HllDataType)*ptr++; + // first byte : type. is_valid() above already rejected every byte that is not one of the + // HllDataType values, so this conversion can never produce an out-of-range enum value. + _type = static_cast(*ptr++); switch (_type) { case HLL_DATA_EMPTY: break; From ec7d5db490f9b05307a146bdcee0991c37c30743 Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Sun, 1 Nov 2026 03:21:59 +0800 Subject: [PATCH 09/10] fix(test): restore bitmap configs after the binary conversion tests --- .../exprs/function/function_bitmap_test.cpp | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/be/test/exprs/function/function_bitmap_test.cpp b/be/test/exprs/function/function_bitmap_test.cpp index 9d1b6a94c39dfa..92dc6b579962e2 100644 --- a/be/test/exprs/function/function_bitmap_test.cpp +++ b/be/test/exprs/function/function_bitmap_test.cpp @@ -31,6 +31,7 @@ #include "core/types.h" #include "core/value/bitmap_value.h" #include "exprs/function/function_test_util.h" +#include "util/defer_op.h" #include "util/url_coding.h" namespace doris { @@ -615,6 +616,17 @@ ut_type::VARBINARY serialize_bitmap_to_binary(const BitmapValue& bitmap) { } // namespace TEST(function_bitmap_test, function_bitmap_to_binary) { + // This test flips two global configs. Restore them on every exit path: the UT binary runs + // many suites in one process, and a leaked enable_set_in_bitmap_value=false changes how a + // later suite's BITMAP column is serialized (SET vs BITMAP), which breaks the checked-in + // Segment goldens of SegmentFlusherTransformFormatTest. + const auto old_enable_set_in_bitmap_value = config::enable_set_in_bitmap_value; + const auto old_bitmap_serialize_version = config::bitmap_serialize_version; + Defer restore_bitmap_config {[&] { + config::enable_set_in_bitmap_value = old_enable_set_in_bitmap_value; + config::bitmap_serialize_version = old_bitmap_serialize_version; + }}; + config::Register::Field field("bool", "enable_set_in_bitmap_value", &config::enable_set_in_bitmap_value, "false", false); config::Register::_s_field_map->insert( @@ -674,6 +686,14 @@ TEST(function_bitmap_test, function_bitmap_to_binary) { } TEST(function_bitmap_test, function_bitmap_from_binary) { + // Same global-config hygiene as function_bitmap_to_binary above. + const auto old_enable_set_in_bitmap_value = config::enable_set_in_bitmap_value; + const auto old_bitmap_serialize_version = config::bitmap_serialize_version; + Defer restore_bitmap_config {[&] { + config::enable_set_in_bitmap_value = old_enable_set_in_bitmap_value; + config::bitmap_serialize_version = old_bitmap_serialize_version; + }}; + config::Register::Field field("bool", "enable_set_in_bitmap_value", &config::enable_set_in_bitmap_value, "false", false); config::Register::_s_field_map->insert( From 09cc06cf5f49fa1b00ed664b5b4b7ed6652a5cfd Mon Sep 17 00:00:00 2001 From: Zhigao Hong Date: Mon, 2 Nov 2026 01:29:03 +0800 Subject: [PATCH 10/10] test(bitmap): enhance unit tests for the bounded bitmap/HLL binary readers --- be/test/core/value/bitmap_value_test.cpp | 193 ++++++++++++++++++ .../exprs/function/function_bitmap_test.cpp | 17 ++ be/test/exprs/function/function_hll_test.cpp | 17 ++ 3 files changed, 227 insertions(+) diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 88226c5a7e36cb..5892dc34985ce6 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -989,6 +990,198 @@ TEST(BitmapValueTest, valid_and_deserialize) { config::enable_set_in_bitmap_value = false; } +// A BITMAP64 payload whose varint64 map size never terminates must be rejected instead of +// letting the decoder run off the end of the buffer looking for the last varint byte. +TEST(BitmapValueTest, valid_and_deserialize_bitmap64_bad_varint) { + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP64); + bytes.append(10, '\x80'); // 10 continuation bytes: decode_varint64_ptr gives up + + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(deserialized.cardinality(), 0); +} + +// Every map entry needs at least a 4-byte key, so a corrupted varint64 map size must be +// rejected up front instead of driving an (almost) unbounded loop over the payload. +TEST(BitmapValueTest, valid_and_deserialize_bitmap64_map_size_too_large) { + std::string bytes; + bytes += static_cast(BitmapTypeCode::BITMAP64); + bytes += static_cast(5); // varint map size = 5 + bytes.append(4, '\0'); // room for a single 4-byte key, not five + + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(deserialized.cardinality(), 0); +} + +// bitmap_serialize_version=2 writes the non-portable roaring format, whose first byte is a +// tag: ARRAY_UINT32 (1) when a uint32 array is smaller than the portable form, CONTAINER (2) +// otherwise. The reader must advance by the bytes the ORIGINAL encoding occupies for both. +TEST(BitmapValueTest, valid_and_deserialize_bitmap_v2_array_uint32) { + const bool old_enable_set = config::enable_set_in_bitmap_value; + const int16_t old_serialize_version = config::bitmap_serialize_version; + config::enable_set_in_bitmap_value = false; + config::bitmap_serialize_version = 2; + + // Two single-element containers: the portable form (20 bytes) is larger than the uint32 + // array form (12 bytes), so roaring_bitmap_serialize picks ARRAY_UINT32. + BitmapValue bitmap({0, 65536}); + std::string bytes = serialize_for_valid_and_deserialize(bitmap); + EXPECT_EQ(BitmapTypeCode::BITMAP32_V2, static_cast(bytes[0])); + EXPECT_EQ(1, static_cast(bytes[1])); // CROARING_SERIALIZATION_ARRAY_UINT32 + + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + check_bitmap_equal(deserialized, bitmap); + + // trailing bytes are rejected for the ARRAY_UINT32 form too + bytes += '\x7f'; + BitmapValue rejected; + EXPECT_FALSE(rejected.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(rejected.cardinality(), 0); + + config::enable_set_in_bitmap_value = old_enable_set; + config::bitmap_serialize_version = old_serialize_version; +} + +// With enable_set_in_bitmap_value on, a SINGLE32/SINGLE64 payload is materialized as a SET so +// that a later add() does not have to convert it. +TEST(BitmapValueTest, valid_and_deserialize_single_with_set_enabled) { + const bool old_enable_set = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + + BitmapValue single32(1); + std::string bytes32 = serialize_for_valid_and_deserialize(single32); + EXPECT_EQ(BitmapTypeCode::SINGLE32, static_cast(bytes32[0])); + BitmapValue deserialized32; + EXPECT_TRUE(deserialized32.valid_and_deserialize(bytes32.data(), bytes32.size())); + EXPECT_EQ(BitmapTypeCode::SET, deserialized32.get_type_code()); + EXPECT_EQ(1u, deserialized32.cardinality()); + EXPECT_TRUE(deserialized32.contains(1)); + + BitmapValue single64((uint64_t)4294967296); + std::string bytes64 = serialize_for_valid_and_deserialize(single64); + EXPECT_EQ(BitmapTypeCode::SINGLE64, static_cast(bytes64[0])); + BitmapValue deserialized64; + EXPECT_TRUE(deserialized64.valid_and_deserialize(bytes64.data(), bytes64.size())); + EXPECT_EQ(BitmapTypeCode::SET, deserialized64.get_type_code()); + EXPECT_EQ(1u, deserialized64.cardinality()); + EXPECT_TRUE(deserialized64.contains(4294967296)); + + config::enable_set_in_bitmap_value = old_enable_set; +} + +// SET payload edge cases: a payload with no count byte, a count above the SET threshold, and +// duplicated values (which the trusted deserialize() path also treats as corruption). +TEST(BitmapValueTest, valid_and_deserialize_set_rejects_malformed_payload) { + // only the type code: the count byte is missing + { + std::string bytes(1, static_cast(BitmapTypeCode::SET)); + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + } + // count above SET_TYPE_THRESHOLD (32): such a payload must be written as SET_V2 + { + std::string bytes; + bytes += static_cast(BitmapTypeCode::SET); + bytes += static_cast(33); + bytes.append(33 * sizeof(uint64_t), '\0'); + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + } + // duplicated values cannot come from a valid writer + { + std::string bytes; + bytes += static_cast(BitmapTypeCode::SET); + bytes += static_cast(2); + for (uint64_t value : {uint64_t(5), uint64_t(5)}) { + bytes.append(reinterpret_cast(&value), sizeof(value)); + } + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(deserialized.cardinality(), 0); + } +} + +// SET_V2 (type code 10) carries a uint32 count instead of a single byte. It is written by other +// engines, so the bounded reader must accept it and honour enable_set_in_bitmap_value. +TEST(BitmapValueTest, valid_and_deserialize_set_v2) { + const bool old_enable_set = config::enable_set_in_bitmap_value; + auto make_set_v2 = [](const std::vector& values) { + std::string bytes; + bytes += static_cast(BitmapTypeCode::SET_V2); + auto count = static_cast(values.size()); + bytes.append(reinterpret_cast(&count), sizeof(count)); + for (uint64_t value : values) { + bytes.append(reinterpret_cast(&value), sizeof(value)); + } + return bytes; + }; + + // set enabled and a count within the SET threshold: the value stays a SET + config::enable_set_in_bitmap_value = true; + { + std::string bytes = make_set_v2({1, 9999999}); + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(BitmapTypeCode::SET, deserialized.get_type_code()); + EXPECT_EQ(2u, deserialized.cardinality()); + EXPECT_TRUE(deserialized.contains(1)); + EXPECT_TRUE(deserialized.contains(9999999)); + } + // a count above the SET threshold converts to BITMAP even with set enabled + { + std::vector values(40); + std::iota(values.begin(), values.end(), 0); + std::string bytes = make_set_v2(values); + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(BitmapTypeCode::BITMAP32, deserialized.get_type_code()); + EXPECT_EQ(values.size(), deserialized.cardinality()); + } + // count == 0 needs no value bytes at all + { + std::string bytes = make_set_v2({}); + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(0u, deserialized.cardinality()); + } + // set disabled: the same payload converts to BITMAP with equal contents + config::enable_set_in_bitmap_value = false; + { + std::string bytes = make_set_v2({1, 9999999}); + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(BitmapTypeCode::BITMAP32, deserialized.get_type_code()); + EXPECT_EQ(2u, deserialized.cardinality()); + EXPECT_TRUE(deserialized.contains(9999999)); + } + { + std::string bytes = make_set_v2({}); + BitmapValue deserialized; + EXPECT_TRUE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(0u, deserialized.cardinality()); + } + // truncated: the uint32 count does not fit in the buffer + { + std::string bytes = make_set_v2({1, 2}); + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), 3)); + EXPECT_EQ(0u, deserialized.cardinality()); + } + // count does not match the payload length + { + std::string bytes = make_set_v2({1, 2}); + bytes.resize(bytes.size() - 1); + BitmapValue deserialized; + EXPECT_FALSE(deserialized.valid_and_deserialize(bytes.data(), bytes.size())); + EXPECT_EQ(0u, deserialized.cardinality()); + } + + config::enable_set_in_bitmap_value = old_enable_set; +} + TEST(BitmapValueTest, set_to_string) { config::enable_set_in_bitmap_value = true; BitmapValue bitmap; diff --git a/be/test/exprs/function/function_bitmap_test.cpp b/be/test/exprs/function/function_bitmap_test.cpp index 92dc6b579962e2..78ac42899336e1 100644 --- a/be/test/exprs/function/function_bitmap_test.cpp +++ b/be/test/exprs/function/function_bitmap_test.cpp @@ -24,6 +24,7 @@ #include "common/config.h" #include "common/status.h" +#include "core/column/column_string.h" #include "core/data_type/data_type_bitmap.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" @@ -816,4 +817,20 @@ TEST(function_bitmap_test, function_bitmap_from_binary) { EXPECT_TRUE(config::set_config("bitmap_serialize_version", "1", false, true).ok()); } +// bitmap_to_binary is registered for exactly one BITMAP argument. A BITMAP-typed slot that +// actually holds a string column can only come from a planner bug, and the function must +// report it instead of reinterpreting the column as a ColumnBitmap. +TEST(function_bitmap_test, function_bitmap_to_binary_rejects_non_bitmap_column) { + Block block; + block.insert({ColumnString::create(), std::make_shared(), "c0"}); + FunctionBasePtr function = SimpleFunctionFactory::instance().get_function( + "bitmap_to_binary", block.get_columns_with_type_and_name(), + std::make_shared()); + ASSERT_TRUE(function != nullptr); + block.insert({nullptr, std::make_shared(), "result"}); + ColumnNumbers arguments {0}; + auto status = function->execute(nullptr, block, arguments, block.columns() - 1, 1); + EXPECT_TRUE(status.is()); +} + } // namespace doris diff --git a/be/test/exprs/function/function_hll_test.cpp b/be/test/exprs/function/function_hll_test.cpp index 635f1ecb109dc3..032e51c73c23f5 100644 --- a/be/test/exprs/function/function_hll_test.cpp +++ b/be/test/exprs/function/function_hll_test.cpp @@ -22,6 +22,7 @@ #include #include +#include "core/column/column_string.h" #include "core/data_type/data_type_hll.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_varbinary.h" @@ -257,4 +258,20 @@ TEST(function_hll_test, function_hll_from_binary_test) { static_cast(check_function(func_name, input_types, data_set)); } } + +// hll_to_binary is registered for exactly one HLL argument. An HLL-typed slot that actually +// holds a string column can only come from a planner bug, and the function must report it +// instead of reinterpreting the column as a ColumnHLL. +TEST(function_hll_test, function_hll_to_binary_rejects_non_hll_column) { + Block block; + block.insert({ColumnString::create(), std::make_shared(), "c0"}); + FunctionBasePtr function = SimpleFunctionFactory::instance().get_function( + "hll_to_binary", block.get_columns_with_type_and_name(), + std::make_shared()); + ASSERT_TRUE(function != nullptr); + block.insert({nullptr, std::make_shared(), "result"}); + ColumnNumbers arguments {0}; + auto status = function->execute(nullptr, block, arguments, block.columns() - 1, 1); + EXPECT_TRUE(status.is()); +} } // namespace doris