From b869b2f579a21be7e87bf8b5112bbf83be1d8938 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 25 Nov 2024 21:16:01 +0800 Subject: [PATCH 01/61] new hash Signed-off-by: guo-shaoge --- dbms/src/Common/HashTable/Hash.h | 125 +++++++++++++++++++++++++++++ dbms/src/Interpreters/Aggregator.h | 20 ++--- 2 files changed, 135 insertions(+), 10 deletions(-) diff --git a/dbms/src/Common/HashTable/Hash.h b/dbms/src/Common/HashTable/Hash.h index b4f5d2c0a04..3f25f64bc74 100644 --- a/dbms/src/Common/HashTable/Hash.h +++ b/dbms/src/Common/HashTable/Hash.h @@ -416,3 +416,128 @@ struct IntHash32, void>> } } }; + +inline uint64_t umul128(uint64_t v, uint64_t kmul, uint64_t * high) +{ + DB::Int128 res = static_cast(v) * static_cast(kmul); + *high = static_cast(res >> 64); + return static_cast(res); +} + +template +inline void hash_combine(uint64_t & seed, const T & val) +{ + // from: https://github.com/HowardHinnant/hash_append/issues/7#issuecomment-629414712 + seed ^= std::hash{}(val) + 0x9e3779b97f4a7c15LLU + (seed << 12) + (seed >> 4); +} + +inline uint64_t hash_int128(uint64_t seed, const DB::Int128 & v) +{ + auto low = static_cast(v); + auto high = static_cast(v >> 64); + hash_combine(seed, low); + hash_combine(seed, high); + return seed; +} + +inline uint64_t hash_uint128(uint64_t seed, const DB::UInt128 & v) +{ + hash_combine(seed, v.low); + hash_combine(seed, v.high); + return seed; +} + +inline uint64_t hash_int256(uint64_t seed, const DB::Int256 & v) +{ + const auto & backend_value = v.backend(); + for (size_t i = 0; i < backend_value.size(); ++i) + { + hash_combine(seed, backend_value.limbs()[i]); + } + return seed; +} + +inline uint64_t hash_uint256(uint64_t seed, const DB::UInt256 & v) +{ + hash_combine(seed, v.a); + hash_combine(seed, v.b); + hash_combine(seed, v.c); + hash_combine(seed, v.d); + return seed; +} + +template +struct HashWithMixSeedHelper +{ + inline size_t operator()(size_t) const; +}; + +template <> +struct HashWithMixSeedHelper<4> +{ + inline size_t operator()(size_t v) const + { + // from: https://github.com/aappleby/smhasher/blob/0ff96f7835817a27d0487325b6c16033e2992eb5/src/MurmurHash3.cpp#L102 + static constexpr uint64_t kmul = 0xcc9e2d51UL; + uint64_t mul = v * kmul; + return static_cast(mul ^ (mul >> 32u)); + } +}; + +template <> +struct HashWithMixSeedHelper<8> +{ + inline size_t operator()(size_t v) const + { + // from: https://github.com/martinus/robin-hood-hashing/blob/b21730713f4b5296bec411917c46919f7b38b178/src/include/robin_hood.h#L735 + static constexpr uint64_t kmul = 0xde5fb9d2630458e9ULL; + uint64_t high = 0; + uint64_t low = umul128(v, kmul, &high); + return static_cast(high + low); + } +}; + +template +struct HashWithMixSeed +{ + inline size_t operator()(const T & v) const + { + return HashWithMixSeedHelper()(std::hash()(v)); + } +}; + +template <> +struct HashWithMixSeed +{ + inline size_t operator()(const DB::Int128 & v) const + { + return HashWithMixSeedHelper()(hash_int128(0, v)); + } +}; + +template <> +struct HashWithMixSeed +{ + inline size_t operator()(const DB::UInt128 & v) const + { + return HashWithMixSeedHelper()(hash_uint128(0, v)); + } +}; + +template <> +struct HashWithMixSeed +{ + inline size_t operator()(const DB::Int256 & v) const + { + return HashWithMixSeedHelper()(hash_int256(0, v)); + } +}; + +template <> +struct HashWithMixSeed +{ + inline size_t operator()(const DB::UInt256 & v) const + { + return HashWithMixSeedHelper()(hash_uint256(0, v)); + } +}; diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 381bfba8462..9515782793a 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -77,27 +77,27 @@ using AggregatedDataWithoutKey = AggregateDataPtr; using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize; using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap; -using AggregatedDataWithUInt32Key = HashMap>; -using AggregatedDataWithUInt64Key = HashMap>; +using AggregatedDataWithUInt32Key = HashMap>; +using AggregatedDataWithUInt64Key = HashMap>; using AggregatedDataWithShortStringKey = StringHashMap; using AggregatedDataWithStringKey = HashMapWithSavedHash; -using AggregatedDataWithInt256Key = HashMap>; +using AggregatedDataWithInt256Key = HashMap>; -using AggregatedDataWithKeys128 = HashMap>; -using AggregatedDataWithKeys256 = HashMap>; +using AggregatedDataWithKeys128 = HashMap>; +using AggregatedDataWithKeys256 = HashMap>; -using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap; using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash; -using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; -using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; /** Variants with better hash function, using more than 32 bits for hash. * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion, From e8a2df81cb2bfc4a5eb3b2660f5b2aa4c5de4d97 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 26 Nov 2024 17:31:24 +0800 Subject: [PATCH 02/61] prefetch done Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 19 ++++ dbms/src/Common/ColumnsHashingImpl.h | 62 ++++++++++--- dbms/src/Common/HashTable/FixedHashTable.h | 3 +- dbms/src/Common/HashTable/Hash.h | 28 +++--- dbms/src/Common/HashTable/HashTable.h | 11 +++ dbms/src/Common/HashTable/SmallTable.h | 1 + dbms/src/Common/HashTable/StringHashMap.h | 9 +- dbms/src/Common/HashTable/StringHashTable.h | 92 ++++++++++++------- dbms/src/Common/HashTable/TwoLevelHashTable.h | 13 +++ .../HashTable/TwoLevelStringHashTable.h | 53 +++++++++-- dbms/src/Interpreters/Aggregator.cpp | 63 +++++++++++-- dbms/src/Interpreters/Aggregator.h | 7 +- 12 files changed, 275 insertions(+), 86 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 398d6605e60..e14a793567c 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -49,14 +49,17 @@ struct HashMethodOneNumber using Base = columns_hashing_impl::HashMethodBase; const FieldType * vec; + const size_t total_rows; /// If the keys of a fixed length then key_sizes contains their lengths, empty otherwise. HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) + : total_rows(key_columns[0]->size()) { vec = &static_cast *>(key_columns[0])->getData()[0]; } explicit HashMethodOneNumber(const IColumn * column) + : total_rows(column->size()) { vec = &static_cast *>(column)->getData()[0]; } @@ -82,6 +85,8 @@ struct HashMethodOneNumber } const FieldType * getKeyData() const { return vec; } + + size_t getTotalRows() const { return total_rows; } }; @@ -97,11 +102,13 @@ struct HashMethodString const IColumn::Offset * offsets; const UInt8 * chars; TiDB::TiDBCollatorPtr collator = nullptr; + const size_t total_rows; HashMethodString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators & collators) + : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); @@ -149,8 +156,10 @@ struct HashMethodStringBin const IColumn::Offset * offsets; const UInt8 * chars; + const size_t total_rows; HashMethodStringBin(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) + : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); @@ -346,10 +355,12 @@ struct HashMethodFastPathTwoKeysSerialized Key1Desc key_1_desc; Key2Desc key_2_desc; + const size_t total_rows; HashMethodFastPathTwoKeysSerialized(const ColumnRawPtrs & key_columns, const Sizes &, const TiDB::TiDBCollators &) : key_1_desc(key_columns[0]) , key_2_desc(key_columns[1]) + , total_rows(key_columns[0]->size()) {} ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const @@ -384,11 +395,13 @@ struct HashMethodFixedString size_t n; const ColumnFixedString::Chars_t * chars; TiDB::TiDBCollatorPtr collator = nullptr; + const size_t total_rows; HashMethodFixedString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators & collators) + : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); @@ -442,6 +455,7 @@ struct HashMethodKeysFixed Sizes key_sizes; size_t keys_size; + const size_t total_rows; /// SSSE3 shuffle method can be used. Shuffle masks will be calculated and stored here. #if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) @@ -467,6 +481,7 @@ struct HashMethodKeysFixed : Base(key_columns) , key_sizes(std::move(key_sizes_)) , keys_size(key_columns.size()) + , total_rows(key_columns[0]->size()) { if (usePreparedKeys(key_sizes)) { @@ -596,6 +611,7 @@ struct HashMethodSerialized ColumnRawPtrs key_columns; size_t keys_size; TiDB::TiDBCollators collators; + const size_t total_rows; HashMethodSerialized( const ColumnRawPtrs & key_columns_, @@ -604,6 +620,7 @@ struct HashMethodSerialized : key_columns(key_columns_) , keys_size(key_columns_.size()) , collators(collators_) + , total_rows(key_columns_[0]->size()) {} ALWAYS_INLINE inline SerializedKeyHolder getKeyHolder( @@ -631,10 +648,12 @@ struct HashMethodHashed ColumnRawPtrs key_columns; TiDB::TiDBCollators collators; + const size_t total_rows; HashMethodHashed(ColumnRawPtrs key_columns_, const Sizes &, const TiDB::TiDBCollators & collators_) : key_columns(std::move(key_columns_)) , collators(collators_) + , total_rows(key_columns[0]->size()) {} ALWAYS_INLINE inline Key getKeyHolder(size_t row, Arena *, std::vector & sort_key_containers) const diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index d4f4143015d..24574ed40a4 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -127,27 +127,53 @@ class HashMethodBase using FindResult = FindResultImpl; static constexpr bool has_mapped = !std::is_same::value; using Cache = LastElementCache; + static constexpr size_t prefetch_step = 16; - template + template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, size_t row, Arena & pool, - std::vector & sort_key_containers) + std::vector & sort_key_containers, + const std::vector & hashvals = {}) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - return emplaceImpl(key_holder, data); + if constexpr (enable_prefetch) + { + const auto idx = row + prefetch_step; + if (idx < hashvals.size()) + data.prefetch(hashvals[idx]); + + return emplaceImpl(key_holder, data, hashvals[row]); + } + else + { + return emplaceImpl(key_holder, data, 0); + } } - template + template ALWAYS_INLINE inline FindResult findKey( Data & data, size_t row, Arena & pool, - std::vector & sort_key_containers) + std::vector & sort_key_containers, + const std::vector & hashvals = {}) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - return findKeyImpl(keyHolderGetKey(key_holder), data); + if constexpr (enable_prefetch) + { + const auto idx = row + prefetch_step; + if (idx < hashvals.size()) + data.prefetch(hashvals[idx]); + + return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); + } + else + { + return findKeyImpl(keyHolderGetKey(key_holder), data, 0); + } + } template @@ -155,9 +181,9 @@ class HashMethodBase const Data & data, size_t row, Arena & pool, - std::vector & sort_key_containers) + std::vector & sort_key_containers) const { - auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); + auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); return data.hash(keyHolderGetKey(key_holder)); } @@ -179,8 +205,8 @@ class HashMethodBase } } - template - ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data) + template + ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { if constexpr (Cache::consecutive_keys_optimization) { @@ -195,7 +221,11 @@ class HashMethodBase typename Data::LookupResult it; bool inserted = false; - data.emplace(key_holder, it, inserted); + + if constexpr (enable_prefetch) + data.emplace(key_holder, it, inserted, hashval); + else + data.emplace(key_holder, it, inserted); [[maybe_unused]] Mapped * cached = nullptr; if constexpr (has_mapped) @@ -232,8 +262,8 @@ class HashMethodBase return EmplaceResult(inserted); } - template - ALWAYS_INLINE inline FindResult findKeyImpl(Key key, Data & data) + template + ALWAYS_INLINE inline FindResult findKeyImpl(Key key, Data & data, size_t hashval) { if constexpr (Cache::consecutive_keys_optimization) { @@ -246,7 +276,11 @@ class HashMethodBase } } - auto it = data.find(key); + typename Data::LookupResult it; + if constexpr (enable_prefetch) + it = data.find(key, hashval); + else + it = data.find(key); if constexpr (consecutive_keys_optimization) { diff --git a/dbms/src/Common/HashTable/FixedHashTable.h b/dbms/src/Common/HashTable/FixedHashTable.h index 259e90684fc..cfa562667dc 100644 --- a/dbms/src/Common/HashTable/FixedHashTable.h +++ b/dbms/src/Common/HashTable/FixedHashTable.h @@ -212,7 +212,6 @@ class FixedHashTable typename cell_type::CellExt cell; }; - public: using key_type = Key; using mapped_type = typename Cell::mapped_type; @@ -352,6 +351,8 @@ class FixedHashTable iterator end() { return iterator(this, buf ? buf + NUM_CELLS : buf); } + inline void prefetch(size_t) {} + /// The last parameter is unused but exists for compatibility with HashTable interface. void ALWAYS_INLINE emplace(const Key & x, LookupResult & it, bool & inserted, size_t /* hash */ = 0) { diff --git a/dbms/src/Common/HashTable/Hash.h b/dbms/src/Common/HashTable/Hash.h index 3f25f64bc74..883ec8ab6ff 100644 --- a/dbms/src/Common/HashTable/Hash.h +++ b/dbms/src/Common/HashTable/Hash.h @@ -469,13 +469,13 @@ inline uint64_t hash_uint256(uint64_t seed, const DB::UInt256 & v) template struct HashWithMixSeedHelper { - inline size_t operator()(size_t) const; + static inline size_t operator()(size_t); }; template <> struct HashWithMixSeedHelper<4> { - inline size_t operator()(size_t v) const + static inline size_t operator()(size_t v) { // from: https://github.com/aappleby/smhasher/blob/0ff96f7835817a27d0487325b6c16033e2992eb5/src/MurmurHash3.cpp#L102 static constexpr uint64_t kmul = 0xcc9e2d51UL; @@ -487,7 +487,7 @@ struct HashWithMixSeedHelper<4> template <> struct HashWithMixSeedHelper<8> { - inline size_t operator()(size_t v) const + static inline size_t operator()(size_t v) { // from: https://github.com/martinus/robin-hood-hashing/blob/b21730713f4b5296bec411917c46919f7b38b178/src/include/robin_hood.h#L735 static constexpr uint64_t kmul = 0xde5fb9d2630458e9ULL; @@ -500,44 +500,44 @@ struct HashWithMixSeedHelper<8> template struct HashWithMixSeed { - inline size_t operator()(const T & v) const + static size_t operator()(const T & v) { - return HashWithMixSeedHelper()(std::hash()(v)); + return HashWithMixSeedHelper::operator()(std::hash()(v)); } }; template <> struct HashWithMixSeed { - inline size_t operator()(const DB::Int128 & v) const + static size_t operator()(const DB::Int128 & v) { - return HashWithMixSeedHelper()(hash_int128(0, v)); + return HashWithMixSeedHelper::operator()(hash_int128(0, v)); } }; template <> struct HashWithMixSeed { - inline size_t operator()(const DB::UInt128 & v) const + static inline size_t operator()(const DB::UInt128 & v) { - return HashWithMixSeedHelper()(hash_uint128(0, v)); + return HashWithMixSeedHelper::operator()(hash_uint128(0, v)); } }; template <> struct HashWithMixSeed { - inline size_t operator()(const DB::Int256 & v) const + static inline size_t operator()(const DB::Int256 & v) { - return HashWithMixSeedHelper()(hash_int256(0, v)); + return HashWithMixSeedHelper::operator()(hash_int256(0, v)); } }; template <> struct HashWithMixSeed -{ - inline size_t operator()(const DB::UInt256 & v) const +{ + static inline size_t operator()(const DB::UInt256 & v) { - return HashWithMixSeedHelper()(hash_uint256(0, v)); + return HashWithMixSeedHelper::operator()(hash_uint256(0, v)); } }; diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index a4f0fe3be03..4f037f60019 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -851,6 +851,17 @@ class HashTable iterator end() { return iterator(this, buf ? buf + grower.bufSize() : buf); } + void ALWAYS_INLINE prefetch(size_t hashval) const + { + (void)hashval; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + size_t place_value = grower.place(hashval); + __mm_prefetch((const char*)(&buf[place_value]), _MM_HINT_NTA); +#elif defined(__GNUC__) + size_t place_value = grower.place(hashval); + __builtin_prefetch(static_cast(&buf[place_value])); +#endif + } protected: const_iterator iteratorTo(const Cell * ptr) const { return const_iterator(this, ptr); } diff --git a/dbms/src/Common/HashTable/SmallTable.h b/dbms/src/Common/HashTable/SmallTable.h index fa40b479430..a032ae76cff 100644 --- a/dbms/src/Common/HashTable/SmallTable.h +++ b/dbms/src/Common/HashTable/SmallTable.h @@ -296,6 +296,7 @@ class SmallTable iterator ALWAYS_INLINE find(Key x) { return iteratorTo(findCell(x)); } const_iterator ALWAYS_INLINE find(Key x) const { return iteratorTo(findCell(x)); } + void ALWAYS_INLINE prefetch(size_t) {} void write(DB::WriteBuffer & wb) const { diff --git a/dbms/src/Common/HashTable/StringHashMap.h b/dbms/src/Common/HashTable/StringHashMap.h index 6f7e668e1d9..cad653907fa 100644 --- a/dbms/src/Common/HashTable/StringHashMap.h +++ b/dbms/src/Common/HashTable/StringHashMap.h @@ -90,29 +90,30 @@ struct StringHashMapCell template struct StringHashMapSubMaps { + using Hash = StringHashTableHash; using T0 = StringHashTableEmpty>; using T1 = HashMapTable< StringKey8, StringHashMapCell, - StringHashTableHash, + Hash, StringHashTableGrower<>, Allocator>; using T2 = HashMapTable< StringKey16, StringHashMapCell, - StringHashTableHash, + Hash, StringHashTableGrower<>, Allocator>; using T3 = HashMapTable< StringKey24, StringHashMapCell, - StringHashTableHash, + Hash, StringHashTableGrower<>, Allocator>; using Ts = HashMapTable< StringRef, StringHashMapCell, - StringHashTableHash, + Hash, StringHashTableGrower<>, Allocator>; }; diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index aa4825f171a..e11972d0795 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -20,7 +20,6 @@ #include #include - using StringKey8 = UInt64; using StringKey16 = DB::UInt128; struct StringKey24 @@ -48,45 +47,38 @@ inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n) return {reinterpret_cast(&n), 24ul - (__builtin_clzll(n.c) >> 3)}; } -struct StringHashTableHash +inline size_t hash_string_key_24(uint64_t seed, const StringKey24 & v) { -#if defined(__SSE4_2__) - size_t ALWAYS_INLINE operator()(StringKey8 key) const - { - size_t res = -1ULL; - res = _mm_crc32_u64(res, key); - return res; - } - size_t ALWAYS_INLINE operator()(const StringKey16 & key) const - { - size_t res = -1ULL; - res = _mm_crc32_u64(res, key.low); - res = _mm_crc32_u64(res, key.high); - return res; - } - size_t ALWAYS_INLINE operator()(const StringKey24 & key) const + hash_combine(seed, v.a); + hash_combine(seed, v.b); + hash_combine(seed, v.c); + return seed; +} + +template <> +struct HashWithMixSeed +{ + static inline size_t operator()(const StringKey24 & v) { - size_t res = -1ULL; - res = _mm_crc32_u64(res, key.a); - res = _mm_crc32_u64(res, key.b); - res = _mm_crc32_u64(res, key.c); - return res; + return HashWithMixSeedHelper::operator()(hash_string_key_24(0, v)); } -#else - size_t ALWAYS_INLINE operator()(StringKey8 key) const +}; + +struct StringHashTableHash +{ + static size_t ALWAYS_INLINE operator()(StringKey8 key) { - return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 8); + return HashWithMixSeed::operator()(key); } - size_t ALWAYS_INLINE operator()(const StringKey16 & key) const + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { - return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 16); + return HashWithMixSeed::operator()(key); } - size_t ALWAYS_INLINE operator()(const StringKey24 & key) const + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { - return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 24); + return HashWithMixSeed::operator()(key); } -#endif - size_t ALWAYS_INLINE operator()(StringRef key) const { return StringRefHash()(key); } + static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHash()(key); } }; template @@ -150,6 +142,8 @@ struct StringHashTableEmpty //-V730 return hasZero() ? zeroValue() : nullptr; } + void ALWAYS_INLINE prefetch(size_t) {} + void write(DB::WriteBuffer & wb) const { zeroValue()->write(wb); } void writeText(DB::WriteBuffer & wb) const { zeroValue()->writeText(wb); } void read(DB::ReadBuffer & rb) { zeroValue()->read(rb); } @@ -157,6 +151,7 @@ struct StringHashTableEmpty //-V730 size_t size() const { return hasZero() ? 1 : 0; } bool empty() const { return !hasZero(); } size_t getBufferSizeInBytes() const { return sizeof(Cell); } + size_t getBufferSizeInCells() const { return 1; } void setResizeCallback(const ResizeCallback &) {} size_t getCollisions() const { return 0; } }; @@ -364,6 +359,13 @@ class StringHashTable : private boost::noncopyable this->dispatch(*this, key_holder, EmplaceCallable(it, inserted)); } + // TODO del + template + void ALWAYS_INLINE emplace(KeyHolder &&, LookupResult &, bool &, size_t) + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::emplace instead"); + } + struct FindCallable { // find() doesn't need any key memory management, so we don't work with @@ -380,12 +382,35 @@ class StringHashTable : private boost::noncopyable } }; + // We will not prefetch StringHashTable directly, instead caller should call specific submap's prefetch. + // Because StringHashTable doesn't know which submap to prefetch. + void prefetch(size_t) const + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::prefetch instead"); + } + LookupResult ALWAYS_INLINE find(const Key & x) { return dispatch(*this, x, FindCallable{}); } ConstLookupResult ALWAYS_INLINE find(const Key & x) const { return dispatch(*this, x, FindCallable{}); } + // TODO del + LookupResult ALWAYS_INLINE find(const Key &, size_t) + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); + } + ConstLookupResult ALWAYS_INLINE find(const Key &, size_t) const + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); + } + bool ALWAYS_INLINE has(const Key & x, size_t = 0) const { return dispatch(*this, x, FindCallable{}) != nullptr; } + template + size_t ALWAYS_INLINE hash(const HashKeyType & key) const + { + return SubMaps::Hash::operator()(key); + } + void write(DB::WriteBuffer & wb) const { m0.write(wb); @@ -434,6 +459,11 @@ class StringHashTable : private boost::noncopyable bool empty() const { return m0.empty() && m1.empty() && m2.empty() && m3.empty() && ms.empty(); } + size_t getBufferSizeInCells() const + { + return m0.getBufferSizeInCells() + m1.getBufferSizeInCells() + m2.getBufferSizeInCells() + + m3.getBufferSizeInCells() + ms.getBufferSizeInCells(); + } size_t getBufferSizeInBytes() const { return m0.getBufferSizeInBytes() + m1.getBufferSizeInBytes() + m2.getBufferSizeInBytes() diff --git a/dbms/src/Common/HashTable/TwoLevelHashTable.h b/dbms/src/Common/HashTable/TwoLevelHashTable.h index 6778cd4a3e8..01c14dd07c2 100644 --- a/dbms/src/Common/HashTable/TwoLevelHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelHashTable.h @@ -285,6 +285,12 @@ class TwoLevelHashTable : private boost::noncopyable impls[buck].emplace(key_holder, it, inserted, hash_value); } + void ALWAYS_INLINE prefetch(size_t hashval) const + { + size_t buck = getBucketFromHash(hashval); + impls[buck].prefetch(hashval); + } + LookupResult ALWAYS_INLINE find(Key x, size_t hash_value) { size_t buck = getBucketFromHash(hash_value); @@ -352,6 +358,13 @@ class TwoLevelHashTable : private boost::noncopyable return true; } + size_t getBufferSizeInCells() const + { + size_t res = 0; + for (const auto & impl : impls) + res += impl.getBufferSizeInCells(); + return res; + } size_t getBufferSizeInBytes() const { size_t res = 0; diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 5bdb24a3d13..5608d0fd0f8 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -30,8 +30,20 @@ class TwoLevelStringHashTable : private boost::noncopyable static constexpr size_t NUM_BUCKETS = 1ULL << BITS_FOR_BUCKET; static constexpr size_t MAX_BUCKET = NUM_BUCKETS - 1; + template + size_t ALWAYS_INLINE hash(const HashKeyType & key) const + { + return SubMaps::Hash::operator()(key); + } + + // Same reason as StringHashTable::prefetch. + void prefetch(size_t) const + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::prefetch instead"); + } + // TODO: currently hashing contains redundant computations when doing distributed or external aggregations - size_t hash(const Key & x) const + size_t hashStringRef(const Key & x) const { return const_cast(*this).dispatch(*this, x, [&](const auto &, const auto &, size_t hash) { return hash; @@ -44,7 +56,7 @@ class TwoLevelStringHashTable : private boost::noncopyable impl.setResizeCallback(resize_callback); } - size_t operator()(const Key & x) const { return hash(x); } + size_t operator()(const Key & x) const { return hashStringRef(x); } /// NOTE Bad for hash tables with more than 2^32 cells. static size_t getBucketFromHash(size_t hash_value) { return (hash_value >> (32 - BITS_FOR_BUCKET)) & MAX_BUCKET; } @@ -104,7 +116,6 @@ class TwoLevelStringHashTable : private boost::noncopyable #endif dispatch(Self & self, KeyHolder && key_holder, Func && func) { - StringHashTableHash hash; const StringRef & x = keyHolderGetKey(key_holder); const size_t sz = x.size; if (sz == 0) @@ -117,7 +128,7 @@ class TwoLevelStringHashTable : private boost::noncopyable { // Strings with trailing zeros are not representable as fixed-size // string keys. Put them to the generic table. - auto res = hash(x); + auto res = SubMaps::Hash::operator()(x); auto buck = getBucketFromHash(res); return func(self.impls[buck].ms, std::forward(key_holder), res); } @@ -154,7 +165,7 @@ class TwoLevelStringHashTable : private boost::noncopyable else n[0] <<= s; } - auto res = hash(k8); + auto res = SubMaps::Hash::operator()(k8); auto buck = getBucketFromHash(res); keyHolderDiscardKey(key_holder); return func(self.impls[buck].m1, k8, res); @@ -168,7 +179,7 @@ class TwoLevelStringHashTable : private boost::noncopyable n[1] >>= s; else n[1] <<= s; - auto res = hash(k16); + auto res = SubMaps::Hash::operator()(k16); auto buck = getBucketFromHash(res); keyHolderDiscardKey(key_holder); return func(self.impls[buck].m2, k16, res); @@ -182,14 +193,14 @@ class TwoLevelStringHashTable : private boost::noncopyable n[2] >>= s; else n[2] <<= s; - auto res = hash(k24); + auto res = SubMaps::Hash::operator()(k24); auto buck = getBucketFromHash(res); keyHolderDiscardKey(key_holder); return func(self.impls[buck].m3, k24, res); } default: { - auto res = hash(x); + auto res = SubMaps::Hash::operator()(x); auto buck = getBucketFromHash(res); return func(self.impls[buck].ms, std::forward(key_holder), res); } @@ -202,12 +213,27 @@ class TwoLevelStringHashTable : private boost::noncopyable dispatch(*this, key_holder, typename Impl::EmplaceCallable{it, inserted}); } - LookupResult ALWAYS_INLINE find(const Key x) { return dispatch(*this, x, typename Impl::FindCallable{}); } + template + void ALWAYS_INLINE emplace(KeyHolder &&, LookupResult &, bool &, size_t) + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::emplace instead"); + } + + LookupResult ALWAYS_INLINE find(const Key & x) { return dispatch(*this, x, typename Impl::FindCallable{}); } - ConstLookupResult ALWAYS_INLINE find(const Key x) const + ConstLookupResult ALWAYS_INLINE find(const Key & x) const { return dispatch(*this, x, typename Impl::FindCallable{}); } + LookupResult ALWAYS_INLINE find(const Key &, size_t) + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); + } + + ConstLookupResult ALWAYS_INLINE find(const Key &, size_t) const + { + RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); + } void write(DB::WriteBuffer & wb) const { @@ -259,6 +285,13 @@ class TwoLevelStringHashTable : private boost::noncopyable return true; } + size_t getBufferSizeInCells() const + { + size_t res = 0; + for (const auto & impl : impls) + res = impl.getBufferSizeInCells(); + return res; + } size_t getBufferSizeInBytes() const { size_t res = 0; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index f25c22717e8..180799bd7ed 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -665,23 +665,43 @@ void NO_INLINE Aggregator::executeImpl( { typename Method::State state(agg_process_info.key_columns, key_sizes, collators); - executeImplBatch(method, state, aggregates_pool, agg_process_info); + if (method.data.getBufferSizeInCells() < 8192) + executeImplBatch(method, state, aggregates_pool, agg_process_info); + else + executeImplBatch(method, state, aggregates_pool, agg_process_info); +} + +template +std::vector getHashVals(size_t start_row, size_t end_row, const Data & data, const State & state, + std::vector & sort_key_containers, Arena * pool) +{ + std::vector hashvals(state.total_rows, 0); + for (size_t i = start_row; i < end_row; ++i) + { + hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); + } + return hashvals; } -template +template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, typename Method::State & state, size_t index, Arena & aggregates_pool, - std::vector & sort_key_containers) const + std::vector & sort_key_containers, + const std::vector & hashvals) const { try { if constexpr (only_lookup) - return state.findKey(method.data, index, aggregates_pool, sort_key_containers); + { + return state.template findKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); + } else - return state.emplaceKey(method.data, index, aggregates_pool, sort_key_containers); + { + return state.template emplaceKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); + } } catch (ResizeException &) { @@ -689,7 +709,7 @@ std::optional::Res } } -template +template ALWAYS_INLINE void Aggregator::executeImplBatch( Method & method, typename Method::State & state, @@ -712,14 +732,28 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( { /// For all rows. AggregateDataPtr place = aggregates_pool->alloc(0); + std::vector hashvals; + if constexpr (enable_prefetch) + { + hashvals = getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool); + + } + for (size_t i = 0; i < agg_size; ++i) { - auto emplace_result_hold = emplaceOrFindKey( + auto emplace_result_hold = emplaceOrFindKey( method, state, agg_process_info.start_row, *aggregates_pool, - sort_key_containers); + sort_key_containers, + hashvals); if likely (emplace_result_hold.has_value()) { if constexpr (collect_hit_rate) @@ -784,13 +818,24 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( std::unique_ptr places(new AggregateDataPtr[agg_size]); std::optional processed_rows; + std::vector hashvals; + if constexpr (enable_prefetch) + { + hashvals = getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool); + } for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + agg_size; ++i) { AggregateDataPtr aggregate_data = nullptr; auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); if unlikely (!emplace_result_holder.has_value()) { LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 9515782793a..0f1365694ac 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1454,20 +1454,21 @@ class Aggregator AggProcessInfo & agg_process_info, TiDB::TiDBCollators & collators) const; - template + template void executeImplBatch( Method & method, typename Method::State & state, Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template + template std::optional::ResultType> emplaceOrFindKey( Method & method, typename Method::State & state, size_t index, Arena & aggregates_pool, - std::vector & sort_key_containers) const; + std::vector & sort_key_containers, + const std::vector & hashvals) const; /// For case when there are no keys (all aggregate into one row). static void executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena); From b3141662d11d44633e2fa9ac8dbf040b674718b5 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 27 Nov 2024 10:57:36 +0800 Subject: [PATCH 03/61] executeImplBatchStringHashMap done Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 54 +++- dbms/src/Common/HashTable/FixedHashTable.h | 2 + dbms/src/Common/HashTable/HashTable.h | 3 + dbms/src/Common/HashTable/SmallTable.h | 3 + dbms/src/Common/HashTable/StringHashTable.h | 164 +++++++++++- dbms/src/Common/HashTable/TwoLevelHashTable.h | 3 + .../HashTable/TwoLevelStringHashTable.h | 66 +++++ dbms/src/Interpreters/Aggregator.cpp | 237 ++++++++++++++++-- dbms/src/Interpreters/Aggregator.h | 17 ++ libs/libcommon/include/common/StringRef.h | 2 +- 10 files changed, 524 insertions(+), 27 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 24574ed40a4..0c8d0bc1a49 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -144,11 +145,11 @@ class HashMethodBase if (idx < hashvals.size()) data.prefetch(hashvals[idx]); - return emplaceImpl(key_holder, data, hashvals[row]); + return emplaceImpl(key_holder, data, hashvals[row]); } else { - return emplaceImpl(key_holder, data, 0); + return emplaceImpl(key_holder, data, 0); } } @@ -167,15 +168,52 @@ class HashMethodBase if (idx < hashvals.size()) data.prefetch(hashvals[idx]); - return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); + return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); } else { - return findKeyImpl(keyHolderGetKey(key_holder), data, 0); + return findKeyImpl(keyHolderGetKey(key_holder), data, 0); } } + template + ALWAYS_INLINE inline EmplaceResult emplaceStringKey( + Data & data, + size_t idx, + const std::vector & datas, + const std::vector & hashvals) + { + auto & submap = typename StringHashTableSubMapSelector>::getSubMap(data); + if constexpr (enable_prefetch) + { + const auto prefetch_idx = idx + prefetch_step; + if (prefetch_idx < hashvals.size()) + submap.prefetch(hashvals[prefetch_idx]); + } + + return emplaceImpl(datas[idx], submap, hashvals[idx]); + } + + // TODO Macro with emplaceStringKey + template + ALWAYS_INLINE inline FindResult findStringKey( + Data & data, + size_t idx, + const std::vector & datas, + const std::vector & hashvals) + { + auto & submap = typename StringHashTableSubMapSelector>::getSubMap(data); + if constexpr (enable_prefetch) + { + const auto prefetch_idx = idx + prefetch_step; + if (prefetch_idx < hashvals.size()) + submap.prefetch(hashvals[prefetch_idx]); + } + + return findKeyImpl(datas[idx], submap, hashvals[idx]); + } + template ALWAYS_INLINE inline size_t getHash( const Data & data, @@ -205,7 +243,7 @@ class HashMethodBase } } - template + template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { if constexpr (Cache::consecutive_keys_optimization) @@ -222,7 +260,7 @@ class HashMethodBase typename Data::LookupResult it; bool inserted = false; - if constexpr (enable_prefetch) + if constexpr (use_hashval) data.emplace(key_holder, it, inserted, hashval); else data.emplace(key_holder, it, inserted); @@ -262,7 +300,7 @@ class HashMethodBase return EmplaceResult(inserted); } - template + template ALWAYS_INLINE inline FindResult findKeyImpl(Key key, Data & data, size_t hashval) { if constexpr (Cache::consecutive_keys_optimization) @@ -277,7 +315,7 @@ class HashMethodBase } typename Data::LookupResult it; - if constexpr (enable_prefetch) + if constexpr (use_hashval) it = data.find(key, hashval); else it = data.find(key); diff --git a/dbms/src/Common/HashTable/FixedHashTable.h b/dbms/src/Common/HashTable/FixedHashTable.h index cfa562667dc..8b0b721aa8c 100644 --- a/dbms/src/Common/HashTable/FixedHashTable.h +++ b/dbms/src/Common/HashTable/FixedHashTable.h @@ -221,6 +221,8 @@ class FixedHashTable using LookupResult = Cell *; using ConstLookupResult = const Cell *; + static constexpr bool is_string_hash_map = false; + static constexpr bool is_two_level = false; size_t hash(const Key & x) const { return x; } diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index 4f037f60019..12ebc49756c 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -402,6 +402,9 @@ class HashTable using Grower = GrowerType; using Allocator = AllocatorType; + static constexpr bool is_string_hash_map = false; + static constexpr bool is_two_level = false; + protected: friend class const_iterator; friend class iterator; diff --git a/dbms/src/Common/HashTable/SmallTable.h b/dbms/src/Common/HashTable/SmallTable.h index a032ae76cff..1292a4205da 100644 --- a/dbms/src/Common/HashTable/SmallTable.h +++ b/dbms/src/Common/HashTable/SmallTable.h @@ -85,6 +85,9 @@ class SmallTable using value_type = typename Cell::value_type; using cell_type = Cell; + static constexpr bool is_string_hash_map = false; + static constexpr bool is_two_level = false; + class Reader final : private Cell::State { public: diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index e11972d0795..f906b043a9e 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -66,19 +67,24 @@ struct HashWithMixSeed struct StringHashTableHash { + using StringKey8Hasher = HashWithMixSeed; + using StringKey16Hasher = HashWithMixSeed; + using StringKey24Hasher = HashWithMixSeed; + using StringRefHasher = StringRefHash; + static size_t ALWAYS_INLINE operator()(StringKey8 key) { - return HashWithMixSeed::operator()(key); + return StringKey8Hasher::operator()(key); } static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { - return HashWithMixSeed::operator()(key); + return StringKey16Hasher::operator()(key); } static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { - return HashWithMixSeed::operator()(key); + return StringKey24Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHash()(key); } + static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } }; template @@ -185,6 +191,92 @@ struct StringHashTableLookupResult friend bool operator!=(const std::nullptr_t &, const StringHashTableLookupResult & b) { return b.mapped_ptr; } }; + template + static auto +#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) + NO_INLINE NO_SANITIZE_ADDRESS NO_SANITIZE_THREAD +#else + ALWAYS_INLINE +#endif + dispatchStringHashTable(size_t row, KeyHolder && key_holder, Func0 && func0, Func8 && func8, Func16 && func16, Func24 && func24, FuncStr && func_str) + { + const StringRef & x = keyHolderGetKey(key_holder); + const size_t sz = x.size; + if (sz == 0) + { + return func0(x, row); + } + + if (x.data[sz - 1] == 0) + { + // Strings with trailing zeros are not representable as fixed-size + // string keys. Put them to the generic table. + return func_str(key_holder, row); + } + + const char * p = x.data; + // pending bits that needs to be shifted out + const char s = (-sz & 7) * 8; + union + { + StringKey8 k8; + StringKey16 k16; + StringKey24 k24; + UInt64 n[3]; + }; + switch ((sz - 1) >> 3) + { + case 0: // 1..8 bytes + { + // first half page + if ((reinterpret_cast(p) & 2048) == 0) + { + memcpy(&n[0], p, 8); + if constexpr (DB::isLittleEndian()) + n[0] &= (-1ULL >> s); + else + n[0] &= (-1ULL << s); + } + else + { + const char * lp = x.data + x.size - 8; + memcpy(&n[0], lp, 8); + if constexpr (DB::isLittleEndian()) + n[0] >>= s; + else + n[0] <<= s; + } + return func8(k8, row); + } + case 1: // 9..16 bytes + { + memcpy(&n[0], p, 8); + const char * lp = x.data + x.size - 8; + memcpy(&n[1], lp, 8); + if constexpr (DB::isLittleEndian()) + n[1] >>= s; + else + n[1] <<= s; + return func16(k16, row); + } + case 2: // 17..24 bytes + { + memcpy(&n[0], p, 16); + const char * lp = x.data + x.size - 8; + memcpy(&n[2], lp, 8); + if constexpr (DB::isLittleEndian()) + n[2] >>= s; + else + n[2] <<= s; + return func24(k24, row); + } + default: // >= 25 bytes + { + return func_str(key_holder, row); + } + } + } + template class StringHashTable : private boost::noncopyable { @@ -221,6 +313,9 @@ class StringHashTable : private boost::noncopyable using LookupResult = StringHashTableLookupResult; using ConstLookupResult = StringHashTableLookupResult; + static constexpr bool is_string_hash_map = true; + static constexpr bool is_two_level = false; + StringHashTable() = default; explicit StringHashTable(size_t reserve_for_num_elements) @@ -488,3 +583,64 @@ class StringHashTable : private boost::noncopyable ms.clearAndShrink(); } }; + +template +struct StringHashTableSubMapSelector; + +template +struct StringHashTableSubMapSelector<0, false, Data> +{ + struct Hash + { + static ALWAYS_INLINE size_t operator()(const StringRef & ) { return 0; } + }; + + typename Data::T0 & getSubMap(size_t, Data & data) + { + return data.m0; + } +}; + +template +struct StringHashTableSubMapSelector<1, false, Data> +{ + using Hash = StringHashTableHash::StringKey8Hasher; + + typename Data::T1 & getSubMap(size_t, Data & data) + { + return data.m1; + } +}; + +template +struct StringHashTableSubMapSelector<2, false, Data> +{ + using Hash = StringHashTableHash::StringKey16Hasher; + + typename Data::T2 & getSubMap(size_t, Data & data) + { + return data.m2; + } +}; + +template +struct StringHashTableSubMapSelector<3, false, Data> +{ + using Hash = StringHashTableHash::StringKey24Hasher; + + typename Data::T3 & getSubMap(size_t, Data & data) + { + return data.m3; + } +}; + +template +struct StringHashTableSubMapSelector<4, false, Data> +{ + using Hash = StringHashTableHash::StringRefHasher; + + typename Data::Ts & getSubMap(size_t, Data & data) + { + return data.ms; + } +}; diff --git a/dbms/src/Common/HashTable/TwoLevelHashTable.h b/dbms/src/Common/HashTable/TwoLevelHashTable.h index 01c14dd07c2..75a5402363d 100644 --- a/dbms/src/Common/HashTable/TwoLevelHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelHashTable.h @@ -60,6 +60,9 @@ class TwoLevelHashTable : private boost::noncopyable static constexpr size_t NUM_BUCKETS = 1ULL << BITS_FOR_BUCKET; static constexpr size_t MAX_BUCKET = NUM_BUCKETS - 1; + static constexpr bool is_string_hash_map = false; + static constexpr bool is_two_level = true; + size_t hash(const Key & x) const { return Hash::operator()(x); } /// NOTE Bad for hash tables with more than 2^32 cells. diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 5608d0fd0f8..d217e0c0260 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -30,6 +30,9 @@ class TwoLevelStringHashTable : private boost::noncopyable static constexpr size_t NUM_BUCKETS = 1ULL << BITS_FOR_BUCKET; static constexpr size_t MAX_BUCKET = NUM_BUCKETS - 1; + static constexpr bool is_string_hash_map = true; + static constexpr bool is_two_level = true; + template size_t ALWAYS_INLINE hash(const HashKeyType & key) const { @@ -301,3 +304,66 @@ class TwoLevelStringHashTable : private boost::noncopyable return res; } }; + +template +struct StringHashTableSubMapSelector<0, true, Data> +{ + struct Hash + { + static ALWAYS_INLINE size_t operator()(const StringRef & ) { return 0; } + }; + + typename Data::T0 & getSubMap(size_t hashval, Data & data) + { + const auto bucket = Data::getBucketFromHash(hashval); + return data.impls[bucket].m0; + } +}; + +template +struct StringHashTableSubMapSelector<1, true, Data> +{ + using Hash = StringHashTableHash::StringKey8Hasher; + + typename Data::T1 & getSubMap(size_t hashval, Data & data) + { + const auto bucket = Data::getBucketFromHash(hashval); + return data.impls[bucket].m1; + } +}; + +template +struct StringHashTableSubMapSelector<2, true, Data> +{ + using Hash = StringHashTableHash::StringKey16Hasher; + + typename Data::T2 & getSubMap(size_t hashval, Data & data) + { + const auto bucket = Data::getBucketFromHash(hashval); + return data.impls[bucket].m2; + } +}; + +template +struct StringHashTableSubMapSelector<3, true, Data> +{ + using Hash = StringHashTableHash::StringKey24Hasher; + + typename Data::T3 & getSubMap(size_t hashval, Data & data) + { + const auto bucket = Data::getBucketFromHash(hashval); + return data.impls[bucket].m3; + } +}; + +template +struct StringHashTableSubMapSelector<4, true, Data> +{ + using Hash = StringHashTableHash::StringRefHasher; + + typename Data::Ts & getSubMap(size_t hashval, Data & data) + { + const auto bucket = Data::getBucketFromHash(hashval); + return data.impls[bucket].ms; + } +}; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 180799bd7ed..54cf52c673d 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -695,13 +695,9 @@ std::optional::Res try { if constexpr (only_lookup) - { return state.template findKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); - } else - { return state.template emplaceKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); - } } catch (ResizeException &) { @@ -709,6 +705,73 @@ std::optional::Res } } +// StringKeyType can be StringRef/StringKey8/StringKey16/StringKey24/ArenaKeyHolder. +// return true when resize exception happens. +template +bool Aggregator::emplaceOrFindStringKey( + typename Method::Data & data, + typename Method::State & state, + const std::vector & key_infos, + const std::vector & key_datas, + Arena & aggregates_pool, + std::vector & places, + AggProcessInfo & agg_process_info) const +{ + RUNTIME_CHECK(key_infos.size() == key_datas.size()); + + using Hash = typename StringHashTableSubMapSelector>::Hash; + std::vector hashvals(key_infos.size(), 0); + for (size_t i = 0; i < key_infos.size(); ++i) + { + hashvals[i] = Hash::operator()(keyHolderGetKey(key_datas[0])); + } + + AggregateDataPtr agg_state = nullptr; + for (size_t i = 0; i < key_infos.size(); ++i) + { + try + { + if constexpr (only_lookup) + { + auto find_result = state.template findStringKey(data, i, key_datas, hashvals); + if (find_result.isFound()) + { + agg_state = find_result.getMapped(); + } + else + { + agg_process_info.not_found_rows.push_back(key_infos[i]); + } + } + else + { + auto emplace_result = state.template emplaceStringKey(data, i, key_datas, hashvals); + if (emplace_result.isInserted()) + { + emplace_result.setMapped(nullptr); + + agg_state = aggregates_pool.alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(agg_state); + + emplace_result.setMapped(agg_state); + } + else + { + agg_state = emplace_result.getMapped(); + } + places.push_back(agg_state); + } + } + catch (ResizeException &) + { + // agg_process_info.set + // TODO handle exception + return true; + } + } + return false; +} + template ALWAYS_INLINE void Aggregator::executeImplBatch( Method & method, @@ -721,10 +784,10 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); - size_t agg_size = agg_process_info.end_row - agg_process_info.start_row; + size_t rows = agg_process_info.end_row - agg_process_info.start_row; fiu_do_on(FailPoints::force_agg_on_partial_block, { - if (agg_size > 0 && agg_process_info.start_row == 0) - agg_size = std::max(agg_size / 2, 1); + if (rows > 0 && agg_process_info.start_row == 0) + rows = std::max(rows / 2, 1); }); /// Optimization for special case when there are no aggregate functions. @@ -745,7 +808,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( } - for (size_t i = 0; i < agg_size; ++i) + for (size_t i = 0; i < rows; ++i) { auto emplace_result_hold = emplaceOrFindKey( method, @@ -789,7 +852,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( { inst->batch_that->addBatchLookupTable8( agg_process_info.start_row, - agg_size, + rows, reinterpret_cast(method.data.data()), inst->state_offset, [&](AggregateDataPtr & aggregate_data) { @@ -801,12 +864,12 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( inst->batch_arguments, aggregates_pool); } - agg_process_info.start_row += agg_size; + agg_process_info.start_row += rows; // For key8, assume all rows are hit. No need to do state switch for auto pass through hashagg. // Because HashMap of key8 is basically a vector of size 256. if constexpr (collect_hit_rate) - agg_process_info.hit_row_cnt = agg_size; + agg_process_info.hit_row_cnt = rows; // Because all rows are hit, so state will not switch to Selective. if constexpr (only_lookup) @@ -815,8 +878,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( } /// Generic case. - - std::unique_ptr places(new AggregateDataPtr[agg_size]); + std::unique_ptr places(new AggregateDataPtr[rows]); std::optional processed_rows; std::vector hashvals; if constexpr (enable_prefetch) @@ -830,7 +892,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( aggregates_pool); } - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + agg_size; ++i) + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { AggregateDataPtr aggregate_data = nullptr; @@ -899,6 +961,153 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( } } +// Emplace key into StringHashMap/TwoLevelStringHashMap is seperated from other situations, +// because it's easy to implement prefetch submap directly. +// TODO not support resize execption +template +ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + AggProcessInfo & agg_process_info) const +{ + // collect_hit_rate and only_lookup cannot be true at the same time. + static_assert(!(collect_hit_rate && only_lookup)); + static_assert(Method::Data::isStringHashMap); + + std::vector sort_key_containers; + sort_key_containers.resize(params.keys_size, ""); + + const size_t rows = agg_process_info.end_row = agg_process_info.start_row; + RUNTIME_CHECK_MSG(rows == state.total_rows, "executeImplBatchStringHashMap only handle resize exception for each Block instead of row"); + const size_t reserve_size = rows / 4; + + std::vector key0_infos; + std::vector key0_datas; + key0_infos.reserve(reserve_size); + key0_datas.reserve(reserve_size); + + std::vector key8_infos; + std::vector key8_datas; + key8_infos.reserve(reserve_size); + key8_datas.reserve(reserve_size); + + std::vector key16_infos; + std::vector key16_datas; + key16_infos.reserve(reserve_size); + key16_datas.reserve(reserve_size); + + std::vector key24_infos; + std::vector key24_datas; + key24_infos.reserve(reserve_size); + key24_datas.reserve(reserve_size); + + std::vector key_str_infos; + std::vector key_str_datas; + key_str_infos.reserve(reserve_size); + key_str_datas.reserve(reserve_size); + + auto dispatch_callback_key0 = [&key0_infos, &key0_datas](const StringRef & key, size_t row) { + key0_infos.push_back(row); + key0_datas.push_back(key); + }; + auto dispatch_callback_key8 = [&key8_infos, &key8_datas](const StringKey8 & key, size_t row) { + key8_infos.push_back(row); + key8_datas.push_back(key); + }; + auto dispatch_callback_key16 = [&key16_infos, &key16_datas](const StringKey16 & key, size_t row) { + key16_infos.push_back(row); + key16_datas.push_back(key); + }; + auto dispatch_callback_key24 = [&key24_infos, &key24_datas](const StringKey24 & key, size_t row) { + key24_infos.push_back(row); + key24_datas.push_back(key); + }; + // Argument type is ArenaKeyHolder instead of StringRef, + // because it will only be persisted when insert into HashTable. + auto dispatch_callback_key_str = [&key_str_infos, &key_str_datas](const ArenaKeyHolder & key, size_t row) { + key_str_infos.push_back(row); + key_str_datas.push_back(key); + }; + for (size_t i = 0; i < rows; ++i) + { + auto key_holder = state.getKeyHolder(i, aggregates_pool, sort_key_containers); + dispatchStringHashTable(key_holder, + dispatch_callback_key0, + dispatch_callback_key8, + dispatch_callback_key16, + dispatch_callback_key24, + dispatch_callback_key_str); + } + + std::vector key0_places; + key0_places.reserve(key0_infos.size()); + + std::vector key8_places; + key8_places.reserve(key8_infos.size()); + + std::vector key16_places; + key16_places.reserve(key16_infos.size()); + + std::vector key24_places; + key24_places.reserve(key24_infos.size()); + + std::vector key_str_places; + key_str_places.reserve(key_str_infos.size()); + + if (!key0_infos.empty()) + { + emplaceOrFindStringKey<0, false>(method.data, state, key0_infos, key0_datas, aggregates_pool, key0_places, agg_process_info); + } + +#define M(INDEX, INFO, DATA, PLACES) \ + if (!(INFO).empty()) \ + { \ + if constexpr (enable_prefetch) \ + emplaceOrFindStringKey(method.data, state, INFO, DATA, aggregates_pool, PLACES, agg_process_info); \ + else \ + emplaceOrFindStringKey(method.data, state, INFO, DATA, aggregates_pool, PLACES, agg_process_info); \ + } + + M(1, key8_infos, key8_datas, key8_places) + M(2, key16_infos, key16_datas, key16_places) + M(3, key24_infos, key24_datas, key24_places) + M(4, key_str_infos, key_str_datas, key_str_places) +#undef M + + RUNTIME_CHECK(rows == key0_places.size() + key8_places.size() + key16_places.size() + key24_places.size() + key_str_places.size()); + + std::vector places(rows, nullptr); + +#define M(INFO, PLACES) \ + for (size_t i = 0; i < (INFO).size(); ++i) \ + { \ + const auto row = (INFO)[i]; \ + places[row] = (PLACES)[i]; \ + } + + M(key0_infos, key0_places) + M(key8_infos, key8_places) + M(key16_infos, key16_places) + M(key24_infos, key24_places) + M(key_str_infos, key_str_places) +#undef M + + + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + ++inst) + { + inst->batch_that->addBatch( + agg_process_info.start_row, + rows, + &places[0], + inst->state_offset, + inst->batch_arguments, + aggregates_pool); + } + agg_process_info.start_row = rows; +} + void NO_INLINE Aggregator::executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 0f1365694ac..6cbbde71b41 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1461,6 +1461,13 @@ class Aggregator Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; + template + void executeImplBatchStringHashMap( + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + AggProcessInfo & agg_process_info) const; + template std::optional::ResultType> emplaceOrFindKey( Method & method, @@ -1470,6 +1477,16 @@ class Aggregator std::vector & sort_key_containers, const std::vector & hashvals) const; + template + bool emplaceOrFindStringKey( + typename Method::Data & data, + typename Method::State & state, + const std::vector & key_infos, + const std::vector & key_datas, + Arena & aggregates_pool, + std::vector & places, + AggProcessInfo & agg_process_info) const; + /// For case when there are no keys (all aggregate into one row). static void executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena); diff --git a/libs/libcommon/include/common/StringRef.h b/libs/libcommon/include/common/StringRef.h index a87b54a7670..bf1ab026a49 100644 --- a/libs/libcommon/include/common/StringRef.h +++ b/libs/libcommon/include/common/StringRef.h @@ -180,7 +180,7 @@ inline size_t hashLessThan8(const char * data, size_t size) struct CRC32Hash { - size_t operator()(StringRef x) const + static size_t operator()(const StringRef & x) { const char * pos = x.data; size_t size = x.size; From ec6e89231d74bee245cecd6b46f1254bc45b28fe Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 27 Nov 2024 15:21:31 +0800 Subject: [PATCH 04/61] handle resize exception done Signed-off-by: guo-shaoge --- .../AggregateFunctionGroupUniqArray.h | 6 +- .../src/AggregateFunctions/KeyHolderHelpers.h | 2 +- dbms/src/Common/ColumnsHashing.h | 6 +- dbms/src/Common/ColumnsHashingImpl.h | 32 +- dbms/src/Common/HashTable/Hash.h | 40 +- dbms/src/Common/HashTable/HashTable.h | 2 +- .../src/Common/HashTable/HashTableKeyHolder.h | 8 +- dbms/src/Common/HashTable/StringHashMap.h | 31 +- dbms/src/Common/HashTable/StringHashTable.h | 190 +++++----- .../HashTable/TwoLevelStringHashTable.h | 12 +- dbms/src/Interpreters/Aggregator.cpp | 356 +++++++++++------- dbms/src/Interpreters/Aggregator.h | 52 ++- 12 files changed, 414 insertions(+), 323 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index 06dd57edf66..d3cbea74195 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -182,18 +182,18 @@ class AggregateFunctionGroupUniqArrayGeneric { // We have to copy the keys to our arena. assert(arena != nullptr); - cur_set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted); + cur_set.emplace(ArenaKeyHolder{rhs_elem.getValue(), arena}, it, inserted); } } void insertResultInto(ConstAggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - ColumnArray & arr_to = assert_cast(to); + auto & arr_to = assert_cast(to); ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); IColumn & data_to = arr_to.getData(); auto & set = this->data(place).value; - offsets_to.push_back((offsets_to.size() == 0 ? 0 : offsets_to.back()) + set.size()); + offsets_to.push_back((offsets_to.empty() ? 0 : offsets_to.back()) + set.size()); for (auto & elem : set) deserializeAndInsert(elem.getValue(), data_to); diff --git a/dbms/src/AggregateFunctions/KeyHolderHelpers.h b/dbms/src/AggregateFunctions/KeyHolderHelpers.h index 6677866f0d3..b8a4ee0def3 100644 --- a/dbms/src/AggregateFunctions/KeyHolderHelpers.h +++ b/dbms/src/AggregateFunctions/KeyHolderHelpers.h @@ -24,7 +24,7 @@ inline auto getKeyHolder(const IColumn & column, size_t row_num, Arena & arena) { if constexpr (is_plain_column) { - return ArenaKeyHolder{column.getDataAt(row_num), arena}; + return ArenaKeyHolder{column.getDataAt(row_num), &arena}; } else { diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index e14a793567c..aabe0733f8c 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -135,7 +135,7 @@ struct HashMethodString { if (likely(collator)) key = collator->sortKey(key.data, key.size, sort_key_containers[0]); - return ArenaKeyHolder{key, *pool}; + return ArenaKeyHolder{key, pool}; } else { @@ -172,7 +172,7 @@ struct HashMethodStringBin auto last_offset = row == 0 ? 0 : offsets[row - 1]; StringRef key(chars + last_offset, offsets[row] - last_offset - 1); key = BinCollatorSortKey(key.data, key.size); - return ArenaKeyHolder{key, *pool}; + return ArenaKeyHolder{key, pool}; } protected: @@ -425,7 +425,7 @@ struct HashMethodFixedString if constexpr (place_string_to_arena) { - return ArenaKeyHolder{key, *pool}; + return ArenaKeyHolder{key, pool}; } else { diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 0c8d0bc1a49..aa583f1a722 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -16,8 +16,8 @@ #include #include -#include #include +#include #include #include #include @@ -174,17 +174,19 @@ class HashMethodBase { return findKeyImpl(keyHolderGetKey(key_holder), data, 0); } - } + // TODO emplaceStringKey merge with emplaceKey? template ALWAYS_INLINE inline EmplaceResult emplaceStringKey( - Data & data, - size_t idx, - const std::vector & datas, - const std::vector & hashvals) + Data & data, + size_t idx, + std::vector & datas, // TODO const + const std::vector & hashvals) { - auto & submap = typename StringHashTableSubMapSelector>::getSubMap(data); + auto & submap = StringHashTableSubMapSelector>::getSubMap( + hashvals[idx], + data); if constexpr (enable_prefetch) { const auto prefetch_idx = idx + prefetch_step; @@ -198,12 +200,14 @@ class HashMethodBase // TODO Macro with emplaceStringKey template ALWAYS_INLINE inline FindResult findStringKey( - Data & data, - size_t idx, - const std::vector & datas, - const std::vector & hashvals) + Data & data, + size_t idx, + std::vector & datas, // TODO const + const std::vector & hashvals) { - auto & submap = typename StringHashTableSubMapSelector>::getSubMap(data); + auto & submap = StringHashTableSubMapSelector>::getSubMap( + hashvals[idx], + data); if constexpr (enable_prefetch) { const auto prefetch_idx = idx + prefetch_step; @@ -211,7 +215,7 @@ class HashMethodBase submap.prefetch(hashvals[prefetch_idx]); } - return findKeyImpl(datas[idx], submap, hashvals[idx]); + return findKeyImpl(keyHolderGetKey(datas[idx]), submap, hashvals[idx]); } template @@ -301,7 +305,7 @@ class HashMethodBase } template - ALWAYS_INLINE inline FindResult findKeyImpl(Key key, Data & data, size_t hashval) + ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data, size_t hashval) { if constexpr (Cache::consecutive_keys_optimization) { diff --git a/dbms/src/Common/HashTable/Hash.h b/dbms/src/Common/HashTable/Hash.h index 883ec8ab6ff..207919a347e 100644 --- a/dbms/src/Common/HashTable/Hash.h +++ b/dbms/src/Common/HashTable/Hash.h @@ -130,8 +130,8 @@ inline DB::UInt64 wideIntHashCRC32(const T & x, DB::UInt64 updated_value) return updated_value; } static_assert( - DB::IsDecimal< - T> || is_boost_number_v || std::is_same_v || std::is_same_v || std::is_same_v); + DB::IsDecimal || is_boost_number_v || std::is_same_v || std::is_same_v + || std::is_same_v); __builtin_unreachable(); } @@ -244,8 +244,8 @@ inline size_t defaultHash64(const std::enable_if_t, T> & key return boost::multiprecision::hash_value(key); } static_assert( - is_boost_number_v< - T> || std::is_same_v || std::is_same_v || std::is_same_v); + is_boost_number_v || std::is_same_v || std::is_same_v + || std::is_same_v); __builtin_unreachable(); } @@ -297,20 +297,26 @@ inline size_t hashCRC32(const std::enable_if_t, T> & key) template struct HashCRC32; -#define DEFINE_HASH(T) \ - template <> \ - struct HashCRC32 \ - { \ - static_assert(is_fit_register); \ - size_t operator()(T key) const { return hashCRC32(key); } \ +#define DEFINE_HASH(T) \ + template <> \ + struct HashCRC32 \ + { \ + static_assert(is_fit_register); \ + size_t operator()(T key) const \ + { \ + return hashCRC32(key); \ + } \ }; -#define DEFINE_HASH_WIDE(T) \ - template <> \ - struct HashCRC32 \ - { \ - static_assert(!is_fit_register); \ - size_t operator()(const T & key) const { return hashCRC32(key); } \ +#define DEFINE_HASH_WIDE(T) \ + template <> \ + struct HashCRC32 \ + { \ + static_assert(!is_fit_register); \ + size_t operator()(const T & key) const \ + { \ + return hashCRC32(key); \ + } \ }; DEFINE_HASH(DB::UInt8) @@ -535,7 +541,7 @@ struct HashWithMixSeed template <> struct HashWithMixSeed -{ +{ static inline size_t operator()(const DB::UInt256 & v) { return HashWithMixSeedHelper::operator()(hash_uint256(0, v)); diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index 12ebc49756c..f8d44e8c406 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -859,7 +859,7 @@ class HashTable (void)hashval; #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) size_t place_value = grower.place(hashval); - __mm_prefetch((const char*)(&buf[place_value]), _MM_HINT_NTA); + __mm_prefetch((const char *)(&buf[place_value]), _MM_HINT_NTA); #elif defined(__GNUC__) size_t place_value = grower.place(hashval); __builtin_prefetch(static_cast(&buf[place_value])); diff --git a/dbms/src/Common/HashTable/HashTableKeyHolder.h b/dbms/src/Common/HashTable/HashTableKeyHolder.h index 01b06dce87d..dd8a4b53376 100644 --- a/dbms/src/Common/HashTable/HashTableKeyHolder.h +++ b/dbms/src/Common/HashTable/HashTableKeyHolder.h @@ -91,8 +91,8 @@ namespace DB */ struct ArenaKeyHolder { - StringRef key; - Arena & pool; + StringRef key{}; + Arena * pool = nullptr; }; } // namespace DB @@ -111,14 +111,14 @@ inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder & holder) { // Hash table shouldn't ask us to persist a zero key assert(holder.key.size > 0); - holder.key.data = holder.pool.insert(holder.key.data, holder.key.size); + holder.key.data = holder.pool->insert(holder.key.data, holder.key.size); } inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder && holder) { // Hash table shouldn't ask us to persist a zero key assert(holder.key.size > 0); - holder.key.data = holder.pool.insert(holder.key.data, holder.key.size); + holder.key.data = holder.pool->insert(holder.key.data, holder.key.size); } inline void ALWAYS_INLINE keyHolderDiscardKey(DB::ArenaKeyHolder &) {} diff --git a/dbms/src/Common/HashTable/StringHashMap.h b/dbms/src/Common/HashTable/StringHashMap.h index cad653907fa..a070f0ef0a9 100644 --- a/dbms/src/Common/HashTable/StringHashMap.h +++ b/dbms/src/Common/HashTable/StringHashMap.h @@ -92,30 +92,13 @@ struct StringHashMapSubMaps { using Hash = StringHashTableHash; using T0 = StringHashTableEmpty>; - using T1 = HashMapTable< - StringKey8, - StringHashMapCell, - Hash, - StringHashTableGrower<>, - Allocator>; - using T2 = HashMapTable< - StringKey16, - StringHashMapCell, - Hash, - StringHashTableGrower<>, - Allocator>; - using T3 = HashMapTable< - StringKey24, - StringHashMapCell, - Hash, - StringHashTableGrower<>, - Allocator>; - using Ts = HashMapTable< - StringRef, - StringHashMapCell, - Hash, - StringHashTableGrower<>, - Allocator>; + using T1 + = HashMapTable, Hash, StringHashTableGrower<>, Allocator>; + using T2 + = HashMapTable, Hash, StringHashTableGrower<>, Allocator>; + using T3 + = HashMapTable, Hash, StringHashTableGrower<>, Allocator>; + using Ts = HashMapTable, Hash, StringHashTableGrower<>, Allocator>; }; template diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index f906b043a9e..a511ce47671 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -72,18 +72,9 @@ struct StringHashTableHash using StringKey24Hasher = HashWithMixSeed; using StringRefHasher = StringRefHash; - static size_t ALWAYS_INLINE operator()(StringKey8 key) - { - return StringKey8Hasher::operator()(key); - } - static size_t ALWAYS_INLINE operator()(const StringKey16 & key) - { - return StringKey16Hasher::operator()(key); - } - static size_t ALWAYS_INLINE operator()(const StringKey24 & key) - { - return StringKey24Hasher::operator()(key); - } + static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } }; @@ -191,97 +182,106 @@ struct StringHashTableLookupResult friend bool operator!=(const std::nullptr_t &, const StringHashTableLookupResult & b) { return b.mapped_ptr; } }; - template - static auto +template +static auto #if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) - NO_INLINE NO_SANITIZE_ADDRESS NO_SANITIZE_THREAD + NO_INLINE NO_SANITIZE_ADDRESS NO_SANITIZE_THREAD #else - ALWAYS_INLINE + ALWAYS_INLINE #endif - dispatchStringHashTable(size_t row, KeyHolder && key_holder, Func0 && func0, Func8 && func8, Func16 && func16, Func24 && func24, FuncStr && func_str) + dispatchStringHashTable( + size_t row, + KeyHolder && key_holder, + Func0 && func0, + Func8 && func8, + Func16 && func16, + Func24 && func24, + FuncStr && func_str) +{ + const StringRef & x = keyHolderGetKey(key_holder); + const size_t sz = x.size; + if (sz == 0) { - const StringRef & x = keyHolderGetKey(key_holder); - const size_t sz = x.size; - if (sz == 0) - { - return func0(x, row); - } + return func0(x, row); + } - if (x.data[sz - 1] == 0) - { - // Strings with trailing zeros are not representable as fixed-size - // string keys. Put them to the generic table. - return func_str(key_holder, row); - } + if (x.data[sz - 1] == 0) + { + // Strings with trailing zeros are not representable as fixed-size + // string keys. Put them to the generic table. + return func_str(key_holder, row); + } - const char * p = x.data; - // pending bits that needs to be shifted out - const char s = (-sz & 7) * 8; - union - { - StringKey8 k8; - StringKey16 k16; - StringKey24 k24; - UInt64 n[3]; - }; - switch ((sz - 1) >> 3) - { - case 0: // 1..8 bytes - { - // first half page - if ((reinterpret_cast(p) & 2048) == 0) - { - memcpy(&n[0], p, 8); - if constexpr (DB::isLittleEndian()) - n[0] &= (-1ULL >> s); - else - n[0] &= (-1ULL << s); - } - else - { - const char * lp = x.data + x.size - 8; - memcpy(&n[0], lp, 8); - if constexpr (DB::isLittleEndian()) - n[0] >>= s; - else - n[0] <<= s; - } - return func8(k8, row); - } - case 1: // 9..16 bytes + const char * p = x.data; + // pending bits that needs to be shifted out + const char s = (-sz & 7) * 8; + union + { + StringKey8 k8; + StringKey16 k16; + StringKey24 k24; + UInt64 n[3]; + }; + switch ((sz - 1) >> 3) + { + case 0: // 1..8 bytes + { + // first half page + if ((reinterpret_cast(p) & 2048) == 0) { memcpy(&n[0], p, 8); - const char * lp = x.data + x.size - 8; - memcpy(&n[1], lp, 8); if constexpr (DB::isLittleEndian()) - n[1] >>= s; + n[0] &= (-1ULL >> s); else - n[1] <<= s; - return func16(k16, row); + n[0] &= (-1ULL << s); } - case 2: // 17..24 bytes + else { - memcpy(&n[0], p, 16); const char * lp = x.data + x.size - 8; - memcpy(&n[2], lp, 8); + memcpy(&n[0], lp, 8); if constexpr (DB::isLittleEndian()) - n[2] >>= s; + n[0] >>= s; else - n[2] <<= s; - return func24(k24, row); - } - default: // >= 25 bytes - { - return func_str(key_holder, row); - } + n[0] <<= s; } + return func8(k8, row); + } + case 1: // 9..16 bytes + { + memcpy(&n[0], p, 8); + const char * lp = x.data + x.size - 8; + memcpy(&n[1], lp, 8); + if constexpr (DB::isLittleEndian()) + n[1] >>= s; + else + n[1] <<= s; + return func16(k16, row); + } + case 2: // 17..24 bytes + { + memcpy(&n[0], p, 16); + const char * lp = x.data + x.size - 8; + memcpy(&n[2], lp, 8); + if constexpr (DB::isLittleEndian()) + n[2] >>= s; + else + n[2] <<= s; + return func24(k24, row); + } + default: // >= 25 bytes + { + return func_str(key_holder, row); } + } +} template class StringHashTable : private boost::noncopyable { protected: static constexpr size_t NUM_MAPS = 5; + using Self = StringHashTable; + // Map for storing empty string using T0 = typename SubMaps::T0; @@ -292,10 +292,11 @@ class StringHashTable : private boost::noncopyable // Long strings are stored as StringRef along with saved hash using Ts = typename SubMaps::Ts; - using Self = StringHashTable; template friend class TwoLevelStringHashTable; + template + friend struct StringHashTableSubMapSelector; T0 m0; T1 m1; @@ -592,13 +593,10 @@ struct StringHashTableSubMapSelector<0, false, Data> { struct Hash { - static ALWAYS_INLINE size_t operator()(const StringRef & ) { return 0; } + static ALWAYS_INLINE size_t operator()(const StringRef &) { return 0; } }; - typename Data::T0 & getSubMap(size_t, Data & data) - { - return data.m0; - } + static typename Data::T0 & getSubMap(size_t, Data & data) { return data.m0; } }; template @@ -606,10 +604,7 @@ struct StringHashTableSubMapSelector<1, false, Data> { using Hash = StringHashTableHash::StringKey8Hasher; - typename Data::T1 & getSubMap(size_t, Data & data) - { - return data.m1; - } + static typename Data::T1 & getSubMap(size_t, Data & data) { return data.m1; } }; template @@ -617,10 +612,7 @@ struct StringHashTableSubMapSelector<2, false, Data> { using Hash = StringHashTableHash::StringKey16Hasher; - typename Data::T2 & getSubMap(size_t, Data & data) - { - return data.m2; - } + static typename Data::T2 & getSubMap(size_t, Data & data) { return data.m2; } }; template @@ -628,10 +620,7 @@ struct StringHashTableSubMapSelector<3, false, Data> { using Hash = StringHashTableHash::StringKey24Hasher; - typename Data::T3 & getSubMap(size_t, Data & data) - { - return data.m3; - } + static typename Data::T3 & getSubMap(size_t, Data & data) { return data.m3; } }; template @@ -639,8 +628,5 @@ struct StringHashTableSubMapSelector<4, false, Data> { using Hash = StringHashTableHash::StringRefHasher; - typename Data::Ts & getSubMap(size_t, Data & data) - { - return data.ms; - } + static typename Data::Ts & getSubMap(size_t, Data & data) { return data.ms; } }; diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index d217e0c0260..e7ea1bb8fce 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -310,10 +310,10 @@ struct StringHashTableSubMapSelector<0, true, Data> { struct Hash { - static ALWAYS_INLINE size_t operator()(const StringRef & ) { return 0; } + static ALWAYS_INLINE size_t operator()(const StringRef &) { return 0; } }; - typename Data::T0 & getSubMap(size_t hashval, Data & data) + static typename Data::Impl::T0 & getSubMap(size_t hashval, Data & data) { const auto bucket = Data::getBucketFromHash(hashval); return data.impls[bucket].m0; @@ -325,7 +325,7 @@ struct StringHashTableSubMapSelector<1, true, Data> { using Hash = StringHashTableHash::StringKey8Hasher; - typename Data::T1 & getSubMap(size_t hashval, Data & data) + static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) { const auto bucket = Data::getBucketFromHash(hashval); return data.impls[bucket].m1; @@ -337,7 +337,7 @@ struct StringHashTableSubMapSelector<2, true, Data> { using Hash = StringHashTableHash::StringKey16Hasher; - typename Data::T2 & getSubMap(size_t hashval, Data & data) + static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) { const auto bucket = Data::getBucketFromHash(hashval); return data.impls[bucket].m2; @@ -349,7 +349,7 @@ struct StringHashTableSubMapSelector<3, true, Data> { using Hash = StringHashTableHash::StringKey24Hasher; - typename Data::T3 & getSubMap(size_t hashval, Data & data) + static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) { const auto bucket = Data::getBucketFromHash(hashval); return data.impls[bucket].m3; @@ -361,7 +361,7 @@ struct StringHashTableSubMapSelector<4, true, Data> { using Hash = StringHashTableHash::StringRefHasher; - typename Data::Ts & getSubMap(size_t hashval, Data & data) + static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) { const auto bucket = Data::getBucketFromHash(hashval); return data.impls[bucket].ms; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 54cf52c673d..4faec37ce9d 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -666,14 +666,37 @@ void NO_INLINE Aggregator::executeImpl( typename Method::State state(agg_process_info.key_columns, key_sizes, collators); if (method.data.getBufferSizeInCells() < 8192) - executeImplBatch(method, state, aggregates_pool, agg_process_info); + { + if constexpr (Method::Data::is_string_hash_map) + executeImplBatchStringHashMap( + method, + state, + aggregates_pool, + agg_process_info); + else + executeImplBatch(method, state, aggregates_pool, agg_process_info); + } else - executeImplBatch(method, state, aggregates_pool, agg_process_info); + { + if constexpr (Method::Data::is_string_hash_map) + executeImplBatchStringHashMap( + method, + state, + aggregates_pool, + agg_process_info); + else + executeImplBatch(method, state, aggregates_pool, agg_process_info); + } } template -std::vector getHashVals(size_t start_row, size_t end_row, const Data & data, const State & state, - std::vector & sort_key_containers, Arena * pool) +std::vector getHashVals( + size_t start_row, + size_t end_row, + const Data & data, + const State & state, + std::vector & sort_key_containers, + Arena * pool) { std::vector hashvals(state.total_rows, 0); for (size_t i = start_row; i < end_row; ++i) @@ -695,9 +718,15 @@ std::optional::Res try { if constexpr (only_lookup) - return state.template findKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); + return state + .template findKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); else - return state.template emplaceKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); + return state.template emplaceKey( + method.data, + index, + aggregates_pool, + sort_key_containers, + hashvals); } catch (ResizeException &) { @@ -707,19 +736,27 @@ std::optional::Res // StringKeyType can be StringRef/StringKey8/StringKey16/StringKey24/ArenaKeyHolder. // return true when resize exception happens. -template -bool Aggregator::emplaceOrFindStringKey( - typename Method::Data & data, - typename Method::State & state, - const std::vector & key_infos, - const std::vector & key_datas, - Arena & aggregates_pool, - std::vector & places, - AggProcessInfo & agg_process_info) const +template < + size_t SubMapIndex, + bool collect_hit_rate, + bool only_lookup, + bool enable_prefetch, + typename Data, + typename State, + typename StringKeyType> +size_t Aggregator::emplaceOrFindStringKey( + Data & data, + State & state, + const std::vector & key_infos, + std::vector & key_datas, // TODO const + Arena & aggregates_pool, + std::vector & places, + AggProcessInfo & agg_process_info) const { + static_assert(!(collect_hit_rate && only_lookup)); RUNTIME_CHECK(key_infos.size() == key_datas.size()); - using Hash = typename StringHashTableSubMapSelector>::Hash; + using Hash = typename StringHashTableSubMapSelector>::Hash; std::vector hashvals(key_infos.size(), 0); for (size_t i = 0; i < key_infos.size(); ++i) { @@ -733,7 +770,8 @@ bool Aggregator::emplaceOrFindStringKey( { if constexpr (only_lookup) { - auto find_result = state.template findStringKey(data, i, key_datas, hashvals); + auto find_result + = state.template findStringKey(data, i, key_datas, hashvals); if (find_result.isFound()) { agg_state = find_result.getMapped(); @@ -745,7 +783,8 @@ bool Aggregator::emplaceOrFindStringKey( } else { - auto emplace_result = state.template emplaceStringKey(data, i, key_datas, hashvals); + auto emplace_result + = state.template emplaceStringKey(data, i, key_datas, hashvals); if (emplace_result.isInserted()) { emplace_result.setMapped(nullptr); @@ -758,18 +797,19 @@ bool Aggregator::emplaceOrFindStringKey( else { agg_state = emplace_result.getMapped(); + + if constexpr (collect_hit_rate) + ++agg_process_info.hit_row_cnt; } - places.push_back(agg_state); + places[i] = agg_state; } } catch (ResizeException &) { - // agg_process_info.set - // TODO handle exception - return true; + return i; } } - return false; + return key_infos.size(); } template @@ -799,13 +839,12 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( if constexpr (enable_prefetch) { hashvals = getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool); - + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool); } for (size_t i = 0; i < rows; ++i) @@ -884,20 +923,25 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( if constexpr (enable_prefetch) { hashvals = getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool); + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool); } for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { AggregateDataPtr aggregate_data = nullptr; - auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); + auto emplace_result_holder = emplaceOrFindKey( + method, + state, + i, + *aggregates_pool, + sort_key_containers, + hashvals); if unlikely (!emplace_result_holder.has_value()) { LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); @@ -961,129 +1005,174 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( } } +#define M(SUBMAPINDEX) \ + template \ + void setupExceptionRecoveryInfoForStringHashTable( \ + Aggregator::AggProcessInfo & agg_process_info, \ + size_t row, \ + const std::vector & key_infos, \ + const std::vector & key_datas, \ + std::integral_constant) \ + { \ + agg_process_info.submap_m##SUBMAPINDEX##_infos \ + = std::vector(key_infos.begin() + row, key_infos.end()); \ + agg_process_info.submap_m##SUBMAPINDEX##_datas \ + = std::vector(key_datas.begin() + row, key_datas.end()); \ + } + +M(0) +M(1) +M(2) +M(3) +M(4) + +#undef M + // Emplace key into StringHashMap/TwoLevelStringHashMap is seperated from other situations, // because it's easy to implement prefetch submap directly. -// TODO not support resize execption template ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - AggProcessInfo & agg_process_info) const + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + AggProcessInfo & agg_process_info) const { // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); - static_assert(Method::Data::isStringHashMap); + static_assert(Method::Data::is_string_hash_map); + +#define M(SUBMAPINDEX) \ + RUNTIME_CHECK( \ + agg_process_info.submap_m##SUBMAPINDEX##_infos.size() \ + == agg_process_info.submap_m##SUBMAPINDEX##_datas.size()); + + M(0) + M(1) + M(2) + M(3) + M(4) +#undef M std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); - const size_t rows = agg_process_info.end_row = agg_process_info.start_row; - RUNTIME_CHECK_MSG(rows == state.total_rows, "executeImplBatchStringHashMap only handle resize exception for each Block instead of row"); - const size_t reserve_size = rows / 4; - - std::vector key0_infos; - std::vector key0_datas; - key0_infos.reserve(reserve_size); - key0_datas.reserve(reserve_size); - - std::vector key8_infos; - std::vector key8_datas; - key8_infos.reserve(reserve_size); - key8_datas.reserve(reserve_size); - - std::vector key16_infos; - std::vector key16_datas; - key16_infos.reserve(reserve_size); - key16_datas.reserve(reserve_size); - - std::vector key24_infos; - std::vector key24_datas; - key24_infos.reserve(reserve_size); - key24_datas.reserve(reserve_size); - - std::vector key_str_infos; - std::vector key_str_datas; - key_str_infos.reserve(reserve_size); - key_str_datas.reserve(reserve_size); - - auto dispatch_callback_key0 = [&key0_infos, &key0_datas](const StringRef & key, size_t row) { - key0_infos.push_back(row); - key0_datas.push_back(key); - }; - auto dispatch_callback_key8 = [&key8_infos, &key8_datas](const StringKey8 & key, size_t row) { - key8_infos.push_back(row); - key8_datas.push_back(key); - }; - auto dispatch_callback_key16 = [&key16_infos, &key16_datas](const StringKey16 & key, size_t row) { - key16_infos.push_back(row); - key16_datas.push_back(key); - }; - auto dispatch_callback_key24 = [&key24_infos, &key24_datas](const StringKey24 & key, size_t row) { - key24_infos.push_back(row); - key24_datas.push_back(key); - }; - // Argument type is ArenaKeyHolder instead of StringRef, - // because it will only be persisted when insert into HashTable. - auto dispatch_callback_key_str = [&key_str_infos, &key_str_datas](const ArenaKeyHolder & key, size_t row) { - key_str_infos.push_back(row); - key_str_datas.push_back(key); - }; - for (size_t i = 0; i < rows; ++i) +#define M(INFO, DATA, KEYTYPE) \ + std::vector(INFO); \ + std::vector(DATA); + + M(key0_infos, key0_datas, StringRef) + M(key8_infos, key8_datas, StringKey8) + M(key16_infos, key16_datas, StringKey16) + M(key24_infos, key24_datas, StringKey24) + M(key_str_infos, key_str_datas, ArenaKeyHolder) +#undef M + + const size_t rows = agg_process_info.end_row - agg_process_info.start_row; + + if likely (agg_process_info.allBlockDataHandled()) { - auto key_holder = state.getKeyHolder(i, aggregates_pool, sort_key_containers); - dispatchStringHashTable(key_holder, + // No resize exception happens, so this is a new Block. + RUNTIME_CHECK(agg_process_info.start_row == 0); + RUNTIME_CHECK_MSG( + rows == state.total_rows, + "executeImplBatchStringHashMap only handle resize exception for each Block instead of row"); + const size_t reserve_size = rows / 4; + +#define M(INFO, DATA, SUBMAPINDEX, KEYTYPE) \ + (INFO).reserve(reserve_size); \ + (DATA).reserve(reserve_size); \ + auto dispatch_callback_key##SUBMAPINDEX = [&INFO, &DATA](const KEYTYPE & key, size_t row) { \ + (INFO).push_back(row); \ + (DATA).push_back(key); \ + }; + + M(key0_infos, key0_datas, 0, StringRef) + M(key8_infos, key8_datas, 8, StringKey8) + M(key16_infos, key16_datas, 16, StringKey16) + M(key24_infos, key24_datas, 24, StringKey24) + // Argument type is ArenaKeyHolder instead of StringRef, + // because it will only be persisted when insert into HashTable. + M(key_str_infos, key_str_datas, str, ArenaKeyHolder) +#undef M + + for (size_t i = 0; i < rows; ++i) + { + auto key_holder = state.getKeyHolder(i, aggregates_pool, sort_key_containers); + dispatchStringHashTable( + i, + key_holder, dispatch_callback_key0, dispatch_callback_key8, dispatch_callback_key16, dispatch_callback_key24, - dispatch_callback_key_str); + dispatch_callback_keystr); + } } - - std::vector key0_places; - key0_places.reserve(key0_infos.size()); - - std::vector key8_places; - key8_places.reserve(key8_infos.size()); - - std::vector key16_places; - key16_places.reserve(key16_infos.size()); - - std::vector key24_places; - key24_places.reserve(key24_infos.size()); - - std::vector key_str_places; - key_str_places.reserve(key_str_infos.size()); - - if (!key0_infos.empty()) + else { - emplaceOrFindStringKey<0, false>(method.data, state, key0_infos, key0_datas, aggregates_pool, key0_places, agg_process_info); - } +#define M(INFO, DATA, SUBMAPINDEX) \ + (INFO) = agg_process_info.submap_m##SUBMAPINDEX##_infos; \ + (DATA) = agg_process_info.submap_m##SUBMAPINDEX##_datas; -#define M(INDEX, INFO, DATA, PLACES) \ - if (!(INFO).empty()) \ - { \ - if constexpr (enable_prefetch) \ - emplaceOrFindStringKey(method.data, state, INFO, DATA, aggregates_pool, PLACES, agg_process_info); \ - else \ - emplaceOrFindStringKey(method.data, state, INFO, DATA, aggregates_pool, PLACES, agg_process_info); \ + M(key0_infos, key0_datas, 0) + M(key8_infos, key8_datas, 1) + M(key16_infos, key16_datas, 2) + M(key24_infos, key24_datas, 3) + M(key_str_infos, key_str_datas, 4) +#undef M } + std::vector key0_places(key0_infos.size(), nullptr); + std::vector key8_places(key8_infos.size(), nullptr); + std::vector key16_places(key16_infos.size(), nullptr); + std::vector key24_places(key24_infos.size(), nullptr); + std::vector key_str_places(key_str_infos.size(), nullptr); + + bool got_resize_exception = false; + size_t emplaced_index = 0; + +#define M(INDEX, INFO, DATA, PLACES) \ + if unlikely (got_resize_exception) \ + { \ + emplaced_index = 0; \ + } \ + else if (!(INFO).empty()) \ + { \ + emplaced_index = emplaceOrFindStringKey( \ + method.data, \ + state, \ + (INFO), \ + (DATA), \ + *aggregates_pool, \ + (PLACES), \ + agg_process_info); \ + if unlikely (emplaced_index != (INFO).size()) \ + got_resize_exception = true; \ + } \ + setupExceptionRecoveryInfoForStringHashTable( \ + agg_process_info, \ + emplaced_index, \ + INFO, \ + DATA, \ + std::integral_constant{}); + + M(0, key0_infos, key0_datas, key0_places) M(1, key8_infos, key8_datas, key8_places) M(2, key16_infos, key16_datas, key16_places) M(3, key24_infos, key24_datas, key24_places) M(4, key_str_infos, key_str_datas, key_str_places) #undef M - RUNTIME_CHECK(rows == key0_places.size() + key8_places.size() + key16_places.size() + key24_places.size() + key_str_places.size()); + RUNTIME_CHECK( + rows + == key0_places.size() + key8_places.size() + key16_places.size() + key24_places.size() + key_str_places.size()); std::vector places(rows, nullptr); - -#define M(INFO, PLACES) \ +#define M(INFO, PLACES) \ for (size_t i = 0; i < (INFO).size(); ++i) \ - { \ - const auto row = (INFO)[i]; \ - places[row] = (PLACES)[i]; \ + { \ + const auto row = (INFO)[i]; \ + places[row] = (PLACES)[i]; \ } M(key0_infos, key0_places) @@ -1093,7 +1182,6 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( M(key_str_infos, key_str_places) #undef M - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; ++inst) { @@ -1105,7 +1193,8 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( inst->batch_arguments, aggregates_pool); } - agg_process_info.start_row = rows; + // For StringHashTable, start_row is meanless, instead submap_mx_infos/submap_mx_datas are used. + agg_process_info.start_row = got_resize_exception ? 0 : rows; } void NO_INLINE @@ -1130,7 +1219,6 @@ Aggregator::executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo agg_process_info.start_row += agg_size; } - void Aggregator::prepareAggregateInstructions( Columns columns, AggregateColumns & aggregate_columns, diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 6cbbde71b41..7142077c8ea 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1319,11 +1319,28 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; + // For StringHashTable resize exception. + std::vector submap_m0_infos{}; + std::vector submap_m1_infos{}; + std::vector submap_m2_infos{}; + std::vector submap_m3_infos{}; + std::vector submap_m4_infos{}; + + std::vector submap_m0_datas{}; + std::vector submap_m1_datas{}; + std::vector submap_m2_datas{}; + std::vector submap_m3_datas{}; + std::vector submap_m4_datas{}; + void prepareForAgg(); bool allBlockDataHandled() const { assert(start_row <= end_row); - return start_row == end_row || aggregator->isCancelled(); + // submap_mx_infos.size() and submap_mx_datas.size() are always equal. + // So only need to check submap_m0_infos is enough. + return (start_row == end_row && !submap_m0_infos.empty() && !submap_m1_infos.empty() + && !submap_m3_infos.empty() && !submap_m4_infos.empty()) + || aggregator->isCancelled(); } void resetBlock(const Block & block_) { @@ -1463,10 +1480,10 @@ class Aggregator template void executeImplBatchStringHashMap( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - AggProcessInfo & agg_process_info) const; + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + AggProcessInfo & agg_process_info) const; template std::optional::ResultType> emplaceOrFindKey( @@ -1477,15 +1494,22 @@ class Aggregator std::vector & sort_key_containers, const std::vector & hashvals) const; - template - bool emplaceOrFindStringKey( - typename Method::Data & data, - typename Method::State & state, - const std::vector & key_infos, - const std::vector & key_datas, - Arena & aggregates_pool, - std::vector & places, - AggProcessInfo & agg_process_info) const; + template < + size_t SubMapIndex, + bool collect_hit_rate, + bool only_lookup, + bool enable_prefetch, + typename Data, + typename State, + typename StringKeyType> + size_t emplaceOrFindStringKey( + Data & data, + State & state, + const std::vector & key_infos, + std::vector & key_datas, + Arena & aggregates_pool, + std::vector & places, + AggProcessInfo & agg_process_info) const; /// For case when there are no keys (all aggregate into one row). static void executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena); From 9dc702dac5e4d8baba90e127607d5e015562fac3 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 27 Nov 2024 22:37:37 +0800 Subject: [PATCH 05/61] tmp save Signed-off-by: guo-shaoge --- dbms/src/Common/Arena.h | 5 ++ dbms/src/Common/ColumnsHashing.h | 80 +++++++++++++++++++------ dbms/src/Interpreters/Aggregator.cpp | 88 ++++++++++++++++++---------- dbms/src/Interpreters/Aggregator.h | 1 + 4 files changed, 126 insertions(+), 48 deletions(-) diff --git a/dbms/src/Common/Arena.h b/dbms/src/Common/Arena.h index b9999f6b179..eb86e1c283c 100644 --- a/dbms/src/Common/Arena.h +++ b/dbms/src/Common/Arena.h @@ -212,5 +212,10 @@ class Arena : private boost::noncopyable using ArenaPtr = std::shared_ptr; using Arenas = std::vector; +size_t alignOf16(size_t l) +{ + return (l + 15) & ~15; +} + } // namespace DB diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index aabe0733f8c..0d8e8d60ef7 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -52,7 +52,7 @@ struct HashMethodOneNumber const size_t total_rows; /// If the keys of a fixed length then key_sizes contains their lengths, empty otherwise. - HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) + HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &, Arena *) : total_rows(key_columns[0]->size()) { vec = &static_cast *>(key_columns[0])->getData()[0]; @@ -107,7 +107,8 @@ struct HashMethodString HashMethodString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, - const TiDB::TiDBCollators & collators) + const TiDB::TiDBCollators & collators, + Arena *) : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; @@ -158,7 +159,7 @@ struct HashMethodStringBin const UInt8 * chars; const size_t total_rows; - HashMethodStringBin(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) + HashMethodStringBin(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &, Arena *) : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; @@ -344,6 +345,43 @@ struct KeyDescStringBinPadding : KeyDescStringBin } }; +void serializeColumnToBuffer(Arena * pool, + const ColumnRawPtrs & key_columns, + PaddedPODArray & pos, + PaddedPODArray & sizes) +{ + RUNTIME_CHECK(!key_columns.empty()); + RUNTIME_CHECK(pos.empty() && sizes.empty()); + + const auto rows = key_columns[0]->size(); + pos.resize(rows, nullptr); + sizes.resize(rows, 0); + + for (const auto * col_ptr : key_columns) + col_ptr->countSerializeByteSize(sizes); + + std::vector aligned_sizes; + aligned_sizes.reserve(sizes.size()); + + size_t total_byte_size = 0; + for (auto size : sizes) + { + auto aligned = alignOf16(size); + total_byte_size += aligned; + aligned_sizes.push_back(aligned); + } + + auto * buffer = pool->alloc(total_byte_size); + for (size_t i = 0; i < aligned_sizes.size(); ++i) + { + pos[i] = buffer; + buffer += aligned_sizes[i]; + } + + for (const auto * col_ptr : key_columns) + col_ptr->serializeToPos(pos, 0, rows, col_ptr->isColumnNullable()); +} + /// For the case when there are 2 keys. template struct HashMethodFastPathTwoKeysSerialized @@ -356,12 +394,16 @@ struct HashMethodFastPathTwoKeysSerialized Key1Desc key_1_desc; Key2Desc key_2_desc; const size_t total_rows; + PaddedPODArray pos; + PaddedPODArray sizes; - HashMethodFastPathTwoKeysSerialized(const ColumnRawPtrs & key_columns, const Sizes &, const TiDB::TiDBCollators &) + HashMethodFastPathTwoKeysSerialized(const ColumnRawPtrs & key_columns, const Sizes &, const TiDB::TiDBCollators &, Arena * pool) : key_1_desc(key_columns[0]) , key_2_desc(key_columns[1]) , total_rows(key_columns[0]->size()) - {} + { + serializeColumnToBuffer(pool, key_columns, pos, sizes); + } ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const { @@ -400,7 +442,8 @@ struct HashMethodFixedString HashMethodFixedString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, - const TiDB::TiDBCollators & collators) + const TiDB::TiDBCollators & collators, + Arena *) : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; @@ -477,7 +520,7 @@ struct HashMethodKeysFixed return true; } - HashMethodKeysFixed(const ColumnRawPtrs & key_columns, const Sizes & key_sizes_, const TiDB::TiDBCollators &) + HashMethodKeysFixed(const ColumnRawPtrs & key_columns, const Sizes & key_sizes_, const TiDB::TiDBCollators &, Arena *) : Base(key_columns) , key_sizes(std::move(key_sizes_)) , keys_size(key_columns.size()) @@ -612,25 +655,28 @@ struct HashMethodSerialized size_t keys_size; TiDB::TiDBCollators collators; const size_t total_rows; + PaddedPODArray pos; + PaddedPODArray sizes; HashMethodSerialized( const ColumnRawPtrs & key_columns_, const Sizes & /*key_sizes*/, - const TiDB::TiDBCollators & collators_) + const TiDB::TiDBCollators & collators_, + Arena * pool) : key_columns(key_columns_) , keys_size(key_columns_.size()) , collators(collators_) , total_rows(key_columns_[0]->size()) - {} + { + serializeColumnToBuffer(pool, key_columns_, pos, sizes); + } - ALWAYS_INLINE inline SerializedKeyHolder getKeyHolder( - size_t row, - Arena * pool, - std::vector & sort_key_containers) const + ALWAYS_INLINE inline StringRef getKeyHolder( + size_t row, + Arena *, + std::vector &) const { - return SerializedKeyHolder{ - serializeKeysToPoolContiguous(row, keys_size, key_columns, collators, sort_key_containers, *pool), - *pool}; + return StringRef(pos[row], sizes[row]); } protected: @@ -650,7 +696,7 @@ struct HashMethodHashed TiDB::TiDBCollators collators; const size_t total_rows; - HashMethodHashed(ColumnRawPtrs key_columns_, const Sizes &, const TiDB::TiDBCollators & collators_) + HashMethodHashed(ColumnRawPtrs key_columns_, const Sizes &, const TiDB::TiDBCollators & collators_, Arena *) : key_columns(std::move(key_columns_)) , collators(collators_) , total_rows(key_columns[0]->size()) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 4faec37ce9d..1738121b665 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -741,6 +741,7 @@ template < bool collect_hit_rate, bool only_lookup, bool enable_prefetch, + bool zero_agg_func_size, typename Data, typename State, typename StringKeyType> @@ -763,7 +764,8 @@ size_t Aggregator::emplaceOrFindStringKey( hashvals[i] = Hash::operator()(keyHolderGetKey(key_datas[0])); } - AggregateDataPtr agg_state = nullptr; + // alloc 0 bytes is useful when agg func size is zero. + AggregateDataPtr agg_state = aggregates_pool.alloc(0); for (size_t i = 0; i < key_infos.size(); ++i) { try @@ -787,21 +789,31 @@ size_t Aggregator::emplaceOrFindStringKey( = state.template emplaceStringKey(data, i, key_datas, hashvals); if (emplace_result.isInserted()) { - emplace_result.setMapped(nullptr); + if constexpr (zero_agg_func_size) + { + emplace_result.setMapped(agg_state); + } + else + { + emplace_result.setMapped(nullptr); - agg_state = aggregates_pool.alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(agg_state); + agg_state + = aggregates_pool.alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(agg_state); - emplace_result.setMapped(agg_state); + emplace_result.setMapped(agg_state); + } } else { - agg_state = emplace_result.getMapped(); + if constexpr (!zero_agg_func_size) + agg_state = emplace_result.getMapped(); if constexpr (collect_hit_rate) ++agg_process_info.hit_row_cnt; } - places[i] = agg_state; + if constexpr (!zero_agg_func_size) + places[i] = agg_state; } } catch (ResizeException &) @@ -1130,30 +1142,41 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( bool got_resize_exception = false; size_t emplaced_index = 0; - -#define M(INDEX, INFO, DATA, PLACES) \ - if unlikely (got_resize_exception) \ - { \ - emplaced_index = 0; \ - } \ - else if (!(INFO).empty()) \ - { \ - emplaced_index = emplaceOrFindStringKey( \ - method.data, \ - state, \ - (INFO), \ - (DATA), \ - *aggregates_pool, \ - (PLACES), \ - agg_process_info); \ - if unlikely (emplaced_index != (INFO).size()) \ - got_resize_exception = true; \ - } \ - setupExceptionRecoveryInfoForStringHashTable( \ - agg_process_info, \ - emplaced_index, \ - INFO, \ - DATA, \ + bool zero_agg_func_size = (params.aggregates_size == 0); + +#define M(INDEX, INFO, DATA, PLACES) \ + if unlikely (got_resize_exception) \ + { \ + emplaced_index = 0; \ + } \ + else if (!(INFO).empty()) \ + { \ + if (zero_agg_func_size) \ + emplaced_index = emplaceOrFindStringKey( \ + method.data, \ + state, \ + (INFO), \ + (DATA), \ + *aggregates_pool, \ + (PLACES), \ + agg_process_info); \ + else \ + emplaced_index = emplaceOrFindStringKey( \ + method.data, \ + state, \ + (INFO), \ + (DATA), \ + *aggregates_pool, \ + (PLACES), \ + agg_process_info); \ + if unlikely (emplaced_index != (INFO).size()) \ + got_resize_exception = true; \ + } \ + setupExceptionRecoveryInfoForStringHashTable( \ + agg_process_info, \ + emplaced_index, \ + INFO, \ + DATA, \ std::integral_constant{}); M(0, key0_infos, key0_datas, key0_places) @@ -1163,6 +1186,9 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( M(4, key_str_infos, key_str_datas, key_str_places) #undef M + if (zero_agg_func_size) + return; + RUNTIME_CHECK( rows == key0_places.size() + key8_places.size() + key16_places.size() + key24_places.size() + key_str_places.size()); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 7142077c8ea..cc2dcd2a408 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1499,6 +1499,7 @@ class Aggregator bool collect_hit_rate, bool only_lookup, bool enable_prefetch, + bool zero_agg_func_size, typename Data, typename State, typename StringKeyType> From ce1f76754e7a454d380c98c9330273481d170556 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 28 Nov 2024 16:29:05 +0800 Subject: [PATCH 06/61] revert Serialized Key changes Signed-off-by: guo-shaoge --- dbms/src/Common/Arena.h | 5 -- dbms/src/Common/ColumnsHashing.h | 80 +++++++------------------------- 2 files changed, 17 insertions(+), 68 deletions(-) diff --git a/dbms/src/Common/Arena.h b/dbms/src/Common/Arena.h index eb86e1c283c..b9999f6b179 100644 --- a/dbms/src/Common/Arena.h +++ b/dbms/src/Common/Arena.h @@ -212,10 +212,5 @@ class Arena : private boost::noncopyable using ArenaPtr = std::shared_ptr; using Arenas = std::vector; -size_t alignOf16(size_t l) -{ - return (l + 15) & ~15; -} - } // namespace DB diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 0d8e8d60ef7..aabe0733f8c 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -52,7 +52,7 @@ struct HashMethodOneNumber const size_t total_rows; /// If the keys of a fixed length then key_sizes contains their lengths, empty otherwise. - HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &, Arena *) + HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) : total_rows(key_columns[0]->size()) { vec = &static_cast *>(key_columns[0])->getData()[0]; @@ -107,8 +107,7 @@ struct HashMethodString HashMethodString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, - const TiDB::TiDBCollators & collators, - Arena *) + const TiDB::TiDBCollators & collators) : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; @@ -159,7 +158,7 @@ struct HashMethodStringBin const UInt8 * chars; const size_t total_rows; - HashMethodStringBin(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &, Arena *) + HashMethodStringBin(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; @@ -345,43 +344,6 @@ struct KeyDescStringBinPadding : KeyDescStringBin } }; -void serializeColumnToBuffer(Arena * pool, - const ColumnRawPtrs & key_columns, - PaddedPODArray & pos, - PaddedPODArray & sizes) -{ - RUNTIME_CHECK(!key_columns.empty()); - RUNTIME_CHECK(pos.empty() && sizes.empty()); - - const auto rows = key_columns[0]->size(); - pos.resize(rows, nullptr); - sizes.resize(rows, 0); - - for (const auto * col_ptr : key_columns) - col_ptr->countSerializeByteSize(sizes); - - std::vector aligned_sizes; - aligned_sizes.reserve(sizes.size()); - - size_t total_byte_size = 0; - for (auto size : sizes) - { - auto aligned = alignOf16(size); - total_byte_size += aligned; - aligned_sizes.push_back(aligned); - } - - auto * buffer = pool->alloc(total_byte_size); - for (size_t i = 0; i < aligned_sizes.size(); ++i) - { - pos[i] = buffer; - buffer += aligned_sizes[i]; - } - - for (const auto * col_ptr : key_columns) - col_ptr->serializeToPos(pos, 0, rows, col_ptr->isColumnNullable()); -} - /// For the case when there are 2 keys. template struct HashMethodFastPathTwoKeysSerialized @@ -394,16 +356,12 @@ struct HashMethodFastPathTwoKeysSerialized Key1Desc key_1_desc; Key2Desc key_2_desc; const size_t total_rows; - PaddedPODArray pos; - PaddedPODArray sizes; - HashMethodFastPathTwoKeysSerialized(const ColumnRawPtrs & key_columns, const Sizes &, const TiDB::TiDBCollators &, Arena * pool) + HashMethodFastPathTwoKeysSerialized(const ColumnRawPtrs & key_columns, const Sizes &, const TiDB::TiDBCollators &) : key_1_desc(key_columns[0]) , key_2_desc(key_columns[1]) , total_rows(key_columns[0]->size()) - { - serializeColumnToBuffer(pool, key_columns, pos, sizes); - } + {} ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const { @@ -442,8 +400,7 @@ struct HashMethodFixedString HashMethodFixedString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, - const TiDB::TiDBCollators & collators, - Arena *) + const TiDB::TiDBCollators & collators) : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; @@ -520,7 +477,7 @@ struct HashMethodKeysFixed return true; } - HashMethodKeysFixed(const ColumnRawPtrs & key_columns, const Sizes & key_sizes_, const TiDB::TiDBCollators &, Arena *) + HashMethodKeysFixed(const ColumnRawPtrs & key_columns, const Sizes & key_sizes_, const TiDB::TiDBCollators &) : Base(key_columns) , key_sizes(std::move(key_sizes_)) , keys_size(key_columns.size()) @@ -655,28 +612,25 @@ struct HashMethodSerialized size_t keys_size; TiDB::TiDBCollators collators; const size_t total_rows; - PaddedPODArray pos; - PaddedPODArray sizes; HashMethodSerialized( const ColumnRawPtrs & key_columns_, const Sizes & /*key_sizes*/, - const TiDB::TiDBCollators & collators_, - Arena * pool) + const TiDB::TiDBCollators & collators_) : key_columns(key_columns_) , keys_size(key_columns_.size()) , collators(collators_) , total_rows(key_columns_[0]->size()) - { - serializeColumnToBuffer(pool, key_columns_, pos, sizes); - } + {} - ALWAYS_INLINE inline StringRef getKeyHolder( - size_t row, - Arena *, - std::vector &) const + ALWAYS_INLINE inline SerializedKeyHolder getKeyHolder( + size_t row, + Arena * pool, + std::vector & sort_key_containers) const { - return StringRef(pos[row], sizes[row]); + return SerializedKeyHolder{ + serializeKeysToPoolContiguous(row, keys_size, key_columns, collators, sort_key_containers, *pool), + *pool}; } protected: @@ -696,7 +650,7 @@ struct HashMethodHashed TiDB::TiDBCollators collators; const size_t total_rows; - HashMethodHashed(ColumnRawPtrs key_columns_, const Sizes &, const TiDB::TiDBCollators & collators_, Arena *) + HashMethodHashed(ColumnRawPtrs key_columns_, const Sizes &, const TiDB::TiDBCollators & collators_) : key_columns(std::move(key_columns_)) , collators(collators_) , total_rows(key_columns[0]->size()) From 8ac8bebe1280ba49a570fb20cef9da78c4d0454f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 28 Nov 2024 17:50:30 +0800 Subject: [PATCH 07/61] refine Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 38 ++++++++-------- dbms/src/Common/HashTable/StringHashTable.h | 44 +++---------------- .../HashTable/TwoLevelStringHashTable.h | 31 +------------ dbms/src/Interpreters/Aggregator.h | 9 ++-- 4 files changed, 31 insertions(+), 91 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index aa583f1a722..ffaffdcd758 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -130,6 +130,14 @@ class HashMethodBase using Cache = LastElementCache; static constexpr size_t prefetch_step = 16; + template + static ALWAYS_INLINE inline void prefetch(Map & map, size_t idx, const std::vector & hashvals) + { + const auto prefetch_idx = idx + prefetch_step; + if likely (prefetch_idx < hashvals.size()) + map.prefetch(hashvals[prefetch_idx]); + } + template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, @@ -141,10 +149,8 @@ class HashMethodBase auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); if constexpr (enable_prefetch) { - const auto idx = row + prefetch_step; - if (idx < hashvals.size()) - data.prefetch(hashvals[idx]); - + assert(hashvals.size() == static_cast(*this).total_rows); + prefetch(data, row, hashvals); return emplaceImpl(key_holder, data, hashvals[row]); } else @@ -164,10 +170,8 @@ class HashMethodBase auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); if constexpr (enable_prefetch) { - const auto idx = row + prefetch_step; - if (idx < hashvals.size()) - data.prefetch(hashvals[idx]); - + assert(hashvals.size() == static_cast(*this).total_rows); + prefetch(data, row, hashvals); return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); } else @@ -176,7 +180,6 @@ class HashMethodBase } } - // TODO emplaceStringKey merge with emplaceKey? template ALWAYS_INLINE inline EmplaceResult emplaceStringKey( Data & data, @@ -184,20 +187,17 @@ class HashMethodBase std::vector & datas, // TODO const const std::vector & hashvals) { + assert(hashvals.size() == static_cast(*this).total_rows); + auto & submap = StringHashTableSubMapSelector>::getSubMap( hashvals[idx], data); if constexpr (enable_prefetch) - { - const auto prefetch_idx = idx + prefetch_step; - if (prefetch_idx < hashvals.size()) - submap.prefetch(hashvals[prefetch_idx]); - } + prefetch(submap, idx, hashvals); return emplaceImpl(datas[idx], submap, hashvals[idx]); } - // TODO Macro with emplaceStringKey template ALWAYS_INLINE inline FindResult findStringKey( Data & data, @@ -205,15 +205,13 @@ class HashMethodBase std::vector & datas, // TODO const const std::vector & hashvals) { + assert(hashvals.size() == static_cast(*this).total_rows); + auto & submap = StringHashTableSubMapSelector>::getSubMap( hashvals[idx], data); if constexpr (enable_prefetch) - { - const auto prefetch_idx = idx + prefetch_step; - if (prefetch_idx < hashvals.size()) - submap.prefetch(hashvals[prefetch_idx]); - } + prefetch(submap, idx, hashvals); return findKeyImpl(keyHolderGetKey(datas[idx]), submap, hashvals[idx]); } diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index a511ce47671..ef668864120 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -139,8 +139,7 @@ struct StringHashTableEmpty //-V730 return hasZero() ? zeroValue() : nullptr; } - void ALWAYS_INLINE prefetch(size_t) {} - + ALWAYS_INLINE inline void prefetch() {} void write(DB::WriteBuffer & wb) const { zeroValue()->write(wb); } void writeText(DB::WriteBuffer & wb) const { zeroValue()->writeText(wb); } void read(DB::ReadBuffer & rb) { zeroValue()->read(rb); } @@ -348,7 +347,6 @@ class StringHashTable : private boost::noncopyable #endif dispatch(Self & self, KeyHolder && key_holder, Func && func) { - StringHashTableHash hash; const StringRef & x = keyHolderGetKey(key_holder); const size_t sz = x.size; if (sz == 0) @@ -361,7 +359,7 @@ class StringHashTable : private boost::noncopyable { // Strings with trailing zeros are not representable as fixed-size // string keys. Put them to the generic table. - return func(self.ms, std::forward(key_holder), hash(x)); + return func(self.ms, std::forward(key_holder), StringHashTableHash::operator()(x)); } const char * p = x.data; @@ -397,7 +395,7 @@ class StringHashTable : private boost::noncopyable n[0] <<= s; } keyHolderDiscardKey(key_holder); - return func(self.m1, k8, hash(k8)); + return func(self.m1, k8, StringHashTableHash::operator()(k8)); } case 1: // 9..16 bytes { @@ -409,7 +407,7 @@ class StringHashTable : private boost::noncopyable else n[1] <<= s; keyHolderDiscardKey(key_holder); - return func(self.m2, k16, hash(k16)); + return func(self.m2, k16, StringHashTableHash::operator()(k16)); } case 2: // 17..24 bytes { @@ -421,11 +419,11 @@ class StringHashTable : private boost::noncopyable else n[2] <<= s; keyHolderDiscardKey(key_holder); - return func(self.m3, k24, hash(k24)); + return func(self.m3, k24, StringHashTableHash::operator()(k24)); } default: // >= 25 bytes { - return func(self.ms, std::forward(key_holder), hash(x)); + return func(self.ms, std::forward(key_holder), StringHashTableHash::operator()(x)); } } } @@ -455,13 +453,6 @@ class StringHashTable : private boost::noncopyable this->dispatch(*this, key_holder, EmplaceCallable(it, inserted)); } - // TODO del - template - void ALWAYS_INLINE emplace(KeyHolder &&, LookupResult &, bool &, size_t) - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::emplace instead"); - } - struct FindCallable { // find() doesn't need any key memory management, so we don't work with @@ -478,35 +469,12 @@ class StringHashTable : private boost::noncopyable } }; - // We will not prefetch StringHashTable directly, instead caller should call specific submap's prefetch. - // Because StringHashTable doesn't know which submap to prefetch. - void prefetch(size_t) const - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::prefetch instead"); - } - LookupResult ALWAYS_INLINE find(const Key & x) { return dispatch(*this, x, FindCallable{}); } ConstLookupResult ALWAYS_INLINE find(const Key & x) const { return dispatch(*this, x, FindCallable{}); } - // TODO del - LookupResult ALWAYS_INLINE find(const Key &, size_t) - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); - } - ConstLookupResult ALWAYS_INLINE find(const Key &, size_t) const - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); - } - bool ALWAYS_INLINE has(const Key & x, size_t = 0) const { return dispatch(*this, x, FindCallable{}) != nullptr; } - template - size_t ALWAYS_INLINE hash(const HashKeyType & key) const - { - return SubMaps::Hash::operator()(key); - } - void write(DB::WriteBuffer & wb) const { m0.write(wb); diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index e7ea1bb8fce..5ea460769ab 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -33,20 +33,8 @@ class TwoLevelStringHashTable : private boost::noncopyable static constexpr bool is_string_hash_map = true; static constexpr bool is_two_level = true; - template - size_t ALWAYS_INLINE hash(const HashKeyType & key) const - { - return SubMaps::Hash::operator()(key); - } - - // Same reason as StringHashTable::prefetch. - void prefetch(size_t) const - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::prefetch instead"); - } - // TODO: currently hashing contains redundant computations when doing distributed or external aggregations - size_t hashStringRef(const Key & x) const + size_t hash(const Key & x) const { return const_cast(*this).dispatch(*this, x, [&](const auto &, const auto &, size_t hash) { return hash; @@ -59,7 +47,7 @@ class TwoLevelStringHashTable : private boost::noncopyable impl.setResizeCallback(resize_callback); } - size_t operator()(const Key & x) const { return hashStringRef(x); } + size_t operator()(const Key & x) const { return hash(x); } /// NOTE Bad for hash tables with more than 2^32 cells. static size_t getBucketFromHash(size_t hash_value) { return (hash_value >> (32 - BITS_FOR_BUCKET)) & MAX_BUCKET; } @@ -216,27 +204,12 @@ class TwoLevelStringHashTable : private boost::noncopyable dispatch(*this, key_holder, typename Impl::EmplaceCallable{it, inserted}); } - template - void ALWAYS_INLINE emplace(KeyHolder &&, LookupResult &, bool &, size_t) - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::emplace instead"); - } - LookupResult ALWAYS_INLINE find(const Key & x) { return dispatch(*this, x, typename Impl::FindCallable{}); } ConstLookupResult ALWAYS_INLINE find(const Key & x) const { return dispatch(*this, x, typename Impl::FindCallable{}); } - LookupResult ALWAYS_INLINE find(const Key &, size_t) - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); - } - - ConstLookupResult ALWAYS_INLINE find(const Key &, size_t) const - { - RUNTIME_CHECK_MSG(false, "shouldn't reach here, you should use submap::find instead"); - } void write(DB::WriteBuffer & wb) const { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index cc2dcd2a408..53bc989dcbc 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1319,7 +1319,8 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; - // For StringHashTable resize exception. + // For StringHashTable, when resize exception happens, the process will be interrupted. + // So we need these infos to continue. std::vector submap_m0_infos{}; std::vector submap_m1_infos{}; std::vector submap_m2_infos{}; @@ -1337,9 +1338,9 @@ class Aggregator { assert(start_row <= end_row); // submap_mx_infos.size() and submap_mx_datas.size() are always equal. - // So only need to check submap_m0_infos is enough. - return (start_row == end_row && !submap_m0_infos.empty() && !submap_m1_infos.empty() - && !submap_m3_infos.empty() && !submap_m4_infos.empty()) + // So only need to check submap_mx_infos is enough. + return (start_row == end_row && submap_m0_infos.empty() && submap_m1_infos.empty() + && submap_m3_infos.empty() && submap_m4_infos.empty()) || aggregator->isCancelled(); } void resetBlock(const Block & block_) From 0053ce8a82f19cd8772ce5f00cea160b20934250 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 28 Nov 2024 18:03:50 +0800 Subject: [PATCH 08/61] refine Signed-off-by: guo-shaoge --- dbms/src/Common/HashTable/StringHashTable.h | 2 +- dbms/src/Interpreters/Aggregator.cpp | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index ef668864120..fde0f810ae6 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -139,7 +139,7 @@ struct StringHashTableEmpty //-V730 return hasZero() ? zeroValue() : nullptr; } - ALWAYS_INLINE inline void prefetch() {} + ALWAYS_INLINE inline void prefetch(size_t) {} void write(DB::WriteBuffer & wb) const { zeroValue()->write(wb); } void writeText(DB::WriteBuffer & wb) const { zeroValue()->writeText(wb); } void read(DB::ReadBuffer & rb) { zeroValue()->read(rb); } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 1738121b665..d8210fc9a9c 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -1090,13 +1090,14 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( "executeImplBatchStringHashMap only handle resize exception for each Block instead of row"); const size_t reserve_size = rows / 4; -#define M(INFO, DATA, SUBMAPINDEX, KEYTYPE) \ - (INFO).reserve(reserve_size); \ - (DATA).reserve(reserve_size); \ - auto dispatch_callback_key##SUBMAPINDEX = [&INFO, &DATA](const KEYTYPE & key, size_t row) { \ - (INFO).push_back(row); \ - (DATA).push_back(key); \ - }; +#define M(INFO, DATA, SUBMAPINDEX, KEYTYPE) \ + (INFO).reserve(reserve_size); \ + (DATA).reserve(reserve_size); \ + auto dispatch_callback_key##SUBMAPINDEX \ + = [&INFO, &DATA](const KEYTYPE & key, size_t row) { /* NOLINT(bugprone-macro-parentheses) */ \ + (INFO).push_back(row); \ + (DATA).push_back(key); \ + }; M(key0_infos, key0_datas, 0, StringRef) M(key8_infos, key8_datas, 8, StringKey8) From fcf8ed2b49ebc8e7d3c8a0c44e035bc4477c5663 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 29 Nov 2024 17:18:00 +0800 Subject: [PATCH 09/61] fix unit test Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 6 ++++-- dbms/src/Interpreters/Aggregator.cpp | 30 ++++++++++++---------------- dbms/src/Interpreters/Aggregator.h | 9 ++++++--- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index ffaffdcd758..1f4e3dbaedf 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -187,7 +187,9 @@ class HashMethodBase std::vector & datas, // TODO const const std::vector & hashvals) { - assert(hashvals.size() == static_cast(*this).total_rows); + // For spill, hashvals.size() will be le to total_rows. + // Because only remaining rows that didn't insert into HashMap will be handled here. + assert(hashvals.size() <= static_cast(*this).total_rows); auto & submap = StringHashTableSubMapSelector>::getSubMap( hashvals[idx], @@ -205,7 +207,7 @@ class HashMethodBase std::vector & datas, // TODO const const std::vector & hashvals) { - assert(hashvals.size() == static_cast(*this).total_rows); + assert(hashvals.size() <= static_cast(*this).total_rows); auto & submap = StringHashTableSubMapSelector>::getSubMap( hashvals[idx], diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index d8210fc9a9c..041f77fc155 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -760,9 +760,7 @@ size_t Aggregator::emplaceOrFindStringKey( using Hash = typename StringHashTableSubMapSelector>::Hash; std::vector hashvals(key_infos.size(), 0); for (size_t i = 0; i < key_infos.size(); ++i) - { - hashvals[i] = Hash::operator()(keyHolderGetKey(key_datas[0])); - } + hashvals[i] = Hash::operator()(keyHolderGetKey(key_datas[i])); // alloc 0 bytes is useful when agg func size is zero. AggregateDataPtr agg_state = aggregates_pool.alloc(0); @@ -1080,14 +1078,12 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( #undef M const size_t rows = agg_process_info.end_row - agg_process_info.start_row; + // If no resize exception happens, so this is a new Block. + // If resize exception happens, start_row also set as zero. + RUNTIME_CHECK(agg_process_info.start_row == 0); - if likely (agg_process_info.allBlockDataHandled()) + if likely (agg_process_info.stringHashTableRecoveryInfoEmpty()) { - // No resize exception happens, so this is a new Block. - RUNTIME_CHECK(agg_process_info.start_row == 0); - RUNTIME_CHECK_MSG( - rows == state.total_rows, - "executeImplBatchStringHashMap only handle resize exception for each Block instead of row"); const size_t reserve_size = rows / 4; #define M(INFO, DATA, SUBMAPINDEX, KEYTYPE) \ @@ -1146,11 +1142,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( bool zero_agg_func_size = (params.aggregates_size == 0); #define M(INDEX, INFO, DATA, PLACES) \ - if unlikely (got_resize_exception) \ - { \ - emplaced_index = 0; \ - } \ - else if (!(INFO).empty()) \ + if (!(INFO).empty()) \ { \ if (zero_agg_func_size) \ emplaced_index = emplaceOrFindStringKey( \ @@ -1173,11 +1165,15 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( if unlikely (emplaced_index != (INFO).size()) \ got_resize_exception = true; \ } \ + else \ + { \ + emplaced_index = 0; \ + } \ setupExceptionRecoveryInfoForStringHashTable( \ agg_process_info, \ emplaced_index, \ - INFO, \ - DATA, \ + (INFO), \ + (DATA), \ std::integral_constant{}); M(0, key0_infos, key0_datas, key0_places) @@ -1221,7 +1217,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( aggregates_pool); } // For StringHashTable, start_row is meanless, instead submap_mx_infos/submap_mx_datas are used. - agg_process_info.start_row = got_resize_exception ? 0 : rows; + agg_process_info.start_row = got_resize_exception ? 0 : agg_process_info.end_row; } void NO_INLINE diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 53bc989dcbc..f5217058ff8 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1339,9 +1339,12 @@ class Aggregator assert(start_row <= end_row); // submap_mx_infos.size() and submap_mx_datas.size() are always equal. // So only need to check submap_mx_infos is enough. - return (start_row == end_row && submap_m0_infos.empty() && submap_m1_infos.empty() - && submap_m3_infos.empty() && submap_m4_infos.empty()) - || aggregator->isCancelled(); + return (start_row == end_row && stringHashTableRecoveryInfoEmpty()) || aggregator->isCancelled(); + } + bool stringHashTableRecoveryInfoEmpty() const + { + return submap_m0_infos.empty() && submap_m1_infos.empty() && + submap_m3_infos.empty() && submap_m4_infos.empty(); } void resetBlock(const Block & block_) { From ae7b969f3f83b9e6dd88f99b0bb478926895be56 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 2 Dec 2024 15:13:55 +0800 Subject: [PATCH 10/61] refine Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 90 ++++++++-------- dbms/src/Common/HashTable/StringHashTable.h | 2 + dbms/src/Interpreters/Aggregator.cpp | 45 ++++---- dbms/src/Interpreters/Aggregator.h | 13 +-- dbms/src/Interpreters/JoinPartition.cpp | 28 ++--- dbms/src/Interpreters/SetVariants.h | 4 +- dbms/src/TiDB/Collation/Collator.cpp | 112 ++++++++++++++++++++ dbms/src/TiDB/Collation/Collator.h | 10 ++ 8 files changed, 221 insertions(+), 83 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index aabe0733f8c..94526714250 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -91,12 +91,11 @@ struct HashMethodOneNumber /// For the case when there is one string key. -template +template struct HashMethodString - : public columns_hashing_impl:: - HashMethodBase, Value, Mapped, use_cache> + : public columns_hashing_impl::HashMethodBase, Value, Mapped, use_cache> { - using Self = HashMethodString; + using Self = HashMethodString; using Base = columns_hashing_impl::HashMethodBase; const IColumn::Offset * offsets; @@ -115,36 +114,40 @@ struct HashMethodString offsets = column_string.getOffsets().data(); chars = column_string.getChars().data(); if (!collators.empty()) - { - if constexpr (!place_string_to_arena) - throw Exception("String with collator must be placed on arena.", ErrorCodes::LOGICAL_ERROR); collator = collators[0]; - } } - ALWAYS_INLINE inline auto getKeyHolder( + ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder( ssize_t row, [[maybe_unused]] Arena * pool, - std::vector & sort_key_containers) const + [[maybe_unused]] std::vector & sort_key_containers) const { - auto last_offset = row == 0 ? 0 : offsets[row - 1]; - // Remove last zero byte. - StringRef key(chars + last_offset, offsets[row] - last_offset - 1); + auto key = getKey(row); + if (likely(collator)) + key = collator->sortKey(key.data, key.size, sort_key_containers[0]); - if constexpr (place_string_to_arena) - { - if (likely(collator)) - key = collator->sortKey(key.data, key.size, sort_key_containers[0]); - return ArenaKeyHolder{key, pool}; - } - else - { - return key; - } + return ArenaKeyHolder{key, pool}; + } + + ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder(ssize_t row, Arena * pool, Arena * sort_key_pool) const + { + auto key = getKey(row); + if (likely(collator)) + key = collator->sortKey(key.data, key.size, *sort_key_pool); + + return ArenaKeyHolder{key, pool}; } protected: friend class columns_hashing_impl::HashMethodBase; + +private: + ALWAYS_INLINE inline StringRef getKey(size_t row) const + { + auto last_offset = row == 0 ? 0 : offsets[row - 1]; + // Remove last zero byte. + return StringRef(chars + last_offset, offsets[row] - last_offset - 1); + } }; template @@ -168,6 +171,11 @@ struct HashMethodStringBin } ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const + { + return getKeyHolder(row, pool, nullptr); + } + + ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, Arena *) const { auto last_offset = row == 0 ? 0 : offsets[row - 1]; StringRef key(chars + last_offset, offsets[row] - last_offset - 1); @@ -381,15 +389,12 @@ struct HashMethodFastPathTwoKeysSerialized /// For the case when there is one fixed-length string key. -template +template struct HashMethodFixedString - : public columns_hashing_impl::HashMethodBase< - HashMethodFixedString, - Value, - Mapped, - use_cache> + : public columns_hashing_impl:: + HashMethodBase, Value, Mapped, use_cache> { - using Self = HashMethodFixedString; + using Self = HashMethodFixedString; using Base = columns_hashing_impl::HashMethodBase; size_t n; @@ -411,26 +416,25 @@ struct HashMethodFixedString collator = collators[0]; } - ALWAYS_INLINE inline auto getKeyHolder( + ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder( size_t row, - [[maybe_unused]] Arena * pool, + Arena * pool, std::vector & sort_key_containers) const { StringRef key(&(*chars)[row * n], n); - if (collator) - { key = collator->sortKeyFastPath(key.data, key.size, sort_key_containers[0]); - } - if constexpr (place_string_to_arena) - { - return ArenaKeyHolder{key, pool}; - } - else - { - return key; - } + return ArenaKeyHolder{key, pool}; + } + + ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder(size_t row, Arena * pool, Arena * sort_key_pool) const + { + StringRef key(&(*chars)[row * n], n); + if (collator) + key = collator->sortKeyFastPath(key.data, key.size, *sort_key_pool); + + return ArenaKeyHolder{key, pool}; } protected: diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index fde0f810ae6..a43f35fdbbf 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -16,7 +16,9 @@ #include #include +#include #include +#include #include #include diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 041f77fc155..543885b6248 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -755,7 +755,7 @@ size_t Aggregator::emplaceOrFindStringKey( AggProcessInfo & agg_process_info) const { static_assert(!(collect_hit_rate && only_lookup)); - RUNTIME_CHECK(key_infos.size() == key_datas.size()); + assert(key_infos.size() == key_datas.size()); using Hash = typename StringHashTableSubMapSelector>::Hash; std::vector hashvals(key_infos.size(), 0); @@ -1017,7 +1017,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( #define M(SUBMAPINDEX) \ template \ - void setupExceptionRecoveryInfoForStringHashTable( \ + ALWAYS_INLINE inline void setupExceptionRecoveryInfoForStringHashTable( \ Aggregator::AggProcessInfo & agg_process_info, \ size_t row, \ const std::vector & key_infos, \ @@ -1038,8 +1038,10 @@ M(4) #undef M -// Emplace key into StringHashMap/TwoLevelStringHashMap is seperated from other situations, -// because it's easy to implement prefetch submap directly. +// In this function, we will prefetch/empalce each specifix submap directly instead of accessing StringHashMap interface, +// which is good for performance. +// NOTE: this function is column-wise, which means sort key buffer cannot be reused. +// This buffer will not be release until this block is processed done. template ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( Method & method, @@ -1063,8 +1065,9 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( M(4) #undef M + const size_t rows = agg_process_info.end_row - agg_process_info.start_row; + auto sort_key_pool = std::make_unique(); std::vector sort_key_containers; - sort_key_containers.resize(params.keys_size, ""); #define M(INFO, DATA, KEYTYPE) \ std::vector(INFO); \ @@ -1077,13 +1080,15 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( M(key_str_infos, key_str_datas, ArenaKeyHolder) #undef M - const size_t rows = agg_process_info.end_row - agg_process_info.start_row; // If no resize exception happens, so this is a new Block. // If resize exception happens, start_row also set as zero. RUNTIME_CHECK(agg_process_info.start_row == 0); if likely (agg_process_info.stringHashTableRecoveryInfoEmpty()) { + // sort_key_pool should already been reset by AggProcessInfo::restBlock() + RUNTIME_CHECK(!agg_process_info.sort_key_pool); + const size_t reserve_size = rows / 4; #define M(INFO, DATA, SUBMAPINDEX, KEYTYPE) \ @@ -1106,7 +1111,9 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( for (size_t i = 0; i < rows; ++i) { - auto key_holder = state.getKeyHolder(i, aggregates_pool, sort_key_containers); + // Use Arena for collation sort key, because we are doing agg in column-wise way. + // So a big arena is needed to store decoded key, and we can avoid resize std::string by using Arena. + auto key_holder = state.getKeyHolder(i, aggregates_pool, sort_key_pool.get()); dispatchStringHashTable( i, key_holder, @@ -1142,7 +1149,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( bool zero_agg_func_size = (params.aggregates_size == 0); #define M(INDEX, INFO, DATA, PLACES) \ - if (!(INFO).empty()) \ + if (!got_resize_exception && !(INFO).empty()) \ { \ if (zero_agg_func_size) \ emplaced_index = emplaceOrFindStringKey( \ @@ -1165,15 +1172,15 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( if unlikely (emplaced_index != (INFO).size()) \ got_resize_exception = true; \ } \ - else \ - { \ - emplaced_index = 0; \ - } \ + else \ + { \ + emplaced_index = 0; \ + } \ setupExceptionRecoveryInfoForStringHashTable( \ agg_process_info, \ emplaced_index, \ - (INFO), \ - (DATA), \ + (INFO), \ + (DATA), \ std::integral_constant{}); M(0, key0_infos, key0_datas, key0_places) @@ -1186,10 +1193,6 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( if (zero_agg_func_size) return; - RUNTIME_CHECK( - rows - == key0_places.size() + key8_places.size() + key16_places.size() + key24_places.size() + key_str_places.size()); - std::vector places(rows, nullptr); #define M(INFO, PLACES) \ for (size_t i = 0; i < (INFO).size(); ++i) \ @@ -1218,6 +1221,12 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( } // For StringHashTable, start_row is meanless, instead submap_mx_infos/submap_mx_datas are used. agg_process_info.start_row = got_resize_exception ? 0 : agg_process_info.end_row; + + if unlikely (got_resize_exception) + { + RUNTIME_CHECK(!agg_process_info.stringHashTableRecoveryInfoEmpty()); + agg_process_info.sort_key_pool = std::move(sort_key_pool); + } } void NO_INLINE diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index f5217058ff8..c6e78fb5618 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -231,8 +231,7 @@ struct AggregationMethodStringNoCache : data(other.data) {} - using State = ColumnsHashing:: - HashMethodString; + using State = ColumnsHashing::HashMethodString; template struct EmplaceOrFindKeyResult { @@ -528,7 +527,7 @@ struct AggregationMethodFixedStringNoCache : data(other.data) {} - using State = ColumnsHashing::HashMethodFixedString; + using State = ColumnsHashing::HashMethodFixedString; template struct EmplaceOrFindKeyResult { @@ -1326,12 +1325,12 @@ class Aggregator std::vector submap_m2_infos{}; std::vector submap_m3_infos{}; std::vector submap_m4_infos{}; - std::vector submap_m0_datas{}; std::vector submap_m1_datas{}; std::vector submap_m2_datas{}; std::vector submap_m3_datas{}; std::vector submap_m4_datas{}; + std::unique_ptr sort_key_pool; void prepareForAgg(); bool allBlockDataHandled() const @@ -1343,8 +1342,8 @@ class Aggregator } bool stringHashTableRecoveryInfoEmpty() const { - return submap_m0_infos.empty() && submap_m1_infos.empty() && - submap_m3_infos.empty() && submap_m4_infos.empty(); + return submap_m0_infos.empty() && submap_m1_infos.empty() && submap_m3_infos.empty() + && submap_m4_infos.empty(); } void resetBlock(const Block & block_) { @@ -1358,6 +1357,8 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); + + sort_key_pool.reset(); } }; diff --git a/dbms/src/Interpreters/JoinPartition.cpp b/dbms/src/Interpreters/JoinPartition.cpp index a060878c4f7..294c72c19a3 100644 --- a/dbms/src/Interpreters/JoinPartition.cpp +++ b/dbms/src/Interpreters/JoinPartition.cpp @@ -412,7 +412,7 @@ struct KeyGetterForTypeImpl template struct KeyGetterForTypeImpl { - using Type = ColumnsHashing::HashMethodString; + using Type = ColumnsHashing::HashMethodString; }; template struct KeyGetterForTypeImpl @@ -427,7 +427,7 @@ struct KeyGetterForTypeImpl template struct KeyGetterForTypeImpl { - using Type = ColumnsHashing::HashMethodFixedString; + using Type = ColumnsHashing::HashMethodFixedString; }; template struct KeyGetterForTypeImpl @@ -652,18 +652,18 @@ void NO_INLINE insertBlockIntoMapsTypeCase( insert_indexes.emplace_back(insert_index); } -#define INSERT_TO_MAP(join_partition, segment_index) \ - auto & current_map = (join_partition)->getHashMap(); \ - for (auto & s_i : (segment_index)) \ - { \ - Inserter::insert( \ - current_map, \ - key_getter, \ - stored_block, \ - s_i, \ - pool, \ - sort_key_containers, \ - probe_cache_column_threshold); \ +#define INSERT_TO_MAP(join_partition, segment_index) \ + auto & current_map = (join_partition) -> getHashMap(); \ + for (auto & s_i : (segment_index)) \ + { \ + Inserter::insert( \ + current_map, \ + key_getter, \ + stored_block, \ + s_i, \ + pool, \ + sort_key_containers, \ + probe_cache_column_threshold); \ } #define INSERT_TO_NOT_INSERTED_MAP \ diff --git a/dbms/src/Interpreters/SetVariants.h b/dbms/src/Interpreters/SetVariants.h index a1591f8c13a..5c503240b7b 100644 --- a/dbms/src/Interpreters/SetVariants.h +++ b/dbms/src/Interpreters/SetVariants.h @@ -54,7 +54,7 @@ struct SetMethodString Data data; - using State = ColumnsHashing::HashMethodString; + using State = ColumnsHashing::HashMethodString; }; template @@ -77,7 +77,7 @@ struct SetMethodFixedString Data data; - using State = ColumnsHashing::HashMethodFixedString; + using State = ColumnsHashing::HashMethodFixedString; }; namespace set_impl diff --git a/dbms/src/TiDB/Collation/Collator.cpp b/dbms/src/TiDB/Collation/Collator.cpp index bf27400f8c4..4365f1f0988 100644 --- a/dbms/src/TiDB/Collation/Collator.cpp +++ b/dbms/src/TiDB/Collation/Collator.cpp @@ -192,6 +192,11 @@ class BinCollator final : public ITiDBCollator return DB::BinCollatorSortKey(s, length); } + StringRef sortKey(const char * s, size_t length, DB::Arena &) const override + { + return DB::BinCollatorSortKey(s, length); + } + StringRef sortKeyNoTrim(const char * s, size_t length, std::string &) const override { return convertForBinCollator(s, length, nullptr); @@ -273,11 +278,54 @@ class GeneralCICollator final : public ITiDBCollator return convertImpl(s, length, container, nullptr); } + StringRef sortKey(const char * s, size_t length, DB::Arena & pool) const override + { + return convertImpl(s, length, pool, nullptr); + } + StringRef sortKeyNoTrim(const char * s, size_t length, std::string & container) const override { return convertImpl(s, length, container, nullptr); } + template + StringRef convertImpl(const char * s, size_t length, DB::Arena & pool, std::vector * lens) const + { + std::string_view v; + + if constexpr (need_trim) + v = rtrim(s, length); + else + v = std::string_view(s, length); + + const size_t size = length * sizeof(WeightType); + auto * buffer = pool.alignedAlloc(size, 16); + + size_t offset = 0; + size_t total_size = 0; + size_t v_length = v.length(); + + if constexpr (need_len) + { + if (lens->capacity() < v_length) + lens->reserve(v_length); + lens->resize(0); + } + + while (offset < v_length) + { + auto c = decodeChar(s, offset); + auto sk = weight(c); + buffer[total_size++] = static_cast(sk >> 8); + buffer[total_size++] = static_cast(sk); + + if constexpr (need_len) + lens->push_back(2); + } + + return StringRef(buffer, total_size); + } + template StringRef convertImpl(const char * s, size_t length, std::string & container, std::vector * lens) const { @@ -479,11 +527,65 @@ class UCACICollator final : public ITiDBCollator return convertImpl(s, length, container, nullptr); } + StringRef sortKey(const char * s, size_t length, DB::Arena & pool) const override + { + return convertImpl(s, length, pool, nullptr); + } + StringRef sortKeyNoTrim(const char * s, size_t length, std::string & container) const override { return convertImpl(s, length, container, nullptr); } + // Use Arena to store decoded string. Normally it's used by column-wise Agg/Join, + // because column-wise process cannot reuse string container. + template + StringRef convertImpl(const char * s, size_t length, DB::Arena & pool, std::vector * lens) const + { + std::string_view v; + + if constexpr (need_trim) + v = preprocess(s, length); + else + v = std::string_view(s, length); + + // every char have 8 uint16 at most. + const auto size = 8 * length * sizeof(uint16_t); + auto * buffer = pool.alignedAlloc(size, 16); + + size_t offset = 0; + size_t total_size = 0; + size_t v_length = v.length(); + + uint64_t first = 0, second = 0; + + if constexpr (need_len) + { + if (lens->capacity() < v_length) + lens->reserve(v_length); + lens->resize(0); + } + + while (offset < v_length) + { + weight(first, second, offset, v_length, s); + + if constexpr (need_len) + lens->push_back(total_size); + + writeResult(first, buffer, total_size); + writeResult(second, buffer, total_size); + + if constexpr (need_len) + { + size_t end_idx = lens->size() - 1; + (*lens)[end_idx] = total_size - (*lens)[end_idx]; + } + } + + return StringRef(buffer, total_size); + } + template StringRef convertImpl(const char * s, size_t length, std::string & container, std::vector * lens) const { @@ -550,6 +652,16 @@ class UCACICollator final : public ITiDBCollator } } + static inline void writeResult(uint64_t & w, char * buffer, size_t & total_size) + { + while (w != 0) + { + buffer[total_size++] = static_cast(w >> 8); + buffer[total_size++] = static_cast(w); + w >>= 16; + } + } + static inline bool regexEq(CharType a, CharType b) { return T::regexEq(a, b); } static inline void weight(uint64_t & first, uint64_t & second, size_t & offset, size_t length, const char * s) diff --git a/dbms/src/TiDB/Collation/Collator.h b/dbms/src/TiDB/Collation/Collator.h index 6bb87883ef1..08c017ba57d 100644 --- a/dbms/src/TiDB/Collation/Collator.h +++ b/dbms/src/TiDB/Collation/Collator.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -101,6 +102,7 @@ class ITiDBCollator = 0; virtual StringRef sortKeyNoTrim(const char * s, size_t length, std::string & container) const = 0; virtual StringRef sortKey(const char * s, size_t length, std::string & container) const = 0; + virtual StringRef sortKey(const char * s, size_t length, DB::Arena &) const = 0; virtual std::unique_ptr pattern() const = 0; int32_t getCollatorId() const { return collator_id; } CollatorType getCollatorType() const { return collator_type; } @@ -135,6 +137,14 @@ class ITiDBCollator } return sortKey(s, length, container); } + ALWAYS_INLINE inline StringRef sortKeyFastPath(const char * s, size_t length, DB::Arena & pool) const + { + if (likely(isPaddingBinary())) + { + return DB::BinCollatorSortKey(s, length); + } + return sortKey(s, length, pool); + } protected: explicit ITiDBCollator(int32_t collator_id_); From 3a86617a9c870be81bf28247d5b6d033cc9b6a1d Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 2 Dec 2024 17:16:26 +0800 Subject: [PATCH 11/61] unit test Signed-off-by: guo-shaoge --- dbms/src/Common/FailPoint.cpp | 1 + .../tests/gtest_aggregation_executor.cpp | 111 +++++++++++------- dbms/src/Flash/tests/gtest_compute_server.cpp | 4 + dbms/src/Interpreters/Aggregator.cpp | 9 +- 4 files changed, 79 insertions(+), 46 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index f6025741325..f73f273dd48 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -114,6 +114,7 @@ namespace DB M(force_set_parallel_prehandle_threshold) \ M(force_raise_prehandle_exception) \ M(force_agg_on_partial_block) \ + M(force_agg_prefetch) \ M(force_set_fap_candidate_store_id) \ M(force_not_clean_fap_on_destroy) \ M(force_fap_worker_throw) \ diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index 7193f24eddb..8c7f5277916 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -24,6 +24,7 @@ namespace DB namespace FailPoints { extern const char force_agg_on_partial_block[]; +extern const char force_agg_prefetch[]; extern const char force_agg_two_level_hash_table_before_merge[]; } // namespace FailPoints namespace tests @@ -238,16 +239,22 @@ class AggExecutorTestRunner : public ExecutorTest ColumnWithUInt64 col_pr{1, 2, 0, 3290124, 968933, 3125, 31236, 4327, 80000}; }; -#define WRAP_FOR_AGG_PARTIAL_BLOCK_START \ - std::vector partial_blocks{true, false}; \ - for (auto partial_block : partial_blocks) \ - { \ - if (partial_block) \ - FailPointHelper::enableFailPoint(FailPoints::force_agg_on_partial_block); \ - else \ - FailPointHelper::disableFailPoint(FailPoints::force_agg_on_partial_block); +#define WRAP_FOR_AGG_FAILPOINTS_START \ + std::vector enables{true, false}; \ + for (auto enable : enables) \ + { \ + if (enable) \ + { \ + FailPointHelper::enableFailPoint(FailPoints::force_agg_on_partial_block); \ + FailPointHelper::enableFailPoint(FailPoints::force_agg_prefetch); \ + } \ + else \ + { \ + FailPointHelper::disableFailPoint(FailPoints::force_agg_on_partial_block); \ + FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); \ + } -#define WRAP_FOR_AGG_PARTIAL_BLOCK_END } +#define WRAP_FOR_AGG_FAILPOINTS_END } /// Guarantee the correctness of group by TEST_F(AggExecutorTestRunner, GroupBy) @@ -363,9 +370,9 @@ try FailPointHelper::enableFailPoint(FailPoints::force_agg_two_level_hash_table_before_merge); else FailPointHelper::disableFailPoint(FailPoints::force_agg_two_level_hash_table_before_merge); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } } @@ -429,9 +436,9 @@ try FailPointHelper::enableFailPoint(FailPoints::force_agg_two_level_hash_table_before_merge); else FailPointHelper::disableFailPoint(FailPoints::force_agg_two_level_hash_table_before_merge); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } } @@ -464,9 +471,9 @@ try for (size_t i = 0; i < test_num; ++i) { request = buildDAGRequest(std::make_pair(db_name, table_name), agg_funcs[i], group_by_exprs[i], projections[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } /// Min function tests @@ -485,9 +492,9 @@ try for (size_t i = 0; i < test_num; ++i) { request = buildDAGRequest(std::make_pair(db_name, table_name), agg_funcs[i], group_by_exprs[i], projections[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } CATCH @@ -545,9 +552,9 @@ try { request = buildDAGRequest(std::make_pair(db_name, table_name), {agg_funcs[i]}, group_by_exprs[i], projections[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } CATCH @@ -615,9 +622,9 @@ try {agg_func}, group_by_exprs[i], projections[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } { @@ -629,9 +636,9 @@ try {agg_func}, group_by_exprs[i], projections[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } for (auto collation_id : {0, static_cast(TiDB::ITiDBCollator::BINARY)}) @@ -668,9 +675,9 @@ try {agg_func}, group_by_exprs[i], projections[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect_cols[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } } @@ -683,9 +690,9 @@ try executeAndAssertColumnsEqual(request, {{toNullableVec({"banana"})}}); request = context.scan("aggnull_test", "t1").aggregation({}, {col("s1")}).build(context); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, {{toNullableVec("s1", {{}, "banana"})}}); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } CATCH @@ -697,9 +704,9 @@ try = {toNullableVec({3}), toNullableVec({1}), toVec({6})}; auto test_single_function = [&](size_t index) { auto request = context.scan("test_db", "test_table").aggregation({functions[index]}, {}).build(context); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, {functions_result[index]}); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END }; for (size_t i = 0; i < functions.size(); ++i) test_single_function(i); @@ -720,9 +727,9 @@ try results.push_back(functions_result[k]); auto request = context.scan("test_db", "test_table").aggregation(funcs, {}).build(context); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, results); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END funcs.pop_back(); results.pop_back(); @@ -758,9 +765,9 @@ try context.context->setSetting( "group_by_two_level_threshold", Field(static_cast(two_level_threshold))); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expect); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } } @@ -791,7 +798,7 @@ try "group_by_two_level_threshold", Field(static_cast(two_level_threshold))); context.context->setSetting("max_block_size", Field(static_cast(block_size))); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START auto blocks = getExecuteStreamsReturnBlocks(request, concurrency); size_t actual_row = 0; for (auto & block : blocks) @@ -800,7 +807,7 @@ try actual_row += block.rows(); } ASSERT_EQ(actual_row, expect_rows[i]); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } } @@ -914,7 +921,7 @@ try "group_by_two_level_threshold", Field(static_cast(two_level_threshold))); context.context->setSetting("max_block_size", Field(static_cast(block_size))); - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START auto blocks = getExecuteStreamsReturnBlocks(request, concurrency); for (auto & block : blocks) { @@ -939,7 +946,7 @@ try vstackBlocks(std::move(blocks)).getColumnsWithTypeAndName(), false)); } - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } } @@ -967,18 +974,18 @@ try request = context.receive("empty_recv", 5).aggregation({Max(col("s1"))}, {col("s2")}, 5).build(context); { - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, {}); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } request = context.scan("test_db", "empty_table") .aggregation({Count(lit(Field(static_cast(1))))}, {}) .build(context); { - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, {toVec({0})}); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } CATCH @@ -1049,7 +1056,9 @@ try toNullableVec("first_row(col_tinyint)", ColumnWithNullableInt8{0, 1, 2, 3}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3}), toVec("col_tinyint", ColumnWithInt8{0, 1, 2, 3})}; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } { @@ -1065,7 +1074,9 @@ try = {toVec("count(1)", ColumnWithUInt64{rows_per_type, rows_per_type, rows_per_type, rows_per_type}), toNullableVec("first_row(col_int)", ColumnWithNullableInt32{0, 1, 2, 3}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3})}; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } { @@ -1082,7 +1093,9 @@ try toNullableVec("first_row(col_string_no_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_no_collator", ColumnWithString{"a", "b", "c", "d"}), }; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } { @@ -1099,7 +1112,9 @@ try toNullableVec("first_row(col_string_with_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_with_collator", ColumnWithString{"a", "b", "c", "d"}), }; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } { @@ -1116,7 +1131,9 @@ try toVec("count(1)", ColumnWithUInt64{rows_per_type, rows_per_type, rows_per_type, rows_per_type}), toVec("first_row(col_string_with_collator)", ColumnWithString{"a", "b", "c", "d"}), }; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } // case-5: none @@ -1138,7 +1155,9 @@ try toVec("col_int", ColumnWithInt32{0, 1, 2, 3}), toVec("col_string_no_collator", ColumnWithString{"a", "b", "c", "d"}), }; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } { @@ -1155,7 +1174,9 @@ try toNullableVec("first_row(col_string_with_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_with_collator", ColumnWithString{"a", "b", "c", "d"}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3})}; + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); + WRAP_FOR_AGG_FAILPOINTS_END } } CATCH @@ -1205,15 +1226,15 @@ try auto baseline = executeStreams(gen_request(1), 1); for (size_t exchange_concurrency : exchange_receiver_concurrency) { - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(gen_request(exchange_concurrency), baseline); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END } } CATCH -#undef WRAP_FOR_AGG_PARTIAL_BLOCK_START -#undef WRAP_FOR_AGG_PARTIAL_BLOCK_END +#undef WRAP_FOR_AGG_FAILPOINTS_START +#undef WRAP_FOR_AGG_FAILPOINTS_END } // namespace tests } // namespace DB diff --git a/dbms/src/Flash/tests/gtest_compute_server.cpp b/dbms/src/Flash/tests/gtest_compute_server.cpp index 69b2242df3d..3c4020db45e 100644 --- a/dbms/src/Flash/tests/gtest_compute_server.cpp +++ b/dbms/src/Flash/tests/gtest_compute_server.cpp @@ -39,6 +39,7 @@ extern const char exception_before_mpp_root_task_run[]; extern const char exception_during_mpp_non_root_task_run[]; extern const char exception_during_mpp_root_task_run[]; extern const char exception_during_query_run[]; +extern const char force_agg_prefetch[]; } // namespace FailPoints namespace tests @@ -1369,6 +1370,7 @@ try FailPoints::exception_during_mpp_non_root_task_run, FailPoints::exception_during_mpp_root_task_run, FailPoints::exception_during_query_run, + FailPoints::force_agg_prefetch, }; size_t query_index = 0; for (const auto & failpoint : failpoint_names) @@ -1843,6 +1845,7 @@ try auto_pass_through_test_data.nullable_high_ndv_tbl_name, auto_pass_through_test_data.nullable_medium_ndv_tbl_name, }; + FailPointHelper::enableFailPoint(FailPoints::force_agg_prefetch); for (const auto & tbl_name : workloads) { const String db_name = auto_pass_through_test_data.db_name; @@ -1868,6 +1871,7 @@ try res_no_pass_through); WRAP_FOR_SERVER_TEST_END } + FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); } CATCH diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 543885b6248..6e3262c15d4 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -43,6 +43,7 @@ extern const char random_aggregate_create_state_failpoint[]; extern const char random_aggregate_merge_failpoint[]; extern const char force_agg_on_partial_block[]; extern const char random_fail_in_resize_callback[]; +extern const char force_agg_prefetch[]; } // namespace FailPoints #define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME @@ -665,7 +666,13 @@ void NO_INLINE Aggregator::executeImpl( { typename Method::State state(agg_process_info.key_columns, key_sizes, collators); - if (method.data.getBufferSizeInCells() < 8192) +#ifndef NDEBUG + bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); + fiu_do_on(FailPoints::force_agg_prefetch, { disable_prefetch = false; }); +#else + const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); +#endif + if (disable_prefetch) { if constexpr (Method::Data::is_string_hash_map) executeImplBatchStringHashMap( From 623fef57f5a160e201e5f765f22430eef88f8300 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 2 Dec 2024 20:14:41 +0800 Subject: [PATCH 12/61] prefetch Signed-off-by: guo-shaoge --- dbms/src/Common/HashTable/HashTable.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index f8d44e8c406..c0f066edbb0 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -856,14 +856,8 @@ class HashTable void ALWAYS_INLINE prefetch(size_t hashval) const { - (void)hashval; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - size_t place_value = grower.place(hashval); - __mm_prefetch((const char *)(&buf[place_value]), _MM_HINT_NTA); -#elif defined(__GNUC__) - size_t place_value = grower.place(hashval); + const size_t place_value = grower.place(hashval); __builtin_prefetch(static_cast(&buf[place_value])); -#endif } protected: From 19f320daa2b5250ee45456ee0d1401f1cd32fa53 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 2 Dec 2024 20:28:40 +0800 Subject: [PATCH 13/61] fix Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 94526714250..a9817308616 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -85,8 +85,6 @@ struct HashMethodOneNumber } const FieldType * getKeyData() const { return vec; } - - size_t getTotalRows() const { return total_rows; } }; From 3a226dfb518caf0afa1769617d7b800f2c97ca12 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 14:14:14 +0800 Subject: [PATCH 14/61] refine Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 4 +- .../HashTable/TwoLevelStringHashTable.h | 2 +- .../tests/gtest_aggregation_executor.cpp | 53 ++++++++++++------- dbms/src/Interpreters/Aggregator.cpp | 21 +++++--- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 1f4e3dbaedf..b5b61fb8630 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -184,7 +184,7 @@ class HashMethodBase ALWAYS_INLINE inline EmplaceResult emplaceStringKey( Data & data, size_t idx, - std::vector & datas, // TODO const + std::vector & datas, const std::vector & hashvals) { // For spill, hashvals.size() will be le to total_rows. @@ -204,7 +204,7 @@ class HashMethodBase ALWAYS_INLINE inline FindResult findStringKey( Data & data, size_t idx, - std::vector & datas, // TODO const + std::vector & datas, const std::vector & hashvals) { assert(hashvals.size() <= static_cast(*this).total_rows); diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 5ea460769ab..ac2ab483e46 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -265,7 +265,7 @@ class TwoLevelStringHashTable : private boost::noncopyable { size_t res = 0; for (const auto & impl : impls) - res = impl.getBufferSizeInCells(); + res += impl.getBufferSizeInCells(); return res; } size_t getBufferSizeInBytes() const diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index 8c7f5277916..3a79025f244 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -1042,6 +1042,24 @@ try toVec("col_tinyint", col_data_tinyint), }); + std::vector max_block_sizes{1, 2, DEFAULT_BLOCK_SIZE}; + std::vector two_level_thresholds{0, 1}; + + context.context->setSetting("group_by_two_level_threshold_bytes", Field(static_cast(0))); +#define WRAP_FOR_AGG_STRING_TEST_BEGIN \ + for (const auto & max_block_size : max_block_sizes) \ + { \ + for (const auto & two_level_threshold : two_level_thresholds) \ + { \ + context.context->setSetting( \ + "group_by_two_level_threshold", \ + Field(static_cast(two_level_threshold))); \ + context.context->setSetting("max_block_size", Field(static_cast(max_block_size))); +#define WRAP_FOR_AGG_STRING_TEST_END \ + } \ + } + + FailPointHelper::enableFailPoint(FailPoints::force_agg_prefetch); { // case-1: select count(1), col_tinyint from t group by col_int, col_tinyint // agg method: keys64(AggregationMethodKeysFixed) @@ -1056,9 +1074,9 @@ try toNullableVec("first_row(col_tinyint)", ColumnWithNullableInt8{0, 1, 2, 3}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3}), toVec("col_tinyint", ColumnWithInt8{0, 1, 2, 3})}; - WRAP_FOR_AGG_FAILPOINTS_START + WRAP_FOR_AGG_STRING_TEST_BEGIN executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END + WRAP_FOR_AGG_STRING_TEST_END } { @@ -1074,9 +1092,9 @@ try = {toVec("count(1)", ColumnWithUInt64{rows_per_type, rows_per_type, rows_per_type, rows_per_type}), toNullableVec("first_row(col_int)", ColumnWithNullableInt32{0, 1, 2, 3}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3})}; - WRAP_FOR_AGG_FAILPOINTS_START + WRAP_FOR_AGG_STRING_TEST_BEGIN executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END + WRAP_FOR_AGG_STRING_TEST_END } { @@ -1093,9 +1111,7 @@ try toNullableVec("first_row(col_string_no_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_no_collator", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_FAILPOINTS_START executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END } { @@ -1112,9 +1128,9 @@ try toNullableVec("first_row(col_string_with_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_with_collator", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_FAILPOINTS_START + WRAP_FOR_AGG_STRING_TEST_BEGIN executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END + WRAP_FOR_AGG_STRING_TEST_END } { @@ -1131,9 +1147,9 @@ try toVec("count(1)", ColumnWithUInt64{rows_per_type, rows_per_type, rows_per_type, rows_per_type}), toVec("first_row(col_string_with_collator)", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_FAILPOINTS_START + WRAP_FOR_AGG_STRING_TEST_BEGIN executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END + WRAP_FOR_AGG_STRING_TEST_END } // case-5: none @@ -1155,9 +1171,9 @@ try toVec("col_int", ColumnWithInt32{0, 1, 2, 3}), toVec("col_string_no_collator", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_FAILPOINTS_START + WRAP_FOR_AGG_STRING_TEST_BEGIN executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END + WRAP_FOR_AGG_STRING_TEST_END } { @@ -1174,10 +1190,13 @@ try toNullableVec("first_row(col_string_with_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_with_collator", ColumnWithString{"a", "b", "c", "d"}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3})}; - WRAP_FOR_AGG_FAILPOINTS_START + WRAP_FOR_AGG_STRING_TEST_BEGIN executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_FAILPOINTS_END + WRAP_FOR_AGG_STRING_TEST_END } + FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); +#undef WRAP_FOR_AGG_STRING_TEST_BEGIN +#undef WRAP_FOR_AGG_STRING_TEST_END } CATCH @@ -1208,13 +1227,9 @@ try context .addExchangeReceiver("exchange_receiver_1_concurrency", column_infos, column_data, 1, partition_column_infos); - context - .addExchangeReceiver("exchange_receiver_3_concurrency", column_infos, column_data, 3, partition_column_infos); - context - .addExchangeReceiver("exchange_receiver_5_concurrency", column_infos, column_data, 5, partition_column_infos); context .addExchangeReceiver("exchange_receiver_10_concurrency", column_infos, column_data, 10, partition_column_infos); - std::vector exchange_receiver_concurrency = {1, 3, 5, 10}; + std::vector exchange_receiver_concurrency = {1, 10}; auto gen_request = [&](size_t exchange_concurrency) { return context diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 6e3262c15d4..2c1aba6b2af 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -741,8 +741,9 @@ std::optional::Res } } +// This is only used by executeImplBatchStringHashMap. +// It will choose specifix submap of StringHashMap then do emplace/find. // StringKeyType can be StringRef/StringKey8/StringKey16/StringKey24/ArenaKeyHolder. -// return true when resize exception happens. template < size_t SubMapIndex, bool collect_hit_rate, @@ -756,7 +757,7 @@ size_t Aggregator::emplaceOrFindStringKey( Data & data, State & state, const std::vector & key_infos, - std::vector & key_datas, // TODO const + std::vector & key_datas, Arena & aggregates_pool, std::vector & places, AggProcessInfo & agg_process_info) const @@ -1045,8 +1046,8 @@ M(4) #undef M -// In this function, we will prefetch/empalce each specifix submap directly instead of accessing StringHashMap interface, -// which is good for performance. +// prefetch/empalce each specifix submap directly instead of accessing StringHashMap interface, +// which is better for performance. // NOTE: this function is column-wise, which means sort key buffer cannot be reused. // This buffer will not be release until this block is processed done. template @@ -1088,7 +1089,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( #undef M // If no resize exception happens, so this is a new Block. - // If resize exception happens, start_row also set as zero. + // If resize exception happens, start_row has already been set to zero at the end of this function. RUNTIME_CHECK(agg_process_info.start_row == 0); if likely (agg_process_info.stringHashTableRecoveryInfoEmpty()) @@ -1226,13 +1227,17 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( inst->batch_arguments, aggregates_pool); } - // For StringHashTable, start_row is meanless, instead submap_mx_infos/submap_mx_datas are used. - agg_process_info.start_row = got_resize_exception ? 0 : agg_process_info.end_row; - if unlikely (got_resize_exception) { RUNTIME_CHECK(!agg_process_info.stringHashTableRecoveryInfoEmpty()); agg_process_info.sort_key_pool = std::move(sort_key_pool); + // For StringHashTable, start_row is meanless, instead submap_mx_infos/submap_mx_datas are used. + // So set it to zero when got_resize_exception. + agg_process_info.start_row = 0; + } + else + { + agg_process_info.start_row = agg_process_info.end_row; } } From c44ace7030558dff7952fde6aae9aedb6c2bd400 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 15:50:56 +0800 Subject: [PATCH 15/61] refine Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 2c1aba6b2af..e5e214c7791 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -674,13 +674,13 @@ void NO_INLINE Aggregator::executeImpl( #endif if (disable_prefetch) { - if constexpr (Method::Data::is_string_hash_map) - executeImplBatchStringHashMap( - method, - state, - aggregates_pool, - agg_process_info); - else + // if constexpr (Method::Data::is_string_hash_map) + // executeImplBatchStringHashMap( + // method, + // state, + // aggregates_pool, + // agg_process_info); + // else executeImplBatch(method, state, aggregates_pool, agg_process_info); } else From 3e30f9561e38d09c39d1b0dc73735b6dcffb7991 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 16:08:34 +0800 Subject: [PATCH 16/61] revert new hasher Signed-off-by: guo-shaoge --- dbms/src/Common/HashTable/StringHashTable.h | 66 +++++++++++++++---- .../HashTable/TwoLevelStringHashTable.h | 8 +-- dbms/src/Interpreters/Aggregator.h | 20 +++--- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index a43f35fdbbf..322523388cc 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -67,17 +67,57 @@ struct HashWithMixSeed } }; +// struct StringHashTableHash +// { +// using StringKey8Hasher = HashWithMixSeed; +// using StringKey16Hasher = HashWithMixSeed; +// using StringKey24Hasher = HashWithMixSeed; +// using StringRefHasher = StringRefHash; +// +// static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } +// static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } +// static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } +// static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } +// }; struct StringHashTableHash { - using StringKey8Hasher = HashWithMixSeed; - using StringKey16Hasher = HashWithMixSeed; - using StringKey24Hasher = HashWithMixSeed; - using StringRefHasher = StringRefHash; - - static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } +#if defined(__SSE4_2__) + static size_t ALWAYS_INLINE operator()(StringKey8 key) + { + size_t res = -1ULL; + res = _mm_crc32_u64(res, key); + return res; + } + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) + { + size_t res = -1ULL; + res = _mm_crc32_u64(res, key.low); + res = _mm_crc32_u64(res, key.high); + return res; + } + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) + { + size_t res = -1ULL; + res = _mm_crc32_u64(res, key.a); + res = _mm_crc32_u64(res, key.b); + res = _mm_crc32_u64(res, key.c); + return res; + } +#else + static size_t ALWAYS_INLINE operator()(StringKey8 key) + { + return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 8); + } + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) + { + return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 16); + } + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) + { + return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 24); + } +#endif + static size_t ALWAYS_INLINE operator()(StringRef key){ return StringRefHash()(key); } }; template @@ -572,7 +612,7 @@ struct StringHashTableSubMapSelector<0, false, Data> template struct StringHashTableSubMapSelector<1, false, Data> { - using Hash = StringHashTableHash::StringKey8Hasher; + using Hash = StringHashTableHash; static typename Data::T1 & getSubMap(size_t, Data & data) { return data.m1; } }; @@ -580,7 +620,7 @@ struct StringHashTableSubMapSelector<1, false, Data> template struct StringHashTableSubMapSelector<2, false, Data> { - using Hash = StringHashTableHash::StringKey16Hasher; + using Hash = StringHashTableHash; static typename Data::T2 & getSubMap(size_t, Data & data) { return data.m2; } }; @@ -588,7 +628,7 @@ struct StringHashTableSubMapSelector<2, false, Data> template struct StringHashTableSubMapSelector<3, false, Data> { - using Hash = StringHashTableHash::StringKey24Hasher; + using Hash = StringHashTableHash; static typename Data::T3 & getSubMap(size_t, Data & data) { return data.m3; } }; @@ -596,7 +636,7 @@ struct StringHashTableSubMapSelector<3, false, Data> template struct StringHashTableSubMapSelector<4, false, Data> { - using Hash = StringHashTableHash::StringRefHasher; + using Hash = StringHashTableHash; static typename Data::Ts & getSubMap(size_t, Data & data) { return data.ms; } }; diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index ac2ab483e46..403b8d3941c 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -296,7 +296,7 @@ struct StringHashTableSubMapSelector<0, true, Data> template struct StringHashTableSubMapSelector<1, true, Data> { - using Hash = StringHashTableHash::StringKey8Hasher; + using Hash = StringHashTableHash; static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) { @@ -308,7 +308,7 @@ struct StringHashTableSubMapSelector<1, true, Data> template struct StringHashTableSubMapSelector<2, true, Data> { - using Hash = StringHashTableHash::StringKey16Hasher; + using Hash = StringHashTableHash; static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) { @@ -320,7 +320,7 @@ struct StringHashTableSubMapSelector<2, true, Data> template struct StringHashTableSubMapSelector<3, true, Data> { - using Hash = StringHashTableHash::StringKey24Hasher; + using Hash = StringHashTableHash; static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) { @@ -332,7 +332,7 @@ struct StringHashTableSubMapSelector<3, true, Data> template struct StringHashTableSubMapSelector<4, true, Data> { - using Hash = StringHashTableHash::StringRefHasher; + using Hash = StringHashTableHash; static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index c6e78fb5618..eb68bc50ae9 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -77,27 +77,27 @@ using AggregatedDataWithoutKey = AggregateDataPtr; using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize; using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap; -using AggregatedDataWithUInt32Key = HashMap>; -using AggregatedDataWithUInt64Key = HashMap>; +using AggregatedDataWithUInt32Key = HashMap>; +using AggregatedDataWithUInt64Key = HashMap>; using AggregatedDataWithShortStringKey = StringHashMap; using AggregatedDataWithStringKey = HashMapWithSavedHash; -using AggregatedDataWithInt256Key = HashMap>; +using AggregatedDataWithInt256Key = HashMap>; -using AggregatedDataWithKeys128 = HashMap>; -using AggregatedDataWithKeys256 = HashMap>; +using AggregatedDataWithKeys128 = HashMap>; +using AggregatedDataWithKeys256 = HashMap>; -using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap; using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash; -using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; -using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; /** Variants with better hash function, using more than 32 bits for hash. * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion, From ea85d19ff8dc4e97abbbf71bb05a424632b17684 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 17:10:09 +0800 Subject: [PATCH 17/61] debug low distinct value Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 121 ++++++++++++++++++++- dbms/src/Interpreters/Aggregator.cpp | 154 ++++++++++++++++++++------- dbms/src/Interpreters/Aggregator.h | 8 ++ 3 files changed, 244 insertions(+), 39 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index b5b61fb8630..f5cc03d82c8 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -138,13 +138,35 @@ class HashMethodBase map.prefetch(hashvals[prefetch_idx]); } + template + ALWAYS_INLINE inline EmplaceResult emplaceKey( + Data & data, + size_t row, + Arena & pool, + std::vector & sort_key_containers) + { + auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); + return emplaceImpl(key_holder, data); + } + + template + ALWAYS_INLINE inline FindResult findKey( + Data & data, + size_t row, + Arena & pool, + std::vector & sort_key_containers) + { + auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); + return findKeyImpl(keyHolderGetKey(key_holder), data, 0); + } + template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, size_t row, Arena & pool, std::vector & sort_key_containers, - const std::vector & hashvals = {}) + const std::vector & hashvals) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); if constexpr (enable_prefetch) @@ -165,7 +187,7 @@ class HashMethodBase size_t row, Arena & pool, std::vector & sort_key_containers, - const std::vector & hashvals = {}) + const std::vector & hashvals) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); if constexpr (enable_prefetch) @@ -247,6 +269,60 @@ class HashMethodBase } } + template + ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data) + { + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.found && cache.check(keyHolderGetKey(key_holder))) + { + if constexpr (has_mapped) + return EmplaceResult(cache.value.second, cache.value.second, false); + else + return EmplaceResult(false); + } + } + + typename Data::LookupResult it; + bool inserted = false; + + data.emplace(key_holder, it, inserted); + + [[maybe_unused]] Mapped * cached = nullptr; + if constexpr (has_mapped) + cached = &it->getMapped(); + + if (inserted) + { + if constexpr (has_mapped) + { + new (&it->getMapped()) Mapped(); + } + } + + if constexpr (consecutive_keys_optimization) + { + cache.found = true; + cache.empty = false; + + if constexpr (has_mapped) + { + cache.value.first = it->getKey(); + cache.value.second = it->getMapped(); + cached = &cache.value.second; + } + else + { + cache.value = it->getKey(); + } + } + + if constexpr (has_mapped) + return EmplaceResult(it->getMapped(), *cached, inserted); + else + return EmplaceResult(inserted); + } + template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { @@ -304,6 +380,47 @@ class HashMethodBase return EmplaceResult(inserted); } + template + ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data) + { + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.check(key)) + { + if constexpr (has_mapped) + return FindResult(&cache.value.second, cache.found); + else + return FindResult(cache.found); + } + } + + typename Data::LookupResult it; + it = data.find(key); + + if constexpr (consecutive_keys_optimization) + { + cache.found = it != nullptr; + cache.empty = false; + + if constexpr (has_mapped) + { + cache.value.first = key; + if (it) + { + cache.value.second = it->getMapped(); + } + } + else + { + cache.value = key; + } + } + + if constexpr (has_mapped) + return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); + else + return FindResult(it != nullptr); + } template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data, size_t hashval) { diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index e5e214c7791..3368a3e9bfe 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -741,6 +741,27 @@ std::optional::Res } } +template +std::optional::ResultType> Aggregator::emplaceOrFindKey( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers) const +{ + try + { + if constexpr (only_lookup) + return state.findKey(method.data, index, aggregates_pool, sort_key_containers); + else + return state.emplaceKey(method.data, index, aggregates_pool, sort_key_containers); + } + catch (ResizeException &) + { + return {}; + } +} + // This is only used by executeImplBatchStringHashMap. // It will choose specifix submap of StringHashMap then do emplace/find. // StringKeyType can be StringRef/StringKey8/StringKey16/StringKey24/ArenaKeyHolder. @@ -937,9 +958,9 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( /// Generic case. std::unique_ptr places(new AggregateDataPtr[rows]); std::optional processed_rows; - std::vector hashvals; if constexpr (enable_prefetch) { + std::vector hashvals; hashvals = getHashVals( agg_process_info.start_row, agg_process_info.end_row, @@ -947,64 +968,123 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( state, sort_key_containers, aggregates_pool); - } - - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - auto emplace_result_holder = emplaceOrFindKey( - method, - state, - i, - *aggregates_pool, - sort_key_containers, - hashvals); - if unlikely (!emplace_result_holder.has_value()) + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); - break; - } + AggregateDataPtr aggregate_data = nullptr; - auto & emplace_result = emplace_result_holder.value(); + auto emplace_result_holder = emplaceOrFindKey( + method, + state, + i, + *aggregates_pool, + sort_key_containers, + hashvals); + if unlikely (!emplace_result_holder.has_value()) + { + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); + break; + } - if constexpr (only_lookup) - { - if (emplace_result.isFound()) + auto & emplace_result = emplace_result_holder.value(); + + if constexpr (only_lookup) { - aggregate_data = emplace_result.getMapped(); + if (emplace_result.isFound()) + { + aggregate_data = emplace_result.getMapped(); + } + else + { + agg_process_info.not_found_rows.push_back(i); + } } else { - agg_process_info.not_found_rows.push_back(i); + /// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key. + if (emplace_result.isInserted()) + { + /// exception-safety - if you can not allocate memory or create states, then destructors will not be called. + emplace_result.setMapped(nullptr); + + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + + emplace_result.setMapped(aggregate_data); + } + else + { + aggregate_data = emplace_result.getMapped(); + + if constexpr (collect_hit_rate) + ++agg_process_info.hit_row_cnt; + } } + + places[i - agg_process_info.start_row] = aggregate_data; + processed_rows = i; } - else + } + else + { + LOG_DEBUG(log, "gjt debug original path"); + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { - /// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key. - if (emplace_result.isInserted()) + AggregateDataPtr aggregate_data = nullptr; + + auto emplace_result_holder = emplaceOrFindKey( + method, + state, + i, + *aggregates_pool, + sort_key_containers); + if unlikely (!emplace_result_holder.has_value()) { - /// exception-safety - if you can not allocate memory or create states, then destructors will not be called. - emplace_result.setMapped(nullptr); + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); + break; + } - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); + auto & emplace_result = emplace_result_holder.value(); - emplace_result.setMapped(aggregate_data); + if constexpr (only_lookup) + { + if (emplace_result.isFound()) + { + aggregate_data = emplace_result.getMapped(); + } + else + { + agg_process_info.not_found_rows.push_back(i); + } } else { - aggregate_data = emplace_result.getMapped(); + /// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key. + if (emplace_result.isInserted()) + { + /// exception-safety - if you can not allocate memory or create states, then destructors will not be called. + emplace_result.setMapped(nullptr); - if constexpr (collect_hit_rate) - ++agg_process_info.hit_row_cnt; + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + + emplace_result.setMapped(aggregate_data); + } + else + { + aggregate_data = emplace_result.getMapped(); + + if constexpr (collect_hit_rate) + ++agg_process_info.hit_row_cnt; + } } - } - places[i - agg_process_info.start_row] = aggregate_data; - processed_rows = i; + places[i - agg_process_info.start_row] = aggregate_data; + processed_rows = i; + } } + if (processed_rows) { /// Add values to the aggregate functions. diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index eb68bc50ae9..729ba863130 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1499,6 +1499,14 @@ class Aggregator std::vector & sort_key_containers, const std::vector & hashvals) const; + template + std::optional::ResultType> emplaceOrFindKey( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers) const; + template < size_t SubMapIndex, bool collect_hit_rate, From 16937ff2cdd5e3fb7cde826887c0bfd252940bf3 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 17:36:18 +0800 Subject: [PATCH 18/61] Revert "revert new hasher" This reverts commit 3e30f9561e38d09c39d1b0dc73735b6dcffb7991. --- dbms/src/Common/HashTable/StringHashTable.h | 66 ++++--------------- .../HashTable/TwoLevelStringHashTable.h | 8 +-- dbms/src/Interpreters/Aggregator.h | 20 +++--- 3 files changed, 27 insertions(+), 67 deletions(-) diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index 322523388cc..a43f35fdbbf 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -67,57 +67,17 @@ struct HashWithMixSeed } }; -// struct StringHashTableHash -// { -// using StringKey8Hasher = HashWithMixSeed; -// using StringKey16Hasher = HashWithMixSeed; -// using StringKey24Hasher = HashWithMixSeed; -// using StringRefHasher = StringRefHash; -// -// static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } -// static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } -// static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } -// static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } -// }; struct StringHashTableHash { -#if defined(__SSE4_2__) - static size_t ALWAYS_INLINE operator()(StringKey8 key) - { - size_t res = -1ULL; - res = _mm_crc32_u64(res, key); - return res; - } - static size_t ALWAYS_INLINE operator()(const StringKey16 & key) - { - size_t res = -1ULL; - res = _mm_crc32_u64(res, key.low); - res = _mm_crc32_u64(res, key.high); - return res; - } - static size_t ALWAYS_INLINE operator()(const StringKey24 & key) - { - size_t res = -1ULL; - res = _mm_crc32_u64(res, key.a); - res = _mm_crc32_u64(res, key.b); - res = _mm_crc32_u64(res, key.c); - return res; - } -#else - static size_t ALWAYS_INLINE operator()(StringKey8 key) - { - return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 8); - } - static size_t ALWAYS_INLINE operator()(const StringKey16 & key) - { - return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 16); - } - static size_t ALWAYS_INLINE operator()(const StringKey24 & key) - { - return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 24); - } -#endif - static size_t ALWAYS_INLINE operator()(StringRef key){ return StringRefHash()(key); } + using StringKey8Hasher = HashWithMixSeed; + using StringKey16Hasher = HashWithMixSeed; + using StringKey24Hasher = HashWithMixSeed; + using StringRefHasher = StringRefHash; + + static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } + static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } }; template @@ -612,7 +572,7 @@ struct StringHashTableSubMapSelector<0, false, Data> template struct StringHashTableSubMapSelector<1, false, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringKey8Hasher; static typename Data::T1 & getSubMap(size_t, Data & data) { return data.m1; } }; @@ -620,7 +580,7 @@ struct StringHashTableSubMapSelector<1, false, Data> template struct StringHashTableSubMapSelector<2, false, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringKey16Hasher; static typename Data::T2 & getSubMap(size_t, Data & data) { return data.m2; } }; @@ -628,7 +588,7 @@ struct StringHashTableSubMapSelector<2, false, Data> template struct StringHashTableSubMapSelector<3, false, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringKey24Hasher; static typename Data::T3 & getSubMap(size_t, Data & data) { return data.m3; } }; @@ -636,7 +596,7 @@ struct StringHashTableSubMapSelector<3, false, Data> template struct StringHashTableSubMapSelector<4, false, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringRefHasher; static typename Data::Ts & getSubMap(size_t, Data & data) { return data.ms; } }; diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 403b8d3941c..ac2ab483e46 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -296,7 +296,7 @@ struct StringHashTableSubMapSelector<0, true, Data> template struct StringHashTableSubMapSelector<1, true, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringKey8Hasher; static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) { @@ -308,7 +308,7 @@ struct StringHashTableSubMapSelector<1, true, Data> template struct StringHashTableSubMapSelector<2, true, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringKey16Hasher; static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) { @@ -320,7 +320,7 @@ struct StringHashTableSubMapSelector<2, true, Data> template struct StringHashTableSubMapSelector<3, true, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringKey24Hasher; static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) { @@ -332,7 +332,7 @@ struct StringHashTableSubMapSelector<3, true, Data> template struct StringHashTableSubMapSelector<4, true, Data> { - using Hash = StringHashTableHash; + using Hash = StringHashTableHash::StringRefHasher; static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 729ba863130..e3666b3d187 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -77,27 +77,27 @@ using AggregatedDataWithoutKey = AggregateDataPtr; using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize; using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap; -using AggregatedDataWithUInt32Key = HashMap>; -using AggregatedDataWithUInt64Key = HashMap>; +using AggregatedDataWithUInt32Key = HashMap>; +using AggregatedDataWithUInt64Key = HashMap>; using AggregatedDataWithShortStringKey = StringHashMap; using AggregatedDataWithStringKey = HashMapWithSavedHash; -using AggregatedDataWithInt256Key = HashMap>; +using AggregatedDataWithInt256Key = HashMap>; -using AggregatedDataWithKeys128 = HashMap>; -using AggregatedDataWithKeys256 = HashMap>; +using AggregatedDataWithKeys128 = HashMap>; +using AggregatedDataWithKeys256 = HashMap>; -using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap; using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash; -using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; -using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; /** Variants with better hash function, using more than 32 bits for hash. * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion, From d2fba576ce82eaa6cd97620f1d4f17a173edc1d3 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 20:07:18 +0800 Subject: [PATCH 19/61] refine original code path Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 302 ++++++++++----------------- dbms/src/Interpreters/Aggregator.cpp | 206 +++++++----------- dbms/src/Interpreters/Aggregator.h | 2 +- 3 files changed, 192 insertions(+), 318 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index f5cc03d82c8..3c4fd601487 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -157,7 +157,7 @@ class HashMethodBase std::vector & sort_key_containers) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - return findKeyImpl(keyHolderGetKey(key_holder), data, 0); + return findKeyImpl(keyHolderGetKey(key_holder), data); } template @@ -173,11 +173,11 @@ class HashMethodBase { assert(hashvals.size() == static_cast(*this).total_rows); prefetch(data, row, hashvals); - return emplaceImpl(key_holder, data, hashvals[row]); + return emplaceImpl(key_holder, data, hashvals[row]); } else { - return emplaceImpl(key_holder, data, 0); + return emplaceImpl(key_holder, data); } } @@ -194,11 +194,11 @@ class HashMethodBase { assert(hashvals.size() == static_cast(*this).total_rows); prefetch(data, row, hashvals); - return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); + return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); } else { - return findKeyImpl(keyHolderGetKey(key_holder), data, 0); + return findKeyImpl(keyHolderGetKey(key_holder), data); } } @@ -219,7 +219,7 @@ class HashMethodBase if constexpr (enable_prefetch) prefetch(submap, idx, hashvals); - return emplaceImpl(datas[idx], submap, hashvals[idx]); + return emplaceImpl(datas[idx], submap, hashvals[idx]); } template @@ -237,7 +237,7 @@ class HashMethodBase if constexpr (enable_prefetch) prefetch(submap, idx, hashvals); - return findKeyImpl(keyHolderGetKey(datas[idx]), submap, hashvals[idx]); + return findKeyImpl(keyHolderGetKey(datas[idx]), submap, hashvals[idx]); } template @@ -269,202 +269,128 @@ class HashMethodBase } } +#define DEFINE_EMPLACE_IMPL_BEGIN \ + if constexpr (Cache::consecutive_keys_optimization) \ + { \ + if (cache.found && cache.check(keyHolderGetKey(key_holder))) \ + { \ + if constexpr (has_mapped) \ + return EmplaceResult(cache.value.second, cache.value.second, false); \ + else \ + return EmplaceResult(false); \ + } \ + } \ + typename Data::LookupResult it; \ + bool inserted = false; + +#define DEFINE_EMPLACE_IMPL_END \ + [[maybe_unused]] Mapped * cached = nullptr; \ + if constexpr (has_mapped) \ + cached = &it->getMapped(); \ + \ + if (inserted) \ + { \ + if constexpr (has_mapped) \ + { \ + new (&it->getMapped()) Mapped(); \ + } \ + } \ + \ + if constexpr (consecutive_keys_optimization) \ + { \ + cache.found = true; \ + cache.empty = false; \ + \ + if constexpr (has_mapped) \ + { \ + cache.value.first = it->getKey(); \ + cache.value.second = it->getMapped(); \ + cached = &cache.value.second; \ + } \ + else \ + { \ + cache.value = it->getKey(); \ + } \ + } \ + \ + if constexpr (has_mapped) \ + return EmplaceResult(it->getMapped(), *cached, inserted); \ + else \ + return EmplaceResult(inserted); + template - ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data) + ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.found && cache.check(keyHolderGetKey(key_holder))) - { - if constexpr (has_mapped) - return EmplaceResult(cache.value.second, cache.value.second, false); - else - return EmplaceResult(false); - } - } - - typename Data::LookupResult it; - bool inserted = false; - - data.emplace(key_holder, it, inserted); - - [[maybe_unused]] Mapped * cached = nullptr; - if constexpr (has_mapped) - cached = &it->getMapped(); - - if (inserted) - { - if constexpr (has_mapped) - { - new (&it->getMapped()) Mapped(); - } - } - - if constexpr (consecutive_keys_optimization) - { - cache.found = true; - cache.empty = false; - - if constexpr (has_mapped) - { - cache.value.first = it->getKey(); - cache.value.second = it->getMapped(); - cached = &cache.value.second; - } - else - { - cache.value = it->getKey(); - } - } - - if constexpr (has_mapped) - return EmplaceResult(it->getMapped(), *cached, inserted); - else - return EmplaceResult(inserted); + DEFINE_EMPLACE_IMPL_BEGIN + data.emplace(key_holder, it, inserted, hashval); + DEFINE_EMPLACE_IMPL_END } - template - ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) + template + ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.found && cache.check(keyHolderGetKey(key_holder))) - { - if constexpr (has_mapped) - return EmplaceResult(cache.value.second, cache.value.second, false); - else - return EmplaceResult(false); - } - } - - typename Data::LookupResult it; - bool inserted = false; - - if constexpr (use_hashval) - data.emplace(key_holder, it, inserted, hashval); - else - data.emplace(key_holder, it, inserted); - - [[maybe_unused]] Mapped * cached = nullptr; - if constexpr (has_mapped) - cached = &it->getMapped(); - - if (inserted) - { - if constexpr (has_mapped) - { - new (&it->getMapped()) Mapped(); - } - } - - if constexpr (consecutive_keys_optimization) - { - cache.found = true; - cache.empty = false; - - if constexpr (has_mapped) - { - cache.value.first = it->getKey(); - cache.value.second = it->getMapped(); - cached = &cache.value.second; - } - else - { - cache.value = it->getKey(); - } - } - - if constexpr (has_mapped) - return EmplaceResult(it->getMapped(), *cached, inserted); - else - return EmplaceResult(inserted); + DEFINE_EMPLACE_IMPL_BEGIN + data.emplace(key_holder, it, inserted); + DEFINE_EMPLACE_IMPL_END } +#undef DEFINE_EMPLACE_IMPL_BEGIN +#undef DEFINE_EMPLACE_IMPL_END + +#define DEFINE_FIND_IMPL_BEGIN \ + if constexpr (Cache::consecutive_keys_optimization) \ + { \ + if (cache.check(key)) \ + { \ + if constexpr (has_mapped) \ + return FindResult(&cache.value.second, cache.found); \ + else \ + return FindResult(cache.found); \ + } \ + } \ + typename Data::LookupResult it; + +#define DEFINE_FIND_IMPL_END \ + if constexpr (consecutive_keys_optimization) \ + { \ + cache.found = it != nullptr; \ + cache.empty = false; \ + \ + if constexpr (has_mapped) \ + { \ + cache.value.first = key; \ + if (it) \ + { \ + cache.value.second = it->getMapped(); \ + } \ + } \ + else \ + { \ + cache.value = key; \ + } \ + } \ + \ + if constexpr (has_mapped) \ + return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); \ + else \ + return FindResult(it != nullptr); template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.check(key)) - { - if constexpr (has_mapped) - return FindResult(&cache.value.second, cache.found); - else - return FindResult(cache.found); - } - } - - typename Data::LookupResult it; + DEFINE_FIND_IMPL_BEGIN it = data.find(key); - - if constexpr (consecutive_keys_optimization) - { - cache.found = it != nullptr; - cache.empty = false; - - if constexpr (has_mapped) - { - cache.value.first = key; - if (it) - { - cache.value.second = it->getMapped(); - } - } - else - { - cache.value = key; - } - } - - if constexpr (has_mapped) - return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); - else - return FindResult(it != nullptr); + DEFINE_FIND_IMPL_END } - template + + template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data, size_t hashval) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.check(key)) - { - if constexpr (has_mapped) - return FindResult(&cache.value.second, cache.found); - else - return FindResult(cache.found); - } - } - - typename Data::LookupResult it; - if constexpr (use_hashval) - it = data.find(key, hashval); - else - it = data.find(key); - - if constexpr (consecutive_keys_optimization) - { - cache.found = it != nullptr; - cache.empty = false; - - if constexpr (has_mapped) - { - cache.value.first = key; - if (it) - { - cache.value.second = it->getMapped(); - } - } - else - { - cache.value = key; - } - } - - if constexpr (has_mapped) - return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); - else - return FindResult(it != nullptr); + DEFINE_FIND_IMPL_BEGIN + it = data.find(key, hashval); + DEFINE_FIND_IMPL_END } +#undef DEFINE_FIND_IMPL_BEGIN +#undef DEFINE_FIND_IMPL_END }; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 3368a3e9bfe..8e12e7383ab 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -681,7 +681,7 @@ void NO_INLINE Aggregator::executeImpl( // aggregates_pool, // agg_process_info); // else - executeImplBatch(method, state, aggregates_pool, agg_process_info); + executeImplBatch(method, state, aggregates_pool, agg_process_info); } else { @@ -713,7 +713,7 @@ std::vector getHashVals( return hashvals; } -template +template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, typename Method::State & state, @@ -725,10 +725,14 @@ std::optional::Res try { if constexpr (only_lookup) - return state - .template findKey(method.data, index, aggregates_pool, sort_key_containers, hashvals); + return state.template findKey( + method.data, + index, + aggregates_pool, + sort_key_containers, + hashvals); else - return state.template emplaceKey( + return state.template emplaceKey( method.data, index, aggregates_pool, @@ -875,26 +879,16 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( /// For all rows. AggregateDataPtr place = aggregates_pool->alloc(0); std::vector hashvals; - if constexpr (enable_prefetch) - { - hashvals = getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool); - } for (size_t i = 0; i < rows; ++i) { - auto emplace_result_hold = emplaceOrFindKey( + // TODO prefetch + auto emplace_result_hold = emplaceOrFindKey( method, state, agg_process_info.start_row, *aggregates_pool, - sort_key_containers, - hashvals); + sort_key_containers); if likely (emplace_result_hold.has_value()) { if constexpr (collect_hit_rate) @@ -958,6 +952,56 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( /// Generic case. std::unique_ptr places(new AggregateDataPtr[rows]); std::optional processed_rows; + +#define WRAP_EMPLACE_AGG_KEY_BEGIN \ + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) \ + { \ + AggregateDataPtr aggregate_data = nullptr; + +#define WRAP_EMPLACE_AGG_KEY_END \ + if unlikely (!emplace_result_holder.has_value()) \ + { \ + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ + break; \ + } \ + \ + auto & emplace_result = emplace_result_holder.value(); \ + \ + if constexpr (only_lookup) \ + { \ + if (emplace_result.isFound()) \ + { \ + aggregate_data = emplace_result.getMapped(); \ + } \ + else \ + { \ + agg_process_info.not_found_rows.push_back(i); \ + } \ + } \ + else \ + { \ + if (emplace_result.isInserted()) \ + { \ + emplace_result.setMapped(nullptr); \ + \ + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); \ + createAggregateStates(aggregate_data); \ + \ + emplace_result.setMapped(aggregate_data); \ + } \ + else \ + { \ + aggregate_data = emplace_result.getMapped(); \ + \ + if constexpr (collect_hit_rate) \ + ++agg_process_info.hit_row_cnt; \ + } \ + } \ + \ + places[i - agg_process_info.start_row] = aggregate_data; \ + processed_rows = i; \ + } + if constexpr (enable_prefetch) { std::vector hashvals; @@ -969,121 +1013,25 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( sort_key_containers, aggregates_pool); - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - - auto emplace_result_holder = emplaceOrFindKey( - method, - state, - i, - *aggregates_pool, - sort_key_containers, - hashvals); - if unlikely (!emplace_result_holder.has_value()) - { - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); - break; - } - - auto & emplace_result = emplace_result_holder.value(); - - if constexpr (only_lookup) - { - if (emplace_result.isFound()) - { - aggregate_data = emplace_result.getMapped(); - } - else - { - agg_process_info.not_found_rows.push_back(i); - } - } - else - { - /// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key. - if (emplace_result.isInserted()) - { - /// exception-safety - if you can not allocate memory or create states, then destructors will not be called. - emplace_result.setMapped(nullptr); - - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); - - emplace_result.setMapped(aggregate_data); - } - else - { - aggregate_data = emplace_result.getMapped(); - - if constexpr (collect_hit_rate) - ++agg_process_info.hit_row_cnt; - } - } - - places[i - agg_process_info.start_row] = aggregate_data; - processed_rows = i; - } + WRAP_EMPLACE_AGG_KEY_BEGIN + auto emplace_result_holder = emplaceOrFindKey( + method, + state, + i, + *aggregates_pool, + sort_key_containers, + hashvals); + WRAP_EMPLACE_AGG_KEY_END } else { - LOG_DEBUG(log, "gjt debug original path"); - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - - auto emplace_result_holder = emplaceOrFindKey( - method, - state, - i, - *aggregates_pool, - sort_key_containers); - if unlikely (!emplace_result_holder.has_value()) - { - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); - break; - } - - auto & emplace_result = emplace_result_holder.value(); - - if constexpr (only_lookup) - { - if (emplace_result.isFound()) - { - aggregate_data = emplace_result.getMapped(); - } - else - { - agg_process_info.not_found_rows.push_back(i); - } - } - else - { - /// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key. - if (emplace_result.isInserted()) - { - /// exception-safety - if you can not allocate memory or create states, then destructors will not be called. - emplace_result.setMapped(nullptr); - - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); - - emplace_result.setMapped(aggregate_data); - } - else - { - aggregate_data = emplace_result.getMapped(); - - if constexpr (collect_hit_rate) - ++agg_process_info.hit_row_cnt; - } - } - - places[i - agg_process_info.start_row] = aggregate_data; - processed_rows = i; - } + WRAP_EMPLACE_AGG_KEY_BEGIN + auto emplace_result_holder + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + WRAP_EMPLACE_AGG_KEY_END } - +#undef WRAP_EMPLACE_AGG_KEY_BEGIN +#undef WRAP_EMPLACE_AGG_KEY_END if (processed_rows) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index e3666b3d187..eb6dfed68f9 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1490,7 +1490,7 @@ class Aggregator Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template + template std::optional::ResultType> emplaceOrFindKey( Method & method, typename Method::State & state, From 71b6ecd5fa3c664a32a47513683dd90c96c3c54e Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 20:58:50 +0800 Subject: [PATCH 20/61] Reapply "revert new hasher" This reverts commit 16937ff2cdd5e3fb7cde826887c0bfd252940bf3. --- dbms/src/Common/HashTable/StringHashTable.h | 66 +++++++++++++++---- .../HashTable/TwoLevelStringHashTable.h | 8 +-- dbms/src/Interpreters/Aggregator.h | 20 +++--- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index a43f35fdbbf..322523388cc 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -67,17 +67,57 @@ struct HashWithMixSeed } }; +// struct StringHashTableHash +// { +// using StringKey8Hasher = HashWithMixSeed; +// using StringKey16Hasher = HashWithMixSeed; +// using StringKey24Hasher = HashWithMixSeed; +// using StringRefHasher = StringRefHash; +// +// static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } +// static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } +// static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } +// static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } +// }; struct StringHashTableHash { - using StringKey8Hasher = HashWithMixSeed; - using StringKey16Hasher = HashWithMixSeed; - using StringKey24Hasher = HashWithMixSeed; - using StringRefHasher = StringRefHash; - - static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } - static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } +#if defined(__SSE4_2__) + static size_t ALWAYS_INLINE operator()(StringKey8 key) + { + size_t res = -1ULL; + res = _mm_crc32_u64(res, key); + return res; + } + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) + { + size_t res = -1ULL; + res = _mm_crc32_u64(res, key.low); + res = _mm_crc32_u64(res, key.high); + return res; + } + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) + { + size_t res = -1ULL; + res = _mm_crc32_u64(res, key.a); + res = _mm_crc32_u64(res, key.b); + res = _mm_crc32_u64(res, key.c); + return res; + } +#else + static size_t ALWAYS_INLINE operator()(StringKey8 key) + { + return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 8); + } + static size_t ALWAYS_INLINE operator()(const StringKey16 & key) + { + return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 16); + } + static size_t ALWAYS_INLINE operator()(const StringKey24 & key) + { + return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 24); + } +#endif + static size_t ALWAYS_INLINE operator()(StringRef key){ return StringRefHash()(key); } }; template @@ -572,7 +612,7 @@ struct StringHashTableSubMapSelector<0, false, Data> template struct StringHashTableSubMapSelector<1, false, Data> { - using Hash = StringHashTableHash::StringKey8Hasher; + using Hash = StringHashTableHash; static typename Data::T1 & getSubMap(size_t, Data & data) { return data.m1; } }; @@ -580,7 +620,7 @@ struct StringHashTableSubMapSelector<1, false, Data> template struct StringHashTableSubMapSelector<2, false, Data> { - using Hash = StringHashTableHash::StringKey16Hasher; + using Hash = StringHashTableHash; static typename Data::T2 & getSubMap(size_t, Data & data) { return data.m2; } }; @@ -588,7 +628,7 @@ struct StringHashTableSubMapSelector<2, false, Data> template struct StringHashTableSubMapSelector<3, false, Data> { - using Hash = StringHashTableHash::StringKey24Hasher; + using Hash = StringHashTableHash; static typename Data::T3 & getSubMap(size_t, Data & data) { return data.m3; } }; @@ -596,7 +636,7 @@ struct StringHashTableSubMapSelector<3, false, Data> template struct StringHashTableSubMapSelector<4, false, Data> { - using Hash = StringHashTableHash::StringRefHasher; + using Hash = StringHashTableHash; static typename Data::Ts & getSubMap(size_t, Data & data) { return data.ms; } }; diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index ac2ab483e46..403b8d3941c 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -296,7 +296,7 @@ struct StringHashTableSubMapSelector<0, true, Data> template struct StringHashTableSubMapSelector<1, true, Data> { - using Hash = StringHashTableHash::StringKey8Hasher; + using Hash = StringHashTableHash; static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) { @@ -308,7 +308,7 @@ struct StringHashTableSubMapSelector<1, true, Data> template struct StringHashTableSubMapSelector<2, true, Data> { - using Hash = StringHashTableHash::StringKey16Hasher; + using Hash = StringHashTableHash; static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) { @@ -320,7 +320,7 @@ struct StringHashTableSubMapSelector<2, true, Data> template struct StringHashTableSubMapSelector<3, true, Data> { - using Hash = StringHashTableHash::StringKey24Hasher; + using Hash = StringHashTableHash; static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) { @@ -332,7 +332,7 @@ struct StringHashTableSubMapSelector<3, true, Data> template struct StringHashTableSubMapSelector<4, true, Data> { - using Hash = StringHashTableHash::StringRefHasher; + using Hash = StringHashTableHash; static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index eb6dfed68f9..81252b8b3c6 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -77,27 +77,27 @@ using AggregatedDataWithoutKey = AggregateDataPtr; using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize; using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap; -using AggregatedDataWithUInt32Key = HashMap>; -using AggregatedDataWithUInt64Key = HashMap>; +using AggregatedDataWithUInt32Key = HashMap>; +using AggregatedDataWithUInt64Key = HashMap>; using AggregatedDataWithShortStringKey = StringHashMap; using AggregatedDataWithStringKey = HashMapWithSavedHash; -using AggregatedDataWithInt256Key = HashMap>; +using AggregatedDataWithInt256Key = HashMap>; -using AggregatedDataWithKeys128 = HashMap>; -using AggregatedDataWithKeys256 = HashMap>; +using AggregatedDataWithKeys128 = HashMap>; +using AggregatedDataWithKeys256 = HashMap>; -using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap; using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash; -using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; -using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; /** Variants with better hash function, using more than 32 bits for hash. * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion, From 40ceb089d4535895456461465279cdd6a09c975f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 3 Dec 2024 21:23:36 +0800 Subject: [PATCH 21/61] one level old hash; two level new hash Signed-off-by: guo-shaoge --- dbms/src/Common/HashTable/HashTable.h | 3 +- dbms/src/Common/HashTable/TwoLevelHashTable.h | 9 +++--- .../HashTable/TwoLevelStringHashTable.h | 30 +++++++++++++------ dbms/src/Interpreters/Aggregator.h | 30 +++++++++---------- dbms/src/Interpreters/Settings.h | 2 +- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index c0f066edbb0..5496f263dde 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -1020,7 +1020,8 @@ class HashTable } /// Copy the cell from another hash table. It is assumed that the cell is not zero, and also that there was no such key in the table yet. - void ALWAYS_INLINE insertUniqueNonZero(const Cell * cell, size_t hash_value) + template + void ALWAYS_INLINE insertUniqueNonZero(const InsertCellType * cell, size_t hash_value) { size_t place_value = findEmptyCell(grower.place(hash_value)); diff --git a/dbms/src/Common/HashTable/TwoLevelHashTable.h b/dbms/src/Common/HashTable/TwoLevelHashTable.h index 75a5402363d..8eb22f851eb 100644 --- a/dbms/src/Common/HashTable/TwoLevelHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelHashTable.h @@ -115,9 +115,9 @@ class TwoLevelHashTable : private boost::noncopyable /// Copy the data from another (normal) hash table. It should have the same hash function. template - explicit TwoLevelHashTable(const Source & src) + explicit TwoLevelHashTable(Source & src) { - typename Source::const_iterator it = src.begin(); + typename Source::iterator it = src.begin(); /// It is assumed that the zero key (stored separately) is first in iteration order. if (it != src.end() && it.getPtr()->isZero(src)) @@ -128,8 +128,9 @@ class TwoLevelHashTable : private boost::noncopyable for (; it != src.end(); ++it) { - const Cell * cell = it.getPtr(); - size_t hash_value = cell->getHash(src); + auto * cell = it.getPtr(); + // size_t hash_value = cell->getHash(src); + size_t hash_value = Hash::operator()(cell->getKey()); size_t buck = getBucketFromHash(hash_value); impls[buck].insertUniqueNonZero(cell, hash_value); } diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 403b8d3941c..526de846fef 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -65,32 +65,40 @@ class TwoLevelStringHashTable : private boost::noncopyable TwoLevelStringHashTable() = default; template - explicit TwoLevelStringHashTable(const Source & src) + explicit TwoLevelStringHashTable(Source & src) { if (src.m0.hasZero()) impls[0].m0.setHasZero(*src.m0.zeroValue()); for (auto & v : src.m1) { - size_t hash_value = v.getHash(src.m1); + // size_t hash_value = v.getHash(src.m1); + const size_t hash_value = ImplTable::T1::Hash::operator()(v.getKey()); + v.setHash(hash_value); size_t buck = getBucketFromHash(hash_value); impls[buck].m1.insertUniqueNonZero(&v, hash_value); } for (auto & v : src.m2) { - size_t hash_value = v.getHash(src.m2); + // size_t hash_value = v.getHash(src.m2); + const size_t hash_value = ImplTable::T2::Hash::operator()(v.getKey()); + v.setHash(hash_value); size_t buck = getBucketFromHash(hash_value); impls[buck].m2.insertUniqueNonZero(&v, hash_value); } for (auto & v : src.m3) { - size_t hash_value = v.getHash(src.m3); + // size_t hash_value = v.getHash(src.m3); + const size_t hash_value = ImplTable::T3::Hash::operator()(v.getKey()); + v.setHash(hash_value); size_t buck = getBucketFromHash(hash_value); impls[buck].m3.insertUniqueNonZero(&v, hash_value); } for (auto & v : src.ms) { - size_t hash_value = v.getHash(src.ms); + // size_t hash_value = v.getHash(src.ms); + const size_t hash_value = ImplTable::Ts::Hash::operator()(v.getKey()); + v.setHash(hash_value); size_t buck = getBucketFromHash(hash_value); impls[buck].ms.insertUniqueNonZero(&v, hash_value); } @@ -296,7 +304,8 @@ struct StringHashTableSubMapSelector<0, true, Data> template struct StringHashTableSubMapSelector<1, true, Data> { - using Hash = StringHashTableHash; + // using Hash = StringHashTableHash; + using Hash = HashWithMixSeed; static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) { @@ -308,7 +317,8 @@ struct StringHashTableSubMapSelector<1, true, Data> template struct StringHashTableSubMapSelector<2, true, Data> { - using Hash = StringHashTableHash; + // using Hash = StringHashTableHash; + using Hash = HashWithMixSeed; static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) { @@ -320,7 +330,8 @@ struct StringHashTableSubMapSelector<2, true, Data> template struct StringHashTableSubMapSelector<3, true, Data> { - using Hash = StringHashTableHash; + // using Hash = StringHashTableHash; + using Hash = HashWithMixSeed; static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) { @@ -332,7 +343,8 @@ struct StringHashTableSubMapSelector<3, true, Data> template struct StringHashTableSubMapSelector<4, true, Data> { - using Hash = StringHashTableHash; + // using Hash = StringHashTableHash; + using Hash = StringRefHash; static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 81252b8b3c6..d900cc3e231 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -88,16 +88,16 @@ using AggregatedDataWithInt256Key = HashMap>; using AggregatedDataWithKeys256 = HashMap>; -using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap; using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash; -using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; -using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; /** Variants with better hash function, using more than 32 bits for hash. * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion, @@ -125,7 +125,7 @@ struct AggregationMethodOneNumber AggregationMethodOneNumber() = default; template - explicit AggregationMethodOneNumber(const Other & other) + explicit AggregationMethodOneNumber(Other & other) : data(other.data) {} @@ -179,7 +179,7 @@ struct AggregationMethodString AggregationMethodString() = default; template - explicit AggregationMethodString(const Other & other) + explicit AggregationMethodString(Other & other) : data(other.data) {} @@ -227,7 +227,7 @@ struct AggregationMethodStringNoCache AggregationMethodStringNoCache() = default; template - explicit AggregationMethodStringNoCache(const Other & other) + explicit AggregationMethodStringNoCache(Other & other) : data(other.data) {} @@ -275,7 +275,7 @@ struct AggregationMethodOneKeyStringNoCache AggregationMethodOneKeyStringNoCache() = default; template - explicit AggregationMethodOneKeyStringNoCache(const Other & other) + explicit AggregationMethodOneKeyStringNoCache(Other & other) : data(other.data) {} @@ -325,7 +325,7 @@ struct AggregationMethodMultiStringNoCache AggregationMethodMultiStringNoCache() = default; template - explicit AggregationMethodMultiStringNoCache(const Other & other) + explicit AggregationMethodMultiStringNoCache(Other & other) : data(other.data) {} @@ -355,7 +355,7 @@ struct AggregationMethodFastPathTwoKeysNoCache AggregationMethodFastPathTwoKeysNoCache() = default; template - explicit AggregationMethodFastPathTwoKeysNoCache(const Other & other) + explicit AggregationMethodFastPathTwoKeysNoCache(Other & other) : data(other.data) {} @@ -475,7 +475,7 @@ struct AggregationMethodFixedString AggregationMethodFixedString() = default; template - explicit AggregationMethodFixedString(const Other & other) + explicit AggregationMethodFixedString(Other & other) : data(other.data) {} @@ -523,7 +523,7 @@ struct AggregationMethodFixedStringNoCache AggregationMethodFixedStringNoCache() = default; template - explicit AggregationMethodFixedStringNoCache(const Other & other) + explicit AggregationMethodFixedStringNoCache(Other & other) : data(other.data) {} @@ -572,7 +572,7 @@ struct AggregationMethodKeysFixed AggregationMethodKeysFixed() = default; template - explicit AggregationMethodKeysFixed(const Other & other) + explicit AggregationMethodKeysFixed(Other & other) : data(other.data) {} @@ -679,7 +679,7 @@ struct AggregationMethodSerialized AggregationMethodSerialized() = default; template - explicit AggregationMethodSerialized(const Other & other) + explicit AggregationMethodSerialized(Other & other) : data(other.data) {} diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 4c2e5dbeca4..5f46d74fd30 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -84,7 +84,7 @@ struct Settings M(SettingLoadBalancing, load_balancing, LoadBalancing::RANDOM, "Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing.") \ \ M(SettingUInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.") \ - M(SettingUInt64, group_by_two_level_threshold_bytes, 100000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. " \ + M(SettingUInt64, group_by_two_level_threshold_bytes, 32000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. " \ "Two-level aggregation is used when at least one of the thresholds is triggered.") \ M(SettingUInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is " \ "consumed. 0 means - same as 'max_threads'.") \ From 4cb24c21fcba3cb700cb6cbdd5d978e7fe214ad3 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 4 Dec 2024 10:40:11 +0800 Subject: [PATCH 22/61] Revert "one level old hash; two level new hash" This reverts commit 40ceb089d4535895456461465279cdd6a09c975f. --- dbms/src/Common/HashTable/HashTable.h | 3 +- dbms/src/Common/HashTable/TwoLevelHashTable.h | 9 +++--- .../HashTable/TwoLevelStringHashTable.h | 30 ++++++------------- dbms/src/Interpreters/Aggregator.h | 30 +++++++++---------- dbms/src/Interpreters/Settings.h | 2 +- 5 files changed, 30 insertions(+), 44 deletions(-) diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index 5496f263dde..c0f066edbb0 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -1020,8 +1020,7 @@ class HashTable } /// Copy the cell from another hash table. It is assumed that the cell is not zero, and also that there was no such key in the table yet. - template - void ALWAYS_INLINE insertUniqueNonZero(const InsertCellType * cell, size_t hash_value) + void ALWAYS_INLINE insertUniqueNonZero(const Cell * cell, size_t hash_value) { size_t place_value = findEmptyCell(grower.place(hash_value)); diff --git a/dbms/src/Common/HashTable/TwoLevelHashTable.h b/dbms/src/Common/HashTable/TwoLevelHashTable.h index 8eb22f851eb..75a5402363d 100644 --- a/dbms/src/Common/HashTable/TwoLevelHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelHashTable.h @@ -115,9 +115,9 @@ class TwoLevelHashTable : private boost::noncopyable /// Copy the data from another (normal) hash table. It should have the same hash function. template - explicit TwoLevelHashTable(Source & src) + explicit TwoLevelHashTable(const Source & src) { - typename Source::iterator it = src.begin(); + typename Source::const_iterator it = src.begin(); /// It is assumed that the zero key (stored separately) is first in iteration order. if (it != src.end() && it.getPtr()->isZero(src)) @@ -128,9 +128,8 @@ class TwoLevelHashTable : private boost::noncopyable for (; it != src.end(); ++it) { - auto * cell = it.getPtr(); - // size_t hash_value = cell->getHash(src); - size_t hash_value = Hash::operator()(cell->getKey()); + const Cell * cell = it.getPtr(); + size_t hash_value = cell->getHash(src); size_t buck = getBucketFromHash(hash_value); impls[buck].insertUniqueNonZero(cell, hash_value); } diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 526de846fef..403b8d3941c 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -65,40 +65,32 @@ class TwoLevelStringHashTable : private boost::noncopyable TwoLevelStringHashTable() = default; template - explicit TwoLevelStringHashTable(Source & src) + explicit TwoLevelStringHashTable(const Source & src) { if (src.m0.hasZero()) impls[0].m0.setHasZero(*src.m0.zeroValue()); for (auto & v : src.m1) { - // size_t hash_value = v.getHash(src.m1); - const size_t hash_value = ImplTable::T1::Hash::operator()(v.getKey()); - v.setHash(hash_value); + size_t hash_value = v.getHash(src.m1); size_t buck = getBucketFromHash(hash_value); impls[buck].m1.insertUniqueNonZero(&v, hash_value); } for (auto & v : src.m2) { - // size_t hash_value = v.getHash(src.m2); - const size_t hash_value = ImplTable::T2::Hash::operator()(v.getKey()); - v.setHash(hash_value); + size_t hash_value = v.getHash(src.m2); size_t buck = getBucketFromHash(hash_value); impls[buck].m2.insertUniqueNonZero(&v, hash_value); } for (auto & v : src.m3) { - // size_t hash_value = v.getHash(src.m3); - const size_t hash_value = ImplTable::T3::Hash::operator()(v.getKey()); - v.setHash(hash_value); + size_t hash_value = v.getHash(src.m3); size_t buck = getBucketFromHash(hash_value); impls[buck].m3.insertUniqueNonZero(&v, hash_value); } for (auto & v : src.ms) { - // size_t hash_value = v.getHash(src.ms); - const size_t hash_value = ImplTable::Ts::Hash::operator()(v.getKey()); - v.setHash(hash_value); + size_t hash_value = v.getHash(src.ms); size_t buck = getBucketFromHash(hash_value); impls[buck].ms.insertUniqueNonZero(&v, hash_value); } @@ -304,8 +296,7 @@ struct StringHashTableSubMapSelector<0, true, Data> template struct StringHashTableSubMapSelector<1, true, Data> { - // using Hash = StringHashTableHash; - using Hash = HashWithMixSeed; + using Hash = StringHashTableHash; static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) { @@ -317,8 +308,7 @@ struct StringHashTableSubMapSelector<1, true, Data> template struct StringHashTableSubMapSelector<2, true, Data> { - // using Hash = StringHashTableHash; - using Hash = HashWithMixSeed; + using Hash = StringHashTableHash; static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) { @@ -330,8 +320,7 @@ struct StringHashTableSubMapSelector<2, true, Data> template struct StringHashTableSubMapSelector<3, true, Data> { - // using Hash = StringHashTableHash; - using Hash = HashWithMixSeed; + using Hash = StringHashTableHash; static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) { @@ -343,8 +332,7 @@ struct StringHashTableSubMapSelector<3, true, Data> template struct StringHashTableSubMapSelector<4, true, Data> { - // using Hash = StringHashTableHash; - using Hash = StringRefHash; + using Hash = StringHashTableHash; static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index d900cc3e231..81252b8b3c6 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -88,16 +88,16 @@ using AggregatedDataWithInt256Key = HashMap>; using AggregatedDataWithKeys256 = HashMap>; -using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap>; -using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; +using AggregatedDataWithInt256KeyTwoLevel = TwoLevelHashMap>; using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap; using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash; -using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; -using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap>; +using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap>; /** Variants with better hash function, using more than 32 bits for hash. * Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion, @@ -125,7 +125,7 @@ struct AggregationMethodOneNumber AggregationMethodOneNumber() = default; template - explicit AggregationMethodOneNumber(Other & other) + explicit AggregationMethodOneNumber(const Other & other) : data(other.data) {} @@ -179,7 +179,7 @@ struct AggregationMethodString AggregationMethodString() = default; template - explicit AggregationMethodString(Other & other) + explicit AggregationMethodString(const Other & other) : data(other.data) {} @@ -227,7 +227,7 @@ struct AggregationMethodStringNoCache AggregationMethodStringNoCache() = default; template - explicit AggregationMethodStringNoCache(Other & other) + explicit AggregationMethodStringNoCache(const Other & other) : data(other.data) {} @@ -275,7 +275,7 @@ struct AggregationMethodOneKeyStringNoCache AggregationMethodOneKeyStringNoCache() = default; template - explicit AggregationMethodOneKeyStringNoCache(Other & other) + explicit AggregationMethodOneKeyStringNoCache(const Other & other) : data(other.data) {} @@ -325,7 +325,7 @@ struct AggregationMethodMultiStringNoCache AggregationMethodMultiStringNoCache() = default; template - explicit AggregationMethodMultiStringNoCache(Other & other) + explicit AggregationMethodMultiStringNoCache(const Other & other) : data(other.data) {} @@ -355,7 +355,7 @@ struct AggregationMethodFastPathTwoKeysNoCache AggregationMethodFastPathTwoKeysNoCache() = default; template - explicit AggregationMethodFastPathTwoKeysNoCache(Other & other) + explicit AggregationMethodFastPathTwoKeysNoCache(const Other & other) : data(other.data) {} @@ -475,7 +475,7 @@ struct AggregationMethodFixedString AggregationMethodFixedString() = default; template - explicit AggregationMethodFixedString(Other & other) + explicit AggregationMethodFixedString(const Other & other) : data(other.data) {} @@ -523,7 +523,7 @@ struct AggregationMethodFixedStringNoCache AggregationMethodFixedStringNoCache() = default; template - explicit AggregationMethodFixedStringNoCache(Other & other) + explicit AggregationMethodFixedStringNoCache(const Other & other) : data(other.data) {} @@ -572,7 +572,7 @@ struct AggregationMethodKeysFixed AggregationMethodKeysFixed() = default; template - explicit AggregationMethodKeysFixed(Other & other) + explicit AggregationMethodKeysFixed(const Other & other) : data(other.data) {} @@ -679,7 +679,7 @@ struct AggregationMethodSerialized AggregationMethodSerialized() = default; template - explicit AggregationMethodSerialized(Other & other) + explicit AggregationMethodSerialized(const Other & other) : data(other.data) {} diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 5f46d74fd30..4c2e5dbeca4 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -84,7 +84,7 @@ struct Settings M(SettingLoadBalancing, load_balancing, LoadBalancing::RANDOM, "Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing.") \ \ M(SettingUInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.") \ - M(SettingUInt64, group_by_two_level_threshold_bytes, 32000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. " \ + M(SettingUInt64, group_by_two_level_threshold_bytes, 100000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. " \ "Two-level aggregation is used when at least one of the thresholds is triggered.") \ M(SettingUInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is " \ "consumed. 0 means - same as 'max_threads'.") \ From c02cf71e90ea12a93dceaea8396e6f1daad49a37 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 4 Dec 2024 10:58:11 +0800 Subject: [PATCH 23/61] revert new hasher; refine original code path Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 3 + dbms/src/Common/HashTable/Hash.h | 125 ----------- dbms/src/Common/HashTable/StringHashTable.h | 31 +-- dbms/src/Interpreters/Aggregator.cpp | 227 ++++++++++---------- 4 files changed, 121 insertions(+), 265 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 3c4fd601487..fcbfc4bc358 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -138,6 +138,7 @@ class HashMethodBase map.prefetch(hashvals[prefetch_idx]); } + // Emplace key without hashval, and this method doesn't support prefetch. template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, @@ -160,6 +161,7 @@ class HashMethodBase return findKeyImpl(keyHolderGetKey(key_holder), data); } + // Emplace key using hashval, you can enable prefetch or not. template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, @@ -318,6 +320,7 @@ class HashMethodBase else \ return EmplaceResult(inserted); + // This method is performance critical, so there are two emplaceImpl to make sure caller can use the one they need. template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { diff --git a/dbms/src/Common/HashTable/Hash.h b/dbms/src/Common/HashTable/Hash.h index 207919a347e..457b4b9f3c0 100644 --- a/dbms/src/Common/HashTable/Hash.h +++ b/dbms/src/Common/HashTable/Hash.h @@ -422,128 +422,3 @@ struct IntHash32, void>> } } }; - -inline uint64_t umul128(uint64_t v, uint64_t kmul, uint64_t * high) -{ - DB::Int128 res = static_cast(v) * static_cast(kmul); - *high = static_cast(res >> 64); - return static_cast(res); -} - -template -inline void hash_combine(uint64_t & seed, const T & val) -{ - // from: https://github.com/HowardHinnant/hash_append/issues/7#issuecomment-629414712 - seed ^= std::hash{}(val) + 0x9e3779b97f4a7c15LLU + (seed << 12) + (seed >> 4); -} - -inline uint64_t hash_int128(uint64_t seed, const DB::Int128 & v) -{ - auto low = static_cast(v); - auto high = static_cast(v >> 64); - hash_combine(seed, low); - hash_combine(seed, high); - return seed; -} - -inline uint64_t hash_uint128(uint64_t seed, const DB::UInt128 & v) -{ - hash_combine(seed, v.low); - hash_combine(seed, v.high); - return seed; -} - -inline uint64_t hash_int256(uint64_t seed, const DB::Int256 & v) -{ - const auto & backend_value = v.backend(); - for (size_t i = 0; i < backend_value.size(); ++i) - { - hash_combine(seed, backend_value.limbs()[i]); - } - return seed; -} - -inline uint64_t hash_uint256(uint64_t seed, const DB::UInt256 & v) -{ - hash_combine(seed, v.a); - hash_combine(seed, v.b); - hash_combine(seed, v.c); - hash_combine(seed, v.d); - return seed; -} - -template -struct HashWithMixSeedHelper -{ - static inline size_t operator()(size_t); -}; - -template <> -struct HashWithMixSeedHelper<4> -{ - static inline size_t operator()(size_t v) - { - // from: https://github.com/aappleby/smhasher/blob/0ff96f7835817a27d0487325b6c16033e2992eb5/src/MurmurHash3.cpp#L102 - static constexpr uint64_t kmul = 0xcc9e2d51UL; - uint64_t mul = v * kmul; - return static_cast(mul ^ (mul >> 32u)); - } -}; - -template <> -struct HashWithMixSeedHelper<8> -{ - static inline size_t operator()(size_t v) - { - // from: https://github.com/martinus/robin-hood-hashing/blob/b21730713f4b5296bec411917c46919f7b38b178/src/include/robin_hood.h#L735 - static constexpr uint64_t kmul = 0xde5fb9d2630458e9ULL; - uint64_t high = 0; - uint64_t low = umul128(v, kmul, &high); - return static_cast(high + low); - } -}; - -template -struct HashWithMixSeed -{ - static size_t operator()(const T & v) - { - return HashWithMixSeedHelper::operator()(std::hash()(v)); - } -}; - -template <> -struct HashWithMixSeed -{ - static size_t operator()(const DB::Int128 & v) - { - return HashWithMixSeedHelper::operator()(hash_int128(0, v)); - } -}; - -template <> -struct HashWithMixSeed -{ - static inline size_t operator()(const DB::UInt128 & v) - { - return HashWithMixSeedHelper::operator()(hash_uint128(0, v)); - } -}; - -template <> -struct HashWithMixSeed -{ - static inline size_t operator()(const DB::Int256 & v) - { - return HashWithMixSeedHelper::operator()(hash_int256(0, v)); - } -}; - -template <> -struct HashWithMixSeed -{ - static inline size_t operator()(const DB::UInt256 & v) - { - return HashWithMixSeedHelper::operator()(hash_uint256(0, v)); - } -}; diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index 322523388cc..9bbdabb91fa 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -50,35 +50,6 @@ inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n) return {reinterpret_cast(&n), 24ul - (__builtin_clzll(n.c) >> 3)}; } -inline size_t hash_string_key_24(uint64_t seed, const StringKey24 & v) -{ - hash_combine(seed, v.a); - hash_combine(seed, v.b); - hash_combine(seed, v.c); - return seed; -} - -template <> -struct HashWithMixSeed -{ - static inline size_t operator()(const StringKey24 & v) - { - return HashWithMixSeedHelper::operator()(hash_string_key_24(0, v)); - } -}; - -// struct StringHashTableHash -// { -// using StringKey8Hasher = HashWithMixSeed; -// using StringKey16Hasher = HashWithMixSeed; -// using StringKey24Hasher = HashWithMixSeed; -// using StringRefHasher = StringRefHash; -// -// static size_t ALWAYS_INLINE operator()(StringKey8 key) { return StringKey8Hasher::operator()(key); } -// static size_t ALWAYS_INLINE operator()(const StringKey16 & key) { return StringKey16Hasher::operator()(key); } -// static size_t ALWAYS_INLINE operator()(const StringKey24 & key) { return StringKey24Hasher::operator()(key); } -// static size_t ALWAYS_INLINE operator()(const StringRef & key) { return StringRefHasher::operator()(key); } -// }; struct StringHashTableHash { #if defined(__SSE4_2__) @@ -117,7 +88,7 @@ struct StringHashTableHash return CityHash_v1_0_2::CityHash64(reinterpret_cast(&key), 24); } #endif - static size_t ALWAYS_INLINE operator()(StringRef key){ return StringRefHash()(key); } + static size_t ALWAYS_INLINE operator()(StringRef key) { return StringRefHash()(key); } }; template diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 8e12e7383ab..b714bacce04 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -674,13 +674,6 @@ void NO_INLINE Aggregator::executeImpl( #endif if (disable_prefetch) { - // if constexpr (Method::Data::is_string_hash_map) - // executeImplBatchStringHashMap( - // method, - // state, - // aggregates_pool, - // agg_process_info); - // else executeImplBatch(method, state, aggregates_pool, agg_process_info); } else @@ -725,14 +718,14 @@ std::optional::Res try { if constexpr (only_lookup) - return state.template findKey( + return state.template findKey( method.data, index, aggregates_pool, sort_key_containers, hashvals); else - return state.template emplaceKey( + return state.template emplaceKey( method.data, index, aggregates_pool, @@ -878,41 +871,64 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( { /// For all rows. AggregateDataPtr place = aggregates_pool->alloc(0); - std::vector hashvals; +#define HANDLE_AGG_EMPLACE_RESULT \ + if likely (emplace_result_hold.has_value()) \ + { \ + if constexpr (collect_hit_rate) \ + { \ + ++agg_process_info.hit_row_cnt; \ + } \ + \ + if constexpr (only_lookup) \ + { \ + if (!emplace_result_hold.value().isFound()) \ + agg_process_info.not_found_rows.push_back(i); \ + } \ + else \ + { \ + emplace_result_hold.value().setMapped(place); \ + } \ + ++agg_process_info.start_row; \ + } \ + else \ + { \ + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ + break; \ + } for (size_t i = 0; i < rows; ++i) { - // TODO prefetch - auto emplace_result_hold = emplaceOrFindKey( - method, - state, - agg_process_info.start_row, - *aggregates_pool, - sort_key_containers); - if likely (emplace_result_hold.has_value()) + if constexpr (enable_prefetch) { - if constexpr (collect_hit_rate) - { - ++agg_process_info.hit_row_cnt; - } - - if constexpr (only_lookup) - { - if (!emplace_result_hold.value().isFound()) - agg_process_info.not_found_rows.push_back(i); - } - else - { - emplace_result_hold.value().setMapped(place); - } - ++agg_process_info.start_row; + auto hashvals = getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool); + + auto emplace_result_hold = emplaceOrFindKey( + method, + state, + agg_process_info.start_row, + *aggregates_pool, + sort_key_containers, + hashvals); + HANDLE_AGG_EMPLACE_RESULT } else { - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); - break; + auto emplace_result_hold = emplaceOrFindKey( + method, + state, + agg_process_info.start_row, + *aggregates_pool, + sort_key_containers); + HANDLE_AGG_EMPLACE_RESULT } } +#undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -953,85 +969,76 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( std::unique_ptr places(new AggregateDataPtr[rows]); std::optional processed_rows; -#define WRAP_EMPLACE_AGG_KEY_BEGIN \ - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) \ - { \ +#define HANDLE_AGG_EMPLACE_RESULT \ + if unlikely (!emplace_result_holder.has_value()) \ + { \ + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ + break; \ + } \ + \ + auto & emplace_result = emplace_result_holder.value(); \ + \ + if constexpr (only_lookup) \ + { \ + if (emplace_result.isFound()) \ + { \ + aggregate_data = emplace_result.getMapped(); \ + } \ + else \ + { \ + agg_process_info.not_found_rows.push_back(i); \ + } \ + } \ + else \ + { \ + if (emplace_result.isInserted()) \ + { \ + emplace_result.setMapped(nullptr); \ + \ + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); \ + createAggregateStates(aggregate_data); \ + \ + emplace_result.setMapped(aggregate_data); \ + } \ + else \ + { \ + aggregate_data = emplace_result.getMapped(); \ + \ + if constexpr (collect_hit_rate) \ + ++agg_process_info.hit_row_cnt; \ + } \ + } \ + \ + places[i - agg_process_info.start_row] = aggregate_data; \ + processed_rows = i; + + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) + { AggregateDataPtr aggregate_data = nullptr; + if constexpr (enable_prefetch) + { + auto hashvals = getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool); -#define WRAP_EMPLACE_AGG_KEY_END \ - if unlikely (!emplace_result_holder.has_value()) \ - { \ - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ - break; \ - } \ - \ - auto & emplace_result = emplace_result_holder.value(); \ - \ - if constexpr (only_lookup) \ - { \ - if (emplace_result.isFound()) \ - { \ - aggregate_data = emplace_result.getMapped(); \ - } \ - else \ - { \ - agg_process_info.not_found_rows.push_back(i); \ - } \ - } \ - else \ - { \ - if (emplace_result.isInserted()) \ - { \ - emplace_result.setMapped(nullptr); \ - \ - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); \ - createAggregateStates(aggregate_data); \ - \ - emplace_result.setMapped(aggregate_data); \ - } \ - else \ - { \ - aggregate_data = emplace_result.getMapped(); \ - \ - if constexpr (collect_hit_rate) \ - ++agg_process_info.hit_row_cnt; \ - } \ - } \ - \ - places[i - agg_process_info.start_row] = aggregate_data; \ - processed_rows = i; \ - } - - if constexpr (enable_prefetch) - { - std::vector hashvals; - hashvals = getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool); + auto emplace_result_holder + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); - WRAP_EMPLACE_AGG_KEY_BEGIN - auto emplace_result_holder = emplaceOrFindKey( - method, - state, - i, - *aggregates_pool, - sort_key_containers, - hashvals); - WRAP_EMPLACE_AGG_KEY_END - } - else - { - WRAP_EMPLACE_AGG_KEY_BEGIN - auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); - WRAP_EMPLACE_AGG_KEY_END + HANDLE_AGG_EMPLACE_RESULT + } + else + { + auto emplace_result_holder + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + + HANDLE_AGG_EMPLACE_RESULT + } } -#undef WRAP_EMPLACE_AGG_KEY_BEGIN -#undef WRAP_EMPLACE_AGG_KEY_END +#undef HANDLE_AGG_EMPLACE_RESULT if (processed_rows) { From 352b710bdebcdc803b2baca3001cc2b32ca4a85f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 5 Dec 2024 11:51:24 +0800 Subject: [PATCH 24/61] fix case Signed-off-by: guo-shaoge --- .../Flash/tests/gtest_spill_aggregation.cpp | 46 +++++----- dbms/src/Interpreters/Aggregator.cpp | 84 ++++++++++++------- dbms/src/Interpreters/Aggregator.h | 4 +- 3 files changed, 82 insertions(+), 52 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_spill_aggregation.cpp b/dbms/src/Flash/tests/gtest_spill_aggregation.cpp index b19aaf03c4c..583e6e038fa 100644 --- a/dbms/src/Flash/tests/gtest_spill_aggregation.cpp +++ b/dbms/src/Flash/tests/gtest_spill_aggregation.cpp @@ -23,6 +23,7 @@ namespace FailPoints { extern const char force_agg_on_partial_block[]; extern const char force_thread_0_no_agg_spill[]; +extern const char force_agg_prefetch[]; } // namespace FailPoints namespace tests @@ -37,16 +38,22 @@ class SpillAggregationTestRunner : public DB::tests::ExecutorTest } }; -#define WRAP_FOR_AGG_PARTIAL_BLOCK_START \ - std::vector partial_blocks{true, false}; \ - for (auto partial_block : partial_blocks) \ - { \ - if (partial_block) \ - FailPointHelper::enableFailPoint(FailPoints::force_agg_on_partial_block); \ - else \ - FailPointHelper::disableFailPoint(FailPoints::force_agg_on_partial_block); +#define WRAP_FOR_AGG_FAILPOINTS_START \ + std::vector enables{true, false}; \ + for (auto enable : enables) \ + { \ + if (enable) \ + { \ + FailPointHelper::enableFailPoint(FailPoints::force_agg_on_partial_block); \ + FailPointHelper::enableFailPoint(FailPoints::force_agg_prefetch); \ + } \ + else \ + { \ + FailPointHelper::disableFailPoint(FailPoints::force_agg_on_partial_block); \ + FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); \ + } -#define WRAP_FOR_AGG_PARTIAL_BLOCK_END } +#define WRAP_FOR_AGG_FAILPOINTS_END } #define WRAP_FOR_AGG_THREAD_0_NO_SPILL_START \ for (auto thread_0_no_spill : {true, false}) \ @@ -114,13 +121,13 @@ try context.context->setSetting("group_by_two_level_threshold_bytes", Field(static_cast(1))); /// don't use `executeAndAssertColumnsEqual` since it takes too long to run /// test single thread aggregation - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START WRAP_FOR_AGG_THREAD_0_NO_SPILL_START ASSERT_COLUMNS_EQ_UR(ref_columns, executeStreams(request, 1)); /// test parallel aggregation ASSERT_COLUMNS_EQ_UR(ref_columns, executeStreams(request, original_max_streams)); WRAP_FOR_AGG_THREAD_0_NO_SPILL_END - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END /// enable spill and use small max_cached_data_bytes_in_spiller context.context->setSetting("max_cached_data_bytes_in_spiller", Field(static_cast(total_data_size / 200))); /// test single thread aggregation @@ -262,7 +269,7 @@ try Field(static_cast(max_bytes_before_external_agg))); context.context->setSetting("max_block_size", Field(static_cast(max_block_size))); WRAP_FOR_SPILL_TEST_BEGIN - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START WRAP_FOR_AGG_THREAD_0_NO_SPILL_START auto blocks = getExecuteStreamsReturnBlocks(request, concurrency); for (auto & block : blocks) @@ -289,7 +296,7 @@ try false)); } WRAP_FOR_AGG_THREAD_0_NO_SPILL_END - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END WRAP_FOR_SPILL_TEST_END } } @@ -369,6 +376,7 @@ try { for (const auto & agg_func : agg_funcs) { + FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); context.setCollation(collator_id); const auto * current_collator = TiDB::ITiDBCollator::getCollator(collator_id); ASSERT_TRUE(current_collator != nullptr); @@ -417,7 +425,7 @@ try Field(static_cast(max_bytes_before_external_agg))); context.context->setSetting("max_block_size", Field(static_cast(max_block_size))); WRAP_FOR_SPILL_TEST_BEGIN - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START WRAP_FOR_AGG_THREAD_0_NO_SPILL_START auto blocks = getExecuteStreamsReturnBlocks(request, concurrency); for (auto & block : blocks) @@ -444,7 +452,7 @@ try false)); } WRAP_FOR_AGG_THREAD_0_NO_SPILL_END - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END WRAP_FOR_SPILL_TEST_END } } @@ -518,9 +526,9 @@ try /// don't use `executeAndAssertColumnsEqual` since it takes too long to run auto request = gen_request(exchange_concurrency); WRAP_FOR_SPILL_TEST_BEGIN - WRAP_FOR_AGG_PARTIAL_BLOCK_START + WRAP_FOR_AGG_FAILPOINTS_START ASSERT_COLUMNS_EQ_UR(baseline, executeStreams(request, exchange_concurrency)); - WRAP_FOR_AGG_PARTIAL_BLOCK_END + WRAP_FOR_AGG_FAILPOINTS_END WRAP_FOR_SPILL_TEST_END } } @@ -528,8 +536,8 @@ CATCH #undef WRAP_FOR_SPILL_TEST_BEGIN #undef WRAP_FOR_SPILL_TEST_END -#undef WRAP_FOR_AGG_PARTIAL_BLOCK_START -#undef WRAP_FOR_AGG_PARTIAL_BLOCK_END +#undef WRAP_FOR_AGG_FAILPOINTS_START +#undef WRAP_FOR_AGG_FAILPOINTS_END } // namespace tests } // namespace DB diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index b714bacce04..537d13d1441 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -666,26 +666,38 @@ void NO_INLINE Aggregator::executeImpl( { typename Method::State state(agg_process_info.key_columns, key_sizes, collators); + // start_row!=0 and stringHashTableRecoveryInfo not empty and cannot be true at the same time. + RUNTIME_CHECK(!(agg_process_info.start_row != 0 && !agg_process_info.stringHashTableRecoveryInfoEmpty())); + #ifndef NDEBUG bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); fiu_do_on(FailPoints::force_agg_prefetch, { disable_prefetch = false; }); #else const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); #endif - if (disable_prefetch) - { - executeImplBatch(method, state, aggregates_pool, agg_process_info); - } - else + + if constexpr (Method::Data::is_string_hash_map) { - if constexpr (Method::Data::is_string_hash_map) - executeImplBatchStringHashMap( + // When will handled by column-wise(executeImplStringHashMapByCol): + // 1. For StringHashMap, which is composed by 5 submaps, needs be handled by column-wise when prefetch is enabled. + // 2. If agg_process_info.start_row != 0, it means the computation process of the current block was interrupted by resize exception in executeImplByRow. + // For clarity and simplicity of implementation, the processing functions for column-wise and row-wise methods handle the entire block independently. + // A block will not be processed first by the row-wise method and then by the column-wise method, or vice-versa. + if (!disable_prefetch && likely(agg_process_info.start_row == 0)) + executeImplStringHashMapByCol( method, state, aggregates_pool, agg_process_info); else - executeImplBatch(method, state, aggregates_pool, agg_process_info); + executeImplByRow(method, state, aggregates_pool, agg_process_info); + } + else + { + if (disable_prefetch) + executeImplByRow(method, state, aggregates_pool, agg_process_info); + else + executeImplByRow(method, state, aggregates_pool, agg_process_info); } } @@ -759,7 +771,7 @@ std::optional::Res } } -// This is only used by executeImplBatchStringHashMap. +// This is only used by executeImplStringHashMapByCol. // It will choose specifix submap of StringHashMap then do emplace/find. // StringKeyType can be StringRef/StringKey8/StringKey16/StringKey24/ArenaKeyHolder. template < @@ -849,7 +861,7 @@ size_t Aggregator::emplaceOrFindStringKey( } template -ALWAYS_INLINE void Aggregator::executeImplBatch( +ALWAYS_INLINE void Aggregator::executeImplByRow( Method & method, typename Method::State & state, Arena * aggregates_pool, @@ -857,6 +869,11 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( { // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); + // If agg_process_info.stringHashTableRecoveryInfoEmpty() is false, it means the current block was + // handled by executeImplStringHashMapByCol(column-wise) before, and resize execption happened. + // This situation is unexpected because for the sake of clarity, we assume that a block will be **fully** processed + // either column-wise or row-wise and cannot be split for processing. + RUNTIME_CHECK(agg_process_info.stringHashTableRecoveryInfoEmpty()); std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); @@ -1086,7 +1103,7 @@ M(4) // NOTE: this function is column-wise, which means sort key buffer cannot be reused. // This buffer will not be release until this block is processed done. template -ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( +ALWAYS_INLINE void Aggregator::executeImplStringHashMapByCol( Method & method, typename Method::State & state, Arena * aggregates_pool, @@ -1125,7 +1142,11 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( // If no resize exception happens, so this is a new Block. // If resize exception happens, start_row has already been set to zero at the end of this function. - RUNTIME_CHECK(agg_process_info.start_row == 0); + RUNTIME_CHECK_MSG( + agg_process_info.start_row == 0, + "unexpected agg_process_info.start_row: {}, end_row: {}", + agg_process_info.start_row, + agg_process_info.end_row); if likely (agg_process_info.stringHashTableRecoveryInfoEmpty()) { @@ -1233,10 +1254,9 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( M(4, key_str_infos, key_str_datas, key_str_places) #undef M - if (zero_agg_func_size) - return; - - std::vector places(rows, nullptr); + if (!zero_agg_func_size) + { + std::vector places(rows, nullptr); #define M(INFO, PLACES) \ for (size_t i = 0; i < (INFO).size(); ++i) \ { \ @@ -1244,24 +1264,26 @@ ALWAYS_INLINE void Aggregator::executeImplBatchStringHashMap( places[row] = (PLACES)[i]; \ } - M(key0_infos, key0_places) - M(key8_infos, key8_places) - M(key16_infos, key16_places) - M(key24_infos, key24_places) - M(key_str_infos, key_str_places) + M(key0_infos, key0_places) + M(key8_infos, key8_places) + M(key16_infos, key16_places) + M(key24_infos, key24_places) + M(key_str_infos, key_str_places) #undef M - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) - { - inst->batch_that->addBatch( - agg_process_info.start_row, - rows, - &places[0], - inst->state_offset, - inst->batch_arguments, - aggregates_pool); + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + ++inst) + { + inst->batch_that->addBatch( + agg_process_info.start_row, + rows, + &places[0], + inst->state_offset, + inst->batch_arguments, + aggregates_pool); + } } + if unlikely (got_resize_exception) { RUNTIME_CHECK(!agg_process_info.stringHashTableRecoveryInfoEmpty()); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 81252b8b3c6..d88a97278f9 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1477,14 +1477,14 @@ class Aggregator TiDB::TiDBCollators & collators) const; template - void executeImplBatch( + void executeImplByRow( Method & method, typename Method::State & state, Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; template - void executeImplBatchStringHashMap( + void executeImplStringHashMapByCol( Method & method, typename Method::State & state, Arena * aggregates_pool, From e5ab87c249daf3b4de8baebb39329b0792b0fab7 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 13 Dec 2024 17:48:02 +0800 Subject: [PATCH 25/61] refine Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 33 +++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 537d13d1441..b06763ee5af 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -675,14 +675,29 @@ void NO_INLINE Aggregator::executeImpl( #else const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); #endif - - if constexpr (Method::Data::is_string_hash_map) - { - // When will handled by column-wise(executeImplStringHashMapByCol): - // 1. For StringHashMap, which is composed by 5 submaps, needs be handled by column-wise when prefetch is enabled. - // 2. If agg_process_info.start_row != 0, it means the computation process of the current block was interrupted by resize exception in executeImplByRow. - // For clarity and simplicity of implementation, the processing functions for column-wise and row-wise methods handle the entire block independently. - // A block will not be processed first by the row-wise method and then by the column-wise method, or vice-versa. + disable_prefetch = true; + + // key_serialized and key_string(StringHashMap) needs column-wise handling for prefetch. + // Because: + // 1. StringHashMap(key_string) is composed by 5 submaps, so prefetch needs to be done for each specific submap. + // 2. getKeyHolder of key_serialized have to copy real data into Arena. + // It means we better getKeyHolder for all Columns once and then use it both for getHash() and emplaceKey(). + // 3. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. + // But getHashVals() still needs to be column-wise. + if constexpr (Method::State::is_serialized_key) + { + // TODO: batch serialize method for Columns is still under development. + // if (!disable_prefetch) + // executeImplSerializedKeyByCol(); + // else + // executeImplByRow(method, state, aggregates_pool, agg_process_info); + executeImplByRow(method, state, aggregates_pool, agg_process_info); + } + else if constexpr (Method::Data::is_string_hash_map) + { + // If agg_process_info.start_row != 0, it means the computation process of the current block was interrupted by resize exception in executeImplByRow. + // For clarity and simplicity of implementation, the processing functions for column-wise and row-wise methods handle the entire block independently. + // A block will not be processed first by the row-wise method and then by the column-wise method, or vice-versa. if (!disable_prefetch && likely(agg_process_info.start_row == 0)) executeImplStringHashMapByCol( method, @@ -867,6 +882,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( Arena * aggregates_pool, AggProcessInfo & agg_process_info) const { + LOG_TRACE(log, "executeImplByRow"); // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); // If agg_process_info.stringHashTableRecoveryInfoEmpty() is false, it means the current block was @@ -1109,6 +1125,7 @@ ALWAYS_INLINE void Aggregator::executeImplStringHashMapByCol( Arena * aggregates_pool, AggProcessInfo & agg_process_info) const { + LOG_TRACE(log, "executeImplStringHashMapByCol"); // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); static_assert(Method::Data::is_string_hash_map); From 30d99be42156f050fdbf591d3373999299002f29 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 15 Dec 2024 12:30:43 +0800 Subject: [PATCH 26/61] is_serialized_key Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index a9817308616..23dd30ecc44 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -48,6 +48,8 @@ struct HashMethodOneNumber using Self = HashMethodOneNumber; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = false; + const FieldType * vec; const size_t total_rows; @@ -96,6 +98,8 @@ struct HashMethodString using Self = HashMethodString; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = false; + const IColumn::Offset * offsets; const UInt8 * chars; TiDB::TiDBCollatorPtr collator = nullptr; @@ -155,6 +159,8 @@ struct HashMethodStringBin using Self = HashMethodStringBin; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = false; + const IColumn::Offset * offsets; const UInt8 * chars; const size_t total_rows; @@ -359,6 +365,8 @@ struct HashMethodFastPathTwoKeysSerialized using Self = HashMethodFastPathTwoKeysSerialized; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = true; + Key1Desc key_1_desc; Key2Desc key_2_desc; const size_t total_rows; @@ -395,6 +403,8 @@ struct HashMethodFixedString using Self = HashMethodFixedString; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = false; + size_t n; const ColumnFixedString::Chars_t * chars; TiDB::TiDBCollatorPtr collator = nullptr; @@ -453,6 +463,7 @@ struct HashMethodKeysFixed using BaseHashed = columns_hashing_impl::HashMethodBase; using Base = columns_hashing_impl::BaseStateKeysFixed; + static constexpr bool is_serialized_key = false; static constexpr bool has_nullable_keys = has_nullable_keys_; Sizes key_sizes; @@ -610,6 +621,8 @@ struct HashMethodSerialized using Self = HashMethodSerialized; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = true; + ColumnRawPtrs key_columns; size_t keys_size; TiDB::TiDBCollators collators; @@ -648,6 +661,8 @@ struct HashMethodHashed using Self = HashMethodHashed; using Base = columns_hashing_impl::HashMethodBase; + static constexpr bool is_serialized_key = false; + ColumnRawPtrs key_columns; TiDB::TiDBCollators collators; const size_t total_rows; From 01cae8078c956265a8930d3504f87822b79ef7a6 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 15 Dec 2024 12:36:51 +0800 Subject: [PATCH 27/61] fix Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index b06763ee5af..f48a5f87741 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -675,7 +675,6 @@ void NO_INLINE Aggregator::executeImpl( #else const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); #endif - disable_prefetch = true; // key_serialized and key_string(StringHashMap) needs column-wise handling for prefetch. // Because: From 95c597de481db7c6c42ff409f6c6cd7d4f4f9b6c Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 15 Dec 2024 16:45:49 +0800 Subject: [PATCH 28/61] fix start_row Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 77 +++++++++++++++------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index f48a5f87741..3fcb1dcde58 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -716,20 +716,20 @@ void NO_INLINE Aggregator::executeImpl( } template -std::vector getHashVals( +void getHashVals( size_t start_row, size_t end_row, const Data & data, const State & state, std::vector & sort_key_containers, - Arena * pool) + Arena * pool, + std::vector & hashvals) { - std::vector hashvals(state.total_rows, 0); + hashvals.resize(state.total_rows); for (size_t i = start_row; i < end_row; ++i) { hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); } - return hashvals; } template @@ -920,7 +920,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( { \ emplace_result_hold.value().setMapped(place); \ } \ - ++agg_process_info.start_row; \ + processed_rows = i; \ } \ else \ { \ @@ -928,38 +928,40 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( break; \ } - for (size_t i = 0; i < rows; ++i) + std::vector hashvals; + std::optional processed_rows; + if constexpr (enable_prefetch) + { + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + hashvals); + } + + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { if constexpr (enable_prefetch) { - auto hashvals = getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool); - - auto emplace_result_hold = emplaceOrFindKey( - method, - state, - agg_process_info.start_row, - *aggregates_pool, - sort_key_containers, - hashvals); + auto emplace_result_hold + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); + HANDLE_AGG_EMPLACE_RESULT } else { - auto emplace_result_hold = emplaceOrFindKey( - method, - state, - agg_process_info.start_row, - *aggregates_pool, - sort_key_containers); + auto emplace_result_hold + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + HANDLE_AGG_EMPLACE_RESULT } } + + if likely (processed_rows) + agg_process_info.start_row = *processed_rows + 1; #undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -1044,19 +1046,24 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( places[i - agg_process_info.start_row] = aggregate_data; \ processed_rows = i; + std::vector hashvals; + if constexpr (enable_prefetch) + { + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + hashvals); + } + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { AggregateDataPtr aggregate_data = nullptr; if constexpr (enable_prefetch) { - auto hashvals = getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool); - auto emplace_result_holder = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); From e0dea797049252cca5e5360678a89c76bf72dec5 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 17 Dec 2024 10:44:30 +0800 Subject: [PATCH 29/61] revert prefetch for StringHashTable Signed-off-by: guo-shaoge --- .../AggregateFunctionGroupUniqArray.h | 2 +- .../src/AggregateFunctions/KeyHolderHelpers.h | 2 +- dbms/src/Common/ColumnsHashing.h | 43 +-- dbms/src/Common/ColumnsHashingImpl.h | 38 -- .../src/Common/HashTable/HashTableKeyHolder.h | 8 +- dbms/src/Common/HashTable/StringHashTable.h | 143 -------- .../HashTable/TwoLevelStringHashTable.h | 63 ---- dbms/src/Interpreters/Aggregator.cpp | 342 +----------------- dbms/src/Interpreters/Aggregator.h | 50 +-- dbms/src/TiDB/Collation/Collator.cpp | 112 ------ dbms/src/TiDB/Collation/Collator.h | 10 - 11 files changed, 19 insertions(+), 794 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index d3cbea74195..f556a009551 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -182,7 +182,7 @@ class AggregateFunctionGroupUniqArrayGeneric { // We have to copy the keys to our arena. assert(arena != nullptr); - cur_set.emplace(ArenaKeyHolder{rhs_elem.getValue(), arena}, it, inserted); + cur_set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted); } } diff --git a/dbms/src/AggregateFunctions/KeyHolderHelpers.h b/dbms/src/AggregateFunctions/KeyHolderHelpers.h index b8a4ee0def3..6677866f0d3 100644 --- a/dbms/src/AggregateFunctions/KeyHolderHelpers.h +++ b/dbms/src/AggregateFunctions/KeyHolderHelpers.h @@ -24,7 +24,7 @@ inline auto getKeyHolder(const IColumn & column, size_t row_num, Arena & arena) { if constexpr (is_plain_column) { - return ArenaKeyHolder{column.getDataAt(row_num), &arena}; + return ArenaKeyHolder{column.getDataAt(row_num), arena}; } else { diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 23dd30ecc44..ac665874969 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -122,34 +122,19 @@ struct HashMethodString ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder( ssize_t row, [[maybe_unused]] Arena * pool, - [[maybe_unused]] std::vector & sort_key_containers) const + std::vector & sort_key_containers) const { - auto key = getKey(row); + auto last_offset = row == 0 ? 0 : offsets[row - 1]; + // Remove last zero byte. + StringRef key(chars + last_offset, offsets[row] - last_offset - 1); if (likely(collator)) key = collator->sortKey(key.data, key.size, sort_key_containers[0]); - return ArenaKeyHolder{key, pool}; - } - - ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder(ssize_t row, Arena * pool, Arena * sort_key_pool) const - { - auto key = getKey(row); - if (likely(collator)) - key = collator->sortKey(key.data, key.size, *sort_key_pool); - - return ArenaKeyHolder{key, pool}; + return ArenaKeyHolder{key, *pool}; } protected: friend class columns_hashing_impl::HashMethodBase; - -private: - ALWAYS_INLINE inline StringRef getKey(size_t row) const - { - auto last_offset = row == 0 ? 0 : offsets[row - 1]; - // Remove last zero byte. - return StringRef(chars + last_offset, offsets[row] - last_offset - 1); - } }; template @@ -175,16 +160,11 @@ struct HashMethodStringBin } ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const - { - return getKeyHolder(row, pool, nullptr); - } - - ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, Arena *) const { auto last_offset = row == 0 ? 0 : offsets[row - 1]; StringRef key(chars + last_offset, offsets[row] - last_offset - 1); key = BinCollatorSortKey(key.data, key.size); - return ArenaKeyHolder{key, pool}; + return ArenaKeyHolder{key, *pool}; } protected: @@ -433,16 +413,7 @@ struct HashMethodFixedString if (collator) key = collator->sortKeyFastPath(key.data, key.size, sort_key_containers[0]); - return ArenaKeyHolder{key, pool}; - } - - ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder(size_t row, Arena * pool, Arena * sort_key_pool) const - { - StringRef key(&(*chars)[row * n], n); - if (collator) - key = collator->sortKeyFastPath(key.data, key.size, *sort_key_pool); - - return ArenaKeyHolder{key, pool}; + return ArenaKeyHolder{key, *pool}; } protected: diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index fcbfc4bc358..09065286dbf 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -204,44 +204,6 @@ class HashMethodBase } } - template - ALWAYS_INLINE inline EmplaceResult emplaceStringKey( - Data & data, - size_t idx, - std::vector & datas, - const std::vector & hashvals) - { - // For spill, hashvals.size() will be le to total_rows. - // Because only remaining rows that didn't insert into HashMap will be handled here. - assert(hashvals.size() <= static_cast(*this).total_rows); - - auto & submap = StringHashTableSubMapSelector>::getSubMap( - hashvals[idx], - data); - if constexpr (enable_prefetch) - prefetch(submap, idx, hashvals); - - return emplaceImpl(datas[idx], submap, hashvals[idx]); - } - - template - ALWAYS_INLINE inline FindResult findStringKey( - Data & data, - size_t idx, - std::vector & datas, - const std::vector & hashvals) - { - assert(hashvals.size() <= static_cast(*this).total_rows); - - auto & submap = StringHashTableSubMapSelector>::getSubMap( - hashvals[idx], - data); - if constexpr (enable_prefetch) - prefetch(submap, idx, hashvals); - - return findKeyImpl(keyHolderGetKey(datas[idx]), submap, hashvals[idx]); - } - template ALWAYS_INLINE inline size_t getHash( const Data & data, diff --git a/dbms/src/Common/HashTable/HashTableKeyHolder.h b/dbms/src/Common/HashTable/HashTableKeyHolder.h index dd8a4b53376..01b06dce87d 100644 --- a/dbms/src/Common/HashTable/HashTableKeyHolder.h +++ b/dbms/src/Common/HashTable/HashTableKeyHolder.h @@ -91,8 +91,8 @@ namespace DB */ struct ArenaKeyHolder { - StringRef key{}; - Arena * pool = nullptr; + StringRef key; + Arena & pool; }; } // namespace DB @@ -111,14 +111,14 @@ inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder & holder) { // Hash table shouldn't ask us to persist a zero key assert(holder.key.size > 0); - holder.key.data = holder.pool->insert(holder.key.data, holder.key.size); + holder.key.data = holder.pool.insert(holder.key.data, holder.key.size); } inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder && holder) { // Hash table shouldn't ask us to persist a zero key assert(holder.key.size > 0); - holder.key.data = holder.pool->insert(holder.key.data, holder.key.size); + holder.key.data = holder.pool.insert(holder.key.data, holder.key.size); } inline void ALWAYS_INLINE keyHolderDiscardKey(DB::ArenaKeyHolder &) {} diff --git a/dbms/src/Common/HashTable/StringHashTable.h b/dbms/src/Common/HashTable/StringHashTable.h index 9bbdabb91fa..e1236caf381 100644 --- a/dbms/src/Common/HashTable/StringHashTable.h +++ b/dbms/src/Common/HashTable/StringHashTable.h @@ -16,9 +16,7 @@ #include #include -#include #include -#include #include #include @@ -194,99 +192,6 @@ struct StringHashTableLookupResult friend bool operator!=(const std::nullptr_t &, const StringHashTableLookupResult & b) { return b.mapped_ptr; } }; -template -static auto -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) - NO_INLINE NO_SANITIZE_ADDRESS NO_SANITIZE_THREAD -#else - ALWAYS_INLINE -#endif - dispatchStringHashTable( - size_t row, - KeyHolder && key_holder, - Func0 && func0, - Func8 && func8, - Func16 && func16, - Func24 && func24, - FuncStr && func_str) -{ - const StringRef & x = keyHolderGetKey(key_holder); - const size_t sz = x.size; - if (sz == 0) - { - return func0(x, row); - } - - if (x.data[sz - 1] == 0) - { - // Strings with trailing zeros are not representable as fixed-size - // string keys. Put them to the generic table. - return func_str(key_holder, row); - } - - const char * p = x.data; - // pending bits that needs to be shifted out - const char s = (-sz & 7) * 8; - union - { - StringKey8 k8; - StringKey16 k16; - StringKey24 k24; - UInt64 n[3]; - }; - switch ((sz - 1) >> 3) - { - case 0: // 1..8 bytes - { - // first half page - if ((reinterpret_cast(p) & 2048) == 0) - { - memcpy(&n[0], p, 8); - if constexpr (DB::isLittleEndian()) - n[0] &= (-1ULL >> s); - else - n[0] &= (-1ULL << s); - } - else - { - const char * lp = x.data + x.size - 8; - memcpy(&n[0], lp, 8); - if constexpr (DB::isLittleEndian()) - n[0] >>= s; - else - n[0] <<= s; - } - return func8(k8, row); - } - case 1: // 9..16 bytes - { - memcpy(&n[0], p, 8); - const char * lp = x.data + x.size - 8; - memcpy(&n[1], lp, 8); - if constexpr (DB::isLittleEndian()) - n[1] >>= s; - else - n[1] <<= s; - return func16(k16, row); - } - case 2: // 17..24 bytes - { - memcpy(&n[0], p, 16); - const char * lp = x.data + x.size - 8; - memcpy(&n[2], lp, 8); - if constexpr (DB::isLittleEndian()) - n[2] >>= s; - else - n[2] <<= s; - return func24(k24, row); - } - default: // >= 25 bytes - { - return func_str(key_holder, row); - } - } -} - template class StringHashTable : private boost::noncopyable { @@ -307,8 +212,6 @@ class StringHashTable : private boost::noncopyable template friend class TwoLevelStringHashTable; - template - friend struct StringHashTableSubMapSelector; T0 m0; T1 m1; @@ -565,49 +468,3 @@ class StringHashTable : private boost::noncopyable ms.clearAndShrink(); } }; - -template -struct StringHashTableSubMapSelector; - -template -struct StringHashTableSubMapSelector<0, false, Data> -{ - struct Hash - { - static ALWAYS_INLINE size_t operator()(const StringRef &) { return 0; } - }; - - static typename Data::T0 & getSubMap(size_t, Data & data) { return data.m0; } -}; - -template -struct StringHashTableSubMapSelector<1, false, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::T1 & getSubMap(size_t, Data & data) { return data.m1; } -}; - -template -struct StringHashTableSubMapSelector<2, false, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::T2 & getSubMap(size_t, Data & data) { return data.m2; } -}; - -template -struct StringHashTableSubMapSelector<3, false, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::T3 & getSubMap(size_t, Data & data) { return data.m3; } -}; - -template -struct StringHashTableSubMapSelector<4, false, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::Ts & getSubMap(size_t, Data & data) { return data.ms; } -}; diff --git a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h index 403b8d3941c..7659b5a73fb 100644 --- a/dbms/src/Common/HashTable/TwoLevelStringHashTable.h +++ b/dbms/src/Common/HashTable/TwoLevelStringHashTable.h @@ -277,66 +277,3 @@ class TwoLevelStringHashTable : private boost::noncopyable return res; } }; - -template -struct StringHashTableSubMapSelector<0, true, Data> -{ - struct Hash - { - static ALWAYS_INLINE size_t operator()(const StringRef &) { return 0; } - }; - - static typename Data::Impl::T0 & getSubMap(size_t hashval, Data & data) - { - const auto bucket = Data::getBucketFromHash(hashval); - return data.impls[bucket].m0; - } -}; - -template -struct StringHashTableSubMapSelector<1, true, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::Impl::T1 & getSubMap(size_t hashval, Data & data) - { - const auto bucket = Data::getBucketFromHash(hashval); - return data.impls[bucket].m1; - } -}; - -template -struct StringHashTableSubMapSelector<2, true, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::Impl::T2 & getSubMap(size_t hashval, Data & data) - { - const auto bucket = Data::getBucketFromHash(hashval); - return data.impls[bucket].m2; - } -}; - -template -struct StringHashTableSubMapSelector<3, true, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::Impl::T3 & getSubMap(size_t hashval, Data & data) - { - const auto bucket = Data::getBucketFromHash(hashval); - return data.impls[bucket].m3; - } -}; - -template -struct StringHashTableSubMapSelector<4, true, Data> -{ - using Hash = StringHashTableHash; - - static typename Data::Impl::Ts & getSubMap(size_t hashval, Data & data) - { - const auto bucket = Data::getBucketFromHash(hashval); - return data.impls[bucket].ms; - } -}; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 3fcb1dcde58..5e00c2e72e6 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -666,9 +666,6 @@ void NO_INLINE Aggregator::executeImpl( { typename Method::State state(agg_process_info.key_columns, key_sizes, collators); - // start_row!=0 and stringHashTableRecoveryInfo not empty and cannot be true at the same time. - RUNTIME_CHECK(!(agg_process_info.start_row != 0 && !agg_process_info.stringHashTableRecoveryInfoEmpty())); - #ifndef NDEBUG bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); fiu_do_on(FailPoints::force_agg_prefetch, { disable_prefetch = false; }); @@ -676,12 +673,11 @@ void NO_INLINE Aggregator::executeImpl( const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); #endif - // key_serialized and key_string(StringHashMap) needs column-wise handling for prefetch. + // key_serialized needs column-wise handling for prefetch. // Because: - // 1. StringHashMap(key_string) is composed by 5 submaps, so prefetch needs to be done for each specific submap. - // 2. getKeyHolder of key_serialized have to copy real data into Arena. + // 1. getKeyHolder of key_serialized have to copy real data into Arena. // It means we better getKeyHolder for all Columns once and then use it both for getHash() and emplaceKey(). - // 3. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. + // 2. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. // But getHashVals() still needs to be column-wise. if constexpr (Method::State::is_serialized_key) { @@ -694,17 +690,8 @@ void NO_INLINE Aggregator::executeImpl( } else if constexpr (Method::Data::is_string_hash_map) { - // If agg_process_info.start_row != 0, it means the computation process of the current block was interrupted by resize exception in executeImplByRow. - // For clarity and simplicity of implementation, the processing functions for column-wise and row-wise methods handle the entire block independently. - // A block will not be processed first by the row-wise method and then by the column-wise method, or vice-versa. - if (!disable_prefetch && likely(agg_process_info.start_row == 0)) - executeImplStringHashMapByCol( - method, - state, - aggregates_pool, - agg_process_info); - else - executeImplByRow(method, state, aggregates_pool, agg_process_info); + // StringHashMap doesn't support prefetch. + executeImplByRow(method, state, aggregates_pool, agg_process_info); } else { @@ -785,95 +772,6 @@ std::optional::Res } } -// This is only used by executeImplStringHashMapByCol. -// It will choose specifix submap of StringHashMap then do emplace/find. -// StringKeyType can be StringRef/StringKey8/StringKey16/StringKey24/ArenaKeyHolder. -template < - size_t SubMapIndex, - bool collect_hit_rate, - bool only_lookup, - bool enable_prefetch, - bool zero_agg_func_size, - typename Data, - typename State, - typename StringKeyType> -size_t Aggregator::emplaceOrFindStringKey( - Data & data, - State & state, - const std::vector & key_infos, - std::vector & key_datas, - Arena & aggregates_pool, - std::vector & places, - AggProcessInfo & agg_process_info) const -{ - static_assert(!(collect_hit_rate && only_lookup)); - assert(key_infos.size() == key_datas.size()); - - using Hash = typename StringHashTableSubMapSelector>::Hash; - std::vector hashvals(key_infos.size(), 0); - for (size_t i = 0; i < key_infos.size(); ++i) - hashvals[i] = Hash::operator()(keyHolderGetKey(key_datas[i])); - - // alloc 0 bytes is useful when agg func size is zero. - AggregateDataPtr agg_state = aggregates_pool.alloc(0); - for (size_t i = 0; i < key_infos.size(); ++i) - { - try - { - if constexpr (only_lookup) - { - auto find_result - = state.template findStringKey(data, i, key_datas, hashvals); - if (find_result.isFound()) - { - agg_state = find_result.getMapped(); - } - else - { - agg_process_info.not_found_rows.push_back(key_infos[i]); - } - } - else - { - auto emplace_result - = state.template emplaceStringKey(data, i, key_datas, hashvals); - if (emplace_result.isInserted()) - { - if constexpr (zero_agg_func_size) - { - emplace_result.setMapped(agg_state); - } - else - { - emplace_result.setMapped(nullptr); - - agg_state - = aggregates_pool.alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(agg_state); - - emplace_result.setMapped(agg_state); - } - } - else - { - if constexpr (!zero_agg_func_size) - agg_state = emplace_result.getMapped(); - - if constexpr (collect_hit_rate) - ++agg_process_info.hit_row_cnt; - } - if constexpr (!zero_agg_func_size) - places[i] = agg_state; - } - } - catch (ResizeException &) - { - return i; - } - } - return key_infos.size(); -} - template ALWAYS_INLINE void Aggregator::executeImplByRow( Method & method, @@ -881,14 +779,8 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( Arena * aggregates_pool, AggProcessInfo & agg_process_info) const { - LOG_TRACE(log, "executeImplByRow"); // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); - // If agg_process_info.stringHashTableRecoveryInfoEmpty() is false, it means the current block was - // handled by executeImplStringHashMapByCol(column-wise) before, and resize execption happened. - // This situation is unexpected because for the sake of clarity, we assume that a block will be **fully** processed - // either column-wise or row-wise and cannot be split for processing. - RUNTIME_CHECK(agg_process_info.stringHashTableRecoveryInfoEmpty()); std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); @@ -1097,230 +989,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } } -#define M(SUBMAPINDEX) \ - template \ - ALWAYS_INLINE inline void setupExceptionRecoveryInfoForStringHashTable( \ - Aggregator::AggProcessInfo & agg_process_info, \ - size_t row, \ - const std::vector & key_infos, \ - const std::vector & key_datas, \ - std::integral_constant) \ - { \ - agg_process_info.submap_m##SUBMAPINDEX##_infos \ - = std::vector(key_infos.begin() + row, key_infos.end()); \ - agg_process_info.submap_m##SUBMAPINDEX##_datas \ - = std::vector(key_datas.begin() + row, key_datas.end()); \ - } - -M(0) -M(1) -M(2) -M(3) -M(4) - -#undef M - -// prefetch/empalce each specifix submap directly instead of accessing StringHashMap interface, -// which is better for performance. -// NOTE: this function is column-wise, which means sort key buffer cannot be reused. -// This buffer will not be release until this block is processed done. -template -ALWAYS_INLINE void Aggregator::executeImplStringHashMapByCol( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - AggProcessInfo & agg_process_info) const -{ - LOG_TRACE(log, "executeImplStringHashMapByCol"); - // collect_hit_rate and only_lookup cannot be true at the same time. - static_assert(!(collect_hit_rate && only_lookup)); - static_assert(Method::Data::is_string_hash_map); - -#define M(SUBMAPINDEX) \ - RUNTIME_CHECK( \ - agg_process_info.submap_m##SUBMAPINDEX##_infos.size() \ - == agg_process_info.submap_m##SUBMAPINDEX##_datas.size()); - - M(0) - M(1) - M(2) - M(3) - M(4) -#undef M - - const size_t rows = agg_process_info.end_row - agg_process_info.start_row; - auto sort_key_pool = std::make_unique(); - std::vector sort_key_containers; - -#define M(INFO, DATA, KEYTYPE) \ - std::vector(INFO); \ - std::vector(DATA); - - M(key0_infos, key0_datas, StringRef) - M(key8_infos, key8_datas, StringKey8) - M(key16_infos, key16_datas, StringKey16) - M(key24_infos, key24_datas, StringKey24) - M(key_str_infos, key_str_datas, ArenaKeyHolder) -#undef M - - // If no resize exception happens, so this is a new Block. - // If resize exception happens, start_row has already been set to zero at the end of this function. - RUNTIME_CHECK_MSG( - agg_process_info.start_row == 0, - "unexpected agg_process_info.start_row: {}, end_row: {}", - agg_process_info.start_row, - agg_process_info.end_row); - - if likely (agg_process_info.stringHashTableRecoveryInfoEmpty()) - { - // sort_key_pool should already been reset by AggProcessInfo::restBlock() - RUNTIME_CHECK(!agg_process_info.sort_key_pool); - - const size_t reserve_size = rows / 4; - -#define M(INFO, DATA, SUBMAPINDEX, KEYTYPE) \ - (INFO).reserve(reserve_size); \ - (DATA).reserve(reserve_size); \ - auto dispatch_callback_key##SUBMAPINDEX \ - = [&INFO, &DATA](const KEYTYPE & key, size_t row) { /* NOLINT(bugprone-macro-parentheses) */ \ - (INFO).push_back(row); \ - (DATA).push_back(key); \ - }; - - M(key0_infos, key0_datas, 0, StringRef) - M(key8_infos, key8_datas, 8, StringKey8) - M(key16_infos, key16_datas, 16, StringKey16) - M(key24_infos, key24_datas, 24, StringKey24) - // Argument type is ArenaKeyHolder instead of StringRef, - // because it will only be persisted when insert into HashTable. - M(key_str_infos, key_str_datas, str, ArenaKeyHolder) -#undef M - - for (size_t i = 0; i < rows; ++i) - { - // Use Arena for collation sort key, because we are doing agg in column-wise way. - // So a big arena is needed to store decoded key, and we can avoid resize std::string by using Arena. - auto key_holder = state.getKeyHolder(i, aggregates_pool, sort_key_pool.get()); - dispatchStringHashTable( - i, - key_holder, - dispatch_callback_key0, - dispatch_callback_key8, - dispatch_callback_key16, - dispatch_callback_key24, - dispatch_callback_keystr); - } - } - else - { -#define M(INFO, DATA, SUBMAPINDEX) \ - (INFO) = agg_process_info.submap_m##SUBMAPINDEX##_infos; \ - (DATA) = agg_process_info.submap_m##SUBMAPINDEX##_datas; - - M(key0_infos, key0_datas, 0) - M(key8_infos, key8_datas, 1) - M(key16_infos, key16_datas, 2) - M(key24_infos, key24_datas, 3) - M(key_str_infos, key_str_datas, 4) -#undef M - } - - std::vector key0_places(key0_infos.size(), nullptr); - std::vector key8_places(key8_infos.size(), nullptr); - std::vector key16_places(key16_infos.size(), nullptr); - std::vector key24_places(key24_infos.size(), nullptr); - std::vector key_str_places(key_str_infos.size(), nullptr); - - bool got_resize_exception = false; - size_t emplaced_index = 0; - bool zero_agg_func_size = (params.aggregates_size == 0); - -#define M(INDEX, INFO, DATA, PLACES) \ - if (!got_resize_exception && !(INFO).empty()) \ - { \ - if (zero_agg_func_size) \ - emplaced_index = emplaceOrFindStringKey( \ - method.data, \ - state, \ - (INFO), \ - (DATA), \ - *aggregates_pool, \ - (PLACES), \ - agg_process_info); \ - else \ - emplaced_index = emplaceOrFindStringKey( \ - method.data, \ - state, \ - (INFO), \ - (DATA), \ - *aggregates_pool, \ - (PLACES), \ - agg_process_info); \ - if unlikely (emplaced_index != (INFO).size()) \ - got_resize_exception = true; \ - } \ - else \ - { \ - emplaced_index = 0; \ - } \ - setupExceptionRecoveryInfoForStringHashTable( \ - agg_process_info, \ - emplaced_index, \ - (INFO), \ - (DATA), \ - std::integral_constant{}); - - M(0, key0_infos, key0_datas, key0_places) - M(1, key8_infos, key8_datas, key8_places) - M(2, key16_infos, key16_datas, key16_places) - M(3, key24_infos, key24_datas, key24_places) - M(4, key_str_infos, key_str_datas, key_str_places) -#undef M - - if (!zero_agg_func_size) - { - std::vector places(rows, nullptr); -#define M(INFO, PLACES) \ - for (size_t i = 0; i < (INFO).size(); ++i) \ - { \ - const auto row = (INFO)[i]; \ - places[row] = (PLACES)[i]; \ - } - - M(key0_infos, key0_places) - M(key8_infos, key8_places) - M(key16_infos, key16_places) - M(key24_infos, key24_places) - M(key_str_infos, key_str_places) -#undef M - - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) - { - inst->batch_that->addBatch( - agg_process_info.start_row, - rows, - &places[0], - inst->state_offset, - inst->batch_arguments, - aggregates_pool); - } - } - - if unlikely (got_resize_exception) - { - RUNTIME_CHECK(!agg_process_info.stringHashTableRecoveryInfoEmpty()); - agg_process_info.sort_key_pool = std::move(sort_key_pool); - // For StringHashTable, start_row is meanless, instead submap_mx_infos/submap_mx_datas are used. - // So set it to zero when got_resize_exception. - agg_process_info.start_row = 0; - } - else - { - agg_process_info.start_row = agg_process_info.end_row; - } -} - void NO_INLINE Aggregator::executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index d88a97278f9..19e6fc7a919 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,32 +1318,11 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; - // For StringHashTable, when resize exception happens, the process will be interrupted. - // So we need these infos to continue. - std::vector submap_m0_infos{}; - std::vector submap_m1_infos{}; - std::vector submap_m2_infos{}; - std::vector submap_m3_infos{}; - std::vector submap_m4_infos{}; - std::vector submap_m0_datas{}; - std::vector submap_m1_datas{}; - std::vector submap_m2_datas{}; - std::vector submap_m3_datas{}; - std::vector submap_m4_datas{}; - std::unique_ptr sort_key_pool; - void prepareForAgg(); bool allBlockDataHandled() const { assert(start_row <= end_row); - // submap_mx_infos.size() and submap_mx_datas.size() are always equal. - // So only need to check submap_mx_infos is enough. - return (start_row == end_row && stringHashTableRecoveryInfoEmpty()) || aggregator->isCancelled(); - } - bool stringHashTableRecoveryInfoEmpty() const - { - return submap_m0_infos.empty() && submap_m1_infos.empty() && submap_m3_infos.empty() - && submap_m4_infos.empty(); + return start_row == end_row || aggregator->isCancelled(); } void resetBlock(const Block & block_) { @@ -1357,8 +1336,6 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); - - sort_key_pool.reset(); } }; @@ -1483,13 +1460,6 @@ class Aggregator Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template - void executeImplStringHashMapByCol( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - AggProcessInfo & agg_process_info) const; - template std::optional::ResultType> emplaceOrFindKey( Method & method, @@ -1507,24 +1477,6 @@ class Aggregator Arena & aggregates_pool, std::vector & sort_key_containers) const; - template < - size_t SubMapIndex, - bool collect_hit_rate, - bool only_lookup, - bool enable_prefetch, - bool zero_agg_func_size, - typename Data, - typename State, - typename StringKeyType> - size_t emplaceOrFindStringKey( - Data & data, - State & state, - const std::vector & key_infos, - std::vector & key_datas, - Arena & aggregates_pool, - std::vector & places, - AggProcessInfo & agg_process_info) const; - /// For case when there are no keys (all aggregate into one row). static void executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena); diff --git a/dbms/src/TiDB/Collation/Collator.cpp b/dbms/src/TiDB/Collation/Collator.cpp index 4365f1f0988..bf27400f8c4 100644 --- a/dbms/src/TiDB/Collation/Collator.cpp +++ b/dbms/src/TiDB/Collation/Collator.cpp @@ -192,11 +192,6 @@ class BinCollator final : public ITiDBCollator return DB::BinCollatorSortKey(s, length); } - StringRef sortKey(const char * s, size_t length, DB::Arena &) const override - { - return DB::BinCollatorSortKey(s, length); - } - StringRef sortKeyNoTrim(const char * s, size_t length, std::string &) const override { return convertForBinCollator(s, length, nullptr); @@ -278,54 +273,11 @@ class GeneralCICollator final : public ITiDBCollator return convertImpl(s, length, container, nullptr); } - StringRef sortKey(const char * s, size_t length, DB::Arena & pool) const override - { - return convertImpl(s, length, pool, nullptr); - } - StringRef sortKeyNoTrim(const char * s, size_t length, std::string & container) const override { return convertImpl(s, length, container, nullptr); } - template - StringRef convertImpl(const char * s, size_t length, DB::Arena & pool, std::vector * lens) const - { - std::string_view v; - - if constexpr (need_trim) - v = rtrim(s, length); - else - v = std::string_view(s, length); - - const size_t size = length * sizeof(WeightType); - auto * buffer = pool.alignedAlloc(size, 16); - - size_t offset = 0; - size_t total_size = 0; - size_t v_length = v.length(); - - if constexpr (need_len) - { - if (lens->capacity() < v_length) - lens->reserve(v_length); - lens->resize(0); - } - - while (offset < v_length) - { - auto c = decodeChar(s, offset); - auto sk = weight(c); - buffer[total_size++] = static_cast(sk >> 8); - buffer[total_size++] = static_cast(sk); - - if constexpr (need_len) - lens->push_back(2); - } - - return StringRef(buffer, total_size); - } - template StringRef convertImpl(const char * s, size_t length, std::string & container, std::vector * lens) const { @@ -527,65 +479,11 @@ class UCACICollator final : public ITiDBCollator return convertImpl(s, length, container, nullptr); } - StringRef sortKey(const char * s, size_t length, DB::Arena & pool) const override - { - return convertImpl(s, length, pool, nullptr); - } - StringRef sortKeyNoTrim(const char * s, size_t length, std::string & container) const override { return convertImpl(s, length, container, nullptr); } - // Use Arena to store decoded string. Normally it's used by column-wise Agg/Join, - // because column-wise process cannot reuse string container. - template - StringRef convertImpl(const char * s, size_t length, DB::Arena & pool, std::vector * lens) const - { - std::string_view v; - - if constexpr (need_trim) - v = preprocess(s, length); - else - v = std::string_view(s, length); - - // every char have 8 uint16 at most. - const auto size = 8 * length * sizeof(uint16_t); - auto * buffer = pool.alignedAlloc(size, 16); - - size_t offset = 0; - size_t total_size = 0; - size_t v_length = v.length(); - - uint64_t first = 0, second = 0; - - if constexpr (need_len) - { - if (lens->capacity() < v_length) - lens->reserve(v_length); - lens->resize(0); - } - - while (offset < v_length) - { - weight(first, second, offset, v_length, s); - - if constexpr (need_len) - lens->push_back(total_size); - - writeResult(first, buffer, total_size); - writeResult(second, buffer, total_size); - - if constexpr (need_len) - { - size_t end_idx = lens->size() - 1; - (*lens)[end_idx] = total_size - (*lens)[end_idx]; - } - } - - return StringRef(buffer, total_size); - } - template StringRef convertImpl(const char * s, size_t length, std::string & container, std::vector * lens) const { @@ -652,16 +550,6 @@ class UCACICollator final : public ITiDBCollator } } - static inline void writeResult(uint64_t & w, char * buffer, size_t & total_size) - { - while (w != 0) - { - buffer[total_size++] = static_cast(w >> 8); - buffer[total_size++] = static_cast(w); - w >>= 16; - } - } - static inline bool regexEq(CharType a, CharType b) { return T::regexEq(a, b); } static inline void weight(uint64_t & first, uint64_t & second, size_t & offset, size_t length, const char * s) diff --git a/dbms/src/TiDB/Collation/Collator.h b/dbms/src/TiDB/Collation/Collator.h index 08c017ba57d..6bb87883ef1 100644 --- a/dbms/src/TiDB/Collation/Collator.h +++ b/dbms/src/TiDB/Collation/Collator.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include @@ -102,7 +101,6 @@ class ITiDBCollator = 0; virtual StringRef sortKeyNoTrim(const char * s, size_t length, std::string & container) const = 0; virtual StringRef sortKey(const char * s, size_t length, std::string & container) const = 0; - virtual StringRef sortKey(const char * s, size_t length, DB::Arena &) const = 0; virtual std::unique_ptr pattern() const = 0; int32_t getCollatorId() const { return collator_id; } CollatorType getCollatorType() const { return collator_type; } @@ -137,14 +135,6 @@ class ITiDBCollator } return sortKey(s, length, container); } - ALWAYS_INLINE inline StringRef sortKeyFastPath(const char * s, size_t length, DB::Arena & pool) const - { - if (likely(isPaddingBinary())) - { - return DB::BinCollatorSortKey(s, length); - } - return sortKey(s, length, pool); - } protected: explicit ITiDBCollator(int32_t collator_id_); From 143fee37e79393d06934b7c15e9ad55bdced4f28 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 17 Dec 2024 19:41:17 +0800 Subject: [PATCH 30/61] comment Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 48 +- dbms/src/Interpreters/Aggregator.cpp.orig | 2973 +++++++++++++++++++++ dbms/src/Interpreters/Aggregator.h | 4 + 3 files changed, 3005 insertions(+), 20 deletions(-) create mode 100644 dbms/src/Interpreters/Aggregator.cpp.orig diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 5e00c2e72e6..23da5d04111 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -820,18 +820,21 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( break; \ } - std::vector hashvals; std::optional processed_rows; if constexpr (enable_prefetch) { - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool, - hashvals); + // hashvals is not empty only when resize exception happened. + if likely (agg_process_info.hashvals.empty()) + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + agg_process_info.hashvals); + + RUNTIME_CHECK(agg_process_info.hashvals.size() == agg_process_info.end_row); } for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) @@ -839,7 +842,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if constexpr (enable_prefetch) { auto emplace_result_hold - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, agg_process_info.hashvals); HANDLE_AGG_EMPLACE_RESULT } @@ -938,17 +941,19 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( places[i - agg_process_info.start_row] = aggregate_data; \ processed_rows = i; - std::vector hashvals; if constexpr (enable_prefetch) { - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool, - hashvals); + if likely (agg_process_info.hashvals.empty()) + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + agg_process_info.hashvals); + + RUNTIME_CHECK(agg_process_info.hashvals.size() == agg_process_info.end_row); } for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) @@ -957,7 +962,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if constexpr (enable_prefetch) { auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, agg_process_info.hashvals); HANDLE_AGG_EMPLACE_RESULT } @@ -987,6 +992,9 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } agg_process_info.start_row = *processed_rows + 1; } + + if likely (agg_process_info.start_row == agg_process_info.end_row) + agg_process_info.hashvals.clear(); } void NO_INLINE diff --git a/dbms/src/Interpreters/Aggregator.cpp.orig b/dbms/src/Interpreters/Aggregator.cpp.orig new file mode 100644 index 00000000000..5e00c2e72e6 --- /dev/null +++ b/dbms/src/Interpreters/Aggregator.cpp.orig @@ -0,0 +1,2973 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int UNKNOWN_AGGREGATED_DATA_VARIANT; +extern const int EMPTY_DATA_PASSED; +extern const int CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS; +extern const int LOGICAL_ERROR; +} // namespace ErrorCodes + +namespace FailPoints +{ +extern const char random_aggregate_create_state_failpoint[]; +extern const char random_aggregate_merge_failpoint[]; +extern const char force_agg_on_partial_block[]; +extern const char random_fail_in_resize_callback[]; +extern const char force_agg_prefetch[]; +} // namespace FailPoints + +#define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME +#define AggregationMethodNameTwoLevel(NAME) AggregatedDataVariants::AggregationMethod_##NAME##_two_level +#define AggregationMethodType(NAME) AggregatedDataVariants::Type::NAME +#define AggregationMethodTypeTwoLevel(NAME) AggregatedDataVariants::Type::NAME##_two_level +#define ToAggregationMethodPtr(NAME, ptr) (reinterpret_cast(ptr)) +#define ToAggregationMethodPtrTwoLevel(NAME, ptr) (reinterpret_cast(ptr)) + +AggregatedDataVariants::~AggregatedDataVariants() +{ + if (aggregator && !aggregator->all_aggregates_has_trivial_destructor) + { + try + { + aggregator->destroyAllAggregateStates(*this); + } + catch (...) + { + tryLogCurrentException(aggregator->log, __PRETTY_FUNCTION__); + } + } + destroyAggregationMethodImpl(); +} + +bool AggregatedDataVariants::tryMarkNeedSpill() +{ + assert(!need_spill); + if (empty()) + return false; + if (!isTwoLevel()) + { + /// Data can only be flushed to disk if a two-level aggregation is supported. + if (!isConvertibleToTwoLevel()) + return false; + convertToTwoLevel(); + } + need_spill = true; + return true; +} + +void AggregatedDataVariants::destroyAggregationMethodImpl() +{ + if (!aggregation_method_impl) + return; + +#define M(NAME, IS_TWO_LEVEL) \ + case AggregationMethodType(NAME): \ + { \ + delete reinterpret_cast(aggregation_method_impl); \ + aggregation_method_impl = nullptr; \ + break; \ + } + switch (type) + { + APPLY_FOR_AGGREGATED_VARIANTS(M) + default: + break; + } +#undef M +} + +void AggregatedDataVariants::init(Type variants_type) +{ + destroyAggregationMethodImpl(); + + switch (variants_type) + { + case Type::EMPTY: + break; + case Type::without_key: + break; + +#define M(NAME, IS_TWO_LEVEL) \ + case AggregationMethodType(NAME): \ + { \ + aggregation_method_impl = std::make_unique().release(); \ + if (aggregator && !aggregator->params.key_ref_agg_func.empty()) \ + RUNTIME_CHECK_MSG( \ + AggregationMethodName(NAME)::canUseKeyRefAggFuncOptimization(), \ + "cannot use key_ref_agg_func optimization for method {}", \ + getMethodName()); \ + break; \ + } + + APPLY_FOR_AGGREGATED_VARIANTS(M) +#undef M + + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } + + type = variants_type; +} + +size_t AggregatedDataVariants::getBucketNumberForTwoLevelHashTable(Type type) +{ + switch (type) + { +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + return AggregationMethodNameTwoLevel(NAME)::Data::NUM_BUCKETS; \ + } + + APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) + +#undef M + + default: + throw Exception("Wrong data variant passed.", ErrorCodes::LOGICAL_ERROR); + } +} + +void AggregatedDataVariants::setResizeCallbackIfNeeded(size_t thread_num) const +{ + // For auto pass through hashagg, no spill should happen. Block will be pass through when need to spill. + // So no need to set callback. Also it's complicated to handle situation when Aggregator didn't process all rows at once. + if (aggregator && !aggregator->is_auto_pass_through) + { + auto agg_spill_context = aggregator->agg_spill_context; + if (agg_spill_context->isSpillEnabled() && agg_spill_context->isInAutoSpillMode()) + { + auto resize_callback = [agg_spill_context, thread_num]() { + if (agg_spill_context->supportFurtherSpill() + && agg_spill_context->isThreadMarkedForAutoSpill(thread_num)) + return false; + bool ret = true; + fiu_do_on(FailPoints::random_fail_in_resize_callback, { + if (agg_spill_context->supportFurtherSpill()) + { + ret = !agg_spill_context->markThreadForAutoSpill(thread_num); + } + }); + return ret; + }; +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + ToAggregationMethodPtr(NAME, aggregation_method_impl)->data.setResizeCallback(resize_callback); \ + break; \ + } + switch (type) + { + APPLY_FOR_VARIANTS_TWO_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } +#undef M + } + } +} + +void AggregatedDataVariants::convertToTwoLevel() +{ + switch (type) + { +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + if (aggregator) \ + LOG_TRACE( \ + aggregator->log, \ + "Converting aggregation data type `{}` to `{}`.", \ + getMethodName(AggregationMethodType(NAME)), \ + getMethodName(AggregationMethodTypeTwoLevel(NAME))); \ + auto ori_ptr = ToAggregationMethodPtr(NAME, aggregation_method_impl); \ + auto two_level = std::make_unique(*ori_ptr); \ + delete ori_ptr; \ + aggregation_method_impl = two_level.release(); \ + type = AggregationMethodTypeTwoLevel(NAME); \ + break; \ + } + + APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) + +#undef M + + default: + throw Exception("Wrong data variant passed.", ErrorCodes::LOGICAL_ERROR); + } + aggregator->useTwoLevelHashTable(); +} + + +Block Aggregator::getHeader(bool final) const +{ + return params.getHeader(final); +} + +/// when there is no input data and current aggregation still need to generate a result(for example, +/// select count(*) from t need to return 0 even if there is no data) the aggregator will use this +/// source header block as the fake input of aggregation +Block Aggregator::getSourceHeader() const +{ + return params.src_header; +} + +Block Aggregator::Params::getHeader( + const Block & src_header, + const ColumnNumbers & keys, + const AggregateDescriptions & aggregates, + const KeyRefAggFuncMap & key_ref_agg_func, + bool final) +{ + Block res; + + for (const auto & key : keys) + { + // For final stage, key optimization is enabled, so no need to output columns. + // An CopyColumn Action will handle this. + const auto & key_col = src_header.safeGetByPosition(key); + if (final && key_ref_agg_func.find(key_col.name) != key_ref_agg_func.end()) + continue; + + res.insert(key_col.cloneEmpty()); + } + + for (const auto & aggregate : aggregates) + { + size_t arguments_size = aggregate.arguments.size(); + DataTypes argument_types(arguments_size); + for (size_t j = 0; j < arguments_size; ++j) + argument_types[j] = src_header.safeGetByPosition(aggregate.arguments[j]).type; + + DataTypePtr type; + if (final) + type = aggregate.function->getReturnType(); + else + type + = std::make_shared(aggregate.function, argument_types, aggregate.parameters); + + res.insert({type, aggregate.column_name}); + } + + return materializeBlock(res); +} + + +Aggregator::Aggregator( + const Params & params_, + const String & req_id, + size_t concurrency, + const RegisterOperatorSpillContext & register_operator_spill_context, + bool is_auto_pass_through_) + : params(params_) + , log(Logger::get(req_id)) + , is_cancelled([]() { return false; }) + , is_auto_pass_through(is_auto_pass_through_) +{ + aggregate_functions.resize(params.aggregates_size); + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i] = params.aggregates[i].function.get(); + + /// Initialize sizes of aggregation states and its offsets. + offsets_of_aggregate_states.resize(params.aggregates_size); + total_size_of_aggregate_states = 0; + all_aggregates_has_trivial_destructor = true; + + // aggreate_states will be aligned as below: + // |<-- state_1 -->|<-- pad_1 -->|<-- state_2 -->|<-- pad_2 -->| ..... + // + // pad_N will be used to match alignment requirement for each next state. + // The address of state_1 is aligned based on maximum alignment requirements in states + for (size_t i = 0; i < params.aggregates_size; ++i) + { + offsets_of_aggregate_states[i] = total_size_of_aggregate_states; + total_size_of_aggregate_states += params.aggregates[i].function->sizeOfData(); + + // aggreate states are aligned based on maximum requirement + align_aggregate_states = std::max(align_aggregate_states, params.aggregates[i].function->alignOfData()); + + // If not the last aggregate_state, we need pad it so that next aggregate_state will be aligned. + if (i + 1 < params.aggregates_size) + { + size_t alignment_of_next_state = params.aggregates[i + 1].function->alignOfData(); + if ((alignment_of_next_state & (alignment_of_next_state - 1)) != 0) + throw Exception("Logical error: alignOfData is not 2^N", ErrorCodes::LOGICAL_ERROR); + + /// Extend total_size to next alignment requirement + /// Add padding by rounding up 'total_size_of_aggregate_states' to be a multiplier of alignment_of_next_state. + total_size_of_aggregate_states = (total_size_of_aggregate_states + alignment_of_next_state - 1) + / alignment_of_next_state * alignment_of_next_state; + } + + if (!params.aggregates[i].function->hasTrivialDestructor()) + all_aggregates_has_trivial_destructor = false; + } + + method_chosen = chooseAggregationMethod(); + RUNTIME_CHECK_MSG(method_chosen != AggregatedDataVariants::Type::EMPTY, "Invalid aggregation method"); + agg_spill_context = std::make_shared( + concurrency, + params.spill_config, + params.getMaxBytesBeforeExternalGroupBy(), + log); + if (agg_spill_context->supportSpill()) + { + bool is_convertible_to_two_level = AggregatedDataVariants::isConvertibleToTwoLevel(method_chosen); + if (!is_convertible_to_two_level) + { + params.setMaxBytesBeforeExternalGroupBy(0); + agg_spill_context->disableSpill(); + if (method_chosen != AggregatedDataVariants::Type::without_key) + LOG_WARNING( + log, + "Aggregation does not support spill because aggregator hash table does not support two level"); + } + } + if (register_operator_spill_context != nullptr) + register_operator_spill_context(agg_spill_context); + if (agg_spill_context->isSpillEnabled()) + { + /// init spiller if needed + /// for aggregation, the input block is sorted by bucket number + /// so it can work with MergingAggregatedMemoryEfficientBlockInputStream + agg_spill_context->buildSpiller(getHeader(false)); + } +} + + +inline bool IsTypeNumber64(const DataTypePtr & type) +{ + return type->isNumber() && type->getSizeOfValueInMemory() == sizeof(uint64_t); +} + +#define APPLY_FOR_AGG_FAST_PATH_TYPES(M) \ + M(Number64) \ + M(StringBin) \ + M(StringBinPadding) + +enum class AggFastPathType +{ +#define M(NAME) NAME, + APPLY_FOR_AGG_FAST_PATH_TYPES(M) +#undef M +}; + +AggregatedDataVariants::Type ChooseAggregationMethodTwoKeys(const AggFastPathType * fast_path_types) +{ + auto tp1 = fast_path_types[0]; + auto tp2 = fast_path_types[1]; + switch (tp1) + { + case AggFastPathType::Number64: + { + switch (tp2) + { + case AggFastPathType::Number64: + return AggregatedDataVariants::Type::serialized; // unreachable. keys64 or keys128 will be used before + case AggFastPathType::StringBin: + return AggregatedDataVariants::Type::two_keys_num64_strbin; + case AggFastPathType::StringBinPadding: + return AggregatedDataVariants::Type::two_keys_num64_strbinpadding; + } + } + case AggFastPathType::StringBin: + { + switch (tp2) + { + case AggFastPathType::Number64: + return AggregatedDataVariants::Type::two_keys_strbin_num64; + case AggFastPathType::StringBin: + return AggregatedDataVariants::Type::two_keys_strbin_strbin; + case AggFastPathType::StringBinPadding: + return AggregatedDataVariants::Type::serialized; // rare case + } + } + case AggFastPathType::StringBinPadding: + { + switch (tp2) + { + case AggFastPathType::Number64: + return AggregatedDataVariants::Type::two_keys_strbinpadding_num64; + case AggFastPathType::StringBin: + return AggregatedDataVariants::Type::serialized; // rare case + case AggFastPathType::StringBinPadding: + return AggregatedDataVariants::Type::two_keys_strbinpadding_strbinpadding; + } + } + } +} + +// return AggregatedDataVariants::Type::serialized if can NOT determine fast path. +AggregatedDataVariants::Type ChooseAggregationMethodFastPath( + size_t keys_size, + const DataTypes & types_not_null, + const TiDB::TiDBCollators & collators) +{ + std::array fast_path_types{}; + + if (keys_size == fast_path_types.max_size()) + { + for (size_t i = 0; i < keys_size; ++i) + { + const auto & type = types_not_null[i]; + if (type->isString()) + { + if (collators.empty() || !collators[i]) + { + // use original way + return AggregatedDataVariants::Type::serialized; + } + else + { + switch (collators[i]->getCollatorType()) + { + case TiDB::ITiDBCollator::CollatorType::UTF8MB4_BIN: + case TiDB::ITiDBCollator::CollatorType::UTF8_BIN: + case TiDB::ITiDBCollator::CollatorType::LATIN1_BIN: + case TiDB::ITiDBCollator::CollatorType::ASCII_BIN: + { + fast_path_types[i] = AggFastPathType::StringBinPadding; + break; + } + case TiDB::ITiDBCollator::CollatorType::BINARY: + { + fast_path_types[i] = AggFastPathType::StringBin; + break; + } + default: + { + // for CI COLLATION, use original way + return AggregatedDataVariants::Type::serialized; + } + } + } + } + else if (IsTypeNumber64(type)) + { + fast_path_types[i] = AggFastPathType::Number64; + } + else + { + return AggregatedDataVariants::Type::serialized; + } + } + return ChooseAggregationMethodTwoKeys(fast_path_types.data()); + } + return AggregatedDataVariants::Type::serialized; +} + +AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() +{ + /// If no keys. All aggregating to single row. + if (params.keys_size == 0) + return AggregatedDataVariants::Type::without_key; + + /// Check if at least one of the specified keys is nullable. + DataTypes types_removed_nullable; + types_removed_nullable.reserve(params.keys.size()); + bool has_nullable_key = false; + + for (const auto & pos : params.keys) + { + const auto & type = params.src_header.safeGetByPosition(pos).type; + + if (type->isNullable()) + { + has_nullable_key = true; + types_removed_nullable.push_back(removeNullable(type)); + } + else + types_removed_nullable.push_back(type); + } + + /** Returns ordinary (not two-level) methods, because we start from them. + * Later, during aggregation process, data may be converted (partitioned) to two-level structure, if cardinality is high. + */ + + size_t keys_bytes = 0; + size_t num_fixed_contiguous_keys = 0; + + key_sizes.resize(params.keys_size); + for (size_t j = 0; j < params.keys_size; ++j) + { + if (types_removed_nullable[j]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion()) + { + if (types_removed_nullable[j]->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion() + && (params.collators.empty() || params.collators[j] == nullptr)) + { + ++num_fixed_contiguous_keys; + key_sizes[j] = types_removed_nullable[j]->getSizeOfValueInMemory(); + keys_bytes += key_sizes[j]; + } + } + } + + if (has_nullable_key) + { + if (params.keys_size == num_fixed_contiguous_keys) + { + /// Pack if possible all the keys along with information about which key values are nulls + /// into a fixed 16- or 32-byte blob. + if (std::tuple_size>::value + keys_bytes <= 16) + return AggregatedDataVariants::Type::nullable_keys128; + if (std::tuple_size>::value + keys_bytes <= 32) + return AggregatedDataVariants::Type::nullable_keys256; + } + + /// Fallback case. + return AggregatedDataVariants::Type::serialized; + } + + /// No key has been found to be nullable. + const DataTypes & types_not_null = types_removed_nullable; + assert(!has_nullable_key); + + /// Single numeric key. + if (params.keys_size == 1 && types_not_null[0]->isValueRepresentedByNumber()) + { + size_t size_of_field = types_not_null[0]->getSizeOfValueInMemory(); + if (size_of_field == 1) + return AggregatedDataVariants::Type::key8; + if (size_of_field == 2) + return AggregatedDataVariants::Type::key16; + if (size_of_field == 4) + return AggregatedDataVariants::Type::key32; + if (size_of_field == 8) + return AggregatedDataVariants::Type::key64; + if (size_of_field == 16) + return AggregatedDataVariants::Type::keys128; + if (size_of_field == 32) + return AggregatedDataVariants::Type::keys256; + if (size_of_field == sizeof(Decimal256)) + return AggregatedDataVariants::Type::key_int256; + throw Exception( + "Logical error: numeric column has sizeOfField not in 1, 2, 4, 8, 16, 32.", + ErrorCodes::LOGICAL_ERROR); + } + + /// If all keys fits in N bits, will use hash table with all keys packed (placed contiguously) to single N-bit key. + if (params.keys_size == num_fixed_contiguous_keys) + { + if (keys_bytes <= 2) + return AggregatedDataVariants::Type::keys16; + if (keys_bytes <= 4) + return AggregatedDataVariants::Type::keys32; + if (keys_bytes <= 8) + return AggregatedDataVariants::Type::keys64; + if (keys_bytes <= 16) + return AggregatedDataVariants::Type::keys128; + if (keys_bytes <= 32) + return AggregatedDataVariants::Type::keys256; + } + + /// If single string key - will use hash table with references to it. Strings itself are stored separately in Arena. + if (params.keys_size == 1 && types_not_null[0]->isString()) + { + if (params.collators.empty() || !params.collators[0]) + { + // use original way. `Type::one_key_strbin` will generate empty column. + return AggregatedDataVariants::Type::key_string; + } + else + { + switch (params.collators[0]->getCollatorType()) + { + case TiDB::ITiDBCollator::CollatorType::UTF8MB4_BIN: + case TiDB::ITiDBCollator::CollatorType::UTF8_BIN: + case TiDB::ITiDBCollator::CollatorType::LATIN1_BIN: + case TiDB::ITiDBCollator::CollatorType::ASCII_BIN: + { + return AggregatedDataVariants::Type::one_key_strbinpadding; + } + case TiDB::ITiDBCollator::CollatorType::BINARY: + { + return AggregatedDataVariants::Type::one_key_strbin; + } + default: + { + // for CI COLLATION, use original way + return AggregatedDataVariants::Type::key_string; + } + } + } + } + + if (params.keys_size == 1 && types_not_null[0]->isFixedString()) + return AggregatedDataVariants::Type::key_fixed_string; + + return ChooseAggregationMethodFastPath(params.keys_size, types_not_null, params.collators); +} + + +void Aggregator::createAggregateStates(AggregateDataPtr & aggregate_data) const +{ + for (size_t j = 0; j < params.aggregates_size; ++j) + { + try + { + /** An exception may occur if there is a shortage of memory. + * In order that then everything is properly destroyed, we "roll back" some of the created states. + * The code is not very convenient. + */ + FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::random_aggregate_create_state_failpoint); + aggregate_functions[j]->create(aggregate_data + offsets_of_aggregate_states[j]); + } + catch (...) + { + for (size_t rollback_j = 0; rollback_j < j; ++rollback_j) + aggregate_functions[rollback_j]->destroy(aggregate_data + offsets_of_aggregate_states[rollback_j]); + + throw; + } + } +} + + +/** It's interesting - if you remove `noinline`, then gcc for some reason will inline this function, and the performance decreases (~ 10%). + * (Probably because after the inline of this function, more internal functions no longer be inlined.) + * Inline does not make sense, since the inner loop is entirely inside this function. + */ +template +void NO_INLINE Aggregator::executeImpl( + Method & method, + Arena * aggregates_pool, + AggProcessInfo & agg_process_info, + TiDB::TiDBCollators & collators) const +{ + typename Method::State state(agg_process_info.key_columns, key_sizes, collators); + +#ifndef NDEBUG + bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); + fiu_do_on(FailPoints::force_agg_prefetch, { disable_prefetch = false; }); +#else + const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); +#endif + + // key_serialized needs column-wise handling for prefetch. + // Because: + // 1. getKeyHolder of key_serialized have to copy real data into Arena. + // It means we better getKeyHolder for all Columns once and then use it both for getHash() and emplaceKey(). + // 2. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. + // But getHashVals() still needs to be column-wise. + if constexpr (Method::State::is_serialized_key) + { + // TODO: batch serialize method for Columns is still under development. + // if (!disable_prefetch) + // executeImplSerializedKeyByCol(); + // else + // executeImplByRow(method, state, aggregates_pool, agg_process_info); + executeImplByRow(method, state, aggregates_pool, agg_process_info); + } + else if constexpr (Method::Data::is_string_hash_map) + { + // StringHashMap doesn't support prefetch. + executeImplByRow(method, state, aggregates_pool, agg_process_info); + } + else + { + if (disable_prefetch) + executeImplByRow(method, state, aggregates_pool, agg_process_info); + else + executeImplByRow(method, state, aggregates_pool, agg_process_info); + } +} + +template +void getHashVals( + size_t start_row, + size_t end_row, + const Data & data, + const State & state, + std::vector & sort_key_containers, + Arena * pool, + std::vector & hashvals) +{ + hashvals.resize(state.total_rows); + for (size_t i = start_row; i < end_row; ++i) + { + hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); + } +} + +template +std::optional::ResultType> Aggregator::emplaceOrFindKey( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers, + const std::vector & hashvals) const +{ + try + { + if constexpr (only_lookup) + return state.template findKey( + method.data, + index, + aggregates_pool, + sort_key_containers, + hashvals); + else + return state.template emplaceKey( + method.data, + index, + aggregates_pool, + sort_key_containers, + hashvals); + } + catch (ResizeException &) + { + return {}; + } +} + +template +std::optional::ResultType> Aggregator::emplaceOrFindKey( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers) const +{ + try + { + if constexpr (only_lookup) + return state.findKey(method.data, index, aggregates_pool, sort_key_containers); + else + return state.emplaceKey(method.data, index, aggregates_pool, sort_key_containers); + } + catch (ResizeException &) + { + return {}; + } +} + +template +ALWAYS_INLINE void Aggregator::executeImplByRow( + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + AggProcessInfo & agg_process_info) const +{ + // collect_hit_rate and only_lookup cannot be true at the same time. + static_assert(!(collect_hit_rate && only_lookup)); + + std::vector sort_key_containers; + sort_key_containers.resize(params.keys_size, ""); + size_t rows = agg_process_info.end_row - agg_process_info.start_row; + fiu_do_on(FailPoints::force_agg_on_partial_block, { + if (rows > 0 && agg_process_info.start_row == 0) + rows = std::max(rows / 2, 1); + }); + + /// Optimization for special case when there are no aggregate functions. + if (params.aggregates_size == 0) + { + /// For all rows. + AggregateDataPtr place = aggregates_pool->alloc(0); +#define HANDLE_AGG_EMPLACE_RESULT \ + if likely (emplace_result_hold.has_value()) \ + { \ + if constexpr (collect_hit_rate) \ + { \ + ++agg_process_info.hit_row_cnt; \ + } \ + \ + if constexpr (only_lookup) \ + { \ + if (!emplace_result_hold.value().isFound()) \ + agg_process_info.not_found_rows.push_back(i); \ + } \ + else \ + { \ + emplace_result_hold.value().setMapped(place); \ + } \ + processed_rows = i; \ + } \ + else \ + { \ + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ + break; \ + } + + std::vector hashvals; + std::optional processed_rows; + if constexpr (enable_prefetch) + { + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + hashvals); + } + + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) + { + if constexpr (enable_prefetch) + { + auto emplace_result_hold + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); + + HANDLE_AGG_EMPLACE_RESULT + } + else + { + auto emplace_result_hold + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + + HANDLE_AGG_EMPLACE_RESULT + } + } + + if likely (processed_rows) + agg_process_info.start_row = *processed_rows + 1; +#undef HANDLE_AGG_EMPLACE_RESULT + return; + } + + /// Optimization for special case when aggregating by 8bit key. + if constexpr (std::is_same_v) + { + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + ++inst) + { + inst->batch_that->addBatchLookupTable8( + agg_process_info.start_row, + rows, + reinterpret_cast(method.data.data()), + inst->state_offset, + [&](AggregateDataPtr & aggregate_data) { + aggregate_data + = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + }, + state.getKeyData(), + inst->batch_arguments, + aggregates_pool); + } + agg_process_info.start_row += rows; + + // For key8, assume all rows are hit. No need to do state switch for auto pass through hashagg. + // Because HashMap of key8 is basically a vector of size 256. + if constexpr (collect_hit_rate) + agg_process_info.hit_row_cnt = rows; + + // Because all rows are hit, so state will not switch to Selective. + if constexpr (only_lookup) + RUNTIME_CHECK_MSG(false, "Aggregator only_lookup should be false for AggregationMethod_key8"); + return; + } + + /// Generic case. + std::unique_ptr places(new AggregateDataPtr[rows]); + std::optional processed_rows; + +#define HANDLE_AGG_EMPLACE_RESULT \ + if unlikely (!emplace_result_holder.has_value()) \ + { \ + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ + break; \ + } \ + \ + auto & emplace_result = emplace_result_holder.value(); \ + \ + if constexpr (only_lookup) \ + { \ + if (emplace_result.isFound()) \ + { \ + aggregate_data = emplace_result.getMapped(); \ + } \ + else \ + { \ + agg_process_info.not_found_rows.push_back(i); \ + } \ + } \ + else \ + { \ + if (emplace_result.isInserted()) \ + { \ + emplace_result.setMapped(nullptr); \ + \ + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); \ + createAggregateStates(aggregate_data); \ + \ + emplace_result.setMapped(aggregate_data); \ + } \ + else \ + { \ + aggregate_data = emplace_result.getMapped(); \ + \ + if constexpr (collect_hit_rate) \ + ++agg_process_info.hit_row_cnt; \ + } \ + } \ + \ + places[i - agg_process_info.start_row] = aggregate_data; \ + processed_rows = i; + + std::vector hashvals; + if constexpr (enable_prefetch) + { + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + hashvals); + } + + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) + { + AggregateDataPtr aggregate_data = nullptr; + if constexpr (enable_prefetch) + { + auto emplace_result_holder + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); + + HANDLE_AGG_EMPLACE_RESULT + } + else + { + auto emplace_result_holder + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + + HANDLE_AGG_EMPLACE_RESULT + } + } +#undef HANDLE_AGG_EMPLACE_RESULT + + if (processed_rows) + { + /// Add values to the aggregate functions. + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + ++inst) + { + inst->batch_that->addBatch( + agg_process_info.start_row, + *processed_rows - agg_process_info.start_row + 1, + places.get(), + inst->state_offset, + inst->batch_arguments, + aggregates_pool); + } + agg_process_info.start_row = *processed_rows + 1; + } +} + +void NO_INLINE +Aggregator::executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena) +{ + size_t agg_size = agg_process_info.end_row - agg_process_info.start_row; + fiu_do_on(FailPoints::force_agg_on_partial_block, { + if (agg_size > 0 && agg_process_info.start_row == 0) + agg_size = std::max(agg_size / 2, 1); + }); + /// Adding values + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + ++inst) + { + inst->batch_that->addBatchSinglePlace( + agg_process_info.start_row, + agg_size, + res + inst->state_offset, + inst->batch_arguments, + arena); + } + agg_process_info.start_row += agg_size; +} + +void Aggregator::prepareAggregateInstructions( + Columns columns, + AggregateColumns & aggregate_columns, + Columns & materialized_columns, + AggregateFunctionInstructions & aggregate_functions_instructions) +{ + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns[i].resize(params.aggregates[i].arguments.size()); + + aggregate_functions_instructions.resize(params.aggregates_size + 1); + aggregate_functions_instructions[params.aggregates_size].that = nullptr; + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + for (size_t j = 0; j < aggregate_columns[i].size(); ++j) + { + aggregate_columns[i][j] = columns.at(params.aggregates[i].arguments[j]).get(); + if (ColumnPtr converted = aggregate_columns[i][j]->convertToFullColumnIfConst()) + { + materialized_columns.push_back(converted); + aggregate_columns[i][j] = materialized_columns.back().get(); + } + } + + aggregate_functions_instructions[i].arguments = aggregate_columns[i].data(); + aggregate_functions_instructions[i].state_offset = offsets_of_aggregate_states[i]; + + auto * that = aggregate_functions[i]; + /// Unnest consecutive trailing -State combinators + while (const auto * func = typeid_cast(that)) + that = func->getNestedFunction().get(); + aggregate_functions_instructions[i].that = that; + + if (const auto * func = typeid_cast(that)) + { + UNUSED(func); + throw Exception("Not support AggregateFunctionArray", ErrorCodes::NOT_IMPLEMENTED); + } + else + aggregate_functions_instructions[i].batch_arguments = aggregate_columns[i].data(); + + aggregate_functions_instructions[i].batch_that = that; + } +} + +void Aggregator::AggProcessInfo::prepareForAgg() +{ + if (prepare_for_agg_done) + return; + RUNTIME_CHECK_MSG(block, "Block must be set before execution aggregation"); + start_row = 0; + end_row = block.rows(); + input_columns = block.getColumns(); + materialized_columns.reserve(aggregator->params.keys_size); + key_columns.resize(aggregator->params.keys_size); + aggregate_columns.resize(aggregator->params.aggregates_size); + + /** Constant columns are not supported directly during aggregation. + * To make them work anyway, we materialize them. + */ + for (size_t i = 0; i < aggregator->params.keys_size; ++i) + { + key_columns[i] = input_columns.at(aggregator->params.keys[i]).get(); + if (ColumnPtr converted = key_columns[i]->convertToFullColumnIfConst()) + { + /// Remember the columns we will work with + materialized_columns.push_back(converted); + key_columns[i] = materialized_columns.back().get(); + } + } + + aggregator->prepareAggregateInstructions( + input_columns, + aggregate_columns, + materialized_columns, + aggregate_functions_instructions); + prepare_for_agg_done = true; +} + +bool Aggregator::executeOnBlock(AggProcessInfo & agg_process_info, AggregatedDataVariants & result, size_t thread_num) +{ + return executeOnBlockImpl(agg_process_info, result, thread_num); +} + +bool Aggregator::executeOnBlockCollectHitRate( + AggProcessInfo & agg_process_info, + AggregatedDataVariants & result, + size_t thread_num) +{ + return executeOnBlockImpl(agg_process_info, result, thread_num); +} + +bool Aggregator::executeOnBlockOnlyLookup( + AggProcessInfo & agg_process_info, + AggregatedDataVariants & result, + size_t thread_num) +{ + return executeOnBlockImpl(agg_process_info, result, thread_num); +} + +template +bool Aggregator::executeOnBlockImpl( + AggProcessInfo & agg_process_info, + AggregatedDataVariants & result, + size_t thread_num) +{ + assert(!result.need_spill); + + if (is_cancelled()) + return true; + + /// `result` will destroy the states of aggregate functions in the destructor + result.aggregator = this; + + /// How to perform the aggregation? + if (!result.inited()) + { + result.init(method_chosen); + result.keys_size = params.keys_size; + result.key_sizes = key_sizes; + LOG_TRACE(log, "Aggregation method: `{}`", result.getMethodName()); + } + + agg_process_info.prepareForAgg(); + + if (is_cancelled()) + return true; + + if (result.type == AggregatedDataVariants::Type::without_key && !result.without_key) + { + AggregateDataPtr place + = result.aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(place); + result.without_key = place; + } + + /// We select one of the aggregation methods and call it. + + assert(agg_process_info.start_row <= agg_process_info.end_row); + /// For the case when there are no keys (all aggregate into one row). + if (result.type == AggregatedDataVariants::Type::without_key) + { + executeWithoutKeyImpl(result.without_key, agg_process_info, result.aggregates_pool); + } + else + { +#define M(NAME, IS_TWO_LEVEL) \ + case AggregationMethodType(NAME): \ + { \ + executeImpl( \ + *ToAggregationMethodPtr(NAME, result.aggregation_method_impl), \ + result.aggregates_pool, \ + agg_process_info, \ + params.collators); \ + break; \ + } + + switch (result.type) + { + APPLY_FOR_AGGREGATED_VARIANTS(M) + default: + break; + } + +#undef M + } + + size_t result_size = result.size(); + auto result_size_bytes = result.bytesCount(); + + /// worth_convert_to_two_level is set to true if + /// 1. some other threads already convert to two level + /// 2. the result size exceeds threshold + bool worth_convert_to_two_level = use_two_level_hash_table + || (group_by_two_level_threshold && result_size >= group_by_two_level_threshold) + || (group_by_two_level_threshold_bytes && result_size_bytes >= group_by_two_level_threshold_bytes); + + /** Converting to a two-level data structure. + * It allows you to make, in the subsequent, an effective merge - either economical from memory or parallel. + */ + if (result.isConvertibleToTwoLevel() && worth_convert_to_two_level) + { + result.convertToTwoLevel(); + result.setResizeCallbackIfNeeded(thread_num); + } + + /** Flush data to disk if too much RAM is consumed. + */ + auto revocable_bytes = result.revocableBytes(); + if (revocable_bytes > 20 * 1024 * 1024) + LOG_TRACE(log, "Revocable bytes after insert one block {}, thread {}", revocable_bytes, thread_num); + if (agg_spill_context->updatePerThreadRevocableMemory(revocable_bytes, thread_num)) + { + assert(!result.empty()); + result.tryMarkNeedSpill(); + } + + return true; +} + + +void Aggregator::finishSpill() +{ + assert(agg_spill_context->getSpiller() != nullptr); + agg_spill_context->getSpiller()->finishSpill(); +} + +BlockInputStreams Aggregator::restoreSpilledData() +{ + assert(agg_spill_context->getSpiller() != nullptr); + return agg_spill_context->getSpiller()->restoreBlocks(0); +} + +void Aggregator::initThresholdByAggregatedDataVariantsSize(size_t aggregated_data_variants_size) +{ + group_by_two_level_threshold = params.getGroupByTwoLevelThreshold(); + group_by_two_level_threshold_bytes + = getAverageThreshold(params.getGroupByTwoLevelThresholdBytes(), aggregated_data_variants_size); +} + +void Aggregator::spill(AggregatedDataVariants & data_variants, size_t thread_num) +{ + assert(data_variants.need_spill); + agg_spill_context->markSpilled(); + /// Flush only two-level data and possibly overflow data. +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + spillImpl(data_variants, *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl), thread_num); \ + break; \ + } + + switch (data_variants.type) + { + APPLY_FOR_VARIANTS_TWO_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } + +#undef M + + /// NOTE Instead of freeing up memory and creating new hash tables and arenas, you can re-use the old ones. + data_variants.init(data_variants.type); + data_variants.setResizeCallbackIfNeeded(thread_num); + data_variants.need_spill = false; + data_variants.aggregates_pools = Arenas(1, std::make_shared()); + data_variants.aggregates_pool = data_variants.aggregates_pools.back().get(); + data_variants.without_key = nullptr; +} + +template +Block Aggregator::convertOneBucketToBlock( + AggregatedDataVariants & data_variants, + Method & method, + Arena * arena, + bool final, + size_t bucket) const +{ +#define FILLER_DEFINE(name, skip_convert_key) \ + auto filler_##name = [bucket, &method, arena, this]( \ + const Sizes & key_sizes, \ + MutableColumns & key_columns, \ + AggregateColumnsData & aggregate_columns, \ + MutableColumns & final_aggregate_columns, \ + bool final_) { \ + using METHOD_TYPE = std::decay_t; \ + using DATA_TYPE = std::decay_t; \ + convertToBlockImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns, \ + aggregate_columns, \ + final_aggregate_columns, \ + arena, \ + final_); \ + } + + FILLER_DEFINE(convert_key, false); + FILLER_DEFINE(skip_convert_key, true); +#undef FILLER_DEFINE + + // Ignore key optimization if in non-final mode(a.k.a. during spilling process). + // Because all keys are needed when insert spilled block back into HashMap during restore process. + size_t convert_key_size = final ? params.keys_size - params.key_ref_agg_func.size() : params.keys_size; + + Block block; + if (final && convert_key_size == 0) + { + block = prepareBlockAndFill( + data_variants, + final, + method.data.impls[bucket].size(), + filler_skip_convert_key, + convert_key_size); + } + else + { + block = prepareBlockAndFill( + data_variants, + final, + method.data.impls[bucket].size(), + filler_convert_key, + convert_key_size); + } + + block.info.bucket_num = bucket; + return block; +} + +template +BlocksList Aggregator::convertOneBucketToBlocks( + AggregatedDataVariants & data_variants, + Method & method, + Arena * arena, + bool final, + size_t bucket) const +{ +#define FILLER_DEFINE(name, skip_convert_key) \ + auto filler_##name = [bucket, &method, arena, this]( \ + const Sizes & key_sizes, \ + std::vector & key_columns_vec, \ + std::vector & aggregate_columns_vec, \ + std::vector & final_aggregate_columns_vec, \ + bool final_) { \ + convertToBlocksImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + arena, \ + final_); \ + }; + + FILLER_DEFINE(convert_key, false); + FILLER_DEFINE(skip_convert_key, true); +#undef FILLER_DEFINE + + BlocksList blocks; + size_t convert_key_size = final ? params.keys_size - params.key_ref_agg_func.size() : params.keys_size; + + if (final && convert_key_size == 0) + { + blocks = prepareBlocksAndFill( + data_variants, + final, + method.data.impls[bucket].size(), + filler_skip_convert_key, + convert_key_size); + } + else + { + blocks = prepareBlocksAndFill( + data_variants, + final, + method.data.impls[bucket].size(), + filler_convert_key, + convert_key_size); + } + + + for (auto & block : blocks) + { + block.info.bucket_num = bucket; + } + + return blocks; +} + + +template +void Aggregator::spillImpl(AggregatedDataVariants & data_variants, Method & method, size_t thread_num) +{ + RUNTIME_ASSERT( + agg_spill_context->getSpiller() != nullptr, + "spiller must not be nullptr in Aggregator when spilling"); + + auto block_input_stream + = std::make_shared>(*this, data_variants, method); + agg_spill_context->getSpiller()->spillBlocksUsingBlockInputStream(block_input_stream, 0, is_cancelled); + agg_spill_context->finishOneSpill(thread_num); + + /// Pass ownership of the aggregate functions states: + /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. + data_variants.aggregator = nullptr; + + auto [max_block_rows, max_block_bytes] = block_input_stream->maxBlockRowAndBytes(); + LOG_TRACE( + log, + "Max size of temporary bucket blocks: {} rows, {:.3f} MiB.", + max_block_rows, + (max_block_bytes / 1048576.0)); +} + + +void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVariants & result, size_t thread_num) +{ + if (is_cancelled()) + return; + + LOG_TRACE(log, "Aggregating"); + + Stopwatch watch; + + size_t src_rows = 0; + size_t src_bytes = 0; + AggProcessInfo agg_process_info(this); + + /// Read all the data + while (Block block = stream->read()) + { + agg_process_info.resetBlock(block); + bool should_stop = false; + do + { + if unlikely (is_cancelled()) + return; + if (!executeOnBlock(agg_process_info, result, thread_num)) + { + should_stop = true; + break; + } + if (result.need_spill) + spill(result, thread_num); + } while (!agg_process_info.allBlockDataHandled()); + + if (should_stop) + break; + + src_rows += block.rows(); + src_bytes += block.bytes(); + } + + /// If there was no data, and we aggregate without keys, and we must return single row with the result of empty aggregation. + /// To do this, we pass a block with zero rows to aggregate. + if (result.empty() && params.keys_size == 0 && !params.empty_result_for_aggregation_by_empty_set) + { + agg_process_info.resetBlock(stream->getHeader()); + executeOnBlock(agg_process_info, result, thread_num); + if (result.need_spill) + spill(result, thread_num); + assert(agg_process_info.allBlockDataHandled()); + } + + double elapsed_seconds = watch.elapsedSeconds(); + size_t rows = result.size(); + LOG_TRACE( + log, + "Aggregated. {} to {} rows (from {:.3f} MiB) in {:.3f} sec. ({:.3f} rows/sec., {:.3f} MiB/sec.)", + src_rows, + rows, + src_bytes / 1048576.0, + elapsed_seconds, + src_rows / elapsed_seconds, + src_bytes / elapsed_seconds / 1048576.0); +} + +template +void Aggregator::convertToBlockImpl( + Method & method, + Table & data, + const Sizes & key_sizes, + MutableColumns & key_columns, + AggregateColumnsData & aggregate_columns, + MutableColumns & final_aggregate_columns, + Arena * arena, + bool final) const +{ + if (data.empty()) + return; + + std::vector raw_key_columns; + raw_key_columns.reserve(key_columns.size()); + for (auto & column : key_columns) + raw_key_columns.push_back(column.get()); + + if (final) + convertToBlockImplFinal( + method, + data, + key_sizes, + std::move(raw_key_columns), + final_aggregate_columns, + arena); + else + convertToBlockImplNotFinal( + method, + data, + key_sizes, + std::move(raw_key_columns), + aggregate_columns); + + /// In order to release memory early. + data.clearAndShrink(); +} + +template +void Aggregator::convertToBlocksImpl( + Method & method, + Table & data, + const Sizes & key_sizes, + std::vector & key_columns_vec, + std::vector & aggregate_columns_vec, + std::vector & final_aggregate_columns_vec, + Arena * arena, + bool final) const +{ + if (data.empty()) + return; + + std::vector> raw_key_columns_vec; + raw_key_columns_vec.reserve(key_columns_vec.size()); + for (auto & key_columns : key_columns_vec) + { + std::vector raw_key_columns; + raw_key_columns.reserve(key_columns.size()); + for (auto & column : key_columns) + { + raw_key_columns.push_back(column.get()); + } + + raw_key_columns_vec.push_back(raw_key_columns); + } + + if (final) + convertToBlocksImplFinal( + method, + data, + key_sizes, + std::move(raw_key_columns_vec), + final_aggregate_columns_vec, + arena); + else + convertToBlocksImplNotFinal( + method, + data, + key_sizes, + std::move(raw_key_columns_vec), + aggregate_columns_vec); + + /// In order to release memory early. + data.clearAndShrink(); +} + + +template +inline void Aggregator::insertAggregatesIntoColumns( + Mapped & mapped, + MutableColumns & final_aggregate_columns, + Arena * arena) const +{ + /** Final values of aggregate functions are inserted to columns. + * Then states of aggregate functions, that are not longer needed, are destroyed. + * + * We mark already destroyed states with "nullptr" in data, + * so they will not be destroyed in destructor of Aggregator + * (other values will be destroyed in destructor in case of exception). + * + * But it becomes tricky, because we have multiple aggregate states pointed by a single pointer in data. + * So, if exception is thrown in the middle of moving states for different aggregate functions, + * we have to catch exceptions and destroy all the states that are no longer needed, + * to keep the data in consistent state. + * + * It is also tricky, because there are aggregate functions with "-State" modifier. + * When we call "insertResultInto" for them, they insert a pointer to the state to ColumnAggregateFunction + * and ColumnAggregateFunction will take ownership of this state. + * So, for aggregate functions with "-State" modifier, the state must not be destroyed + * after it has been transferred to ColumnAggregateFunction. + * But we should mark that the data no longer owns these states. + */ + + size_t insert_i = 0; + std::exception_ptr exception; + + try + { + /// Insert final values of aggregate functions into columns. + for (; insert_i < params.aggregates_size; ++insert_i) + aggregate_functions[insert_i]->insertResultInto( + mapped + offsets_of_aggregate_states[insert_i], + *final_aggregate_columns[insert_i], + arena); + } + catch (...) + { + exception = std::current_exception(); + } + + /** Destroy states that are no longer needed. This loop does not throw. + * + * Don't destroy states for "-State" aggregate functions, + * because the ownership of this state is transferred to ColumnAggregateFunction + * and ColumnAggregateFunction will take care. + * + * But it's only for states that has been transferred to ColumnAggregateFunction + * before exception has been thrown; + */ + for (size_t destroy_i = 0; destroy_i < params.aggregates_size; ++destroy_i) + { + /// If ownership was not transferred to ColumnAggregateFunction. + if (destroy_i >= insert_i || !aggregate_functions[destroy_i]->isState()) + aggregate_functions[destroy_i]->destroy(mapped + offsets_of_aggregate_states[destroy_i]); + } + + /// Mark the cell as destroyed so it will not be destroyed in destructor. + mapped = nullptr; + + if (exception) + std::rethrow_exception(exception); +} + +template +struct AggregatorMethodInitKeyColumnHelper +{ + Method & method; + explicit AggregatorMethodInitKeyColumnHelper(Method & method_) + : method(method_) + {} + ALWAYS_INLINE inline void initAggKeys(size_t, std::vector &) {} + template + ALWAYS_INLINE inline void insertKeyIntoColumns( + const Key & key, + std::vector & key_columns, + const Sizes & sizes, + const TiDB::TiDBCollators & collators) + { + method.insertKeyIntoColumns(key, key_columns, sizes, collators); + } +}; + +template +struct AggregatorMethodInitKeyColumnHelper> +{ + using Method = AggregationMethodFastPathTwoKeysNoCache; + size_t index{}; + std::function &, size_t)> insert_key_into_columns_function_ptr{}; + + Method & method; + explicit AggregatorMethodInitKeyColumnHelper(Method & method_) + : method(method_) + {} + + ALWAYS_INLINE inline void initAggKeys(size_t rows, std::vector & key_columns) + { + index = 0; + if (key_columns.size() == 1) + { + Method::template initAggKeys(rows, key_columns[0]); + insert_key_into_columns_function_ptr + = AggregationMethodFastPathTwoKeysNoCache::insertKeyIntoColumnsOneKey; + } + else if (key_columns.size() == 2) + { + Method::template initAggKeys(rows, key_columns[0]); + Method::template initAggKeys(rows, key_columns[1]); + insert_key_into_columns_function_ptr + = AggregationMethodFastPathTwoKeysNoCache::insertKeyIntoColumnsTwoKey; + } + else + { + throw Exception("unexpected key_columns size for AggMethodFastPathTwoKey: {}", key_columns.size()); + } + } + ALWAYS_INLINE inline void insertKeyIntoColumns( + const StringRef & key, + std::vector & key_columns, + const Sizes &, + const TiDB::TiDBCollators &) + { + assert(insert_key_into_columns_function_ptr); + insert_key_into_columns_function_ptr(key, key_columns, index); + ++index; + } +}; + +template +struct AggregatorMethodInitKeyColumnHelper> +{ + using Method = AggregationMethodOneKeyStringNoCache; + size_t index{}; + + Method & method; + explicit AggregatorMethodInitKeyColumnHelper(Method & method_) + : method(method_) + {} + + void initAggKeys(size_t rows, std::vector & key_columns) + { + index = 0; + RUNTIME_CHECK_MSG( + key_columns.size() == 1, + "unexpected key_columns size for AggMethodOneKeyString: {}", + key_columns.size()); + Method::initAggKeys(rows, key_columns[0]); + } + ALWAYS_INLINE inline void insertKeyIntoColumns( + const StringRef & key, + std::vector & key_columns, + const Sizes &, + const TiDB::TiDBCollators &) + { + method.insertKeyIntoColumns(key, key_columns, index); + ++index; + } +}; + +template +void NO_INLINE Aggregator::convertToBlockImplFinal( + Method & method, + Table & data, + const Sizes & key_sizes, + std::vector key_columns, + MutableColumns & final_aggregate_columns, + Arena * arena) const +{ + assert(key_sizes.size() == key_columns.size()); + Sizes key_sizes_ref = key_sizes; // NOLINT + AggregatorMethodInitKeyColumnHelper agg_keys_helper{method}; + if constexpr (!skip_convert_key) + { + auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns, key_sizes); + if (shuffled_key_sizes) + { + // When key_ref_agg_func is not empty, we may reorder key to skip copying some key from HashMap. + // But this optimization is not compatible with shuffleKeyColumns of AggregationMethodKeysFixed. + RUNTIME_CHECK(params.key_ref_agg_func.empty()); + key_sizes_ref = *shuffled_key_sizes; + } + agg_keys_helper.initAggKeys(data.size(), key_columns); + } + + data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { + if constexpr (!skip_convert_key) + { + agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); + } + + insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); + }); +} + +namespace +{ +template +std::optional shuffleKeyColumnsForKeyColumnsVec( + Method & method, + std::vector> & key_columns_vec, + const Sizes & key_sizes) +{ + auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns_vec[0], key_sizes); + for (size_t i = 1; i < key_columns_vec.size(); ++i) + { + auto new_key_sizes = method.shuffleKeyColumns(key_columns_vec[i], key_sizes); + assert(shuffled_key_sizes == new_key_sizes); + } + return shuffled_key_sizes; +} +template +std::vector>> initAggKeysForKeyColumnsVec( + Method & method, + std::vector> & key_columns_vec, + size_t max_block_size, + size_t total_row_count) +{ + std::vector>> agg_keys_helpers; + size_t block_row_count = max_block_size; + for (size_t i = 0; i < key_columns_vec.size(); ++i) + { + if (i == key_columns_vec.size() - 1 && total_row_count % block_row_count != 0) + /// update block_row_count for the last block + block_row_count = total_row_count % block_row_count; + agg_keys_helpers.push_back(std::make_unique>(method)); + agg_keys_helpers.back()->initAggKeys(block_row_count, key_columns_vec[i]); + } + return agg_keys_helpers; +} +} // namespace + +template +void NO_INLINE Aggregator::convertToBlocksImplFinal( + Method & method, + Table & data, + const Sizes & key_sizes, + std::vector> && key_columns_vec, + std::vector & final_aggregate_columns_vec, + Arena * arena) const +{ + assert(!key_columns_vec.empty()); +#ifndef NDEBUG + for (const auto & key_columns : key_columns_vec) + { + assert(key_columns.size() == key_sizes.size()); + } +#endif + std::vector>>> agg_keys_helpers; + Sizes key_sizes_ref = key_sizes; // NOLINT + if constexpr (!skip_convert_key) + { + auto shuffled_key_sizes = shuffleKeyColumnsForKeyColumnsVec(method, key_columns_vec, key_sizes); + if (shuffled_key_sizes) + { + RUNTIME_CHECK(params.key_ref_agg_func.empty()); + key_sizes_ref = *shuffled_key_sizes; + } + agg_keys_helpers = initAggKeysForKeyColumnsVec(method, key_columns_vec, params.max_block_size, data.size()); + } + + size_t data_index = 0; + data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { + size_t key_columns_vec_index = data_index / params.max_block_size; + if constexpr (!skip_convert_key) + { + agg_keys_helpers[key_columns_vec_index] + ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + } + insertAggregatesIntoColumns(mapped, final_aggregate_columns_vec[key_columns_vec_index], arena); + ++data_index; + }); +} + +template +void NO_INLINE Aggregator::convertToBlockImplNotFinal( + Method & method, + Table & data, + const Sizes & key_sizes, + std::vector key_columns, + AggregateColumnsData & aggregate_columns) const +{ + assert(key_sizes.size() == key_columns.size()); + AggregatorMethodInitKeyColumnHelper agg_keys_helper{method}; + Sizes key_sizes_ref = key_sizes; // NOLINT + if constexpr (!skip_convert_key) + { + auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns, key_sizes); + if (shuffled_key_sizes) + { + RUNTIME_CHECK(params.key_ref_agg_func.empty()); + key_sizes_ref = *shuffled_key_sizes; + } + agg_keys_helper.initAggKeys(data.size(), key_columns); + } + + data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { + if constexpr (!skip_convert_key) + { + agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); + } + + /// reserved, so push_back does not throw exceptions + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); + + mapped = nullptr; + }); +} + +template +void NO_INLINE Aggregator::convertToBlocksImplNotFinal( + Method & method, + Table & data, + const Sizes & key_sizes, + std::vector> && key_columns_vec, + std::vector & aggregate_columns_vec) const +{ +#ifndef NDEBUG + for (const auto & key_columns : key_columns_vec) + { + assert(key_sizes.size() == key_columns.size()); + } +#endif + std::vector>>> agg_keys_helpers; + Sizes key_sizes_ref = key_sizes; // NOLINT + if constexpr (!skip_convert_key) + { + auto shuffled_key_sizes = shuffleKeyColumnsForKeyColumnsVec(method, key_columns_vec, key_sizes); + if (shuffled_key_sizes) + { + RUNTIME_CHECK(params.key_ref_agg_func.empty()); + key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes; + } + agg_keys_helpers = initAggKeysForKeyColumnsVec(method, key_columns_vec, params.max_block_size, data.size()); + } + + size_t data_index = 0; + data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { + size_t key_columns_vec_index = data_index / params.max_block_size; + if constexpr (!skip_convert_key) + { + agg_keys_helpers[key_columns_vec_index] + ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + } + + /// reserved, so push_back does not throw exceptions + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns_vec[key_columns_vec_index][i]->push_back(mapped + offsets_of_aggregate_states[i]); + + ++data_index; + mapped = nullptr; + }); +} + +template +Block Aggregator::prepareBlockAndFill( + AggregatedDataVariants & data_variants, + bool final, + size_t rows, + Filler && filler, + size_t convert_key_size) const +{ + MutableColumns key_columns(convert_key_size); + MutableColumns aggregate_columns(params.aggregates_size); + MutableColumns final_aggregate_columns(params.aggregates_size); + AggregateColumnsData aggregate_columns_data(params.aggregates_size); + // Store size of keys that need to convert. + Sizes new_key_sizes; + new_key_sizes.reserve(convert_key_size); + + Block header = getHeader(final); + + for (size_t i = 0; i < convert_key_size; ++i) + { + key_columns[i] = header.safeGetByPosition(i).type->createColumn(); + key_columns[i]->reserve(rows); + new_key_sizes.push_back(key_sizes[i]); + } + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + if (!final) + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); + + /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. + auto & column_aggregate_func = assert_cast(*aggregate_columns[i]); + + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func.addArena(pool); + + aggregate_columns_data[i] = &column_aggregate_func.getData(); + aggregate_columns_data[i]->reserve(rows); + } + else + { + final_aggregate_columns[i] = aggregate_functions[i]->getReturnType()->createColumn(); + final_aggregate_columns[i]->reserve(rows); + + if (aggregate_functions[i]->isState()) + { + /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. + if (auto * column_aggregate_func + = typeid_cast(final_aggregate_columns[i].get())) + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func->addArena(pool); + } + } + } + + filler(new_key_sizes, key_columns, aggregate_columns_data, final_aggregate_columns, final); + + Block res = header.cloneEmpty(); + + for (size_t i = 0; i < convert_key_size; ++i) + res.getByPosition(i).column = std::move(key_columns[i]); + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + if (final) + res.getByName(aggregate_column_name).column = std::move(final_aggregate_columns[i]); + else + res.getByName(aggregate_column_name).column = std::move(aggregate_columns[i]); + } + + /// Change the size of the columns-constants in the block. + size_t columns = header.columns(); + for (size_t i = 0; i < columns; ++i) + if (res.getByPosition(i).column->isColumnConst()) + res.getByPosition(i).column = res.getByPosition(i).column->cut(0, rows); + + return res; +} + +template +BlocksList Aggregator::prepareBlocksAndFill( + AggregatedDataVariants & data_variants, + bool final, + size_t rows, + Filler && filler, + size_t convert_key_size) const +{ + Block header = getHeader(final); + + size_t block_count = (rows + params.max_block_size - 1) / params.max_block_size; + std::vector key_columns_vec; + std::vector aggregate_columns_data_vec; + std::vector aggregate_columns_vec; + std::vector final_aggregate_columns_vec; + // Store size of keys that need to convert. + Sizes new_key_sizes; + new_key_sizes.reserve(convert_key_size); + for (size_t i = 0; i < convert_key_size; ++i) + { + new_key_sizes.push_back(key_sizes[i]); + } + + size_t block_rows = params.max_block_size; + + for (size_t j = 0; j < block_count; ++j) + { + if (j == (block_count - 1) && rows % block_rows != 0) + { + block_rows = rows % block_rows; + } + + key_columns_vec.push_back(MutableColumns(convert_key_size)); + aggregate_columns_data_vec.push_back(AggregateColumnsData(params.aggregates_size)); + aggregate_columns_vec.push_back(MutableColumns(params.aggregates_size)); + final_aggregate_columns_vec.push_back(MutableColumns(params.aggregates_size)); + + auto & key_columns = key_columns_vec.back(); + auto & aggregate_columns_data = aggregate_columns_data_vec.back(); + auto & aggregate_columns = aggregate_columns_vec.back(); + auto & final_aggregate_columns = final_aggregate_columns_vec.back(); + + for (size_t i = 0; i < convert_key_size; ++i) + { + key_columns[i] = header.safeGetByPosition(i).type->createColumn(); + key_columns[i]->reserve(block_rows); + } + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + if (!final) + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); + + /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. + auto & column_aggregate_func = assert_cast(*aggregate_columns[i]); + + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func.addArena(pool); + + aggregate_columns_data[i] = &column_aggregate_func.getData(); + aggregate_columns_data[i]->reserve(block_rows); + } + else + { + final_aggregate_columns[i] = aggregate_functions[i]->getReturnType()->createColumn(); + final_aggregate_columns[i]->reserve(block_rows); + + if (aggregate_functions[i]->isState()) + { + /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. + if (auto * column_aggregate_func + = typeid_cast(final_aggregate_columns[i].get())) + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func->addArena(pool); + } + } + } + } + + filler(new_key_sizes, key_columns_vec, aggregate_columns_data_vec, final_aggregate_columns_vec, final); + + BlocksList res_list; + block_rows = params.max_block_size; + for (size_t j = 0; j < block_count; ++j) + { + Block res = header.cloneEmpty(); + + for (size_t i = 0; i < convert_key_size; ++i) + res.getByPosition(i).column = std::move(key_columns_vec[j][i]); + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + if (final) + res.getByName(aggregate_column_name).column = std::move(final_aggregate_columns_vec[j][i]); + else + res.getByName(aggregate_column_name).column = std::move(aggregate_columns_vec[j][i]); + } + + if (j == (block_count - 1) && rows % block_rows != 0) + { + block_rows = rows % block_rows; + } + + /// Change the size of the columns-constants in the block. + size_t columns = header.columns(); + for (size_t i = 0; i < columns; ++i) + if (res.getByPosition(i).column->isColumnConst()) + res.getByPosition(i).column = res.getByPosition(i).column->cut(0, block_rows); + + res_list.push_back(res); + } + + if (res_list.empty()) + { + /// at least one valid block must be returned. + res_list.push_back(header.cloneWithColumns(header.cloneEmptyColumns())); + } + return res_list; +} + + +BlocksList Aggregator::prepareBlocksAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final) const +{ + size_t rows = 1; + + auto filler = [&data_variants, this]( + const Sizes &, + std::vector &, + std::vector & aggregate_columns_vec, + std::vector & final_aggregate_columns_vec, + bool final_) { + if (data_variants.type == AggregatedDataVariants::Type::without_key) + { + AggregatedDataWithoutKey & data = data_variants.without_key; + + RUNTIME_CHECK_MSG(data, "Wrong data variant passed."); + + if (!final_) + { + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns_vec[0][i]->push_back(data + offsets_of_aggregate_states[i]); + data = nullptr; + } + else + { + /// Always single-thread. It's safe to pass current arena from 'aggregates_pool'. + insertAggregatesIntoColumns(data, final_aggregate_columns_vec[0], data_variants.aggregates_pool); + } + } + }; + + BlocksList blocks = prepareBlocksAndFill(data_variants, final, rows, filler, /*convert_key_size=*/0); + + if (final) + destroyWithoutKey(data_variants); + + return blocks; +} + +BlocksList Aggregator::prepareBlocksAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const +{ + size_t rows = data_variants.size(); +#define M(NAME, skip_convert_key) \ + case AggregationMethodType(NAME): \ + { \ + auto & tmp_method = *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl); \ + auto & tmp_data = ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl) -> data; \ + convertToBlocksImpl( \ + tmp_method, \ + tmp_data, \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + data_variants.aggregates_pool, \ + final_); \ + break; \ + } + +#define M_skip_convert_key(NAME) M(NAME, true) +#define M_convert_key(NAME) M(NAME, false) + +#define FILLER_DEFINE(name, M_tmp) \ + auto filler_##name = [&data_variants, this]( \ + const Sizes & key_sizes, \ + std::vector & key_columns_vec, \ + std::vector & aggregate_columns_vec, \ + std::vector & final_aggregate_columns_vec, \ + bool final_) { \ + switch (data_variants.type) \ + { \ + APPLY_FOR_VARIANTS_SINGLE_LEVEL(M_tmp) \ + default: \ + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); \ + } \ + } + + FILLER_DEFINE(convert_key, M_convert_key); + FILLER_DEFINE(skip_convert_key, M_skip_convert_key); + +#undef M +#undef M_skip_convert_key +#undef M_convert_key +#undef FILLER_DEFINE + + size_t convert_key_size = final ? params.keys_size - params.key_ref_agg_func.size() : params.keys_size; + + if (final && convert_key_size == 0) + { + return prepareBlocksAndFill(data_variants, final, rows, filler_skip_convert_key, convert_key_size); + } + return prepareBlocksAndFill(data_variants, final, rows, filler_convert_key, convert_key_size); +} + + +template +void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena) const +{ + table_src.mergeToViaEmplace( + table_dst, + [&](AggregateDataPtr & __restrict dst, AggregateDataPtr & __restrict src, bool inserted) { + if (!inserted) + { + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->merge( + dst + offsets_of_aggregate_states[i], + src + offsets_of_aggregate_states[i], + arena); + + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); + } + else + { + dst = src; + } + + src = nullptr; + }); + table_src.clearAndShrink(); +} + +void NO_INLINE Aggregator::mergeWithoutKeyDataImpl(ManyAggregatedDataVariants & non_empty_data) const +{ + AggregatedDataVariantsPtr & res = non_empty_data[0]; + + /// We merge all aggregation results to the first. + for (size_t result_num = 1, size = non_empty_data.size(); result_num < size; ++result_num) + { + AggregatedDataWithoutKey & res_data = res->without_key; + AggregatedDataWithoutKey & current_data = non_empty_data[result_num]->without_key; + + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->merge( + res_data + offsets_of_aggregate_states[i], + current_data + offsets_of_aggregate_states[i], + res->aggregates_pool); + + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->destroy(current_data + offsets_of_aggregate_states[i]); + + current_data = nullptr; + } +} + + +template +void NO_INLINE Aggregator::mergeSingleLevelDataImpl(ManyAggregatedDataVariants & non_empty_data) const +{ + AggregatedDataVariantsPtr & res = non_empty_data[0]; + + /// We merge all aggregation results to the first. + for (size_t result_num = 1, size = non_empty_data.size(); result_num < size; ++result_num) + { + AggregatedDataVariants & current = *non_empty_data[result_num]; + + mergeDataImpl( + getDataVariant(*res).data, + getDataVariant(current).data, + res->aggregates_pool); + + /// `current` will not destroy the states of aggregate functions in the destructor + current.aggregator = nullptr; + } +} + +#define M(NAME) \ + template void NO_INLINE Aggregator::mergeSingleLevelDataImpl( \ + ManyAggregatedDataVariants & non_empty_data) const; +APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) +#undef M + +template +void NO_INLINE Aggregator::mergeBucketImpl(ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena) const +{ + /// We merge all aggregation results to the first. + AggregatedDataVariantsPtr & res = data[0]; + for (size_t result_num = 1, size = data.size(); result_num < size; ++result_num) + { + if (is_cancelled()) + return; + + AggregatedDataVariants & current = *data[result_num]; + + mergeDataImpl( + getDataVariant(*res).data.impls[bucket], + getDataVariant(current).data.impls[bucket], + arena); + } +} + + +MergingBucketsPtr Aggregator::mergeAndConvertToBlocks( + ManyAggregatedDataVariants & data_variants, + bool final, + size_t max_threads) const +{ + if (unlikely(data_variants.empty())) + throw Exception("Empty data passed to Aggregator::mergeAndConvertToBlocks.", ErrorCodes::EMPTY_DATA_PASSED); + + LOG_TRACE(log, "Merging aggregated data"); + + ManyAggregatedDataVariants non_empty_data; + non_empty_data.reserve(data_variants.size()); + for (auto & data : data_variants) + if (!data->empty()) + non_empty_data.push_back(data); + + if (non_empty_data.empty()) + return nullptr; + + if (non_empty_data.size() > 1) + { + /// Sort the states in descending order so that the merge is more efficient (since all states are merged into the first). + std::sort( + non_empty_data.begin(), + non_empty_data.end(), + [](const AggregatedDataVariantsPtr & lhs, const AggregatedDataVariantsPtr & rhs) { + return lhs->size() > rhs->size(); + }); + } + + /// If at least one of the options is two-level, then convert all the options into two-level ones, if there are not such. + /// Note - perhaps it would be more optimal not to convert single-level versions before the merge, but merge them separately, at the end. + + bool has_at_least_one_two_level = false; + for (const auto & variant : non_empty_data) + { + if (variant->isTwoLevel()) + { + has_at_least_one_two_level = true; + break; + } + } + + if (has_at_least_one_two_level) + for (auto & variant : non_empty_data) + if (!variant->isTwoLevel()) + variant->convertToTwoLevel(); + + AggregatedDataVariantsPtr & first = non_empty_data[0]; + + for (size_t i = 1, size = non_empty_data.size(); i < size; ++i) + { + if (unlikely(first->type != non_empty_data[i]->type)) + throw Exception( + "Cannot merge different aggregated data variants.", + ErrorCodes::CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS); + + /** Elements from the remaining sets can be moved to the first data set. + * Therefore, it must own all the arenas of all other sets. + */ + first->aggregates_pools.insert( + first->aggregates_pools.end(), + non_empty_data[i]->aggregates_pools.begin(), + non_empty_data[i]->aggregates_pools.end()); + } + + // for single level merge, concurrency must be 1. + size_t merge_concurrency = has_at_least_one_two_level ? std::max(max_threads, 1) : 1; + return std::make_shared(*this, non_empty_data, final, merge_concurrency); +} + +template +void NO_INLINE Aggregator::mergeStreamsImplCase( + Block & block, + Arena * aggregates_pool, + Method & method [[maybe_unused]], + Table & data) const +{ + ColumnRawPtrs key_columns(params.keys_size); + AggregateColumnsConstData aggregate_columns(params.aggregates_size); + + /// Remember the columns we will work with + for (size_t i = 0; i < params.keys_size; ++i) + key_columns[i] = block.safeGetByPosition(i).column.get(); + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] + = &typeid_cast(*block.getByName(aggregate_column_name).column).getData(); + } + + std::vector sort_key_containers; + sort_key_containers.resize(params.keys_size, ""); + + /// in merge stage, don't need to care about the collator because the key is already the sort_key of original string + typename Method::State state(key_columns, key_sizes, {}); + + /// For all rows. + size_t rows = block.rows(); + std::unique_ptr places(new AggregateDataPtr[rows]); + + for (size_t i = 0; i < rows; ++i) + { + AggregateDataPtr aggregate_data = nullptr; + + auto emplace_result = state.emplaceKey(data, i, *aggregates_pool, sort_key_containers); + if (emplace_result.isInserted()) + { + emplace_result.setMapped(nullptr); + + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + + emplace_result.setMapped(aggregate_data); + } + else + aggregate_data = emplace_result.getMapped(); + + places[i] = aggregate_data; + } + + for (size_t j = 0; j < params.aggregates_size; ++j) + { + /// Merge state of aggregate functions. + aggregate_functions[j]->mergeBatch( + rows, + places.get(), + offsets_of_aggregate_states[j], + aggregate_columns[j]->data(), + aggregates_pool); + } + + + /// Early release memory. + block.clear(); +} + +template +void NO_INLINE Aggregator::mergeStreamsImpl(Block & block, Arena * aggregates_pool, Method & method, Table & data) const +{ + mergeStreamsImplCase(block, aggregates_pool, method, data); +} + + +void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl(Block & block, AggregatedDataVariants & result) const +{ + AggregateColumnsConstData aggregate_columns(params.aggregates_size); + + /// Remember the columns we will work with + for (size_t i = 0; i < params.aggregates_size; ++i) + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] + = &typeid_cast(*block.getByName(aggregate_column_name).column).getData(); + } + + AggregatedDataWithoutKey & res = result.without_key; + if (!res) + { + AggregateDataPtr place + = result.aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(place); + res = place; + } + + if (block.rows() > 0) + { + /// Adding Values + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->merge( + res + offsets_of_aggregate_states[i], + (*aggregate_columns[i])[0], + result.aggregates_pool); + } + + /// Early release memory. + block.clear(); +} + +BlocksList Aggregator::vstackBlocks(BlocksList & blocks, bool final) +{ + RUNTIME_CHECK_MSG(!blocks.empty(), "The input blocks list for Aggregator::vstackBlocks must be non-empty"); + + auto bucket_num = blocks.front().info.bucket_num; + + LOG_TRACE( + log, + "Merging partially aggregated blocks (bucket = {}). Original method `{}`.", + bucket_num, + AggregatedDataVariants::getMethodName(method_chosen)); + Stopwatch watch; + + /** If possible, change 'method' to some_hash64. Otherwise, leave as is. + * Better hash function is needed because during external aggregation, + * we may merge partitions of data with total number of keys far greater than 4 billion. + */ + auto merge_method = method_chosen; + +#define APPLY_FOR_VARIANTS_THAT_MAY_USE_BETTER_HASH_FUNCTION(M) \ + M(key64) \ + M(key_string) \ + M(key_fixed_string) \ + M(keys128) \ + M(keys256) \ + M(serialized) + +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + merge_method = AggregatedDataVariants::Type::NAME##_hash64; \ + break; \ + } + + switch (merge_method) + { + APPLY_FOR_VARIANTS_THAT_MAY_USE_BETTER_HASH_FUNCTION(M) + default: + break; + } +#undef M + +#undef APPLY_FOR_VARIANTS_THAT_MAY_USE_BETTER_HASH_FUNCTION + + /// Temporary data for aggregation. + AggregatedDataVariants result; + + /// result will destroy the states of aggregate functions in the destructor + result.aggregator = this; + + result.init(merge_method); + result.keys_size = params.keys_size; + result.key_sizes = key_sizes; + + for (Block & block : blocks) + { + if (bucket_num >= 0 && block.info.bucket_num != bucket_num) + bucket_num = -1; + + if (result.type == AggregatedDataVariants::Type::without_key) + mergeWithoutKeyStreamsImpl(block, result); + +#define M(NAME, IS_TWO_LEVEL) \ + case AggregationMethodType(NAME): \ + { \ + mergeStreamsImpl( \ + block, \ + result.aggregates_pool, \ + *ToAggregationMethodPtr(NAME, result.aggregation_method_impl), \ + ToAggregationMethodPtr(NAME, result.aggregation_method_impl)->data); \ + break; \ + } + switch (result.type) + { + APPLY_FOR_AGGREGATED_VARIANTS(M) + case AggregatedDataVariants::Type::without_key: + break; + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } +#undef M + } + + BlocksList return_blocks; + if (result.type == AggregatedDataVariants::Type::without_key) + return_blocks = prepareBlocksAndFillWithoutKey(result, final); + else + return_blocks = prepareBlocksAndFillSingleLevel(result, final); + /// NOTE: two-level data is not possible here - chooseAggregationMethod chooses only among single-level methods. + + if (!final) + { + /// Pass ownership of aggregate function states from result to ColumnAggregateFunction objects in the resulting block. + result.aggregator = nullptr; + } + + size_t rows = 0; + size_t bytes = 0; + for (auto block : return_blocks) + { + rows += block.rows(); + bytes += block.bytes(); + block.info.bucket_num = bucket_num; + } + double elapsed_seconds = watch.elapsedSeconds(); + LOG_TRACE( + log, + "Merged partially aggregated blocks. Return {} rows in {} blocks, {:.3f} MiB. in {:.3f} sec. ({:.3f} " + "rows/sec., {:.3f} MiB/sec.)", + rows, + return_blocks.size(), + bytes / 1048576.0, + elapsed_seconds, + rows / elapsed_seconds, + bytes / elapsed_seconds / 1048576.0); + + return return_blocks; +} + + +template +void NO_INLINE Aggregator::convertBlockToTwoLevelImpl( + Method & method, + Arena * pool, + ColumnRawPtrs & key_columns, + const Block & source, + Blocks & destinations) const +{ + typename Method::State state(key_columns, key_sizes, params.collators); + + std::vector sort_key_containers; + sort_key_containers.resize(params.keys_size, ""); + + size_t rows = source.rows(); + size_t columns = source.columns(); + + /// Create a 'selector' that will contain bucket index for every row. It will be used to scatter rows to buckets. + IColumn::Selector selector(rows); + + /// For every row. + for (size_t i = 0; i < rows; ++i) + { + /// Calculate bucket number from row hash. + auto hash = state.getHash(method.data, i, *pool, sort_key_containers); + auto bucket = method.data.getBucketFromHash(hash); + + selector[i] = bucket; + } + + size_t num_buckets = destinations.size(); + + for (size_t column_idx = 0; column_idx < columns; ++column_idx) + { + const ColumnWithTypeAndName & src_col = source.getByPosition(column_idx); + MutableColumns scattered_columns = src_col.column->scatter(num_buckets, selector); + + for (size_t bucket = 0, size = num_buckets; bucket < size; ++bucket) + { + if (!scattered_columns[bucket]->empty()) + { + Block & dst = destinations[bucket]; + dst.info.bucket_num = bucket; + dst.insert({std::move(scattered_columns[bucket]), src_col.type, src_col.name}); + } + + /** Inserted columns of type ColumnAggregateFunction will own states of aggregate functions + * by holding shared_ptr to source column. See ColumnAggregateFunction.h + */ + } + } +} + + +Blocks Aggregator::convertBlockToTwoLevel(const Block & block) +{ + if (!block) + return {}; + + AggregatedDataVariants data; + + ColumnRawPtrs key_columns(params.keys_size); + + /// Remember the columns we will work with + for (size_t i = 0; i < params.keys_size; ++i) + key_columns[i] = block.safeGetByPosition(i).column.get(); + + AggregatedDataVariants::Type type = method_chosen; + data.keys_size = params.keys_size; + data.key_sizes = key_sizes; + +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + type = AggregationMethodTypeTwoLevel(NAME); \ + break; \ + } + + switch (type) + { + APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } +#undef M + + data.init(type); + + size_t num_buckets = 0; + +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + num_buckets = ToAggregationMethodPtr(NAME, data.aggregation_method_impl)->data.NUM_BUCKETS; \ + break; \ + } + + switch (data.type) + { + APPLY_FOR_VARIANTS_TWO_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } + +#undef M + + + Blocks splitted_blocks(num_buckets); + +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + convertBlockToTwoLevelImpl( \ + *ToAggregationMethodPtr(NAME, data.aggregation_method_impl), \ + data.aggregates_pool, \ + key_columns, \ + block, \ + splitted_blocks); \ + break; \ + } + + switch (data.type) + { + APPLY_FOR_VARIANTS_TWO_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } +#undef M + + + return splitted_blocks; +} + + +template +void NO_INLINE Aggregator::destroyImpl(Table & table) const +{ + table.forEachMapped([&](AggregateDataPtr & data) { + /** If an exception (usually a lack of memory, the MemoryTracker throws) arose + * after inserting the key into a hash table, but before creating all states of aggregate functions, + * then data will be equal nullptr. + */ + if (nullptr == data) + return; + + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->destroy(data + offsets_of_aggregate_states[i]); + + data = nullptr; + }); +} + + +void Aggregator::destroyWithoutKey(AggregatedDataVariants & result) const +{ + AggregatedDataWithoutKey & res_data = result.without_key; + + if (nullptr != res_data) + { + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->destroy(res_data + offsets_of_aggregate_states[i]); + + res_data = nullptr; + } +} + + +void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) +{ + if (!result.inited()) + return; + + LOG_TRACE(log, "Destroying aggregate states"); + + /// In what data structure is the data aggregated? + if (result.type == AggregatedDataVariants::Type::without_key) + destroyWithoutKey(result); + +#define M(NAME, IS_TWO_LEVEL) \ + case AggregationMethodType(NAME): \ + { \ + destroyImpl(ToAggregationMethodPtr(NAME, result.aggregation_method_impl)->data); \ + break; \ + } + + switch (result.type) + { + APPLY_FOR_AGGREGATED_VARIANTS(M) + case AggregatedDataVariants::Type::without_key: + break; + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } + +#undef M +} + + +void Aggregator::setCancellationHook(CancellationHook cancellation_hook) +{ + is_cancelled = cancellation_hook; +} + +MergingBuckets::MergingBuckets( + const Aggregator & aggregator_, + const ManyAggregatedDataVariants & data_, + bool final_, + size_t concurrency_) + : log(Logger::get(aggregator_.log ? aggregator_.log->identifier() : "")) + , aggregator(aggregator_) + , data(data_) + , final(final_) + , concurrency(concurrency_) +{ + assert(concurrency > 0); + if (!data.empty()) + { + is_two_level = data[0]->isTwoLevel(); + if (is_two_level) + { + for (size_t i = 0; i < concurrency; ++i) + two_level_parallel_merge_data.push_back(std::make_unique()); + } + else + { + // for single level, concurrency must be 1. + RUNTIME_CHECK(concurrency == 1); + } + + /// At least we need one arena in first data item per concurrency + if (concurrency > data[0]->aggregates_pools.size()) + { + Arenas & first_pool = data[0]->aggregates_pools; + for (size_t j = first_pool.size(); j < concurrency; ++j) + first_pool.emplace_back(std::make_shared()); + } + } +} + +Block MergingBuckets::getHeader() const +{ + return aggregator.getHeader(final); +} + +Block MergingBuckets::getData(size_t concurrency_index) +{ + assert(concurrency_index < concurrency); + + if (unlikely(data.empty())) + return {}; + + FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::random_aggregate_merge_failpoint); + + return is_two_level ? getDataForTwoLevel(concurrency_index) : getDataForSingleLevel(); +} + +Block MergingBuckets::getDataForSingleLevel() +{ + assert(!data.empty()); + + Block out_block = popBlocksListFront(single_level_blocks); + if (likely(out_block)) + { + return out_block; + } + // The bucket number of single level merge can only be 0. + if (current_bucket_num > 0) + return {}; + + AggregatedDataVariantsPtr & first = data[0]; + if (first->type == AggregatedDataVariants::Type::without_key) + { + aggregator.mergeWithoutKeyDataImpl(data); + single_level_blocks = aggregator.prepareBlocksAndFillWithoutKey(*first, final); + } + else + { +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + aggregator.mergeSingleLevelDataImpl(data); \ + break; \ + } + switch (first->type) + { + APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } +#undef M + single_level_blocks = aggregator.prepareBlocksAndFillSingleLevel(*first, final); + } + ++current_bucket_num; + return popBlocksListFront(single_level_blocks); +} + +Block MergingBuckets::getDataForTwoLevel(size_t concurrency_index) +{ + assert(concurrency_index < two_level_parallel_merge_data.size()); + auto & two_level_merge_data = *two_level_parallel_merge_data[concurrency_index]; + + Block out_block = popBlocksListFront(two_level_merge_data); + if (likely(out_block)) + return out_block; + + if (current_bucket_num >= NUM_BUCKETS) + return {}; + while (true) + { + auto local_current_bucket_num = current_bucket_num.fetch_add(1); + if (unlikely(local_current_bucket_num >= NUM_BUCKETS)) + return {}; + + doLevelMerge(local_current_bucket_num, concurrency_index); + Block block = popBlocksListFront(two_level_merge_data); + if (likely(block)) + return block; + } +} + +void MergingBuckets::doLevelMerge(Int32 bucket_num, size_t concurrency_index) +{ + auto & two_level_merge_data = *two_level_parallel_merge_data[concurrency_index]; + assert(two_level_merge_data.empty()); + + assert(!data.empty()); + auto & merged_data = *data[0]; + auto method = merged_data.type; + + /// Select Arena to avoid race conditions + Arena * arena = merged_data.aggregates_pools.at(concurrency_index).get(); + +#define M(NAME) \ + case AggregationMethodType(NAME): \ + { \ + aggregator.mergeBucketImpl(data, bucket_num, arena); \ + two_level_merge_data = aggregator.convertOneBucketToBlocks( \ + merged_data, \ + *ToAggregationMethodPtr(NAME, merged_data.aggregation_method_impl), \ + arena, \ + final, \ + bucket_num); \ + break; \ + } + switch (method) + { + APPLY_FOR_VARIANTS_TWO_LEVEL(M) + default: + throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); + } +#undef M +} + +#undef AggregationMethodName +#undef AggregationMethodNameTwoLevel +#undef AggregationMethodType +#undef AggregationMethodTypeTwoLevel +#undef ToAggregationMethodPtr +#undef ToAggregationMethodPtrTwoLevel + +} // namespace DB diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 19e6fc7a919..d424645e6e7 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,6 +1318,8 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; + std::vector hashvals{}; + void prepareForAgg(); bool allBlockDataHandled() const { @@ -1336,6 +1338,8 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); + + hashvals.clear(); } }; From 6cea464db34b59defb67167c492770589179edc2 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 18 Dec 2024 10:08:33 +0800 Subject: [PATCH 31/61] fmt Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 23da5d04111..aaa118ce0f6 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -841,8 +841,13 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( { if constexpr (enable_prefetch) { - auto emplace_result_hold - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, agg_process_info.hashvals); + auto emplace_result_hold = emplaceOrFindKey( + method, + state, + i, + *aggregates_pool, + sort_key_containers, + agg_process_info.hashvals); HANDLE_AGG_EMPLACE_RESULT } @@ -961,8 +966,13 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( AggregateDataPtr aggregate_data = nullptr; if constexpr (enable_prefetch) { - auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, agg_process_info.hashvals); + auto emplace_result_holder = emplaceOrFindKey( + method, + state, + i, + *aggregates_pool, + sort_key_containers, + agg_process_info.hashvals); HANDLE_AGG_EMPLACE_RESULT } From 6750737cf0397d6aa739600014c27960a6381d02 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 19 Dec 2024 11:04:15 +0800 Subject: [PATCH 32/61] comment Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp.orig | 2973 --------------------- 1 file changed, 2973 deletions(-) delete mode 100644 dbms/src/Interpreters/Aggregator.cpp.orig diff --git a/dbms/src/Interpreters/Aggregator.cpp.orig b/dbms/src/Interpreters/Aggregator.cpp.orig deleted file mode 100644 index 5e00c2e72e6..00000000000 --- a/dbms/src/Interpreters/Aggregator.cpp.orig +++ /dev/null @@ -1,2973 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ -extern const int UNKNOWN_AGGREGATED_DATA_VARIANT; -extern const int EMPTY_DATA_PASSED; -extern const int CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS; -extern const int LOGICAL_ERROR; -} // namespace ErrorCodes - -namespace FailPoints -{ -extern const char random_aggregate_create_state_failpoint[]; -extern const char random_aggregate_merge_failpoint[]; -extern const char force_agg_on_partial_block[]; -extern const char random_fail_in_resize_callback[]; -extern const char force_agg_prefetch[]; -} // namespace FailPoints - -#define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME -#define AggregationMethodNameTwoLevel(NAME) AggregatedDataVariants::AggregationMethod_##NAME##_two_level -#define AggregationMethodType(NAME) AggregatedDataVariants::Type::NAME -#define AggregationMethodTypeTwoLevel(NAME) AggregatedDataVariants::Type::NAME##_two_level -#define ToAggregationMethodPtr(NAME, ptr) (reinterpret_cast(ptr)) -#define ToAggregationMethodPtrTwoLevel(NAME, ptr) (reinterpret_cast(ptr)) - -AggregatedDataVariants::~AggregatedDataVariants() -{ - if (aggregator && !aggregator->all_aggregates_has_trivial_destructor) - { - try - { - aggregator->destroyAllAggregateStates(*this); - } - catch (...) - { - tryLogCurrentException(aggregator->log, __PRETTY_FUNCTION__); - } - } - destroyAggregationMethodImpl(); -} - -bool AggregatedDataVariants::tryMarkNeedSpill() -{ - assert(!need_spill); - if (empty()) - return false; - if (!isTwoLevel()) - { - /// Data can only be flushed to disk if a two-level aggregation is supported. - if (!isConvertibleToTwoLevel()) - return false; - convertToTwoLevel(); - } - need_spill = true; - return true; -} - -void AggregatedDataVariants::destroyAggregationMethodImpl() -{ - if (!aggregation_method_impl) - return; - -#define M(NAME, IS_TWO_LEVEL) \ - case AggregationMethodType(NAME): \ - { \ - delete reinterpret_cast(aggregation_method_impl); \ - aggregation_method_impl = nullptr; \ - break; \ - } - switch (type) - { - APPLY_FOR_AGGREGATED_VARIANTS(M) - default: - break; - } -#undef M -} - -void AggregatedDataVariants::init(Type variants_type) -{ - destroyAggregationMethodImpl(); - - switch (variants_type) - { - case Type::EMPTY: - break; - case Type::without_key: - break; - -#define M(NAME, IS_TWO_LEVEL) \ - case AggregationMethodType(NAME): \ - { \ - aggregation_method_impl = std::make_unique().release(); \ - if (aggregator && !aggregator->params.key_ref_agg_func.empty()) \ - RUNTIME_CHECK_MSG( \ - AggregationMethodName(NAME)::canUseKeyRefAggFuncOptimization(), \ - "cannot use key_ref_agg_func optimization for method {}", \ - getMethodName()); \ - break; \ - } - - APPLY_FOR_AGGREGATED_VARIANTS(M) -#undef M - - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } - - type = variants_type; -} - -size_t AggregatedDataVariants::getBucketNumberForTwoLevelHashTable(Type type) -{ - switch (type) - { -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - return AggregationMethodNameTwoLevel(NAME)::Data::NUM_BUCKETS; \ - } - - APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) - -#undef M - - default: - throw Exception("Wrong data variant passed.", ErrorCodes::LOGICAL_ERROR); - } -} - -void AggregatedDataVariants::setResizeCallbackIfNeeded(size_t thread_num) const -{ - // For auto pass through hashagg, no spill should happen. Block will be pass through when need to spill. - // So no need to set callback. Also it's complicated to handle situation when Aggregator didn't process all rows at once. - if (aggregator && !aggregator->is_auto_pass_through) - { - auto agg_spill_context = aggregator->agg_spill_context; - if (agg_spill_context->isSpillEnabled() && agg_spill_context->isInAutoSpillMode()) - { - auto resize_callback = [agg_spill_context, thread_num]() { - if (agg_spill_context->supportFurtherSpill() - && agg_spill_context->isThreadMarkedForAutoSpill(thread_num)) - return false; - bool ret = true; - fiu_do_on(FailPoints::random_fail_in_resize_callback, { - if (agg_spill_context->supportFurtherSpill()) - { - ret = !agg_spill_context->markThreadForAutoSpill(thread_num); - } - }); - return ret; - }; -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - ToAggregationMethodPtr(NAME, aggregation_method_impl)->data.setResizeCallback(resize_callback); \ - break; \ - } - switch (type) - { - APPLY_FOR_VARIANTS_TWO_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } -#undef M - } - } -} - -void AggregatedDataVariants::convertToTwoLevel() -{ - switch (type) - { -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - if (aggregator) \ - LOG_TRACE( \ - aggregator->log, \ - "Converting aggregation data type `{}` to `{}`.", \ - getMethodName(AggregationMethodType(NAME)), \ - getMethodName(AggregationMethodTypeTwoLevel(NAME))); \ - auto ori_ptr = ToAggregationMethodPtr(NAME, aggregation_method_impl); \ - auto two_level = std::make_unique(*ori_ptr); \ - delete ori_ptr; \ - aggregation_method_impl = two_level.release(); \ - type = AggregationMethodTypeTwoLevel(NAME); \ - break; \ - } - - APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) - -#undef M - - default: - throw Exception("Wrong data variant passed.", ErrorCodes::LOGICAL_ERROR); - } - aggregator->useTwoLevelHashTable(); -} - - -Block Aggregator::getHeader(bool final) const -{ - return params.getHeader(final); -} - -/// when there is no input data and current aggregation still need to generate a result(for example, -/// select count(*) from t need to return 0 even if there is no data) the aggregator will use this -/// source header block as the fake input of aggregation -Block Aggregator::getSourceHeader() const -{ - return params.src_header; -} - -Block Aggregator::Params::getHeader( - const Block & src_header, - const ColumnNumbers & keys, - const AggregateDescriptions & aggregates, - const KeyRefAggFuncMap & key_ref_agg_func, - bool final) -{ - Block res; - - for (const auto & key : keys) - { - // For final stage, key optimization is enabled, so no need to output columns. - // An CopyColumn Action will handle this. - const auto & key_col = src_header.safeGetByPosition(key); - if (final && key_ref_agg_func.find(key_col.name) != key_ref_agg_func.end()) - continue; - - res.insert(key_col.cloneEmpty()); - } - - for (const auto & aggregate : aggregates) - { - size_t arguments_size = aggregate.arguments.size(); - DataTypes argument_types(arguments_size); - for (size_t j = 0; j < arguments_size; ++j) - argument_types[j] = src_header.safeGetByPosition(aggregate.arguments[j]).type; - - DataTypePtr type; - if (final) - type = aggregate.function->getReturnType(); - else - type - = std::make_shared(aggregate.function, argument_types, aggregate.parameters); - - res.insert({type, aggregate.column_name}); - } - - return materializeBlock(res); -} - - -Aggregator::Aggregator( - const Params & params_, - const String & req_id, - size_t concurrency, - const RegisterOperatorSpillContext & register_operator_spill_context, - bool is_auto_pass_through_) - : params(params_) - , log(Logger::get(req_id)) - , is_cancelled([]() { return false; }) - , is_auto_pass_through(is_auto_pass_through_) -{ - aggregate_functions.resize(params.aggregates_size); - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i] = params.aggregates[i].function.get(); - - /// Initialize sizes of aggregation states and its offsets. - offsets_of_aggregate_states.resize(params.aggregates_size); - total_size_of_aggregate_states = 0; - all_aggregates_has_trivial_destructor = true; - - // aggreate_states will be aligned as below: - // |<-- state_1 -->|<-- pad_1 -->|<-- state_2 -->|<-- pad_2 -->| ..... - // - // pad_N will be used to match alignment requirement for each next state. - // The address of state_1 is aligned based on maximum alignment requirements in states - for (size_t i = 0; i < params.aggregates_size; ++i) - { - offsets_of_aggregate_states[i] = total_size_of_aggregate_states; - total_size_of_aggregate_states += params.aggregates[i].function->sizeOfData(); - - // aggreate states are aligned based on maximum requirement - align_aggregate_states = std::max(align_aggregate_states, params.aggregates[i].function->alignOfData()); - - // If not the last aggregate_state, we need pad it so that next aggregate_state will be aligned. - if (i + 1 < params.aggregates_size) - { - size_t alignment_of_next_state = params.aggregates[i + 1].function->alignOfData(); - if ((alignment_of_next_state & (alignment_of_next_state - 1)) != 0) - throw Exception("Logical error: alignOfData is not 2^N", ErrorCodes::LOGICAL_ERROR); - - /// Extend total_size to next alignment requirement - /// Add padding by rounding up 'total_size_of_aggregate_states' to be a multiplier of alignment_of_next_state. - total_size_of_aggregate_states = (total_size_of_aggregate_states + alignment_of_next_state - 1) - / alignment_of_next_state * alignment_of_next_state; - } - - if (!params.aggregates[i].function->hasTrivialDestructor()) - all_aggregates_has_trivial_destructor = false; - } - - method_chosen = chooseAggregationMethod(); - RUNTIME_CHECK_MSG(method_chosen != AggregatedDataVariants::Type::EMPTY, "Invalid aggregation method"); - agg_spill_context = std::make_shared( - concurrency, - params.spill_config, - params.getMaxBytesBeforeExternalGroupBy(), - log); - if (agg_spill_context->supportSpill()) - { - bool is_convertible_to_two_level = AggregatedDataVariants::isConvertibleToTwoLevel(method_chosen); - if (!is_convertible_to_two_level) - { - params.setMaxBytesBeforeExternalGroupBy(0); - agg_spill_context->disableSpill(); - if (method_chosen != AggregatedDataVariants::Type::without_key) - LOG_WARNING( - log, - "Aggregation does not support spill because aggregator hash table does not support two level"); - } - } - if (register_operator_spill_context != nullptr) - register_operator_spill_context(agg_spill_context); - if (agg_spill_context->isSpillEnabled()) - { - /// init spiller if needed - /// for aggregation, the input block is sorted by bucket number - /// so it can work with MergingAggregatedMemoryEfficientBlockInputStream - agg_spill_context->buildSpiller(getHeader(false)); - } -} - - -inline bool IsTypeNumber64(const DataTypePtr & type) -{ - return type->isNumber() && type->getSizeOfValueInMemory() == sizeof(uint64_t); -} - -#define APPLY_FOR_AGG_FAST_PATH_TYPES(M) \ - M(Number64) \ - M(StringBin) \ - M(StringBinPadding) - -enum class AggFastPathType -{ -#define M(NAME) NAME, - APPLY_FOR_AGG_FAST_PATH_TYPES(M) -#undef M -}; - -AggregatedDataVariants::Type ChooseAggregationMethodTwoKeys(const AggFastPathType * fast_path_types) -{ - auto tp1 = fast_path_types[0]; - auto tp2 = fast_path_types[1]; - switch (tp1) - { - case AggFastPathType::Number64: - { - switch (tp2) - { - case AggFastPathType::Number64: - return AggregatedDataVariants::Type::serialized; // unreachable. keys64 or keys128 will be used before - case AggFastPathType::StringBin: - return AggregatedDataVariants::Type::two_keys_num64_strbin; - case AggFastPathType::StringBinPadding: - return AggregatedDataVariants::Type::two_keys_num64_strbinpadding; - } - } - case AggFastPathType::StringBin: - { - switch (tp2) - { - case AggFastPathType::Number64: - return AggregatedDataVariants::Type::two_keys_strbin_num64; - case AggFastPathType::StringBin: - return AggregatedDataVariants::Type::two_keys_strbin_strbin; - case AggFastPathType::StringBinPadding: - return AggregatedDataVariants::Type::serialized; // rare case - } - } - case AggFastPathType::StringBinPadding: - { - switch (tp2) - { - case AggFastPathType::Number64: - return AggregatedDataVariants::Type::two_keys_strbinpadding_num64; - case AggFastPathType::StringBin: - return AggregatedDataVariants::Type::serialized; // rare case - case AggFastPathType::StringBinPadding: - return AggregatedDataVariants::Type::two_keys_strbinpadding_strbinpadding; - } - } - } -} - -// return AggregatedDataVariants::Type::serialized if can NOT determine fast path. -AggregatedDataVariants::Type ChooseAggregationMethodFastPath( - size_t keys_size, - const DataTypes & types_not_null, - const TiDB::TiDBCollators & collators) -{ - std::array fast_path_types{}; - - if (keys_size == fast_path_types.max_size()) - { - for (size_t i = 0; i < keys_size; ++i) - { - const auto & type = types_not_null[i]; - if (type->isString()) - { - if (collators.empty() || !collators[i]) - { - // use original way - return AggregatedDataVariants::Type::serialized; - } - else - { - switch (collators[i]->getCollatorType()) - { - case TiDB::ITiDBCollator::CollatorType::UTF8MB4_BIN: - case TiDB::ITiDBCollator::CollatorType::UTF8_BIN: - case TiDB::ITiDBCollator::CollatorType::LATIN1_BIN: - case TiDB::ITiDBCollator::CollatorType::ASCII_BIN: - { - fast_path_types[i] = AggFastPathType::StringBinPadding; - break; - } - case TiDB::ITiDBCollator::CollatorType::BINARY: - { - fast_path_types[i] = AggFastPathType::StringBin; - break; - } - default: - { - // for CI COLLATION, use original way - return AggregatedDataVariants::Type::serialized; - } - } - } - } - else if (IsTypeNumber64(type)) - { - fast_path_types[i] = AggFastPathType::Number64; - } - else - { - return AggregatedDataVariants::Type::serialized; - } - } - return ChooseAggregationMethodTwoKeys(fast_path_types.data()); - } - return AggregatedDataVariants::Type::serialized; -} - -AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() -{ - /// If no keys. All aggregating to single row. - if (params.keys_size == 0) - return AggregatedDataVariants::Type::without_key; - - /// Check if at least one of the specified keys is nullable. - DataTypes types_removed_nullable; - types_removed_nullable.reserve(params.keys.size()); - bool has_nullable_key = false; - - for (const auto & pos : params.keys) - { - const auto & type = params.src_header.safeGetByPosition(pos).type; - - if (type->isNullable()) - { - has_nullable_key = true; - types_removed_nullable.push_back(removeNullable(type)); - } - else - types_removed_nullable.push_back(type); - } - - /** Returns ordinary (not two-level) methods, because we start from them. - * Later, during aggregation process, data may be converted (partitioned) to two-level structure, if cardinality is high. - */ - - size_t keys_bytes = 0; - size_t num_fixed_contiguous_keys = 0; - - key_sizes.resize(params.keys_size); - for (size_t j = 0; j < params.keys_size; ++j) - { - if (types_removed_nullable[j]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion()) - { - if (types_removed_nullable[j]->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion() - && (params.collators.empty() || params.collators[j] == nullptr)) - { - ++num_fixed_contiguous_keys; - key_sizes[j] = types_removed_nullable[j]->getSizeOfValueInMemory(); - keys_bytes += key_sizes[j]; - } - } - } - - if (has_nullable_key) - { - if (params.keys_size == num_fixed_contiguous_keys) - { - /// Pack if possible all the keys along with information about which key values are nulls - /// into a fixed 16- or 32-byte blob. - if (std::tuple_size>::value + keys_bytes <= 16) - return AggregatedDataVariants::Type::nullable_keys128; - if (std::tuple_size>::value + keys_bytes <= 32) - return AggregatedDataVariants::Type::nullable_keys256; - } - - /// Fallback case. - return AggregatedDataVariants::Type::serialized; - } - - /// No key has been found to be nullable. - const DataTypes & types_not_null = types_removed_nullable; - assert(!has_nullable_key); - - /// Single numeric key. - if (params.keys_size == 1 && types_not_null[0]->isValueRepresentedByNumber()) - { - size_t size_of_field = types_not_null[0]->getSizeOfValueInMemory(); - if (size_of_field == 1) - return AggregatedDataVariants::Type::key8; - if (size_of_field == 2) - return AggregatedDataVariants::Type::key16; - if (size_of_field == 4) - return AggregatedDataVariants::Type::key32; - if (size_of_field == 8) - return AggregatedDataVariants::Type::key64; - if (size_of_field == 16) - return AggregatedDataVariants::Type::keys128; - if (size_of_field == 32) - return AggregatedDataVariants::Type::keys256; - if (size_of_field == sizeof(Decimal256)) - return AggregatedDataVariants::Type::key_int256; - throw Exception( - "Logical error: numeric column has sizeOfField not in 1, 2, 4, 8, 16, 32.", - ErrorCodes::LOGICAL_ERROR); - } - - /// If all keys fits in N bits, will use hash table with all keys packed (placed contiguously) to single N-bit key. - if (params.keys_size == num_fixed_contiguous_keys) - { - if (keys_bytes <= 2) - return AggregatedDataVariants::Type::keys16; - if (keys_bytes <= 4) - return AggregatedDataVariants::Type::keys32; - if (keys_bytes <= 8) - return AggregatedDataVariants::Type::keys64; - if (keys_bytes <= 16) - return AggregatedDataVariants::Type::keys128; - if (keys_bytes <= 32) - return AggregatedDataVariants::Type::keys256; - } - - /// If single string key - will use hash table with references to it. Strings itself are stored separately in Arena. - if (params.keys_size == 1 && types_not_null[0]->isString()) - { - if (params.collators.empty() || !params.collators[0]) - { - // use original way. `Type::one_key_strbin` will generate empty column. - return AggregatedDataVariants::Type::key_string; - } - else - { - switch (params.collators[0]->getCollatorType()) - { - case TiDB::ITiDBCollator::CollatorType::UTF8MB4_BIN: - case TiDB::ITiDBCollator::CollatorType::UTF8_BIN: - case TiDB::ITiDBCollator::CollatorType::LATIN1_BIN: - case TiDB::ITiDBCollator::CollatorType::ASCII_BIN: - { - return AggregatedDataVariants::Type::one_key_strbinpadding; - } - case TiDB::ITiDBCollator::CollatorType::BINARY: - { - return AggregatedDataVariants::Type::one_key_strbin; - } - default: - { - // for CI COLLATION, use original way - return AggregatedDataVariants::Type::key_string; - } - } - } - } - - if (params.keys_size == 1 && types_not_null[0]->isFixedString()) - return AggregatedDataVariants::Type::key_fixed_string; - - return ChooseAggregationMethodFastPath(params.keys_size, types_not_null, params.collators); -} - - -void Aggregator::createAggregateStates(AggregateDataPtr & aggregate_data) const -{ - for (size_t j = 0; j < params.aggregates_size; ++j) - { - try - { - /** An exception may occur if there is a shortage of memory. - * In order that then everything is properly destroyed, we "roll back" some of the created states. - * The code is not very convenient. - */ - FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::random_aggregate_create_state_failpoint); - aggregate_functions[j]->create(aggregate_data + offsets_of_aggregate_states[j]); - } - catch (...) - { - for (size_t rollback_j = 0; rollback_j < j; ++rollback_j) - aggregate_functions[rollback_j]->destroy(aggregate_data + offsets_of_aggregate_states[rollback_j]); - - throw; - } - } -} - - -/** It's interesting - if you remove `noinline`, then gcc for some reason will inline this function, and the performance decreases (~ 10%). - * (Probably because after the inline of this function, more internal functions no longer be inlined.) - * Inline does not make sense, since the inner loop is entirely inside this function. - */ -template -void NO_INLINE Aggregator::executeImpl( - Method & method, - Arena * aggregates_pool, - AggProcessInfo & agg_process_info, - TiDB::TiDBCollators & collators) const -{ - typename Method::State state(agg_process_info.key_columns, key_sizes, collators); - -#ifndef NDEBUG - bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); - fiu_do_on(FailPoints::force_agg_prefetch, { disable_prefetch = false; }); -#else - const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); -#endif - - // key_serialized needs column-wise handling for prefetch. - // Because: - // 1. getKeyHolder of key_serialized have to copy real data into Arena. - // It means we better getKeyHolder for all Columns once and then use it both for getHash() and emplaceKey(). - // 2. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. - // But getHashVals() still needs to be column-wise. - if constexpr (Method::State::is_serialized_key) - { - // TODO: batch serialize method for Columns is still under development. - // if (!disable_prefetch) - // executeImplSerializedKeyByCol(); - // else - // executeImplByRow(method, state, aggregates_pool, agg_process_info); - executeImplByRow(method, state, aggregates_pool, agg_process_info); - } - else if constexpr (Method::Data::is_string_hash_map) - { - // StringHashMap doesn't support prefetch. - executeImplByRow(method, state, aggregates_pool, agg_process_info); - } - else - { - if (disable_prefetch) - executeImplByRow(method, state, aggregates_pool, agg_process_info); - else - executeImplByRow(method, state, aggregates_pool, agg_process_info); - } -} - -template -void getHashVals( - size_t start_row, - size_t end_row, - const Data & data, - const State & state, - std::vector & sort_key_containers, - Arena * pool, - std::vector & hashvals) -{ - hashvals.resize(state.total_rows); - for (size_t i = start_row; i < end_row; ++i) - { - hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); - } -} - -template -std::optional::ResultType> Aggregator::emplaceOrFindKey( - Method & method, - typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers, - const std::vector & hashvals) const -{ - try - { - if constexpr (only_lookup) - return state.template findKey( - method.data, - index, - aggregates_pool, - sort_key_containers, - hashvals); - else - return state.template emplaceKey( - method.data, - index, - aggregates_pool, - sort_key_containers, - hashvals); - } - catch (ResizeException &) - { - return {}; - } -} - -template -std::optional::ResultType> Aggregator::emplaceOrFindKey( - Method & method, - typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers) const -{ - try - { - if constexpr (only_lookup) - return state.findKey(method.data, index, aggregates_pool, sort_key_containers); - else - return state.emplaceKey(method.data, index, aggregates_pool, sort_key_containers); - } - catch (ResizeException &) - { - return {}; - } -} - -template -ALWAYS_INLINE void Aggregator::executeImplByRow( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - AggProcessInfo & agg_process_info) const -{ - // collect_hit_rate and only_lookup cannot be true at the same time. - static_assert(!(collect_hit_rate && only_lookup)); - - std::vector sort_key_containers; - sort_key_containers.resize(params.keys_size, ""); - size_t rows = agg_process_info.end_row - agg_process_info.start_row; - fiu_do_on(FailPoints::force_agg_on_partial_block, { - if (rows > 0 && agg_process_info.start_row == 0) - rows = std::max(rows / 2, 1); - }); - - /// Optimization for special case when there are no aggregate functions. - if (params.aggregates_size == 0) - { - /// For all rows. - AggregateDataPtr place = aggregates_pool->alloc(0); -#define HANDLE_AGG_EMPLACE_RESULT \ - if likely (emplace_result_hold.has_value()) \ - { \ - if constexpr (collect_hit_rate) \ - { \ - ++agg_process_info.hit_row_cnt; \ - } \ - \ - if constexpr (only_lookup) \ - { \ - if (!emplace_result_hold.value().isFound()) \ - agg_process_info.not_found_rows.push_back(i); \ - } \ - else \ - { \ - emplace_result_hold.value().setMapped(place); \ - } \ - processed_rows = i; \ - } \ - else \ - { \ - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ - break; \ - } - - std::vector hashvals; - std::optional processed_rows; - if constexpr (enable_prefetch) - { - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool, - hashvals); - } - - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - { - if constexpr (enable_prefetch) - { - auto emplace_result_hold - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); - - HANDLE_AGG_EMPLACE_RESULT - } - else - { - auto emplace_result_hold - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); - - HANDLE_AGG_EMPLACE_RESULT - } - } - - if likely (processed_rows) - agg_process_info.start_row = *processed_rows + 1; -#undef HANDLE_AGG_EMPLACE_RESULT - return; - } - - /// Optimization for special case when aggregating by 8bit key. - if constexpr (std::is_same_v) - { - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) - { - inst->batch_that->addBatchLookupTable8( - agg_process_info.start_row, - rows, - reinterpret_cast(method.data.data()), - inst->state_offset, - [&](AggregateDataPtr & aggregate_data) { - aggregate_data - = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); - }, - state.getKeyData(), - inst->batch_arguments, - aggregates_pool); - } - agg_process_info.start_row += rows; - - // For key8, assume all rows are hit. No need to do state switch for auto pass through hashagg. - // Because HashMap of key8 is basically a vector of size 256. - if constexpr (collect_hit_rate) - agg_process_info.hit_row_cnt = rows; - - // Because all rows are hit, so state will not switch to Selective. - if constexpr (only_lookup) - RUNTIME_CHECK_MSG(false, "Aggregator only_lookup should be false for AggregationMethod_key8"); - return; - } - - /// Generic case. - std::unique_ptr places(new AggregateDataPtr[rows]); - std::optional processed_rows; - -#define HANDLE_AGG_EMPLACE_RESULT \ - if unlikely (!emplace_result_holder.has_value()) \ - { \ - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ - break; \ - } \ - \ - auto & emplace_result = emplace_result_holder.value(); \ - \ - if constexpr (only_lookup) \ - { \ - if (emplace_result.isFound()) \ - { \ - aggregate_data = emplace_result.getMapped(); \ - } \ - else \ - { \ - agg_process_info.not_found_rows.push_back(i); \ - } \ - } \ - else \ - { \ - if (emplace_result.isInserted()) \ - { \ - emplace_result.setMapped(nullptr); \ - \ - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); \ - createAggregateStates(aggregate_data); \ - \ - emplace_result.setMapped(aggregate_data); \ - } \ - else \ - { \ - aggregate_data = emplace_result.getMapped(); \ - \ - if constexpr (collect_hit_rate) \ - ++agg_process_info.hit_row_cnt; \ - } \ - } \ - \ - places[i - agg_process_info.start_row] = aggregate_data; \ - processed_rows = i; - - std::vector hashvals; - if constexpr (enable_prefetch) - { - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool, - hashvals); - } - - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - if constexpr (enable_prefetch) - { - auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashvals); - - HANDLE_AGG_EMPLACE_RESULT - } - else - { - auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); - - HANDLE_AGG_EMPLACE_RESULT - } - } -#undef HANDLE_AGG_EMPLACE_RESULT - - if (processed_rows) - { - /// Add values to the aggregate functions. - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) - { - inst->batch_that->addBatch( - agg_process_info.start_row, - *processed_rows - agg_process_info.start_row + 1, - places.get(), - inst->state_offset, - inst->batch_arguments, - aggregates_pool); - } - agg_process_info.start_row = *processed_rows + 1; - } -} - -void NO_INLINE -Aggregator::executeWithoutKeyImpl(AggregatedDataWithoutKey & res, AggProcessInfo & agg_process_info, Arena * arena) -{ - size_t agg_size = agg_process_info.end_row - agg_process_info.start_row; - fiu_do_on(FailPoints::force_agg_on_partial_block, { - if (agg_size > 0 && agg_process_info.start_row == 0) - agg_size = std::max(agg_size / 2, 1); - }); - /// Adding values - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) - { - inst->batch_that->addBatchSinglePlace( - agg_process_info.start_row, - agg_size, - res + inst->state_offset, - inst->batch_arguments, - arena); - } - agg_process_info.start_row += agg_size; -} - -void Aggregator::prepareAggregateInstructions( - Columns columns, - AggregateColumns & aggregate_columns, - Columns & materialized_columns, - AggregateFunctionInstructions & aggregate_functions_instructions) -{ - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i].resize(params.aggregates[i].arguments.size()); - - aggregate_functions_instructions.resize(params.aggregates_size + 1); - aggregate_functions_instructions[params.aggregates_size].that = nullptr; - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - for (size_t j = 0; j < aggregate_columns[i].size(); ++j) - { - aggregate_columns[i][j] = columns.at(params.aggregates[i].arguments[j]).get(); - if (ColumnPtr converted = aggregate_columns[i][j]->convertToFullColumnIfConst()) - { - materialized_columns.push_back(converted); - aggregate_columns[i][j] = materialized_columns.back().get(); - } - } - - aggregate_functions_instructions[i].arguments = aggregate_columns[i].data(); - aggregate_functions_instructions[i].state_offset = offsets_of_aggregate_states[i]; - - auto * that = aggregate_functions[i]; - /// Unnest consecutive trailing -State combinators - while (const auto * func = typeid_cast(that)) - that = func->getNestedFunction().get(); - aggregate_functions_instructions[i].that = that; - - if (const auto * func = typeid_cast(that)) - { - UNUSED(func); - throw Exception("Not support AggregateFunctionArray", ErrorCodes::NOT_IMPLEMENTED); - } - else - aggregate_functions_instructions[i].batch_arguments = aggregate_columns[i].data(); - - aggregate_functions_instructions[i].batch_that = that; - } -} - -void Aggregator::AggProcessInfo::prepareForAgg() -{ - if (prepare_for_agg_done) - return; - RUNTIME_CHECK_MSG(block, "Block must be set before execution aggregation"); - start_row = 0; - end_row = block.rows(); - input_columns = block.getColumns(); - materialized_columns.reserve(aggregator->params.keys_size); - key_columns.resize(aggregator->params.keys_size); - aggregate_columns.resize(aggregator->params.aggregates_size); - - /** Constant columns are not supported directly during aggregation. - * To make them work anyway, we materialize them. - */ - for (size_t i = 0; i < aggregator->params.keys_size; ++i) - { - key_columns[i] = input_columns.at(aggregator->params.keys[i]).get(); - if (ColumnPtr converted = key_columns[i]->convertToFullColumnIfConst()) - { - /// Remember the columns we will work with - materialized_columns.push_back(converted); - key_columns[i] = materialized_columns.back().get(); - } - } - - aggregator->prepareAggregateInstructions( - input_columns, - aggregate_columns, - materialized_columns, - aggregate_functions_instructions); - prepare_for_agg_done = true; -} - -bool Aggregator::executeOnBlock(AggProcessInfo & agg_process_info, AggregatedDataVariants & result, size_t thread_num) -{ - return executeOnBlockImpl(agg_process_info, result, thread_num); -} - -bool Aggregator::executeOnBlockCollectHitRate( - AggProcessInfo & agg_process_info, - AggregatedDataVariants & result, - size_t thread_num) -{ - return executeOnBlockImpl(agg_process_info, result, thread_num); -} - -bool Aggregator::executeOnBlockOnlyLookup( - AggProcessInfo & agg_process_info, - AggregatedDataVariants & result, - size_t thread_num) -{ - return executeOnBlockImpl(agg_process_info, result, thread_num); -} - -template -bool Aggregator::executeOnBlockImpl( - AggProcessInfo & agg_process_info, - AggregatedDataVariants & result, - size_t thread_num) -{ - assert(!result.need_spill); - - if (is_cancelled()) - return true; - - /// `result` will destroy the states of aggregate functions in the destructor - result.aggregator = this; - - /// How to perform the aggregation? - if (!result.inited()) - { - result.init(method_chosen); - result.keys_size = params.keys_size; - result.key_sizes = key_sizes; - LOG_TRACE(log, "Aggregation method: `{}`", result.getMethodName()); - } - - agg_process_info.prepareForAgg(); - - if (is_cancelled()) - return true; - - if (result.type == AggregatedDataVariants::Type::without_key && !result.without_key) - { - AggregateDataPtr place - = result.aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(place); - result.without_key = place; - } - - /// We select one of the aggregation methods and call it. - - assert(agg_process_info.start_row <= agg_process_info.end_row); - /// For the case when there are no keys (all aggregate into one row). - if (result.type == AggregatedDataVariants::Type::without_key) - { - executeWithoutKeyImpl(result.without_key, agg_process_info, result.aggregates_pool); - } - else - { -#define M(NAME, IS_TWO_LEVEL) \ - case AggregationMethodType(NAME): \ - { \ - executeImpl( \ - *ToAggregationMethodPtr(NAME, result.aggregation_method_impl), \ - result.aggregates_pool, \ - agg_process_info, \ - params.collators); \ - break; \ - } - - switch (result.type) - { - APPLY_FOR_AGGREGATED_VARIANTS(M) - default: - break; - } - -#undef M - } - - size_t result_size = result.size(); - auto result_size_bytes = result.bytesCount(); - - /// worth_convert_to_two_level is set to true if - /// 1. some other threads already convert to two level - /// 2. the result size exceeds threshold - bool worth_convert_to_two_level = use_two_level_hash_table - || (group_by_two_level_threshold && result_size >= group_by_two_level_threshold) - || (group_by_two_level_threshold_bytes && result_size_bytes >= group_by_two_level_threshold_bytes); - - /** Converting to a two-level data structure. - * It allows you to make, in the subsequent, an effective merge - either economical from memory or parallel. - */ - if (result.isConvertibleToTwoLevel() && worth_convert_to_two_level) - { - result.convertToTwoLevel(); - result.setResizeCallbackIfNeeded(thread_num); - } - - /** Flush data to disk if too much RAM is consumed. - */ - auto revocable_bytes = result.revocableBytes(); - if (revocable_bytes > 20 * 1024 * 1024) - LOG_TRACE(log, "Revocable bytes after insert one block {}, thread {}", revocable_bytes, thread_num); - if (agg_spill_context->updatePerThreadRevocableMemory(revocable_bytes, thread_num)) - { - assert(!result.empty()); - result.tryMarkNeedSpill(); - } - - return true; -} - - -void Aggregator::finishSpill() -{ - assert(agg_spill_context->getSpiller() != nullptr); - agg_spill_context->getSpiller()->finishSpill(); -} - -BlockInputStreams Aggregator::restoreSpilledData() -{ - assert(agg_spill_context->getSpiller() != nullptr); - return agg_spill_context->getSpiller()->restoreBlocks(0); -} - -void Aggregator::initThresholdByAggregatedDataVariantsSize(size_t aggregated_data_variants_size) -{ - group_by_two_level_threshold = params.getGroupByTwoLevelThreshold(); - group_by_two_level_threshold_bytes - = getAverageThreshold(params.getGroupByTwoLevelThresholdBytes(), aggregated_data_variants_size); -} - -void Aggregator::spill(AggregatedDataVariants & data_variants, size_t thread_num) -{ - assert(data_variants.need_spill); - agg_spill_context->markSpilled(); - /// Flush only two-level data and possibly overflow data. -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - spillImpl(data_variants, *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl), thread_num); \ - break; \ - } - - switch (data_variants.type) - { - APPLY_FOR_VARIANTS_TWO_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } - -#undef M - - /// NOTE Instead of freeing up memory and creating new hash tables and arenas, you can re-use the old ones. - data_variants.init(data_variants.type); - data_variants.setResizeCallbackIfNeeded(thread_num); - data_variants.need_spill = false; - data_variants.aggregates_pools = Arenas(1, std::make_shared()); - data_variants.aggregates_pool = data_variants.aggregates_pools.back().get(); - data_variants.without_key = nullptr; -} - -template -Block Aggregator::convertOneBucketToBlock( - AggregatedDataVariants & data_variants, - Method & method, - Arena * arena, - bool final, - size_t bucket) const -{ -#define FILLER_DEFINE(name, skip_convert_key) \ - auto filler_##name = [bucket, &method, arena, this]( \ - const Sizes & key_sizes, \ - MutableColumns & key_columns, \ - AggregateColumnsData & aggregate_columns, \ - MutableColumns & final_aggregate_columns, \ - bool final_) { \ - using METHOD_TYPE = std::decay_t; \ - using DATA_TYPE = std::decay_t; \ - convertToBlockImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns, \ - aggregate_columns, \ - final_aggregate_columns, \ - arena, \ - final_); \ - } - - FILLER_DEFINE(convert_key, false); - FILLER_DEFINE(skip_convert_key, true); -#undef FILLER_DEFINE - - // Ignore key optimization if in non-final mode(a.k.a. during spilling process). - // Because all keys are needed when insert spilled block back into HashMap during restore process. - size_t convert_key_size = final ? params.keys_size - params.key_ref_agg_func.size() : params.keys_size; - - Block block; - if (final && convert_key_size == 0) - { - block = prepareBlockAndFill( - data_variants, - final, - method.data.impls[bucket].size(), - filler_skip_convert_key, - convert_key_size); - } - else - { - block = prepareBlockAndFill( - data_variants, - final, - method.data.impls[bucket].size(), - filler_convert_key, - convert_key_size); - } - - block.info.bucket_num = bucket; - return block; -} - -template -BlocksList Aggregator::convertOneBucketToBlocks( - AggregatedDataVariants & data_variants, - Method & method, - Arena * arena, - bool final, - size_t bucket) const -{ -#define FILLER_DEFINE(name, skip_convert_key) \ - auto filler_##name = [bucket, &method, arena, this]( \ - const Sizes & key_sizes, \ - std::vector & key_columns_vec, \ - std::vector & aggregate_columns_vec, \ - std::vector & final_aggregate_columns_vec, \ - bool final_) { \ - convertToBlocksImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - arena, \ - final_); \ - }; - - FILLER_DEFINE(convert_key, false); - FILLER_DEFINE(skip_convert_key, true); -#undef FILLER_DEFINE - - BlocksList blocks; - size_t convert_key_size = final ? params.keys_size - params.key_ref_agg_func.size() : params.keys_size; - - if (final && convert_key_size == 0) - { - blocks = prepareBlocksAndFill( - data_variants, - final, - method.data.impls[bucket].size(), - filler_skip_convert_key, - convert_key_size); - } - else - { - blocks = prepareBlocksAndFill( - data_variants, - final, - method.data.impls[bucket].size(), - filler_convert_key, - convert_key_size); - } - - - for (auto & block : blocks) - { - block.info.bucket_num = bucket; - } - - return blocks; -} - - -template -void Aggregator::spillImpl(AggregatedDataVariants & data_variants, Method & method, size_t thread_num) -{ - RUNTIME_ASSERT( - agg_spill_context->getSpiller() != nullptr, - "spiller must not be nullptr in Aggregator when spilling"); - - auto block_input_stream - = std::make_shared>(*this, data_variants, method); - agg_spill_context->getSpiller()->spillBlocksUsingBlockInputStream(block_input_stream, 0, is_cancelled); - agg_spill_context->finishOneSpill(thread_num); - - /// Pass ownership of the aggregate functions states: - /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. - data_variants.aggregator = nullptr; - - auto [max_block_rows, max_block_bytes] = block_input_stream->maxBlockRowAndBytes(); - LOG_TRACE( - log, - "Max size of temporary bucket blocks: {} rows, {:.3f} MiB.", - max_block_rows, - (max_block_bytes / 1048576.0)); -} - - -void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVariants & result, size_t thread_num) -{ - if (is_cancelled()) - return; - - LOG_TRACE(log, "Aggregating"); - - Stopwatch watch; - - size_t src_rows = 0; - size_t src_bytes = 0; - AggProcessInfo agg_process_info(this); - - /// Read all the data - while (Block block = stream->read()) - { - agg_process_info.resetBlock(block); - bool should_stop = false; - do - { - if unlikely (is_cancelled()) - return; - if (!executeOnBlock(agg_process_info, result, thread_num)) - { - should_stop = true; - break; - } - if (result.need_spill) - spill(result, thread_num); - } while (!agg_process_info.allBlockDataHandled()); - - if (should_stop) - break; - - src_rows += block.rows(); - src_bytes += block.bytes(); - } - - /// If there was no data, and we aggregate without keys, and we must return single row with the result of empty aggregation. - /// To do this, we pass a block with zero rows to aggregate. - if (result.empty() && params.keys_size == 0 && !params.empty_result_for_aggregation_by_empty_set) - { - agg_process_info.resetBlock(stream->getHeader()); - executeOnBlock(agg_process_info, result, thread_num); - if (result.need_spill) - spill(result, thread_num); - assert(agg_process_info.allBlockDataHandled()); - } - - double elapsed_seconds = watch.elapsedSeconds(); - size_t rows = result.size(); - LOG_TRACE( - log, - "Aggregated. {} to {} rows (from {:.3f} MiB) in {:.3f} sec. ({:.3f} rows/sec., {:.3f} MiB/sec.)", - src_rows, - rows, - src_bytes / 1048576.0, - elapsed_seconds, - src_rows / elapsed_seconds, - src_bytes / elapsed_seconds / 1048576.0); -} - -template -void Aggregator::convertToBlockImpl( - Method & method, - Table & data, - const Sizes & key_sizes, - MutableColumns & key_columns, - AggregateColumnsData & aggregate_columns, - MutableColumns & final_aggregate_columns, - Arena * arena, - bool final) const -{ - if (data.empty()) - return; - - std::vector raw_key_columns; - raw_key_columns.reserve(key_columns.size()); - for (auto & column : key_columns) - raw_key_columns.push_back(column.get()); - - if (final) - convertToBlockImplFinal( - method, - data, - key_sizes, - std::move(raw_key_columns), - final_aggregate_columns, - arena); - else - convertToBlockImplNotFinal( - method, - data, - key_sizes, - std::move(raw_key_columns), - aggregate_columns); - - /// In order to release memory early. - data.clearAndShrink(); -} - -template -void Aggregator::convertToBlocksImpl( - Method & method, - Table & data, - const Sizes & key_sizes, - std::vector & key_columns_vec, - std::vector & aggregate_columns_vec, - std::vector & final_aggregate_columns_vec, - Arena * arena, - bool final) const -{ - if (data.empty()) - return; - - std::vector> raw_key_columns_vec; - raw_key_columns_vec.reserve(key_columns_vec.size()); - for (auto & key_columns : key_columns_vec) - { - std::vector raw_key_columns; - raw_key_columns.reserve(key_columns.size()); - for (auto & column : key_columns) - { - raw_key_columns.push_back(column.get()); - } - - raw_key_columns_vec.push_back(raw_key_columns); - } - - if (final) - convertToBlocksImplFinal( - method, - data, - key_sizes, - std::move(raw_key_columns_vec), - final_aggregate_columns_vec, - arena); - else - convertToBlocksImplNotFinal( - method, - data, - key_sizes, - std::move(raw_key_columns_vec), - aggregate_columns_vec); - - /// In order to release memory early. - data.clearAndShrink(); -} - - -template -inline void Aggregator::insertAggregatesIntoColumns( - Mapped & mapped, - MutableColumns & final_aggregate_columns, - Arena * arena) const -{ - /** Final values of aggregate functions are inserted to columns. - * Then states of aggregate functions, that are not longer needed, are destroyed. - * - * We mark already destroyed states with "nullptr" in data, - * so they will not be destroyed in destructor of Aggregator - * (other values will be destroyed in destructor in case of exception). - * - * But it becomes tricky, because we have multiple aggregate states pointed by a single pointer in data. - * So, if exception is thrown in the middle of moving states for different aggregate functions, - * we have to catch exceptions and destroy all the states that are no longer needed, - * to keep the data in consistent state. - * - * It is also tricky, because there are aggregate functions with "-State" modifier. - * When we call "insertResultInto" for them, they insert a pointer to the state to ColumnAggregateFunction - * and ColumnAggregateFunction will take ownership of this state. - * So, for aggregate functions with "-State" modifier, the state must not be destroyed - * after it has been transferred to ColumnAggregateFunction. - * But we should mark that the data no longer owns these states. - */ - - size_t insert_i = 0; - std::exception_ptr exception; - - try - { - /// Insert final values of aggregate functions into columns. - for (; insert_i < params.aggregates_size; ++insert_i) - aggregate_functions[insert_i]->insertResultInto( - mapped + offsets_of_aggregate_states[insert_i], - *final_aggregate_columns[insert_i], - arena); - } - catch (...) - { - exception = std::current_exception(); - } - - /** Destroy states that are no longer needed. This loop does not throw. - * - * Don't destroy states for "-State" aggregate functions, - * because the ownership of this state is transferred to ColumnAggregateFunction - * and ColumnAggregateFunction will take care. - * - * But it's only for states that has been transferred to ColumnAggregateFunction - * before exception has been thrown; - */ - for (size_t destroy_i = 0; destroy_i < params.aggregates_size; ++destroy_i) - { - /// If ownership was not transferred to ColumnAggregateFunction. - if (destroy_i >= insert_i || !aggregate_functions[destroy_i]->isState()) - aggregate_functions[destroy_i]->destroy(mapped + offsets_of_aggregate_states[destroy_i]); - } - - /// Mark the cell as destroyed so it will not be destroyed in destructor. - mapped = nullptr; - - if (exception) - std::rethrow_exception(exception); -} - -template -struct AggregatorMethodInitKeyColumnHelper -{ - Method & method; - explicit AggregatorMethodInitKeyColumnHelper(Method & method_) - : method(method_) - {} - ALWAYS_INLINE inline void initAggKeys(size_t, std::vector &) {} - template - ALWAYS_INLINE inline void insertKeyIntoColumns( - const Key & key, - std::vector & key_columns, - const Sizes & sizes, - const TiDB::TiDBCollators & collators) - { - method.insertKeyIntoColumns(key, key_columns, sizes, collators); - } -}; - -template -struct AggregatorMethodInitKeyColumnHelper> -{ - using Method = AggregationMethodFastPathTwoKeysNoCache; - size_t index{}; - std::function &, size_t)> insert_key_into_columns_function_ptr{}; - - Method & method; - explicit AggregatorMethodInitKeyColumnHelper(Method & method_) - : method(method_) - {} - - ALWAYS_INLINE inline void initAggKeys(size_t rows, std::vector & key_columns) - { - index = 0; - if (key_columns.size() == 1) - { - Method::template initAggKeys(rows, key_columns[0]); - insert_key_into_columns_function_ptr - = AggregationMethodFastPathTwoKeysNoCache::insertKeyIntoColumnsOneKey; - } - else if (key_columns.size() == 2) - { - Method::template initAggKeys(rows, key_columns[0]); - Method::template initAggKeys(rows, key_columns[1]); - insert_key_into_columns_function_ptr - = AggregationMethodFastPathTwoKeysNoCache::insertKeyIntoColumnsTwoKey; - } - else - { - throw Exception("unexpected key_columns size for AggMethodFastPathTwoKey: {}", key_columns.size()); - } - } - ALWAYS_INLINE inline void insertKeyIntoColumns( - const StringRef & key, - std::vector & key_columns, - const Sizes &, - const TiDB::TiDBCollators &) - { - assert(insert_key_into_columns_function_ptr); - insert_key_into_columns_function_ptr(key, key_columns, index); - ++index; - } -}; - -template -struct AggregatorMethodInitKeyColumnHelper> -{ - using Method = AggregationMethodOneKeyStringNoCache; - size_t index{}; - - Method & method; - explicit AggregatorMethodInitKeyColumnHelper(Method & method_) - : method(method_) - {} - - void initAggKeys(size_t rows, std::vector & key_columns) - { - index = 0; - RUNTIME_CHECK_MSG( - key_columns.size() == 1, - "unexpected key_columns size for AggMethodOneKeyString: {}", - key_columns.size()); - Method::initAggKeys(rows, key_columns[0]); - } - ALWAYS_INLINE inline void insertKeyIntoColumns( - const StringRef & key, - std::vector & key_columns, - const Sizes &, - const TiDB::TiDBCollators &) - { - method.insertKeyIntoColumns(key, key_columns, index); - ++index; - } -}; - -template -void NO_INLINE Aggregator::convertToBlockImplFinal( - Method & method, - Table & data, - const Sizes & key_sizes, - std::vector key_columns, - MutableColumns & final_aggregate_columns, - Arena * arena) const -{ - assert(key_sizes.size() == key_columns.size()); - Sizes key_sizes_ref = key_sizes; // NOLINT - AggregatorMethodInitKeyColumnHelper agg_keys_helper{method}; - if constexpr (!skip_convert_key) - { - auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns, key_sizes); - if (shuffled_key_sizes) - { - // When key_ref_agg_func is not empty, we may reorder key to skip copying some key from HashMap. - // But this optimization is not compatible with shuffleKeyColumns of AggregationMethodKeysFixed. - RUNTIME_CHECK(params.key_ref_agg_func.empty()); - key_sizes_ref = *shuffled_key_sizes; - } - agg_keys_helper.initAggKeys(data.size(), key_columns); - } - - data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { - if constexpr (!skip_convert_key) - { - agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); - } - - insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); - }); -} - -namespace -{ -template -std::optional shuffleKeyColumnsForKeyColumnsVec( - Method & method, - std::vector> & key_columns_vec, - const Sizes & key_sizes) -{ - auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns_vec[0], key_sizes); - for (size_t i = 1; i < key_columns_vec.size(); ++i) - { - auto new_key_sizes = method.shuffleKeyColumns(key_columns_vec[i], key_sizes); - assert(shuffled_key_sizes == new_key_sizes); - } - return shuffled_key_sizes; -} -template -std::vector>> initAggKeysForKeyColumnsVec( - Method & method, - std::vector> & key_columns_vec, - size_t max_block_size, - size_t total_row_count) -{ - std::vector>> agg_keys_helpers; - size_t block_row_count = max_block_size; - for (size_t i = 0; i < key_columns_vec.size(); ++i) - { - if (i == key_columns_vec.size() - 1 && total_row_count % block_row_count != 0) - /// update block_row_count for the last block - block_row_count = total_row_count % block_row_count; - agg_keys_helpers.push_back(std::make_unique>(method)); - agg_keys_helpers.back()->initAggKeys(block_row_count, key_columns_vec[i]); - } - return agg_keys_helpers; -} -} // namespace - -template -void NO_INLINE Aggregator::convertToBlocksImplFinal( - Method & method, - Table & data, - const Sizes & key_sizes, - std::vector> && key_columns_vec, - std::vector & final_aggregate_columns_vec, - Arena * arena) const -{ - assert(!key_columns_vec.empty()); -#ifndef NDEBUG - for (const auto & key_columns : key_columns_vec) - { - assert(key_columns.size() == key_sizes.size()); - } -#endif - std::vector>>> agg_keys_helpers; - Sizes key_sizes_ref = key_sizes; // NOLINT - if constexpr (!skip_convert_key) - { - auto shuffled_key_sizes = shuffleKeyColumnsForKeyColumnsVec(method, key_columns_vec, key_sizes); - if (shuffled_key_sizes) - { - RUNTIME_CHECK(params.key_ref_agg_func.empty()); - key_sizes_ref = *shuffled_key_sizes; - } - agg_keys_helpers = initAggKeysForKeyColumnsVec(method, key_columns_vec, params.max_block_size, data.size()); - } - - size_t data_index = 0; - data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { - size_t key_columns_vec_index = data_index / params.max_block_size; - if constexpr (!skip_convert_key) - { - agg_keys_helpers[key_columns_vec_index] - ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); - } - insertAggregatesIntoColumns(mapped, final_aggregate_columns_vec[key_columns_vec_index], arena); - ++data_index; - }); -} - -template -void NO_INLINE Aggregator::convertToBlockImplNotFinal( - Method & method, - Table & data, - const Sizes & key_sizes, - std::vector key_columns, - AggregateColumnsData & aggregate_columns) const -{ - assert(key_sizes.size() == key_columns.size()); - AggregatorMethodInitKeyColumnHelper agg_keys_helper{method}; - Sizes key_sizes_ref = key_sizes; // NOLINT - if constexpr (!skip_convert_key) - { - auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns, key_sizes); - if (shuffled_key_sizes) - { - RUNTIME_CHECK(params.key_ref_agg_func.empty()); - key_sizes_ref = *shuffled_key_sizes; - } - agg_keys_helper.initAggKeys(data.size(), key_columns); - } - - data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { - if constexpr (!skip_convert_key) - { - agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); - } - - /// reserved, so push_back does not throw exceptions - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); - - mapped = nullptr; - }); -} - -template -void NO_INLINE Aggregator::convertToBlocksImplNotFinal( - Method & method, - Table & data, - const Sizes & key_sizes, - std::vector> && key_columns_vec, - std::vector & aggregate_columns_vec) const -{ -#ifndef NDEBUG - for (const auto & key_columns : key_columns_vec) - { - assert(key_sizes.size() == key_columns.size()); - } -#endif - std::vector>>> agg_keys_helpers; - Sizes key_sizes_ref = key_sizes; // NOLINT - if constexpr (!skip_convert_key) - { - auto shuffled_key_sizes = shuffleKeyColumnsForKeyColumnsVec(method, key_columns_vec, key_sizes); - if (shuffled_key_sizes) - { - RUNTIME_CHECK(params.key_ref_agg_func.empty()); - key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes; - } - agg_keys_helpers = initAggKeysForKeyColumnsVec(method, key_columns_vec, params.max_block_size, data.size()); - } - - size_t data_index = 0; - data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { - size_t key_columns_vec_index = data_index / params.max_block_size; - if constexpr (!skip_convert_key) - { - agg_keys_helpers[key_columns_vec_index] - ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); - } - - /// reserved, so push_back does not throw exceptions - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns_vec[key_columns_vec_index][i]->push_back(mapped + offsets_of_aggregate_states[i]); - - ++data_index; - mapped = nullptr; - }); -} - -template -Block Aggregator::prepareBlockAndFill( - AggregatedDataVariants & data_variants, - bool final, - size_t rows, - Filler && filler, - size_t convert_key_size) const -{ - MutableColumns key_columns(convert_key_size); - MutableColumns aggregate_columns(params.aggregates_size); - MutableColumns final_aggregate_columns(params.aggregates_size); - AggregateColumnsData aggregate_columns_data(params.aggregates_size); - // Store size of keys that need to convert. - Sizes new_key_sizes; - new_key_sizes.reserve(convert_key_size); - - Block header = getHeader(final); - - for (size_t i = 0; i < convert_key_size; ++i) - { - key_columns[i] = header.safeGetByPosition(i).type->createColumn(); - key_columns[i]->reserve(rows); - new_key_sizes.push_back(key_sizes[i]); - } - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - if (!final) - { - const auto & aggregate_column_name = params.aggregates[i].column_name; - aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); - - /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. - auto & column_aggregate_func = assert_cast(*aggregate_columns[i]); - - for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func.addArena(pool); - - aggregate_columns_data[i] = &column_aggregate_func.getData(); - aggregate_columns_data[i]->reserve(rows); - } - else - { - final_aggregate_columns[i] = aggregate_functions[i]->getReturnType()->createColumn(); - final_aggregate_columns[i]->reserve(rows); - - if (aggregate_functions[i]->isState()) - { - /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - if (auto * column_aggregate_func - = typeid_cast(final_aggregate_columns[i].get())) - for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func->addArena(pool); - } - } - } - - filler(new_key_sizes, key_columns, aggregate_columns_data, final_aggregate_columns, final); - - Block res = header.cloneEmpty(); - - for (size_t i = 0; i < convert_key_size; ++i) - res.getByPosition(i).column = std::move(key_columns[i]); - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - const auto & aggregate_column_name = params.aggregates[i].column_name; - if (final) - res.getByName(aggregate_column_name).column = std::move(final_aggregate_columns[i]); - else - res.getByName(aggregate_column_name).column = std::move(aggregate_columns[i]); - } - - /// Change the size of the columns-constants in the block. - size_t columns = header.columns(); - for (size_t i = 0; i < columns; ++i) - if (res.getByPosition(i).column->isColumnConst()) - res.getByPosition(i).column = res.getByPosition(i).column->cut(0, rows); - - return res; -} - -template -BlocksList Aggregator::prepareBlocksAndFill( - AggregatedDataVariants & data_variants, - bool final, - size_t rows, - Filler && filler, - size_t convert_key_size) const -{ - Block header = getHeader(final); - - size_t block_count = (rows + params.max_block_size - 1) / params.max_block_size; - std::vector key_columns_vec; - std::vector aggregate_columns_data_vec; - std::vector aggregate_columns_vec; - std::vector final_aggregate_columns_vec; - // Store size of keys that need to convert. - Sizes new_key_sizes; - new_key_sizes.reserve(convert_key_size); - for (size_t i = 0; i < convert_key_size; ++i) - { - new_key_sizes.push_back(key_sizes[i]); - } - - size_t block_rows = params.max_block_size; - - for (size_t j = 0; j < block_count; ++j) - { - if (j == (block_count - 1) && rows % block_rows != 0) - { - block_rows = rows % block_rows; - } - - key_columns_vec.push_back(MutableColumns(convert_key_size)); - aggregate_columns_data_vec.push_back(AggregateColumnsData(params.aggregates_size)); - aggregate_columns_vec.push_back(MutableColumns(params.aggregates_size)); - final_aggregate_columns_vec.push_back(MutableColumns(params.aggregates_size)); - - auto & key_columns = key_columns_vec.back(); - auto & aggregate_columns_data = aggregate_columns_data_vec.back(); - auto & aggregate_columns = aggregate_columns_vec.back(); - auto & final_aggregate_columns = final_aggregate_columns_vec.back(); - - for (size_t i = 0; i < convert_key_size; ++i) - { - key_columns[i] = header.safeGetByPosition(i).type->createColumn(); - key_columns[i]->reserve(block_rows); - } - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - if (!final) - { - const auto & aggregate_column_name = params.aggregates[i].column_name; - aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); - - /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. - auto & column_aggregate_func = assert_cast(*aggregate_columns[i]); - - for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func.addArena(pool); - - aggregate_columns_data[i] = &column_aggregate_func.getData(); - aggregate_columns_data[i]->reserve(block_rows); - } - else - { - final_aggregate_columns[i] = aggregate_functions[i]->getReturnType()->createColumn(); - final_aggregate_columns[i]->reserve(block_rows); - - if (aggregate_functions[i]->isState()) - { - /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - if (auto * column_aggregate_func - = typeid_cast(final_aggregate_columns[i].get())) - for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func->addArena(pool); - } - } - } - } - - filler(new_key_sizes, key_columns_vec, aggregate_columns_data_vec, final_aggregate_columns_vec, final); - - BlocksList res_list; - block_rows = params.max_block_size; - for (size_t j = 0; j < block_count; ++j) - { - Block res = header.cloneEmpty(); - - for (size_t i = 0; i < convert_key_size; ++i) - res.getByPosition(i).column = std::move(key_columns_vec[j][i]); - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - const auto & aggregate_column_name = params.aggregates[i].column_name; - if (final) - res.getByName(aggregate_column_name).column = std::move(final_aggregate_columns_vec[j][i]); - else - res.getByName(aggregate_column_name).column = std::move(aggregate_columns_vec[j][i]); - } - - if (j == (block_count - 1) && rows % block_rows != 0) - { - block_rows = rows % block_rows; - } - - /// Change the size of the columns-constants in the block. - size_t columns = header.columns(); - for (size_t i = 0; i < columns; ++i) - if (res.getByPosition(i).column->isColumnConst()) - res.getByPosition(i).column = res.getByPosition(i).column->cut(0, block_rows); - - res_list.push_back(res); - } - - if (res_list.empty()) - { - /// at least one valid block must be returned. - res_list.push_back(header.cloneWithColumns(header.cloneEmptyColumns())); - } - return res_list; -} - - -BlocksList Aggregator::prepareBlocksAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final) const -{ - size_t rows = 1; - - auto filler = [&data_variants, this]( - const Sizes &, - std::vector &, - std::vector & aggregate_columns_vec, - std::vector & final_aggregate_columns_vec, - bool final_) { - if (data_variants.type == AggregatedDataVariants::Type::without_key) - { - AggregatedDataWithoutKey & data = data_variants.without_key; - - RUNTIME_CHECK_MSG(data, "Wrong data variant passed."); - - if (!final_) - { - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns_vec[0][i]->push_back(data + offsets_of_aggregate_states[i]); - data = nullptr; - } - else - { - /// Always single-thread. It's safe to pass current arena from 'aggregates_pool'. - insertAggregatesIntoColumns(data, final_aggregate_columns_vec[0], data_variants.aggregates_pool); - } - } - }; - - BlocksList blocks = prepareBlocksAndFill(data_variants, final, rows, filler, /*convert_key_size=*/0); - - if (final) - destroyWithoutKey(data_variants); - - return blocks; -} - -BlocksList Aggregator::prepareBlocksAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const -{ - size_t rows = data_variants.size(); -#define M(NAME, skip_convert_key) \ - case AggregationMethodType(NAME): \ - { \ - auto & tmp_method = *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl); \ - auto & tmp_data = ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl) -> data; \ - convertToBlocksImpl( \ - tmp_method, \ - tmp_data, \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - data_variants.aggregates_pool, \ - final_); \ - break; \ - } - -#define M_skip_convert_key(NAME) M(NAME, true) -#define M_convert_key(NAME) M(NAME, false) - -#define FILLER_DEFINE(name, M_tmp) \ - auto filler_##name = [&data_variants, this]( \ - const Sizes & key_sizes, \ - std::vector & key_columns_vec, \ - std::vector & aggregate_columns_vec, \ - std::vector & final_aggregate_columns_vec, \ - bool final_) { \ - switch (data_variants.type) \ - { \ - APPLY_FOR_VARIANTS_SINGLE_LEVEL(M_tmp) \ - default: \ - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); \ - } \ - } - - FILLER_DEFINE(convert_key, M_convert_key); - FILLER_DEFINE(skip_convert_key, M_skip_convert_key); - -#undef M -#undef M_skip_convert_key -#undef M_convert_key -#undef FILLER_DEFINE - - size_t convert_key_size = final ? params.keys_size - params.key_ref_agg_func.size() : params.keys_size; - - if (final && convert_key_size == 0) - { - return prepareBlocksAndFill(data_variants, final, rows, filler_skip_convert_key, convert_key_size); - } - return prepareBlocksAndFill(data_variants, final, rows, filler_convert_key, convert_key_size); -} - - -template -void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena) const -{ - table_src.mergeToViaEmplace( - table_dst, - [&](AggregateDataPtr & __restrict dst, AggregateDataPtr & __restrict src, bool inserted) { - if (!inserted) - { - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->merge( - dst + offsets_of_aggregate_states[i], - src + offsets_of_aggregate_states[i], - arena); - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); - } - else - { - dst = src; - } - - src = nullptr; - }); - table_src.clearAndShrink(); -} - -void NO_INLINE Aggregator::mergeWithoutKeyDataImpl(ManyAggregatedDataVariants & non_empty_data) const -{ - AggregatedDataVariantsPtr & res = non_empty_data[0]; - - /// We merge all aggregation results to the first. - for (size_t result_num = 1, size = non_empty_data.size(); result_num < size; ++result_num) - { - AggregatedDataWithoutKey & res_data = res->without_key; - AggregatedDataWithoutKey & current_data = non_empty_data[result_num]->without_key; - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->merge( - res_data + offsets_of_aggregate_states[i], - current_data + offsets_of_aggregate_states[i], - res->aggregates_pool); - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(current_data + offsets_of_aggregate_states[i]); - - current_data = nullptr; - } -} - - -template -void NO_INLINE Aggregator::mergeSingleLevelDataImpl(ManyAggregatedDataVariants & non_empty_data) const -{ - AggregatedDataVariantsPtr & res = non_empty_data[0]; - - /// We merge all aggregation results to the first. - for (size_t result_num = 1, size = non_empty_data.size(); result_num < size; ++result_num) - { - AggregatedDataVariants & current = *non_empty_data[result_num]; - - mergeDataImpl( - getDataVariant(*res).data, - getDataVariant(current).data, - res->aggregates_pool); - - /// `current` will not destroy the states of aggregate functions in the destructor - current.aggregator = nullptr; - } -} - -#define M(NAME) \ - template void NO_INLINE Aggregator::mergeSingleLevelDataImpl( \ - ManyAggregatedDataVariants & non_empty_data) const; -APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) -#undef M - -template -void NO_INLINE Aggregator::mergeBucketImpl(ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena) const -{ - /// We merge all aggregation results to the first. - AggregatedDataVariantsPtr & res = data[0]; - for (size_t result_num = 1, size = data.size(); result_num < size; ++result_num) - { - if (is_cancelled()) - return; - - AggregatedDataVariants & current = *data[result_num]; - - mergeDataImpl( - getDataVariant(*res).data.impls[bucket], - getDataVariant(current).data.impls[bucket], - arena); - } -} - - -MergingBucketsPtr Aggregator::mergeAndConvertToBlocks( - ManyAggregatedDataVariants & data_variants, - bool final, - size_t max_threads) const -{ - if (unlikely(data_variants.empty())) - throw Exception("Empty data passed to Aggregator::mergeAndConvertToBlocks.", ErrorCodes::EMPTY_DATA_PASSED); - - LOG_TRACE(log, "Merging aggregated data"); - - ManyAggregatedDataVariants non_empty_data; - non_empty_data.reserve(data_variants.size()); - for (auto & data : data_variants) - if (!data->empty()) - non_empty_data.push_back(data); - - if (non_empty_data.empty()) - return nullptr; - - if (non_empty_data.size() > 1) - { - /// Sort the states in descending order so that the merge is more efficient (since all states are merged into the first). - std::sort( - non_empty_data.begin(), - non_empty_data.end(), - [](const AggregatedDataVariantsPtr & lhs, const AggregatedDataVariantsPtr & rhs) { - return lhs->size() > rhs->size(); - }); - } - - /// If at least one of the options is two-level, then convert all the options into two-level ones, if there are not such. - /// Note - perhaps it would be more optimal not to convert single-level versions before the merge, but merge them separately, at the end. - - bool has_at_least_one_two_level = false; - for (const auto & variant : non_empty_data) - { - if (variant->isTwoLevel()) - { - has_at_least_one_two_level = true; - break; - } - } - - if (has_at_least_one_two_level) - for (auto & variant : non_empty_data) - if (!variant->isTwoLevel()) - variant->convertToTwoLevel(); - - AggregatedDataVariantsPtr & first = non_empty_data[0]; - - for (size_t i = 1, size = non_empty_data.size(); i < size; ++i) - { - if (unlikely(first->type != non_empty_data[i]->type)) - throw Exception( - "Cannot merge different aggregated data variants.", - ErrorCodes::CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS); - - /** Elements from the remaining sets can be moved to the first data set. - * Therefore, it must own all the arenas of all other sets. - */ - first->aggregates_pools.insert( - first->aggregates_pools.end(), - non_empty_data[i]->aggregates_pools.begin(), - non_empty_data[i]->aggregates_pools.end()); - } - - // for single level merge, concurrency must be 1. - size_t merge_concurrency = has_at_least_one_two_level ? std::max(max_threads, 1) : 1; - return std::make_shared(*this, non_empty_data, final, merge_concurrency); -} - -template -void NO_INLINE Aggregator::mergeStreamsImplCase( - Block & block, - Arena * aggregates_pool, - Method & method [[maybe_unused]], - Table & data) const -{ - ColumnRawPtrs key_columns(params.keys_size); - AggregateColumnsConstData aggregate_columns(params.aggregates_size); - - /// Remember the columns we will work with - for (size_t i = 0; i < params.keys_size; ++i) - key_columns[i] = block.safeGetByPosition(i).column.get(); - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - const auto & aggregate_column_name = params.aggregates[i].column_name; - aggregate_columns[i] - = &typeid_cast(*block.getByName(aggregate_column_name).column).getData(); - } - - std::vector sort_key_containers; - sort_key_containers.resize(params.keys_size, ""); - - /// in merge stage, don't need to care about the collator because the key is already the sort_key of original string - typename Method::State state(key_columns, key_sizes, {}); - - /// For all rows. - size_t rows = block.rows(); - std::unique_ptr places(new AggregateDataPtr[rows]); - - for (size_t i = 0; i < rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - - auto emplace_result = state.emplaceKey(data, i, *aggregates_pool, sort_key_containers); - if (emplace_result.isInserted()) - { - emplace_result.setMapped(nullptr); - - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); - - emplace_result.setMapped(aggregate_data); - } - else - aggregate_data = emplace_result.getMapped(); - - places[i] = aggregate_data; - } - - for (size_t j = 0; j < params.aggregates_size; ++j) - { - /// Merge state of aggregate functions. - aggregate_functions[j]->mergeBatch( - rows, - places.get(), - offsets_of_aggregate_states[j], - aggregate_columns[j]->data(), - aggregates_pool); - } - - - /// Early release memory. - block.clear(); -} - -template -void NO_INLINE Aggregator::mergeStreamsImpl(Block & block, Arena * aggregates_pool, Method & method, Table & data) const -{ - mergeStreamsImplCase(block, aggregates_pool, method, data); -} - - -void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl(Block & block, AggregatedDataVariants & result) const -{ - AggregateColumnsConstData aggregate_columns(params.aggregates_size); - - /// Remember the columns we will work with - for (size_t i = 0; i < params.aggregates_size; ++i) - { - const auto & aggregate_column_name = params.aggregates[i].column_name; - aggregate_columns[i] - = &typeid_cast(*block.getByName(aggregate_column_name).column).getData(); - } - - AggregatedDataWithoutKey & res = result.without_key; - if (!res) - { - AggregateDataPtr place - = result.aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(place); - res = place; - } - - if (block.rows() > 0) - { - /// Adding Values - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->merge( - res + offsets_of_aggregate_states[i], - (*aggregate_columns[i])[0], - result.aggregates_pool); - } - - /// Early release memory. - block.clear(); -} - -BlocksList Aggregator::vstackBlocks(BlocksList & blocks, bool final) -{ - RUNTIME_CHECK_MSG(!blocks.empty(), "The input blocks list for Aggregator::vstackBlocks must be non-empty"); - - auto bucket_num = blocks.front().info.bucket_num; - - LOG_TRACE( - log, - "Merging partially aggregated blocks (bucket = {}). Original method `{}`.", - bucket_num, - AggregatedDataVariants::getMethodName(method_chosen)); - Stopwatch watch; - - /** If possible, change 'method' to some_hash64. Otherwise, leave as is. - * Better hash function is needed because during external aggregation, - * we may merge partitions of data with total number of keys far greater than 4 billion. - */ - auto merge_method = method_chosen; - -#define APPLY_FOR_VARIANTS_THAT_MAY_USE_BETTER_HASH_FUNCTION(M) \ - M(key64) \ - M(key_string) \ - M(key_fixed_string) \ - M(keys128) \ - M(keys256) \ - M(serialized) - -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - merge_method = AggregatedDataVariants::Type::NAME##_hash64; \ - break; \ - } - - switch (merge_method) - { - APPLY_FOR_VARIANTS_THAT_MAY_USE_BETTER_HASH_FUNCTION(M) - default: - break; - } -#undef M - -#undef APPLY_FOR_VARIANTS_THAT_MAY_USE_BETTER_HASH_FUNCTION - - /// Temporary data for aggregation. - AggregatedDataVariants result; - - /// result will destroy the states of aggregate functions in the destructor - result.aggregator = this; - - result.init(merge_method); - result.keys_size = params.keys_size; - result.key_sizes = key_sizes; - - for (Block & block : blocks) - { - if (bucket_num >= 0 && block.info.bucket_num != bucket_num) - bucket_num = -1; - - if (result.type == AggregatedDataVariants::Type::without_key) - mergeWithoutKeyStreamsImpl(block, result); - -#define M(NAME, IS_TWO_LEVEL) \ - case AggregationMethodType(NAME): \ - { \ - mergeStreamsImpl( \ - block, \ - result.aggregates_pool, \ - *ToAggregationMethodPtr(NAME, result.aggregation_method_impl), \ - ToAggregationMethodPtr(NAME, result.aggregation_method_impl)->data); \ - break; \ - } - switch (result.type) - { - APPLY_FOR_AGGREGATED_VARIANTS(M) - case AggregatedDataVariants::Type::without_key: - break; - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } -#undef M - } - - BlocksList return_blocks; - if (result.type == AggregatedDataVariants::Type::without_key) - return_blocks = prepareBlocksAndFillWithoutKey(result, final); - else - return_blocks = prepareBlocksAndFillSingleLevel(result, final); - /// NOTE: two-level data is not possible here - chooseAggregationMethod chooses only among single-level methods. - - if (!final) - { - /// Pass ownership of aggregate function states from result to ColumnAggregateFunction objects in the resulting block. - result.aggregator = nullptr; - } - - size_t rows = 0; - size_t bytes = 0; - for (auto block : return_blocks) - { - rows += block.rows(); - bytes += block.bytes(); - block.info.bucket_num = bucket_num; - } - double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE( - log, - "Merged partially aggregated blocks. Return {} rows in {} blocks, {:.3f} MiB. in {:.3f} sec. ({:.3f} " - "rows/sec., {:.3f} MiB/sec.)", - rows, - return_blocks.size(), - bytes / 1048576.0, - elapsed_seconds, - rows / elapsed_seconds, - bytes / elapsed_seconds / 1048576.0); - - return return_blocks; -} - - -template -void NO_INLINE Aggregator::convertBlockToTwoLevelImpl( - Method & method, - Arena * pool, - ColumnRawPtrs & key_columns, - const Block & source, - Blocks & destinations) const -{ - typename Method::State state(key_columns, key_sizes, params.collators); - - std::vector sort_key_containers; - sort_key_containers.resize(params.keys_size, ""); - - size_t rows = source.rows(); - size_t columns = source.columns(); - - /// Create a 'selector' that will contain bucket index for every row. It will be used to scatter rows to buckets. - IColumn::Selector selector(rows); - - /// For every row. - for (size_t i = 0; i < rows; ++i) - { - /// Calculate bucket number from row hash. - auto hash = state.getHash(method.data, i, *pool, sort_key_containers); - auto bucket = method.data.getBucketFromHash(hash); - - selector[i] = bucket; - } - - size_t num_buckets = destinations.size(); - - for (size_t column_idx = 0; column_idx < columns; ++column_idx) - { - const ColumnWithTypeAndName & src_col = source.getByPosition(column_idx); - MutableColumns scattered_columns = src_col.column->scatter(num_buckets, selector); - - for (size_t bucket = 0, size = num_buckets; bucket < size; ++bucket) - { - if (!scattered_columns[bucket]->empty()) - { - Block & dst = destinations[bucket]; - dst.info.bucket_num = bucket; - dst.insert({std::move(scattered_columns[bucket]), src_col.type, src_col.name}); - } - - /** Inserted columns of type ColumnAggregateFunction will own states of aggregate functions - * by holding shared_ptr to source column. See ColumnAggregateFunction.h - */ - } - } -} - - -Blocks Aggregator::convertBlockToTwoLevel(const Block & block) -{ - if (!block) - return {}; - - AggregatedDataVariants data; - - ColumnRawPtrs key_columns(params.keys_size); - - /// Remember the columns we will work with - for (size_t i = 0; i < params.keys_size; ++i) - key_columns[i] = block.safeGetByPosition(i).column.get(); - - AggregatedDataVariants::Type type = method_chosen; - data.keys_size = params.keys_size; - data.key_sizes = key_sizes; - -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - type = AggregationMethodTypeTwoLevel(NAME); \ - break; \ - } - - switch (type) - { - APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } -#undef M - - data.init(type); - - size_t num_buckets = 0; - -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - num_buckets = ToAggregationMethodPtr(NAME, data.aggregation_method_impl)->data.NUM_BUCKETS; \ - break; \ - } - - switch (data.type) - { - APPLY_FOR_VARIANTS_TWO_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } - -#undef M - - - Blocks splitted_blocks(num_buckets); - -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - convertBlockToTwoLevelImpl( \ - *ToAggregationMethodPtr(NAME, data.aggregation_method_impl), \ - data.aggregates_pool, \ - key_columns, \ - block, \ - splitted_blocks); \ - break; \ - } - - switch (data.type) - { - APPLY_FOR_VARIANTS_TWO_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } -#undef M - - - return splitted_blocks; -} - - -template -void NO_INLINE Aggregator::destroyImpl(Table & table) const -{ - table.forEachMapped([&](AggregateDataPtr & data) { - /** If an exception (usually a lack of memory, the MemoryTracker throws) arose - * after inserting the key into a hash table, but before creating all states of aggregate functions, - * then data will be equal nullptr. - */ - if (nullptr == data) - return; - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(data + offsets_of_aggregate_states[i]); - - data = nullptr; - }); -} - - -void Aggregator::destroyWithoutKey(AggregatedDataVariants & result) const -{ - AggregatedDataWithoutKey & res_data = result.without_key; - - if (nullptr != res_data) - { - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(res_data + offsets_of_aggregate_states[i]); - - res_data = nullptr; - } -} - - -void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) -{ - if (!result.inited()) - return; - - LOG_TRACE(log, "Destroying aggregate states"); - - /// In what data structure is the data aggregated? - if (result.type == AggregatedDataVariants::Type::without_key) - destroyWithoutKey(result); - -#define M(NAME, IS_TWO_LEVEL) \ - case AggregationMethodType(NAME): \ - { \ - destroyImpl(ToAggregationMethodPtr(NAME, result.aggregation_method_impl)->data); \ - break; \ - } - - switch (result.type) - { - APPLY_FOR_AGGREGATED_VARIANTS(M) - case AggregatedDataVariants::Type::without_key: - break; - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } - -#undef M -} - - -void Aggregator::setCancellationHook(CancellationHook cancellation_hook) -{ - is_cancelled = cancellation_hook; -} - -MergingBuckets::MergingBuckets( - const Aggregator & aggregator_, - const ManyAggregatedDataVariants & data_, - bool final_, - size_t concurrency_) - : log(Logger::get(aggregator_.log ? aggregator_.log->identifier() : "")) - , aggregator(aggregator_) - , data(data_) - , final(final_) - , concurrency(concurrency_) -{ - assert(concurrency > 0); - if (!data.empty()) - { - is_two_level = data[0]->isTwoLevel(); - if (is_two_level) - { - for (size_t i = 0; i < concurrency; ++i) - two_level_parallel_merge_data.push_back(std::make_unique()); - } - else - { - // for single level, concurrency must be 1. - RUNTIME_CHECK(concurrency == 1); - } - - /// At least we need one arena in first data item per concurrency - if (concurrency > data[0]->aggregates_pools.size()) - { - Arenas & first_pool = data[0]->aggregates_pools; - for (size_t j = first_pool.size(); j < concurrency; ++j) - first_pool.emplace_back(std::make_shared()); - } - } -} - -Block MergingBuckets::getHeader() const -{ - return aggregator.getHeader(final); -} - -Block MergingBuckets::getData(size_t concurrency_index) -{ - assert(concurrency_index < concurrency); - - if (unlikely(data.empty())) - return {}; - - FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::random_aggregate_merge_failpoint); - - return is_two_level ? getDataForTwoLevel(concurrency_index) : getDataForSingleLevel(); -} - -Block MergingBuckets::getDataForSingleLevel() -{ - assert(!data.empty()); - - Block out_block = popBlocksListFront(single_level_blocks); - if (likely(out_block)) - { - return out_block; - } - // The bucket number of single level merge can only be 0. - if (current_bucket_num > 0) - return {}; - - AggregatedDataVariantsPtr & first = data[0]; - if (first->type == AggregatedDataVariants::Type::without_key) - { - aggregator.mergeWithoutKeyDataImpl(data); - single_level_blocks = aggregator.prepareBlocksAndFillWithoutKey(*first, final); - } - else - { -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - aggregator.mergeSingleLevelDataImpl(data); \ - break; \ - } - switch (first->type) - { - APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } -#undef M - single_level_blocks = aggregator.prepareBlocksAndFillSingleLevel(*first, final); - } - ++current_bucket_num; - return popBlocksListFront(single_level_blocks); -} - -Block MergingBuckets::getDataForTwoLevel(size_t concurrency_index) -{ - assert(concurrency_index < two_level_parallel_merge_data.size()); - auto & two_level_merge_data = *two_level_parallel_merge_data[concurrency_index]; - - Block out_block = popBlocksListFront(two_level_merge_data); - if (likely(out_block)) - return out_block; - - if (current_bucket_num >= NUM_BUCKETS) - return {}; - while (true) - { - auto local_current_bucket_num = current_bucket_num.fetch_add(1); - if (unlikely(local_current_bucket_num >= NUM_BUCKETS)) - return {}; - - doLevelMerge(local_current_bucket_num, concurrency_index); - Block block = popBlocksListFront(two_level_merge_data); - if (likely(block)) - return block; - } -} - -void MergingBuckets::doLevelMerge(Int32 bucket_num, size_t concurrency_index) -{ - auto & two_level_merge_data = *two_level_parallel_merge_data[concurrency_index]; - assert(two_level_merge_data.empty()); - - assert(!data.empty()); - auto & merged_data = *data[0]; - auto method = merged_data.type; - - /// Select Arena to avoid race conditions - Arena * arena = merged_data.aggregates_pools.at(concurrency_index).get(); - -#define M(NAME) \ - case AggregationMethodType(NAME): \ - { \ - aggregator.mergeBucketImpl(data, bucket_num, arena); \ - two_level_merge_data = aggregator.convertOneBucketToBlocks( \ - merged_data, \ - *ToAggregationMethodPtr(NAME, merged_data.aggregation_method_impl), \ - arena, \ - final, \ - bucket_num); \ - break; \ - } - switch (method) - { - APPLY_FOR_VARIANTS_TWO_LEVEL(M) - default: - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - } -#undef M -} - -#undef AggregationMethodName -#undef AggregationMethodNameTwoLevel -#undef AggregationMethodType -#undef AggregationMethodTypeTwoLevel -#undef ToAggregationMethodPtr -#undef ToAggregationMethodPtrTwoLevel - -} // namespace DB From 2e94829e7dc038d88a0742bbf47fed37940c130c Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 20 Dec 2024 12:01:51 +0800 Subject: [PATCH 33/61] prefetch agg data Signed-off-by: guo-shaoge --- .../AggregateFunctions/IAggregateFunction.h | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/dbms/src/AggregateFunctions/IAggregateFunction.h b/dbms/src/AggregateFunctions/IAggregateFunction.h index a7927085c3b..2994b80f82c 100644 --- a/dbms/src/AggregateFunctions/IAggregateFunction.h +++ b/dbms/src/AggregateFunctions/IAggregateFunction.h @@ -235,20 +235,42 @@ class IAggregateFunctionHelper : public IAggregateFunction Arena * arena, ssize_t if_argument_pos = -1) const override { + const auto end = start_offset + batch_size; + static constexpr size_t prefetch_step = 16; if (if_argument_pos >= 0) { const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); - for (size_t i = start_offset; i < start_offset + batch_size; ++i) + for (size_t i = start_offset; i < end; ++i) { - if (flags[i] && places[i - start_offset]) - static_cast(this)->add(places[i - start_offset] + place_offset, columns, i, arena); + const auto place_idx = i - start_offset; + const auto prefetch_idx = place_idx + prefetch_step; + + if (flags[i] && places[place_idx]) + { + // Only prefetch first agg function data for simplicity. + // This may not work when there are too many agg functions. + if likely (prefetch_idx < end) + __builtin_prefetch(places[prefetch_idx]); + + static_cast(this)->add(places[place_idx] + place_offset, columns, i, arena); + } } } else { - for (size_t i = start_offset; i < start_offset + batch_size; ++i) - if (places[i - start_offset]) - static_cast(this)->add(places[i - start_offset] + place_offset, columns, i, arena); + for (size_t i = start_offset; i < end; ++i) + { + const auto place_idx = i - start_offset; + const auto prefetch_idx = place_idx + prefetch_step; + + if (places[place_idx]) + { + if likely (prefetch_idx < end) + __builtin_prefetch(places[prefetch_idx]); + + static_cast(this)->add(places[place_idx] + place_offset, columns, i, arena); + } + } } } From dc0973abf3bddac5a031aa3e0ebf49a0b7e4e5c5 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 22 Dec 2024 12:27:35 +0800 Subject: [PATCH 34/61] try fix ci Signed-off-by: guo-shaoge --- .../tests/gtest_aggregation_executor.cpp | 52 +++++++++---------- dbms/src/Flash/tests/gtest_compute_server.cpp | 1 - .../Flash/tests/gtest_spill_aggregation.cpp | 1 - 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index 3a79025f244..e407fcae764 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -1042,22 +1042,29 @@ try toVec("col_tinyint", col_data_tinyint), }); + std::random_device rd; + std::mt19937_64 gen(rd()); + std::vector max_block_sizes{1, 2, DEFAULT_BLOCK_SIZE}; std::vector two_level_thresholds{0, 1}; + std::uniform_int_distribution dist(0, max_block_sizes.size()); + size_t random_block_size = max_block_sizes[dist(gen)]; + + std::uniform_int_distribution dist1(0, two_level_thresholds.size()); + size_t random_two_level_threshold = two_level_thresholds[dist1(gen)]; + LOG_DEBUG( + Logger::get("AggExecutorTestRunner::AggKeyOptimization"), + "max_block_size: {}, two_level_threshold: {}", + random_block_size, + random_two_level_threshold); + context.context->setSetting("group_by_two_level_threshold_bytes", Field(static_cast(0))); -#define WRAP_FOR_AGG_STRING_TEST_BEGIN \ - for (const auto & max_block_size : max_block_sizes) \ - { \ - for (const auto & two_level_threshold : two_level_thresholds) \ - { \ - context.context->setSetting( \ - "group_by_two_level_threshold", \ - Field(static_cast(two_level_threshold))); \ - context.context->setSetting("max_block_size", Field(static_cast(max_block_size))); -#define WRAP_FOR_AGG_STRING_TEST_END \ - } \ - } +#define WRAP_FOR_AGG_CHANGE_SETTINGS \ + context.context->setSetting( \ + "group_by_two_level_threshold", \ + Field(static_cast(random_two_level_threshold))); \ + context.context->setSetting("max_block_size", Field(static_cast(random_block_size))); FailPointHelper::enableFailPoint(FailPoints::force_agg_prefetch); { @@ -1074,9 +1081,8 @@ try toNullableVec("first_row(col_tinyint)", ColumnWithNullableInt8{0, 1, 2, 3}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3}), toVec("col_tinyint", ColumnWithInt8{0, 1, 2, 3})}; - WRAP_FOR_AGG_STRING_TEST_BEGIN + WRAP_FOR_AGG_CHANGE_SETTINGS executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_STRING_TEST_END } { @@ -1092,9 +1098,8 @@ try = {toVec("count(1)", ColumnWithUInt64{rows_per_type, rows_per_type, rows_per_type, rows_per_type}), toNullableVec("first_row(col_int)", ColumnWithNullableInt32{0, 1, 2, 3}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3})}; - WRAP_FOR_AGG_STRING_TEST_BEGIN + WRAP_FOR_AGG_CHANGE_SETTINGS executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_STRING_TEST_END } { @@ -1128,9 +1133,8 @@ try toNullableVec("first_row(col_string_with_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_with_collator", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_STRING_TEST_BEGIN + WRAP_FOR_AGG_CHANGE_SETTINGS executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_STRING_TEST_END } { @@ -1147,9 +1151,8 @@ try toVec("count(1)", ColumnWithUInt64{rows_per_type, rows_per_type, rows_per_type, rows_per_type}), toVec("first_row(col_string_with_collator)", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_STRING_TEST_BEGIN + WRAP_FOR_AGG_CHANGE_SETTINGS executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_STRING_TEST_END } // case-5: none @@ -1171,9 +1174,8 @@ try toVec("col_int", ColumnWithInt32{0, 1, 2, 3}), toVec("col_string_no_collator", ColumnWithString{"a", "b", "c", "d"}), }; - WRAP_FOR_AGG_STRING_TEST_BEGIN + WRAP_FOR_AGG_CHANGE_SETTINGS executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_STRING_TEST_END } { @@ -1190,13 +1192,11 @@ try toNullableVec("first_row(col_string_with_collator)", ColumnWithNullableString{"a", "b", "c", "d"}), toVec("col_string_with_collator", ColumnWithString{"a", "b", "c", "d"}), toVec("col_int", ColumnWithInt32{0, 1, 2, 3})}; - WRAP_FOR_AGG_STRING_TEST_BEGIN + WRAP_FOR_AGG_CHANGE_SETTINGS executeAndAssertColumnsEqual(request, expected); - WRAP_FOR_AGG_STRING_TEST_END } FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); -#undef WRAP_FOR_AGG_STRING_TEST_BEGIN -#undef WRAP_FOR_AGG_STRING_TEST_END +#undef WRAP_FOR_AGG_CHANGE_SETTINGS } CATCH diff --git a/dbms/src/Flash/tests/gtest_compute_server.cpp b/dbms/src/Flash/tests/gtest_compute_server.cpp index 3c4020db45e..ab1680e1697 100644 --- a/dbms/src/Flash/tests/gtest_compute_server.cpp +++ b/dbms/src/Flash/tests/gtest_compute_server.cpp @@ -1370,7 +1370,6 @@ try FailPoints::exception_during_mpp_non_root_task_run, FailPoints::exception_during_mpp_root_task_run, FailPoints::exception_during_query_run, - FailPoints::force_agg_prefetch, }; size_t query_index = 0; for (const auto & failpoint : failpoint_names) diff --git a/dbms/src/Flash/tests/gtest_spill_aggregation.cpp b/dbms/src/Flash/tests/gtest_spill_aggregation.cpp index 583e6e038fa..2f1df6404fd 100644 --- a/dbms/src/Flash/tests/gtest_spill_aggregation.cpp +++ b/dbms/src/Flash/tests/gtest_spill_aggregation.cpp @@ -376,7 +376,6 @@ try { for (const auto & agg_func : agg_funcs) { - FailPointHelper::disableFailPoint(FailPoints::force_agg_prefetch); context.setCollation(collator_id); const auto * current_collator = TiDB::ITiDBCollator::getCollator(collator_id); ASSERT_TRUE(current_collator != nullptr); From 9623368e13c5d71ea0c031752d5af06fce2707bc Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 24 Dec 2024 14:07:13 +0800 Subject: [PATCH 35/61] prefetch threshold(2MB); mini batch; prefetch exists key; prefetch insertResultInto Signed-off-by: guo-shaoge --- .../AggregateFunctions/IAggregateFunction.h | 15 - dbms/src/Common/ColumnsHashingImpl.h | 24 ++ dbms/src/Interpreters/Aggregator.cpp | 262 ++++++++++++++---- dbms/src/Interpreters/Aggregator.h | 13 +- 4 files changed, 244 insertions(+), 70 deletions(-) diff --git a/dbms/src/AggregateFunctions/IAggregateFunction.h b/dbms/src/AggregateFunctions/IAggregateFunction.h index 2994b80f82c..1f0cc6903a2 100644 --- a/dbms/src/AggregateFunctions/IAggregateFunction.h +++ b/dbms/src/AggregateFunctions/IAggregateFunction.h @@ -236,24 +236,15 @@ class IAggregateFunctionHelper : public IAggregateFunction ssize_t if_argument_pos = -1) const override { const auto end = start_offset + batch_size; - static constexpr size_t prefetch_step = 16; if (if_argument_pos >= 0) { const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = start_offset; i < end; ++i) { const auto place_idx = i - start_offset; - const auto prefetch_idx = place_idx + prefetch_step; if (flags[i] && places[place_idx]) - { - // Only prefetch first agg function data for simplicity. - // This may not work when there are too many agg functions. - if likely (prefetch_idx < end) - __builtin_prefetch(places[prefetch_idx]); - static_cast(this)->add(places[place_idx] + place_offset, columns, i, arena); - } } } else @@ -261,15 +252,9 @@ class IAggregateFunctionHelper : public IAggregateFunction for (size_t i = start_offset; i < end; ++i) { const auto place_idx = i - start_offset; - const auto prefetch_idx = place_idx + prefetch_step; if (places[place_idx]) - { - if likely (prefetch_idx < end) - __builtin_prefetch(places[prefetch_idx]); - static_cast(this)->add(places[place_idx] + place_offset, columns, i, arena); - } } } } diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 09065286dbf..61c4e3e581a 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -150,6 +150,18 @@ class HashMethodBase return emplaceImpl(key_holder, data); } + template + ALWAYS_INLINE inline EmplaceResult emplaceKey( + Data & data, + size_t row, + Arena & pool, + std::vector & sort_key_containers, + size_t hashval) + { + auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); + return emplaceImpl(key_holder, data, hashval); + } + template ALWAYS_INLINE inline FindResult findKey( Data & data, @@ -161,6 +173,18 @@ class HashMethodBase return findKeyImpl(keyHolderGetKey(key_holder), data); } + template + ALWAYS_INLINE inline FindResult findKey( + Data & data, + size_t row, + Arena & pool, + std::vector & sort_key_containers, + size_t hashval) + { + auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); + return findKeyImpl(keyHolderGetKey(key_holder), data, hashval); + } + // Emplace key using hashval, you can enable prefetch or not. template ALWAYS_INLINE inline EmplaceResult emplaceKey( diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index aaa118ce0f6..56ed42aeab1 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -666,11 +666,13 @@ void NO_INLINE Aggregator::executeImpl( { typename Method::State state(agg_process_info.key_columns, key_sizes, collators); + // 2MB as prefetch threshold, because normally server L2 cache is 1MB. + static constexpr size_t prefetch_threshold = (2 << 20); #ifndef NDEBUG - bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); + bool disable_prefetch = (method.data.getBufferSizeInBytes() < prefetch_threshold); fiu_do_on(FailPoints::force_agg_prefetch, { disable_prefetch = false; }); #else - const bool disable_prefetch = (method.data.getBufferSizeInCells() < 8192); + const bool disable_prefetch = (method.data.getBufferSizeInBytes() < prefetch_threshold); #endif // key_serialized needs column-wise handling for prefetch. @@ -751,6 +753,38 @@ std::optional::Res } } +template +std::optional::ResultType> Aggregator::emplaceOrFindKey( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers, + size_t hashval) const +{ + try + { + if constexpr (only_lookup) + return state.template findKey( + method.data, + index, + aggregates_pool, + sort_key_containers, + hashval); + else + return state.template emplaceKey( + method.data, + index, + aggregates_pool, + sort_key_containers, + hashval); + } + catch (ResizeException &) + { + return {}; + } +} + template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, @@ -833,8 +867,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( sort_key_containers, aggregates_pool, agg_process_info.hashvals); - - RUNTIME_CHECK(agg_process_info.hashvals.size() == agg_process_info.end_row); } for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) @@ -940,71 +972,183 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( \ if constexpr (collect_hit_rate) \ ++agg_process_info.hit_row_cnt; \ + \ + if constexpr (enable_prefetch) \ + __builtin_prefetch(aggregate_data); \ } \ } \ \ - places[i - agg_process_info.start_row] = aggregate_data; \ - processed_rows = i; + places[j - agg_process_info.start_row] = aggregate_data; \ + processed_rows = j; + const size_t prefetch_step = 16; + std::vector hashvals; if constexpr (enable_prefetch) { - if likely (agg_process_info.hashvals.empty()) - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool, - agg_process_info.hashvals); - - RUNTIME_CHECK(agg_process_info.hashvals.size() == agg_process_info.end_row); + const auto hashvals_size = std::min(prefetch_step, rows); + hashvals.resize(hashvals_size); + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + hashvals_size; ++i) + { + hashvals[i] = state.getHash(method.data, i, *aggregates_pool, sort_key_containers); + } } - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - if constexpr (enable_prefetch) + // for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) + // { + // AggregateDataPtr aggregate_data = nullptr; + // if constexpr (enable_prefetch) + // { + // auto emplace_result_holder = emplaceOrFindKey( + // method, + // state, + // i, + // *aggregates_pool, + // sort_key_containers, + // agg_process_info.hashvals); + + // HANDLE_AGG_EMPLACE_RESULT + // } + // else + // { + // auto emplace_result_holder + // = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + + // HANDLE_AGG_EMPLACE_RESULT + // } + // } + + size_t i = agg_process_info.start_row; + const size_t end = agg_process_info.end_row; + // const size_t end = *processed_rows - agg_process_info.start_row + 1; + const size_t mini_batch = 256; + while (i < end) + { + size_t batch_size = mini_batch; + if unlikely (i + batch_size > end) + batch_size = end - i; + + const auto cur_batch_end = i + batch_size; + for (size_t j = i; j < cur_batch_end; ++j) { - auto emplace_result_holder = emplaceOrFindKey( - method, - state, - i, - *aggregates_pool, - sort_key_containers, - agg_process_info.hashvals); + AggregateDataPtr aggregate_data = nullptr; + if constexpr (enable_prefetch) + { + const auto prefetch_hash_idx = j % prefetch_step; + const size_t hashval = hashvals[prefetch_hash_idx]; + const size_t prefetch_idx = j + prefetch_step; + if likely (prefetch_idx < end) + { + const auto new_hashval = state.getHash(method.data, prefetch_idx, *aggregates_pool, sort_key_containers); + method.data.prefetch(new_hashval); + hashvals[prefetch_hash_idx] = new_hashval; + } - HANDLE_AGG_EMPLACE_RESULT - } - else - { - auto emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + auto emplace_result_holder = emplaceOrFindKey( + method, + state, + j, + *aggregates_pool, + sort_key_containers, + hashval); + + HANDLE_AGG_EMPLACE_RESULT + } + else + { + auto emplace_result_holder + = emplaceOrFindKey(method, state, j, *aggregates_pool, sort_key_containers); - HANDLE_AGG_EMPLACE_RESULT + HANDLE_AGG_EMPLACE_RESULT + } } - } -#undef HANDLE_AGG_EMPLACE_RESULT - if (processed_rows) - { - /// Add values to the aggregate functions. + if unlikely (!processed_rows.has_value()) + break; + + size_t processed_size = *processed_rows - i + 1; + // bool first_inst = true; for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) + ++inst) { + // if (first_inst) + // inst->batch_that->addBatchWithPrefetch( + // i, + // batch_size, + // places.get() + i, + // inst->state_offset, + // inst->batch_arguments, + // aggregates_pool); + // else inst->batch_that->addBatch( - agg_process_info.start_row, - *processed_rows - agg_process_info.start_row + 1, - places.get(), - inst->state_offset, - inst->batch_arguments, - aggregates_pool); + i, + processed_size, + places.get() + i, + inst->state_offset, + inst->batch_arguments, + aggregates_pool); + // first_inst = false; } - agg_process_info.start_row = *processed_rows + 1; + + i += processed_size; + + if unlikely (processed_size != batch_size) + break; } - if likely (agg_process_info.start_row == agg_process_info.end_row) - agg_process_info.hashvals.clear(); + if likely (processed_rows) + agg_process_info.start_row = *processed_rows + 1; + +#undef HANDLE_AGG_EMPLACE_RESULT + + // if (processed_rows) + // { + // // /// Add values to the aggregate functions. + // // for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + // // ++inst) + // // { + // // inst->batch_that->addBatch( + // // agg_process_info.start_row, + // // *processed_rows - agg_process_info.start_row + 1, + // // places.get(), + // // inst->state_offset, + // // inst->batch_arguments, + // // aggregates_pool); + // // } + // size_t i = agg_process_info.start_row; + // const size_t end = *processed_rows - agg_process_info.start_row + 1; + // const size_t step = 256; + // while (i < end) + // { + // size_t batch_size = step; + // if unlikely (i + batch_size > end) + // batch_size = end - i; + + // bool first_inst = true; + // for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + // ++inst) + // { + // if (first_inst) + // inst->batch_that->addBatchWithPrefetch( + // i, + // batch_size, + // places.get() + i, + // inst->state_offset, + // inst->batch_arguments, + // aggregates_pool); + // else + // inst->batch_that->addBatch( + // i, + // batch_size, + // places.get() + i, + // inst->state_offset, + // inst->batch_arguments, + // aggregates_pool); + // first_inst = false; + // } + // i += batch_size; + // } + // agg_process_info.start_row = *processed_rows + 1; + // } } void NO_INLINE @@ -1839,6 +1983,9 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( } size_t data_index = 0; + const auto rows = data.size(); + std::unique_ptr places(new AggregateDataPtr[rows]); + data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { size_t key_columns_vec_index = data_index / params.max_block_size; if constexpr (!skip_convert_key) @@ -1846,9 +1993,22 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( agg_keys_helpers[key_columns_vec_index] ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); } - insertAggregatesIntoColumns(mapped, final_aggregate_columns_vec[key_columns_vec_index], arena); + // insertAggregatesIntoColumns(mapped, final_aggregate_columns_vec[key_columns_vec_index], arena); + places[data_index] = mapped; ++data_index; }); + + auto prefetch_idx = 16; + data_index = 0; + for (size_t i = 0; i < rows; ++i) + { + if (prefetch_idx < rows) + __builtin_prefetch(places[prefetch_idx++]); + + size_t key_columns_vec_index = data_index / params.max_block_size; + insertAggregatesIntoColumns(places[i], final_aggregate_columns_vec[key_columns_vec_index], arena); + ++data_index; + } } template diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index d424645e6e7..379f2b1b611 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,8 +1318,6 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; - std::vector hashvals{}; - void prepareForAgg(); bool allBlockDataHandled() const { @@ -1338,8 +1336,6 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); - - hashvals.clear(); } }; @@ -1473,6 +1469,15 @@ class Aggregator std::vector & sort_key_containers, const std::vector & hashvals) const; + template + std::optional::ResultType> emplaceOrFindKey( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers, + size_t hashval) const; + template std::optional::ResultType> emplaceOrFindKey( Method & method, From 0ef6ba691f3d50561389193a7e459de27123ef1e Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 24 Dec 2024 14:17:33 +0800 Subject: [PATCH 36/61] clean code Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 52 -------- dbms/src/Interpreters/Aggregator.cpp | 184 +++++---------------------- dbms/src/Interpreters/Aggregator.h | 14 +- 3 files changed, 35 insertions(+), 215 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 61c4e3e581a..c7cec18c2db 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -128,15 +128,6 @@ class HashMethodBase using FindResult = FindResultImpl; static constexpr bool has_mapped = !std::is_same::value; using Cache = LastElementCache; - static constexpr size_t prefetch_step = 16; - - template - static ALWAYS_INLINE inline void prefetch(Map & map, size_t idx, const std::vector & hashvals) - { - const auto prefetch_idx = idx + prefetch_step; - if likely (prefetch_idx < hashvals.size()) - map.prefetch(hashvals[prefetch_idx]); - } // Emplace key without hashval, and this method doesn't support prefetch. template @@ -185,49 +176,6 @@ class HashMethodBase return findKeyImpl(keyHolderGetKey(key_holder), data, hashval); } - // Emplace key using hashval, you can enable prefetch or not. - template - ALWAYS_INLINE inline EmplaceResult emplaceKey( - Data & data, - size_t row, - Arena & pool, - std::vector & sort_key_containers, - const std::vector & hashvals) - { - auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - if constexpr (enable_prefetch) - { - assert(hashvals.size() == static_cast(*this).total_rows); - prefetch(data, row, hashvals); - return emplaceImpl(key_holder, data, hashvals[row]); - } - else - { - return emplaceImpl(key_holder, data); - } - } - - template - ALWAYS_INLINE inline FindResult findKey( - Data & data, - size_t row, - Arena & pool, - std::vector & sort_key_containers, - const std::vector & hashvals) - { - auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - if constexpr (enable_prefetch) - { - assert(hashvals.size() == static_cast(*this).total_rows); - prefetch(data, row, hashvals); - return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); - } - else - { - return findKeyImpl(keyHolderGetKey(key_holder), data); - } - } - template ALWAYS_INLINE inline size_t getHash( const Data & data, diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 56ed42aeab1..3eb3e3191a3 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -721,38 +721,6 @@ void getHashVals( } } -template -std::optional::ResultType> Aggregator::emplaceOrFindKey( - Method & method, - typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers, - const std::vector & hashvals) const -{ - try - { - if constexpr (only_lookup) - return state.template findKey( - method.data, - index, - aggregates_pool, - sort_key_containers, - hashvals); - else - return state.template emplaceKey( - method.data, - index, - aggregates_pool, - sort_key_containers, - hashvals); - } - catch (ResizeException &) - { - return {}; - } -} - template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, @@ -765,19 +733,9 @@ std::optional::Res try { if constexpr (only_lookup) - return state.template findKey( - method.data, - index, - aggregates_pool, - sort_key_containers, - hashval); + return state.template findKey(method.data, index, aggregates_pool, sort_key_containers, hashval); else - return state.template emplaceKey( - method.data, - index, - aggregates_pool, - sort_key_containers, - hashval); + return state.template emplaceKey(method.data, index, aggregates_pool, sort_key_containers, hashval); } catch (ResizeException &) { @@ -879,7 +837,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( i, *aggregates_pool, sort_key_containers, - agg_process_info.hashvals); + agg_process_info.hashvals[i]); HANDLE_AGG_EMPLACE_RESULT } @@ -893,7 +851,13 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } if likely (processed_rows) + { agg_process_info.start_row = *processed_rows + 1; + + if likely (agg_process_info.start_row == agg_process_info.end_row) + agg_process_info.hashvals.clear(); + } + #undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -972,9 +936,9 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( \ if constexpr (collect_hit_rate) \ ++agg_process_info.hit_row_cnt; \ - \ - if constexpr (enable_prefetch) \ - __builtin_prefetch(aggregate_data); \ + \ + if constexpr (enable_prefetch) \ + __builtin_prefetch(aggregate_data); \ } \ } \ \ @@ -993,33 +957,8 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } } - // for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) - // { - // AggregateDataPtr aggregate_data = nullptr; - // if constexpr (enable_prefetch) - // { - // auto emplace_result_holder = emplaceOrFindKey( - // method, - // state, - // i, - // *aggregates_pool, - // sort_key_containers, - // agg_process_info.hashvals); - - // HANDLE_AGG_EMPLACE_RESULT - // } - // else - // { - // auto emplace_result_holder - // = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); - - // HANDLE_AGG_EMPLACE_RESULT - // } - // } - size_t i = agg_process_info.start_row; const size_t end = agg_process_info.end_row; - // const size_t end = *processed_rows - agg_process_info.start_row + 1; const size_t mini_batch = 256; while (i < end) { @@ -1028,6 +967,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( batch_size = end - i; const auto cur_batch_end = i + batch_size; + // j is the row index inside each mini batch. for (size_t j = i; j < cur_batch_end; ++j) { AggregateDataPtr aggregate_data = nullptr; @@ -1038,18 +978,14 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( const size_t prefetch_idx = j + prefetch_step; if likely (prefetch_idx < end) { - const auto new_hashval = state.getHash(method.data, prefetch_idx, *aggregates_pool, sort_key_containers); + const auto new_hashval + = state.getHash(method.data, prefetch_idx, *aggregates_pool, sort_key_containers); method.data.prefetch(new_hashval); hashvals[prefetch_hash_idx] = new_hashval; } - auto emplace_result_holder = emplaceOrFindKey( - method, - state, - j, - *aggregates_pool, - sort_key_containers, - hashval); + auto emplace_result_holder + = emplaceOrFindKey(method, state, j, *aggregates_pool, sort_key_containers, hashval); HANDLE_AGG_EMPLACE_RESULT } @@ -1065,90 +1001,29 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if unlikely (!processed_rows.has_value()) break; - size_t processed_size = *processed_rows - i + 1; - // bool first_inst = true; + const size_t processed_size = *processed_rows - i + 1; for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) + ++inst) { - // if (first_inst) - // inst->batch_that->addBatchWithPrefetch( - // i, - // batch_size, - // places.get() + i, - // inst->state_offset, - // inst->batch_arguments, - // aggregates_pool); - // else inst->batch_that->addBatch( - i, - processed_size, - places.get() + i, - inst->state_offset, - inst->batch_arguments, - aggregates_pool); - // first_inst = false; + i, + processed_size, + places.get() + i, + inst->state_offset, + inst->batch_arguments, + aggregates_pool); } - - i += processed_size; if unlikely (processed_size != batch_size) break; + + i = cur_batch_end; } if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; #undef HANDLE_AGG_EMPLACE_RESULT - - // if (processed_rows) - // { - // // /// Add values to the aggregate functions. - // // for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - // // ++inst) - // // { - // // inst->batch_that->addBatch( - // // agg_process_info.start_row, - // // *processed_rows - agg_process_info.start_row + 1, - // // places.get(), - // // inst->state_offset, - // // inst->batch_arguments, - // // aggregates_pool); - // // } - // size_t i = agg_process_info.start_row; - // const size_t end = *processed_rows - agg_process_info.start_row + 1; - // const size_t step = 256; - // while (i < end) - // { - // size_t batch_size = step; - // if unlikely (i + batch_size > end) - // batch_size = end - i; - - // bool first_inst = true; - // for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - // ++inst) - // { - // if (first_inst) - // inst->batch_that->addBatchWithPrefetch( - // i, - // batch_size, - // places.get() + i, - // inst->state_offset, - // inst->batch_arguments, - // aggregates_pool); - // else - // inst->batch_that->addBatch( - // i, - // batch_size, - // places.get() + i, - // inst->state_offset, - // inst->batch_arguments, - // aggregates_pool); - // first_inst = false; - // } - // i += batch_size; - // } - // agg_process_info.start_row = *processed_rows + 1; - // } } void NO_INLINE @@ -1993,12 +1868,13 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( agg_keys_helpers[key_columns_vec_index] ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); } - // insertAggregatesIntoColumns(mapped, final_aggregate_columns_vec[key_columns_vec_index], arena); places[data_index] = mapped; ++data_index; }); - auto prefetch_idx = 16; + // convertToBlockImplFinal didn't prefetch because it's used during spill. + // places vector will occupy extra memory, also it doesn't care performance during spill. + size_t prefetch_idx = 16; data_index = 0; for (size_t i = 0; i < rows; ++i) { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 379f2b1b611..8b1fd82b9fb 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,6 +1318,9 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; + // Used by prefetch. + std::vector hashvals; + void prepareForAgg(); bool allBlockDataHandled() const { @@ -1336,6 +1339,8 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); + + hashvals.clear(); } }; @@ -1460,15 +1465,6 @@ class Aggregator Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template - std::optional::ResultType> emplaceOrFindKey( - Method & method, - typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers, - const std::vector & hashvals) const; - template std::optional::ResultType> emplaceOrFindKey( Method & method, From 49d4188985bb0a2dd92fc9ca3196cd1476a930b4 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 24 Dec 2024 15:40:57 +0800 Subject: [PATCH 37/61] fix prefetch agg func is 0 Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 3eb3e3191a3..f43bbfa9ec1 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -774,6 +774,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); + const size_t prefetch_step = 16; std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); size_t rows = agg_process_info.end_row - agg_process_info.start_row; @@ -827,10 +828,15 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( agg_process_info.hashvals); } - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) + const auto end = agg_process_info.start_row + rows; + for (size_t i = agg_process_info.start_row; i < end; ++i) { if constexpr (enable_prefetch) { + const auto prefetch_idx = i + prefetch_step; + if likely (prefetch_idx < end) + method.data.prefetch(agg_process_info.hashvals[prefetch_idx]); + auto emplace_result_hold = emplaceOrFindKey( method, state, @@ -945,7 +951,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( places[j - agg_process_info.start_row] = aggregate_data; \ processed_rows = j; - const size_t prefetch_step = 16; std::vector hashvals; if constexpr (enable_prefetch) { From b4a1bde52f5e2b579ee1f670b415024071639903 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 24 Dec 2024 18:04:39 +0800 Subject: [PATCH 38/61] fix case Signed-off-by: guo-shaoge --- .../AggregateFunctions/IAggregateFunction.h | 19 ++++++------------ dbms/src/Interpreters/Aggregator.cpp | 20 +++++++++---------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/dbms/src/AggregateFunctions/IAggregateFunction.h b/dbms/src/AggregateFunctions/IAggregateFunction.h index 1f0cc6903a2..a7927085c3b 100644 --- a/dbms/src/AggregateFunctions/IAggregateFunction.h +++ b/dbms/src/AggregateFunctions/IAggregateFunction.h @@ -235,27 +235,20 @@ class IAggregateFunctionHelper : public IAggregateFunction Arena * arena, ssize_t if_argument_pos = -1) const override { - const auto end = start_offset + batch_size; if (if_argument_pos >= 0) { const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); - for (size_t i = start_offset; i < end; ++i) + for (size_t i = start_offset; i < start_offset + batch_size; ++i) { - const auto place_idx = i - start_offset; - - if (flags[i] && places[place_idx]) - static_cast(this)->add(places[place_idx] + place_offset, columns, i, arena); + if (flags[i] && places[i - start_offset]) + static_cast(this)->add(places[i - start_offset] + place_offset, columns, i, arena); } } else { - for (size_t i = start_offset; i < end; ++i) - { - const auto place_idx = i - start_offset; - - if (places[place_idx]) - static_cast(this)->add(places[place_idx] + place_offset, columns, i, arena); - } + for (size_t i = start_offset; i < start_offset + batch_size; ++i) + if (places[i - start_offset]) + static_cast(this)->add(places[i - start_offset] + place_offset, columns, i, arena); } } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index f43bbfa9ec1..dac18d1826f 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -922,7 +922,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } \ else \ { \ - agg_process_info.not_found_rows.push_back(i); \ + agg_process_info.not_found_rows.push_back(j); \ } \ } \ else \ @@ -954,17 +954,19 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( std::vector hashvals; if constexpr (enable_prefetch) { - const auto hashvals_size = std::min(prefetch_step, rows); - hashvals.resize(hashvals_size); - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + hashvals_size; ++i) + hashvals.resize(prefetch_step); + for (size_t i = agg_process_info.start_row; + i < agg_process_info.start_row + prefetch_step && i < agg_process_info.end_row; + ++i) { - hashvals[i] = state.getHash(method.data, i, *aggregates_pool, sort_key_containers); + hashvals[i % prefetch_step] = state.getHash(method.data, i, *aggregates_pool, sort_key_containers); } } size_t i = agg_process_info.start_row; - const size_t end = agg_process_info.end_row; + const size_t end = agg_process_info.start_row + rows; const size_t mini_batch = 256; + // i is the begin row index of each mini batch. while (i < end) { size_t batch_size = mini_batch; @@ -1013,7 +1015,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( inst->batch_that->addBatch( i, processed_size, - places.get() + i, + places.get() + i - agg_process_info.start_row, inst->state_offset, inst->batch_arguments, aggregates_pool); @@ -1880,15 +1882,13 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( // convertToBlockImplFinal didn't prefetch because it's used during spill. // places vector will occupy extra memory, also it doesn't care performance during spill. size_t prefetch_idx = 16; - data_index = 0; for (size_t i = 0; i < rows; ++i) { if (prefetch_idx < rows) __builtin_prefetch(places[prefetch_idx++]); - size_t key_columns_vec_index = data_index / params.max_block_size; + size_t key_columns_vec_index = i / params.max_block_size; insertAggregatesIntoColumns(places[i], final_aggregate_columns_vec[key_columns_vec_index], arena); - ++data_index; } } From 738ab2afb8330c39dc010d3154847106c1d3cbfe Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 25 Dec 2024 10:59:19 +0800 Subject: [PATCH 39/61] refine agg func is zero Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 104 ++++++++++++++------------- dbms/src/Interpreters/Aggregator.h | 5 -- 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index dac18d1826f..107bb9cd48d 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -764,6 +764,31 @@ std::optional::Res } } +template +ALWAYS_INLINE inline size_t getCurrentHashAndSetUpPrefetchHash( + size_t row_idx, + size_t end_idx, + size_t prefetch_step, + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + std::vector & sort_key_containers, + std::vector & hashvals) +{ + assert(hashvals.size() == prefetch_step); + + const auto prefetch_hash_idx = row_idx % prefetch_step; + const size_t hashval = hashvals[prefetch_hash_idx]; + const size_t prefetch_idx = row_idx + prefetch_step; + if likely (prefetch_idx < end_idx) + { + const auto new_hashval = state.getHash(method.data, prefetch_idx, *aggregates_pool, sort_key_containers); + method.data.prefetch(new_hashval); + hashvals[prefetch_hash_idx] = new_hashval; + } + return hashval; +} + template ALWAYS_INLINE void Aggregator::executeImplByRow( Method & method, @@ -783,6 +808,18 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( rows = std::max(rows / 2, 1); }); + std::vector hashvals; + if constexpr (enable_prefetch) + { + hashvals.resize(prefetch_step); + for (size_t i = agg_process_info.start_row; + i < agg_process_info.start_row + prefetch_step && i < agg_process_info.end_row; + ++i) + { + hashvals[i % prefetch_step] = state.getHash(method.data, i, *aggregates_pool, sort_key_containers); + } + } + /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) { @@ -814,36 +851,23 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } std::optional processed_rows; - if constexpr (enable_prefetch) - { - // hashvals is not empty only when resize exception happened. - if likely (agg_process_info.hashvals.empty()) - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, - aggregates_pool, - agg_process_info.hashvals); - } - const auto end = agg_process_info.start_row + rows; for (size_t i = agg_process_info.start_row; i < end; ++i) { if constexpr (enable_prefetch) { - const auto prefetch_idx = i + prefetch_step; - if likely (prefetch_idx < end) - method.data.prefetch(agg_process_info.hashvals[prefetch_idx]); - - auto emplace_result_hold = emplaceOrFindKey( + const size_t hashval = getCurrentHashAndSetUpPrefetchHash( + i, + end, + prefetch_step, method, state, - i, - *aggregates_pool, + aggregates_pool, sort_key_containers, - agg_process_info.hashvals[i]); + hashvals); + + auto emplace_result_hold + = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashval); HANDLE_AGG_EMPLACE_RESULT } @@ -857,13 +881,8 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } if likely (processed_rows) - { agg_process_info.start_row = *processed_rows + 1; - if likely (agg_process_info.start_row == agg_process_info.end_row) - agg_process_info.hashvals.clear(); - } - #undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -951,18 +970,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( places[j - agg_process_info.start_row] = aggregate_data; \ processed_rows = j; - std::vector hashvals; - if constexpr (enable_prefetch) - { - hashvals.resize(prefetch_step); - for (size_t i = agg_process_info.start_row; - i < agg_process_info.start_row + prefetch_step && i < agg_process_info.end_row; - ++i) - { - hashvals[i % prefetch_step] = state.getHash(method.data, i, *aggregates_pool, sort_key_containers); - } - } - size_t i = agg_process_info.start_row; const size_t end = agg_process_info.start_row + rows; const size_t mini_batch = 256; @@ -980,16 +987,15 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( AggregateDataPtr aggregate_data = nullptr; if constexpr (enable_prefetch) { - const auto prefetch_hash_idx = j % prefetch_step; - const size_t hashval = hashvals[prefetch_hash_idx]; - const size_t prefetch_idx = j + prefetch_step; - if likely (prefetch_idx < end) - { - const auto new_hashval - = state.getHash(method.data, prefetch_idx, *aggregates_pool, sort_key_containers); - method.data.prefetch(new_hashval); - hashvals[prefetch_hash_idx] = new_hashval; - } + const size_t hashval = getCurrentHashAndSetUpPrefetchHash( + j, + end, + prefetch_step, + method, + state, + aggregates_pool, + sort_key_containers, + hashvals); auto emplace_result_holder = emplaceOrFindKey(method, state, j, *aggregates_pool, sort_key_containers, hashval); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 8b1fd82b9fb..2372643f6f2 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,9 +1318,6 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; - // Used by prefetch. - std::vector hashvals; - void prepareForAgg(); bool allBlockDataHandled() const { @@ -1339,8 +1336,6 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); - - hashvals.clear(); } }; From 361f988c251534661344cc911a736d70cef8ea0b Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 25 Dec 2024 11:40:31 +0800 Subject: [PATCH 40/61] refine Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 107bb9cd48d..09a8ed3d6cd 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -680,7 +680,6 @@ void NO_INLINE Aggregator::executeImpl( // 1. getKeyHolder of key_serialized have to copy real data into Arena. // It means we better getKeyHolder for all Columns once and then use it both for getHash() and emplaceKey(). // 2. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. - // But getHashVals() still needs to be column-wise. if constexpr (Method::State::is_serialized_key) { // TODO: batch serialize method for Columns is still under development. @@ -704,23 +703,6 @@ void NO_INLINE Aggregator::executeImpl( } } -template -void getHashVals( - size_t start_row, - size_t end_row, - const Data & data, - const State & state, - std::vector & sort_key_containers, - Arena * pool, - std::vector & hashvals) -{ - hashvals.resize(state.total_rows); - for (size_t i = start_row; i < end_row; ++i) - { - hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); - } -} - template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, From 5158885333af675a7f0099b9a6981b266c74aeca Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 25 Dec 2024 15:10:57 +0800 Subject: [PATCH 41/61] refine comment Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 09a8ed3d6cd..e2e56a1d941 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -747,7 +747,7 @@ std::optional::Res } template -ALWAYS_INLINE inline size_t getCurrentHashAndSetUpPrefetchHash( +ALWAYS_INLINE inline size_t getCurrentHashAndDoPrefetch( size_t row_idx, size_t end_idx, size_t prefetch_step, @@ -838,7 +838,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( { if constexpr (enable_prefetch) { - const size_t hashval = getCurrentHashAndSetUpPrefetchHash( + const size_t hashval = getCurrentHashAndDoPrefetch( i, end, prefetch_step, @@ -969,7 +969,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( AggregateDataPtr aggregate_data = nullptr; if constexpr (enable_prefetch) { - const size_t hashval = getCurrentHashAndSetUpPrefetchHash( + const size_t hashval = getCurrentHashAndDoPrefetch( j, end, prefetch_step, @@ -1776,6 +1776,7 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( agg_keys_helper.initAggKeys(data.size(), key_columns); } + // Doesn't prefetch agg data, because places[data.size()] is needed, which can be very large. data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { if constexpr (!skip_convert_key) { @@ -1867,8 +1868,6 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( ++data_index; }); - // convertToBlockImplFinal didn't prefetch because it's used during spill. - // places vector will occupy extra memory, also it doesn't care performance during spill. size_t prefetch_idx = 16; for (size_t i = 0; i < rows; ++i) { From 80b7e228ea124c62d5db12b12d0275d7e971ddb0 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 26 Dec 2024 12:06:18 +0800 Subject: [PATCH 42/61] getHash full Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 25 +++++-- dbms/src/Interpreters/Aggregator.cpp | 106 +++++++++++++-------------- dbms/src/Interpreters/Aggregator.h | 9 ++- 3 files changed, 75 insertions(+), 65 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index c7cec18c2db..89f888aecee 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -129,6 +129,15 @@ class HashMethodBase static constexpr bool has_mapped = !std::is_same::value; using Cache = LastElementCache; + static const size_t prefetch_step = 16; + template + static ALWAYS_INLINE inline void prefetch(Map & map, size_t idx, const std::vector & hashvals) + { + const auto prefetch_idx = idx + prefetch_step; + if likely (prefetch_idx < hashvals.size()) + map.prefetch(hashvals[prefetch_idx]); + } + // Emplace key without hashval, and this method doesn't support prefetch. template ALWAYS_INLINE inline EmplaceResult emplaceKey( @@ -142,15 +151,17 @@ class HashMethodBase } template - ALWAYS_INLINE inline EmplaceResult emplaceKey( + ALWAYS_INLINE inline EmplaceResult emplaceKeyWithPrefetch( Data & data, size_t row, Arena & pool, std::vector & sort_key_containers, - size_t hashval) + const std::vector & hashvals) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - return emplaceImpl(key_holder, data, hashval); + assert(hashvals.size() == static_cast(*this).total_rows); + prefetch(data, row, hashvals); + return emplaceImpl(key_holder, data, hashvals[row]); } template @@ -165,15 +176,17 @@ class HashMethodBase } template - ALWAYS_INLINE inline FindResult findKey( + ALWAYS_INLINE inline FindResult findKeyWithPrefetch( Data & data, size_t row, Arena & pool, std::vector & sort_key_containers, - size_t hashval) + const std::vector & hashvals) { auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - return findKeyImpl(keyHolderGetKey(key_holder), data, hashval); + assert(hashvals.size() == static_cast(*this).total_rows); + prefetch(data, row, hashvals); + return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); } template diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index e2e56a1d941..5155eb4b087 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -704,20 +704,23 @@ void NO_INLINE Aggregator::executeImpl( } template -std::optional::ResultType> Aggregator::emplaceOrFindKey( - Method & method, - typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers, - size_t hashval) const +std::optional::ResultType> Aggregator:: + emplaceOrFindKeyWithPrefetch( + Method & method, + typename Method::State & state, + size_t index, + Arena & aggregates_pool, + std::vector & sort_key_containers, + const std::vector & hashvals) const { try { if constexpr (only_lookup) - return state.template findKey(method.data, index, aggregates_pool, sort_key_containers, hashval); + return state + .template findKeyWithPrefetch(method.data, index, aggregates_pool, sort_key_containers, hashvals); else - return state.template emplaceKey(method.data, index, aggregates_pool, sort_key_containers, hashval); + return state + .template emplaceKeyWithPrefetch(method.data, index, aggregates_pool, sort_key_containers, hashvals); } catch (ResizeException &) { @@ -746,29 +749,21 @@ std::optional::Res } } -template -ALWAYS_INLINE inline size_t getCurrentHashAndDoPrefetch( - size_t row_idx, - size_t end_idx, - size_t prefetch_step, - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - std::vector & sort_key_containers, +template +void getHashVals( + size_t start_row, + size_t end_row, + const Data & data, + const State & state, + std::vector & sort_key_containers, + Arena * pool, std::vector & hashvals) { - assert(hashvals.size() == prefetch_step); - - const auto prefetch_hash_idx = row_idx % prefetch_step; - const size_t hashval = hashvals[prefetch_hash_idx]; - const size_t prefetch_idx = row_idx + prefetch_step; - if likely (prefetch_idx < end_idx) + hashvals.resize(state.total_rows); + for (size_t i = start_row; i < end_row; ++i) { - const auto new_hashval = state.getHash(method.data, prefetch_idx, *aggregates_pool, sort_key_containers); - method.data.prefetch(new_hashval); - hashvals[prefetch_hash_idx] = new_hashval; + hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); } - return hashval; } template @@ -781,7 +776,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); - const size_t prefetch_step = 16; std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); size_t rows = agg_process_info.end_row - agg_process_info.start_row; @@ -790,18 +784,21 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( rows = std::max(rows / 2, 1); }); - std::vector hashvals; if constexpr (enable_prefetch) { - hashvals.resize(prefetch_step); - for (size_t i = agg_process_info.start_row; - i < agg_process_info.start_row + prefetch_step && i < agg_process_info.end_row; - ++i) - { - hashvals[i % prefetch_step] = state.getHash(method.data, i, *aggregates_pool, sort_key_containers); - } + // agg_process_info.hashvals.empty() is false only when resize exception happens. + if likely (agg_process_info.hashvals.empty()) + getHashVals( + agg_process_info.start_row, + agg_process_info.end_row, + method.data, + state, + sort_key_containers, + aggregates_pool, + agg_process_info.hashvals); } + /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) { @@ -833,23 +830,17 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } std::optional processed_rows; - const auto end = agg_process_info.start_row + rows; - for (size_t i = agg_process_info.start_row; i < end; ++i) + for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) { if constexpr (enable_prefetch) { - const size_t hashval = getCurrentHashAndDoPrefetch( - i, - end, - prefetch_step, + auto emplace_result_hold = emplaceOrFindKeyWithPrefetch( method, state, - aggregates_pool, + i, + *aggregates_pool, sort_key_containers, - hashvals); - - auto emplace_result_hold - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers, hashval); + agg_process_info.hashvals); HANDLE_AGG_EMPLACE_RESULT } @@ -865,6 +856,9 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; + if likely (agg_process_info.start_row == agg_process_info.end_row) + agg_process_info.hashvals.clear(); + #undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -969,18 +963,13 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( AggregateDataPtr aggregate_data = nullptr; if constexpr (enable_prefetch) { - const size_t hashval = getCurrentHashAndDoPrefetch( - j, - end, - prefetch_step, + auto emplace_result_holder = emplaceOrFindKeyWithPrefetch( method, state, - aggregates_pool, + j, + *aggregates_pool, sort_key_containers, - hashvals); - - auto emplace_result_holder - = emplaceOrFindKey(method, state, j, *aggregates_pool, sort_key_containers, hashval); + agg_process_info.hashvals); HANDLE_AGG_EMPLACE_RESULT } @@ -1018,6 +1007,9 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; + if likely (agg_process_info.start_row == agg_process_info.end_row) + agg_process_info.hashvals.clear(); + #undef HANDLE_AGG_EMPLACE_RESULT } diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 2372643f6f2..3216ef76209 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,6 +1318,9 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; + // For prefetch. + std::vector hashvals; + void prepareForAgg(); bool allBlockDataHandled() const { @@ -1336,6 +1339,8 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); + + hashvals.clear(); } }; @@ -1461,13 +1466,13 @@ class Aggregator AggProcessInfo & agg_process_info) const; template - std::optional::ResultType> emplaceOrFindKey( + std::optional::ResultType> emplaceOrFindKeyWithPrefetch( Method & method, typename Method::State & state, size_t index, Arena & aggregates_pool, std::vector & sort_key_containers, - size_t hashval) const; + const std::vector & hashvals) const; template std::optional::ResultType> emplaceOrFindKey( From c41dba4011ed0a747e47d87a6d85f73342416a95 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 26 Dec 2024 14:51:54 +0800 Subject: [PATCH 43/61] minor refine Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 5155eb4b087..c8f16c503db 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -798,7 +798,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( agg_process_info.hashvals); } - /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) { @@ -906,9 +905,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ break; \ } \ - \ auto & emplace_result = emplace_result_holder.value(); \ - \ if constexpr (only_lookup) \ { \ if (emplace_result.isFound()) \ @@ -942,7 +939,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( __builtin_prefetch(aggregate_data); \ } \ } \ - \ places[j - agg_process_info.start_row] = aggregate_data; \ processed_rows = j; From ab31bebb2b1dc52f864c7d3bfeca83fc324e1daf Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 26 Dec 2024 18:30:10 +0800 Subject: [PATCH 44/61] hashvals sizeof 16 Signed-off-by: guo-shaoge --- .../AggregateFunctionGroupUniqArray.h | 2 +- .../src/AggregateFunctions/KeyHolderHelpers.h | 4 +- dbms/src/Common/ColumnsHashing.h | 43 +++--- dbms/src/Common/ColumnsHashingImpl.h | 40 ++---- .../src/Common/HashTable/HashTableKeyHolder.h | 12 +- dbms/src/Interpreters/Aggregator.cpp | 122 +++++++++++------- dbms/src/Interpreters/Aggregator.h | 13 +- 7 files changed, 118 insertions(+), 118 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index f556a009551..d3cbea74195 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -182,7 +182,7 @@ class AggregateFunctionGroupUniqArrayGeneric { // We have to copy the keys to our arena. assert(arena != nullptr); - cur_set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted); + cur_set.emplace(ArenaKeyHolder{rhs_elem.getValue(), arena}, it, inserted); } } diff --git a/dbms/src/AggregateFunctions/KeyHolderHelpers.h b/dbms/src/AggregateFunctions/KeyHolderHelpers.h index 6677866f0d3..5c3a617a1cb 100644 --- a/dbms/src/AggregateFunctions/KeyHolderHelpers.h +++ b/dbms/src/AggregateFunctions/KeyHolderHelpers.h @@ -24,14 +24,14 @@ inline auto getKeyHolder(const IColumn & column, size_t row_num, Arena & arena) { if constexpr (is_plain_column) { - return ArenaKeyHolder{column.getDataAt(row_num), arena}; + return ArenaKeyHolder{column.getDataAt(row_num), &arena}; } else { const char * begin = nullptr; StringRef serialized = column.serializeValueIntoArena(row_num, arena, begin); assert(serialized.data != nullptr); - return SerializedKeyHolder{serialized, arena}; + return SerializedKeyHolder{serialized, &arena}; } } diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index ac665874969..15c7ef78a9e 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -47,6 +47,7 @@ struct HashMethodOneNumber { using Self = HashMethodOneNumber; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = FieldType; static constexpr bool is_serialized_key = false; @@ -78,7 +79,7 @@ struct HashMethodOneNumber using Base::getHash; /// (const Data & data, size_t row, Arena & pool) -> size_t /// Is used for default implementation in HashMethodBase. - ALWAYS_INLINE inline FieldType getKeyHolder(size_t row, Arena *, std::vector &) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena *, std::vector &) const { if constexpr (std::is_same_v) return vec[row]; @@ -97,6 +98,7 @@ struct HashMethodString { using Self = HashMethodString; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = ArenaKeyHolder; static constexpr bool is_serialized_key = false; @@ -119,7 +121,7 @@ struct HashMethodString collator = collators[0]; } - ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder( + ALWAYS_INLINE inline KeyHolderType getKeyHolder( ssize_t row, [[maybe_unused]] Arena * pool, std::vector & sort_key_containers) const @@ -130,7 +132,7 @@ struct HashMethodString if (likely(collator)) key = collator->sortKey(key.data, key.size, sort_key_containers[0]); - return ArenaKeyHolder{key, *pool}; + return ArenaKeyHolder{key, pool}; } protected: @@ -143,6 +145,7 @@ struct HashMethodStringBin { using Self = HashMethodStringBin; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = ArenaKeyHolder; static constexpr bool is_serialized_key = false; @@ -159,12 +162,12 @@ struct HashMethodStringBin chars = column_string.getChars().data(); } - ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(ssize_t row, Arena * pool, std::vector &) const { auto last_offset = row == 0 ? 0 : offsets[row - 1]; StringRef key(chars + last_offset, offsets[row] - last_offset - 1); key = BinCollatorSortKey(key.data, key.size); - return ArenaKeyHolder{key, *pool}; + return ArenaKeyHolder{key, pool}; } protected: @@ -344,6 +347,7 @@ struct HashMethodFastPathTwoKeysSerialized { using Self = HashMethodFastPathTwoKeysSerialized; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = SerializedKeyHolder; static constexpr bool is_serialized_key = true; @@ -357,13 +361,13 @@ struct HashMethodFastPathTwoKeysSerialized , total_rows(key_columns[0]->size()) {} - ALWAYS_INLINE inline auto getKeyHolder(ssize_t row, Arena * pool, std::vector &) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(ssize_t row, Arena * pool, std::vector &) const { StringRef key1; StringRef key2; size_t alloc_size = key_1_desc.getKey(row, key1) + key_2_desc.getKey(row, key2); char * start = pool->alloc(alloc_size); - SerializedKeyHolder ret{{start, alloc_size}, *pool}; + SerializedKeyHolder ret{{start, alloc_size}, pool}; Key1Desc::serializeKey(start, key1); Key2Desc::serializeKey(start, key2); return ret; @@ -382,6 +386,7 @@ struct HashMethodFixedString { using Self = HashMethodFixedString; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = ArenaKeyHolder; static constexpr bool is_serialized_key = false; @@ -404,16 +409,14 @@ struct HashMethodFixedString collator = collators[0]; } - ALWAYS_INLINE inline ArenaKeyHolder getKeyHolder( - size_t row, - Arena * pool, - std::vector & sort_key_containers) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena * pool, std::vector & sort_key_containers) + const { StringRef key(&(*chars)[row * n], n); if (collator) key = collator->sortKeyFastPath(key.data, key.size, sort_key_containers[0]); - return ArenaKeyHolder{key, *pool}; + return ArenaKeyHolder{key, pool}; } protected: @@ -433,6 +436,7 @@ struct HashMethodKeysFixed using Self = HashMethodKeysFixed; using BaseHashed = columns_hashing_impl::HashMethodBase; using Base = columns_hashing_impl::BaseStateKeysFixed; + using KeyHolderType = Key; static constexpr bool is_serialized_key = false; static constexpr bool has_nullable_keys = has_nullable_keys_; @@ -525,7 +529,7 @@ struct HashMethodKeysFixed #endif } - ALWAYS_INLINE inline Key getKeyHolder(size_t row, Arena *, std::vector &) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena *, std::vector &) const { if constexpr (has_nullable_keys) { @@ -591,6 +595,7 @@ struct HashMethodSerialized { using Self = HashMethodSerialized; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = SerializedKeyHolder; static constexpr bool is_serialized_key = true; @@ -609,14 +614,12 @@ struct HashMethodSerialized , total_rows(key_columns_[0]->size()) {} - ALWAYS_INLINE inline SerializedKeyHolder getKeyHolder( - size_t row, - Arena * pool, - std::vector & sort_key_containers) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena * pool, std::vector & sort_key_containers) + const { return SerializedKeyHolder{ serializeKeysToPoolContiguous(row, keys_size, key_columns, collators, sort_key_containers, *pool), - *pool}; + pool}; } protected: @@ -631,6 +634,7 @@ struct HashMethodHashed using Key = UInt128; using Self = HashMethodHashed; using Base = columns_hashing_impl::HashMethodBase; + using KeyHolderType = Key; static constexpr bool is_serialized_key = false; @@ -644,7 +648,8 @@ struct HashMethodHashed , total_rows(key_columns[0]->size()) {} - ALWAYS_INLINE inline Key getKeyHolder(size_t row, Arena *, std::vector & sort_key_containers) const + ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena *, std::vector & sort_key_containers) + const { return hash128(row, key_columns.size(), key_columns, collators, sort_key_containers); } diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 89f888aecee..7863033aca5 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -120,7 +120,7 @@ class FindResultImpl bool isFound() const { return found; } }; -template +template class HashMethodBase { public: @@ -128,15 +128,7 @@ class HashMethodBase using FindResult = FindResultImpl; static constexpr bool has_mapped = !std::is_same::value; using Cache = LastElementCache; - - static const size_t prefetch_step = 16; - template - static ALWAYS_INLINE inline void prefetch(Map & map, size_t idx, const std::vector & hashvals) - { - const auto prefetch_idx = idx + prefetch_step; - if likely (prefetch_idx < hashvals.size()) - map.prefetch(hashvals[prefetch_idx]); - } + using Derived = TDerived; // Emplace key without hashval, and this method doesn't support prefetch. template @@ -150,18 +142,10 @@ class HashMethodBase return emplaceImpl(key_holder, data); } - template - ALWAYS_INLINE inline EmplaceResult emplaceKeyWithPrefetch( - Data & data, - size_t row, - Arena & pool, - std::vector & sort_key_containers, - const std::vector & hashvals) + template + ALWAYS_INLINE inline EmplaceResult emplaceKey(Data & data, KeyHolder && key_holder, size_t hashval) { - auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - assert(hashvals.size() == static_cast(*this).total_rows); - prefetch(data, row, hashvals); - return emplaceImpl(key_holder, data, hashvals[row]); + return emplaceImpl(key_holder, data, hashval); } template @@ -175,18 +159,10 @@ class HashMethodBase return findKeyImpl(keyHolderGetKey(key_holder), data); } - template - ALWAYS_INLINE inline FindResult findKeyWithPrefetch( - Data & data, - size_t row, - Arena & pool, - std::vector & sort_key_containers, - const std::vector & hashvals) + template + ALWAYS_INLINE inline FindResult findKey(Data & data, KeyHolder && key_holder, size_t hashval) { - auto key_holder = static_cast(*this).getKeyHolder(row, &pool, sort_key_containers); - assert(hashvals.size() == static_cast(*this).total_rows); - prefetch(data, row, hashvals); - return findKeyImpl(keyHolderGetKey(key_holder), data, hashvals[row]); + return findKeyImpl(keyHolderGetKey(key_holder), data, hashval); } template diff --git a/dbms/src/Common/HashTable/HashTableKeyHolder.h b/dbms/src/Common/HashTable/HashTableKeyHolder.h index 01b06dce87d..2c81b050789 100644 --- a/dbms/src/Common/HashTable/HashTableKeyHolder.h +++ b/dbms/src/Common/HashTable/HashTableKeyHolder.h @@ -92,7 +92,7 @@ namespace DB struct ArenaKeyHolder { StringRef key; - Arena & pool; + Arena * pool; }; } // namespace DB @@ -111,14 +111,14 @@ inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder & holder) { // Hash table shouldn't ask us to persist a zero key assert(holder.key.size > 0); - holder.key.data = holder.pool.insert(holder.key.data, holder.key.size); + holder.key.data = holder.pool->insert(holder.key.data, holder.key.size); } inline void ALWAYS_INLINE keyHolderPersistKey(DB::ArenaKeyHolder && holder) { // Hash table shouldn't ask us to persist a zero key assert(holder.key.size > 0); - holder.key.data = holder.pool.insert(holder.key.data, holder.key.size); + holder.key.data = holder.pool->insert(holder.key.data, holder.key.size); } inline void ALWAYS_INLINE keyHolderDiscardKey(DB::ArenaKeyHolder &) {} @@ -134,7 +134,7 @@ namespace DB struct SerializedKeyHolder { StringRef key; - Arena & pool; + Arena * pool; }; } // namespace DB @@ -157,7 +157,7 @@ inline void ALWAYS_INLINE keyHolderDiscardKey(DB::SerializedKeyHolder & holder) { //[[maybe_unused]] void * new_head = holder.pool.rollback(holder.key.size); //assert(new_head == holder.key.data); - holder.pool.rollback(holder.key.size); + holder.pool->rollback(holder.key.size); holder.key.data = nullptr; holder.key.size = 0; } @@ -166,7 +166,7 @@ inline void ALWAYS_INLINE keyHolderDiscardKey(DB::SerializedKeyHolder && holder) { //[[maybe_unused]] void * new_head = holder.pool.rollback(holder.key.size); //assert(new_head == holder.key.data); - holder.pool.rollback(holder.key.size); + holder.pool->rollback(holder.key.size); holder.key.data = nullptr; holder.key.size = 0; } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index c8f16c503db..0055b241da4 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -46,6 +46,8 @@ extern const char random_fail_in_resize_callback[]; extern const char force_agg_prefetch[]; } // namespace FailPoints +static constexpr size_t agg_prefetch_step = 16; + #define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME #define AggregationMethodNameTwoLevel(NAME) AggregatedDataVariants::AggregationMethod_##NAME##_two_level #define AggregationMethodType(NAME) AggregatedDataVariants::Type::NAME @@ -704,23 +706,18 @@ void NO_INLINE Aggregator::executeImpl( } template -std::optional::ResultType> Aggregator:: - emplaceOrFindKeyWithPrefetch( - Method & method, - typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers, - const std::vector & hashvals) const +std::optional::ResultType> Aggregator::emplaceOrFindKey( + Method & method, + typename Method::State & state, + typename Method::State::Derived::KeyHolderType && key_holder, + size_t hashval) const { try { if constexpr (only_lookup) - return state - .template findKeyWithPrefetch(method.data, index, aggregates_pool, sort_key_containers, hashvals); + return state.template findKey(method.data, std::move(key_holder), hashval); else - return state - .template emplaceKeyWithPrefetch(method.data, index, aggregates_pool, sort_key_containers, hashvals); + return state.template emplaceKey(method.data, std::move(key_holder), hashval); } catch (ResizeException &) { @@ -749,21 +746,39 @@ std::optional::Res } } -template -void getHashVals( - size_t start_row, - size_t end_row, - const Data & data, - const State & state, - std::vector & sort_key_containers, - Arena * pool, - std::vector & hashvals) +template +ALWAYS_INLINE inline std::pair getCurrentHashAndDoPrefetch( + size_t cur_row_idx, + size_t end_row_idx, + Method & method, + typename Method::State & state, + Arena * aggregates_pool, + std::vector & sort_key_containers, + std::vector & hashvals, + std::vector & key_holders) { - hashvals.resize(state.total_rows); - for (size_t i = start_row; i < end_row; ++i) + assert(hashvals.size() == agg_prefetch_step); + + const auto hashvals_idx = cur_row_idx % agg_prefetch_step; + const size_t prefetch_row_idx = cur_row_idx + agg_prefetch_step; + + const size_t cur_hashval = hashvals[hashvals_idx]; + auto cur_key_holder = std::move(key_holders[hashvals_idx]); + + if likely (prefetch_row_idx < end_row_idx) { - hashvals[i] = state.getHash(data, i, *pool, sort_key_containers); + auto key_holder = static_cast(&state)->getKeyHolder( + prefetch_row_idx, + aggregates_pool, + sort_key_containers); + const size_t new_hashval = method.data.hash(keyHolderGetKey(key_holder)); + + key_holders[hashvals_idx] = std::move(key_holder); + hashvals[hashvals_idx] = new_hashval; + + method.data.prefetch(new_hashval); } + return {cur_key_holder, cur_hashval}; } template @@ -784,18 +799,25 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( rows = std::max(rows / 2, 1); }); + std::vector hashvals; + std::vector key_holders; if constexpr (enable_prefetch) { - // agg_process_info.hashvals.empty() is false only when resize exception happens. - if likely (agg_process_info.hashvals.empty()) - getHashVals( - agg_process_info.start_row, - agg_process_info.end_row, - method.data, - state, - sort_key_containers, + hashvals.resize(agg_prefetch_step); + key_holders.resize(agg_prefetch_step); + for (size_t i = agg_process_info.start_row % agg_prefetch_step; + i < agg_prefetch_step && i + agg_process_info.start_row < agg_process_info.end_row; + ++i) + { + auto key_holder = static_cast(&state)->getKeyHolder( + i + agg_process_info.start_row, aggregates_pool, - agg_process_info.hashvals); + sort_key_containers); + const size_t hashval = method.data.hash(keyHolderGetKey(key_holder)); + + key_holders[i] = std::move(key_holder); + hashvals[i] = hashval; + } } /// Optimization for special case when there are no aggregate functions. @@ -829,17 +851,22 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } std::optional processed_rows; - for (size_t i = agg_process_info.start_row; i < agg_process_info.start_row + rows; ++i) + const auto end = agg_process_info.start_row + rows; + for (size_t i = agg_process_info.start_row; i < end; ++i) { if constexpr (enable_prefetch) { - auto emplace_result_hold = emplaceOrFindKeyWithPrefetch( + auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( + i, + end, method, state, - i, - *aggregates_pool, + aggregates_pool, sort_key_containers, - agg_process_info.hashvals); + hashvals, + key_holders); + + auto emplace_result_hold = emplaceOrFindKey(method, state, std::move(key_holder), hashval); HANDLE_AGG_EMPLACE_RESULT } @@ -855,9 +882,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; - if likely (agg_process_info.start_row == agg_process_info.end_row) - agg_process_info.hashvals.clear(); - #undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -959,13 +983,18 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( AggregateDataPtr aggregate_data = nullptr; if constexpr (enable_prefetch) { - auto emplace_result_holder = emplaceOrFindKeyWithPrefetch( + auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( + j, + end, method, state, - j, - *aggregates_pool, + aggregates_pool, sort_key_containers, - agg_process_info.hashvals); + hashvals, + key_holders); + + auto emplace_result_holder + = emplaceOrFindKey(method, state, std::move(key_holder), hashval); HANDLE_AGG_EMPLACE_RESULT } @@ -1003,9 +1032,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; - if likely (agg_process_info.start_row == agg_process_info.end_row) - agg_process_info.hashvals.clear(); - #undef HANDLE_AGG_EMPLACE_RESULT } diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 3216ef76209..c55edcbd85c 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1318,9 +1318,6 @@ class Aggregator size_t hit_row_cnt = 0; std::vector not_found_rows; - // For prefetch. - std::vector hashvals; - void prepareForAgg(); bool allBlockDataHandled() const { @@ -1339,8 +1336,6 @@ class Aggregator hit_row_cnt = 0; not_found_rows.clear(); not_found_rows.reserve(block_.rows() / 2); - - hashvals.clear(); } }; @@ -1466,13 +1461,11 @@ class Aggregator AggProcessInfo & agg_process_info) const; template - std::optional::ResultType> emplaceOrFindKeyWithPrefetch( + std::optional::ResultType> emplaceOrFindKey( Method & method, typename Method::State & state, - size_t index, - Arena & aggregates_pool, - std::vector & sort_key_containers, - const std::vector & hashvals) const; + typename Method::State::Derived::KeyHolderType && key_holder, + size_t hashval) const; template std::optional::ResultType> emplaceOrFindKey( From 00cc20d7e967f3b085b26f7f5223ad94f8ca9d27 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 26 Dec 2024 21:12:15 +0800 Subject: [PATCH 45/61] fix case Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 0055b241da4..850ea56441a 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -749,6 +749,7 @@ std::optional::Res template ALWAYS_INLINE inline std::pair getCurrentHashAndDoPrefetch( size_t cur_row_idx, + size_t start_row_idx, size_t end_row_idx, Method & method, typename Method::State & state, @@ -759,7 +760,7 @@ ALWAYS_INLINE inline std::pair(&state)->getKeyHolder( i + agg_process_info.start_row, @@ -858,6 +857,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( { auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( i, + agg_process_info.start_row, end, method, state, @@ -985,6 +985,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( { auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( j, + agg_process_info.start_row, end, method, state, From 4aec90b1e9730ef2bfa720b60c46890585cf9ac4 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 27 Dec 2024 11:07:00 +0800 Subject: [PATCH 46/61] remove macro Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 16 ++--- dbms/src/Interpreters/Aggregator.cpp | 95 ++++++++++++++-------------- 2 files changed, 54 insertions(+), 57 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 7863033aca5..57a7fa88a59 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -60,24 +60,24 @@ struct LastElementCache template class EmplaceResultImpl { - Mapped & value; - Mapped & cached_value; - bool inserted; + Mapped * value = nullptr; + Mapped * cached_value = nullptr; + bool inserted = false; public: EmplaceResultImpl(Mapped & value_, Mapped & cached_value_, bool inserted_) - : value(value_) - , cached_value(cached_value_) + : value(&value_) + , cached_value(&cached_value_) , inserted(inserted_) {} bool isInserted() const { return inserted; } - auto & getMapped() const { return value; } + auto & getMapped() const { return *value; } void setMapped(const Mapped & mapped) { - cached_value = mapped; - value = mapped; + *cached_value = mapped; + *value = mapped; } }; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 850ea56441a..9edc2b97b2a 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -922,49 +922,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( /// Generic case. std::unique_ptr places(new AggregateDataPtr[rows]); std::optional processed_rows; - -#define HANDLE_AGG_EMPLACE_RESULT \ - if unlikely (!emplace_result_holder.has_value()) \ - { \ - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ - break; \ - } \ - auto & emplace_result = emplace_result_holder.value(); \ - if constexpr (only_lookup) \ - { \ - if (emplace_result.isFound()) \ - { \ - aggregate_data = emplace_result.getMapped(); \ - } \ - else \ - { \ - agg_process_info.not_found_rows.push_back(j); \ - } \ - } \ - else \ - { \ - if (emplace_result.isInserted()) \ - { \ - emplace_result.setMapped(nullptr); \ - \ - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); \ - createAggregateStates(aggregate_data); \ - \ - emplace_result.setMapped(aggregate_data); \ - } \ - else \ - { \ - aggregate_data = emplace_result.getMapped(); \ - \ - if constexpr (collect_hit_rate) \ - ++agg_process_info.hit_row_cnt; \ - \ - if constexpr (enable_prefetch) \ - __builtin_prefetch(aggregate_data); \ - } \ - } \ - places[j - agg_process_info.start_row] = aggregate_data; \ - processed_rows = j; + std::optional::ResultType> emplace_result_holder; size_t i = agg_process_info.start_row; const size_t end = agg_process_info.start_row + rows; @@ -994,18 +952,57 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( hashvals, key_holders); - auto emplace_result_holder + emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); - - HANDLE_AGG_EMPLACE_RESULT } else { - auto emplace_result_holder + emplace_result_holder = emplaceOrFindKey(method, state, j, *aggregates_pool, sort_key_containers); - - HANDLE_AGG_EMPLACE_RESULT } + + if unlikely (!emplace_result_holder.has_value()) + { + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); + break; + } + + auto & emplace_result = emplace_result_holder.value(); + if constexpr (only_lookup) + { + if (emplace_result.isFound()) + { + aggregate_data = emplace_result.getMapped(); + } + else + { + agg_process_info.not_found_rows.push_back(j); + } + } + else + { + if (emplace_result.isInserted()) + { + emplace_result.setMapped(nullptr); + + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + + emplace_result.setMapped(aggregate_data); + } + else + { + aggregate_data = emplace_result.getMapped(); + + if constexpr (collect_hit_rate) + ++agg_process_info.hit_row_cnt; + + if constexpr (enable_prefetch) + __builtin_prefetch(aggregate_data); + } + } + places[j - agg_process_info.start_row] = aggregate_data; + processed_rows = j; } if unlikely (!processed_rows.has_value()) From 606e01360f43efd8f32366291d86d15b89dbc558 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 27 Dec 2024 11:35:49 +0800 Subject: [PATCH 47/61] remove macro 2 Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 226 +++++++++++++++------------ 1 file changed, 127 insertions(+), 99 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 57a7fa88a59..fa3e5de8a9d 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -194,129 +194,157 @@ class HashMethodBase } } -#define DEFINE_EMPLACE_IMPL_BEGIN \ - if constexpr (Cache::consecutive_keys_optimization) \ - { \ - if (cache.found && cache.check(keyHolderGetKey(key_holder))) \ - { \ - if constexpr (has_mapped) \ - return EmplaceResult(cache.value.second, cache.value.second, false); \ - else \ - return EmplaceResult(false); \ - } \ - } \ - typename Data::LookupResult it; \ - bool inserted = false; - -#define DEFINE_EMPLACE_IMPL_END \ - [[maybe_unused]] Mapped * cached = nullptr; \ - if constexpr (has_mapped) \ - cached = &it->getMapped(); \ - \ - if (inserted) \ - { \ - if constexpr (has_mapped) \ - { \ - new (&it->getMapped()) Mapped(); \ - } \ - } \ - \ - if constexpr (consecutive_keys_optimization) \ - { \ - cache.found = true; \ - cache.empty = false; \ - \ - if constexpr (has_mapped) \ - { \ - cache.value.first = it->getKey(); \ - cache.value.second = it->getMapped(); \ - cached = &cache.value.second; \ - } \ - else \ - { \ - cache.value = it->getKey(); \ - } \ - } \ - \ - if constexpr (has_mapped) \ - return EmplaceResult(it->getMapped(), *cached, inserted); \ - else \ - return EmplaceResult(inserted); - // This method is performance critical, so there are two emplaceImpl to make sure caller can use the one they need. template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { - DEFINE_EMPLACE_IMPL_BEGIN + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.found && cache.check(keyHolderGetKey(key_holder))) + { + if constexpr (has_mapped) + return EmplaceResult(cache.value.second, cache.value.second, false); + else + return EmplaceResult(false); + } + } + + typename Data::LookupResult it; + bool inserted = false; + data.emplace(key_holder, it, inserted, hashval); - DEFINE_EMPLACE_IMPL_END + + return handleEmplaceResult(it, inserted); } template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data) { - DEFINE_EMPLACE_IMPL_BEGIN + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.found && cache.check(keyHolderGetKey(key_holder))) + { + if constexpr (has_mapped) + return EmplaceResult(cache.value.second, cache.value.second, false); + else + return EmplaceResult(false); + } + } + + typename Data::LookupResult it; + bool inserted = false; + data.emplace(key_holder, it, inserted); - DEFINE_EMPLACE_IMPL_END + + return handleEmplaceResult(it, inserted); + } + + template + ALWAYS_INLINE inline EmplaceResult handleEmplaceResult(typename Data::LookupResult & it, bool inserted) + { + [[maybe_unused]] Mapped * cached = nullptr; + if constexpr (has_mapped) + cached = &it->getMapped(); + + if (inserted) + { + if constexpr (has_mapped) + { + new (&it->getMapped()) Mapped(); + } + } + + if constexpr (consecutive_keys_optimization) + { + cache.found = true; + cache.empty = false; + + if constexpr (has_mapped) + { + cache.value.first = it->getKey(); + cache.value.second = it->getMapped(); + cached = &cache.value.second; + } + else + { + cache.value = it->getKey(); + } + } + + if constexpr (has_mapped) + return EmplaceResult(it->getMapped(), *cached, inserted); + else + return EmplaceResult(inserted); } -#undef DEFINE_EMPLACE_IMPL_BEGIN -#undef DEFINE_EMPLACE_IMPL_END - -#define DEFINE_FIND_IMPL_BEGIN \ - if constexpr (Cache::consecutive_keys_optimization) \ - { \ - if (cache.check(key)) \ - { \ - if constexpr (has_mapped) \ - return FindResult(&cache.value.second, cache.found); \ - else \ - return FindResult(cache.found); \ - } \ - } \ - typename Data::LookupResult it; - -#define DEFINE_FIND_IMPL_END \ - if constexpr (consecutive_keys_optimization) \ - { \ - cache.found = it != nullptr; \ - cache.empty = false; \ - \ - if constexpr (has_mapped) \ - { \ - cache.value.first = key; \ - if (it) \ - { \ - cache.value.second = it->getMapped(); \ - } \ - } \ - else \ - { \ - cache.value = key; \ - } \ - } \ - \ - if constexpr (has_mapped) \ - return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); \ - else \ - return FindResult(it != nullptr); template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data) { - DEFINE_FIND_IMPL_BEGIN + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.check(key)) + { + if constexpr (has_mapped) + return FindResult(&cache.value.second, cache.found); + else + return FindResult(cache.found); + } + } + + typename Data::LookupResult it; it = data.find(key); - DEFINE_FIND_IMPL_END + + return handleFindResult(key, it); } template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data, size_t hashval) { - DEFINE_FIND_IMPL_BEGIN + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.check(key)) + { + if constexpr (has_mapped) + return FindResult(&cache.value.second, cache.found); + else + return FindResult(cache.found); + } + } + + typename Data::LookupResult it; it = data.find(key, hashval); - DEFINE_FIND_IMPL_END + + return handleFindResult(key, it); } -#undef DEFINE_FIND_IMPL_BEGIN -#undef DEFINE_FIND_IMPL_END + + template + ALWAYS_INLINE inline FindResult handleFindResult(Key & key, typename Data::LookupResult & it) + { + if constexpr (consecutive_keys_optimization) + { + cache.found = it != nullptr; + cache.empty = false; + + if constexpr (has_mapped) + { + cache.value.first = key; + if (it) + { + cache.value.second = it->getMapped(); + } + } + else + { + cache.value = key; + } + } + + if constexpr (has_mapped) + return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); + else + return FindResult(it != nullptr); + } + }; From a7361e10289311fb24cb6a51f7fc5fa2f8f7f0df Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 27 Dec 2024 15:46:50 +0800 Subject: [PATCH 48/61] fix Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 197 +++++++++++++-------------- dbms/src/Interpreters/Aggregator.cpp | 143 ++++++++++--------- 2 files changed, 168 insertions(+), 172 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index fa3e5de8a9d..4797cd07f4c 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -198,18 +198,18 @@ class HashMethodBase template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.found && cache.check(keyHolderGetKey(key_holder))) - { - if constexpr (has_mapped) - return EmplaceResult(cache.value.second, cache.value.second, false); - else - return EmplaceResult(false); - } - } - - typename Data::LookupResult it; + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.found && cache.check(keyHolderGetKey(key_holder))) + { + if constexpr (has_mapped) + return EmplaceResult(cache.value.second, cache.value.second, false); + else + return EmplaceResult(false); + } + } + + typename Data::LookupResult it; bool inserted = false; data.emplace(key_holder, it, inserted, hashval); @@ -220,18 +220,18 @@ class HashMethodBase template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.found && cache.check(keyHolderGetKey(key_holder))) - { - if constexpr (has_mapped) - return EmplaceResult(cache.value.second, cache.value.second, false); - else - return EmplaceResult(false); - } - } - - typename Data::LookupResult it; + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.found && cache.check(keyHolderGetKey(key_holder))) + { + if constexpr (has_mapped) + return EmplaceResult(cache.value.second, cache.value.second, false); + else + return EmplaceResult(false); + } + } + + typename Data::LookupResult it; bool inserted = false; data.emplace(key_holder, it, inserted); @@ -242,54 +242,54 @@ class HashMethodBase template ALWAYS_INLINE inline EmplaceResult handleEmplaceResult(typename Data::LookupResult & it, bool inserted) { - [[maybe_unused]] Mapped * cached = nullptr; - if constexpr (has_mapped) - cached = &it->getMapped(); - - if (inserted) - { - if constexpr (has_mapped) - { - new (&it->getMapped()) Mapped(); - } - } - - if constexpr (consecutive_keys_optimization) - { - cache.found = true; - cache.empty = false; - - if constexpr (has_mapped) - { - cache.value.first = it->getKey(); - cache.value.second = it->getMapped(); - cached = &cache.value.second; - } - else - { - cache.value = it->getKey(); - } - } - - if constexpr (has_mapped) - return EmplaceResult(it->getMapped(), *cached, inserted); - else + [[maybe_unused]] Mapped * cached = nullptr; + if constexpr (has_mapped) + cached = &it->getMapped(); + + if (inserted) + { + if constexpr (has_mapped) + { + new (&it->getMapped()) Mapped(); + } + } + + if constexpr (consecutive_keys_optimization) + { + cache.found = true; + cache.empty = false; + + if constexpr (has_mapped) + { + cache.value.first = it->getKey(); + cache.value.second = it->getMapped(); + cached = &cache.value.second; + } + else + { + cache.value = it->getKey(); + } + } + + if constexpr (has_mapped) + return EmplaceResult(it->getMapped(), *cached, inserted); + else return EmplaceResult(inserted); } template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.check(key)) - { - if constexpr (has_mapped) - return FindResult(&cache.value.second, cache.found); - else - return FindResult(cache.found); - } - } + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.check(key)) + { + if constexpr (has_mapped) + return FindResult(&cache.value.second, cache.found); + else + return FindResult(cache.found); + } + } typename Data::LookupResult it; it = data.find(key); @@ -300,16 +300,16 @@ class HashMethodBase template ALWAYS_INLINE inline FindResult findKeyImpl(Key & key, Data & data, size_t hashval) { - if constexpr (Cache::consecutive_keys_optimization) - { - if (cache.check(key)) - { - if constexpr (has_mapped) - return FindResult(&cache.value.second, cache.found); - else - return FindResult(cache.found); - } - } + if constexpr (Cache::consecutive_keys_optimization) + { + if (cache.check(key)) + { + if constexpr (has_mapped) + return FindResult(&cache.value.second, cache.found); + else + return FindResult(cache.found); + } + } typename Data::LookupResult it; it = data.find(key, hashval); @@ -320,31 +320,30 @@ class HashMethodBase template ALWAYS_INLINE inline FindResult handleFindResult(Key & key, typename Data::LookupResult & it) { - if constexpr (consecutive_keys_optimization) - { - cache.found = it != nullptr; - cache.empty = false; - - if constexpr (has_mapped) - { - cache.value.first = key; - if (it) - { - cache.value.second = it->getMapped(); - } - } - else - { - cache.value = key; - } - } - - if constexpr (has_mapped) - return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); - else + if constexpr (consecutive_keys_optimization) + { + cache.found = it != nullptr; + cache.empty = false; + + if constexpr (has_mapped) + { + cache.value.first = key; + if (it) + { + cache.value.second = it->getMapped(); + } + } + else + { + cache.value = key; + } + } + + if constexpr (has_mapped) + return FindResult(it ? &it->getMapped() : nullptr, it != nullptr); + else return FindResult(it != nullptr); } - }; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 9edc2b97b2a..ebde1cbc7d3 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -824,33 +824,10 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( { /// For all rows. AggregateDataPtr place = aggregates_pool->alloc(0); -#define HANDLE_AGG_EMPLACE_RESULT \ - if likely (emplace_result_hold.has_value()) \ - { \ - if constexpr (collect_hit_rate) \ - { \ - ++agg_process_info.hit_row_cnt; \ - } \ - \ - if constexpr (only_lookup) \ - { \ - if (!emplace_result_hold.value().isFound()) \ - agg_process_info.not_found_rows.push_back(i); \ - } \ - else \ - { \ - emplace_result_hold.value().setMapped(place); \ - } \ - processed_rows = i; \ - } \ - else \ - { \ - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); \ - break; \ - } - std::optional processed_rows; + std::optional::ResultType> emplace_result_holder; const auto end = agg_process_info.start_row + rows; + for (size_t i = agg_process_info.start_row; i < end; ++i) { if constexpr (enable_prefetch) @@ -866,16 +843,36 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( hashvals, key_holders); - auto emplace_result_hold = emplaceOrFindKey(method, state, std::move(key_holder), hashval); - - HANDLE_AGG_EMPLACE_RESULT + emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); } else { - auto emplace_result_hold + emplace_result_holder = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); + } - HANDLE_AGG_EMPLACE_RESULT + if likely (emplace_result_holder.has_value()) + { + if constexpr (collect_hit_rate) + { + ++agg_process_info.hit_row_cnt; + } + + if constexpr (only_lookup) + { + if (!emplace_result_holder.value().isFound()) + agg_process_info.not_found_rows.push_back(i); + } + else + { + emplace_result_holder.value().setMapped(place); + } + processed_rows = i; + } + else + { + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); + break; } } @@ -952,8 +949,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( hashvals, key_holders); - emplace_result_holder - = emplaceOrFindKey(method, state, std::move(key_holder), hashval); + emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); } else { @@ -961,47 +957,48 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( = emplaceOrFindKey(method, state, j, *aggregates_pool, sort_key_containers); } - if unlikely (!emplace_result_holder.has_value()) - { - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); - break; - } - - auto & emplace_result = emplace_result_holder.value(); - if constexpr (only_lookup) - { - if (emplace_result.isFound()) - { - aggregate_data = emplace_result.getMapped(); - } - else - { - agg_process_info.not_found_rows.push_back(j); - } - } - else - { - if (emplace_result.isInserted()) - { - emplace_result.setMapped(nullptr); - - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); - - emplace_result.setMapped(aggregate_data); - } - else - { - aggregate_data = emplace_result.getMapped(); - - if constexpr (collect_hit_rate) - ++agg_process_info.hit_row_cnt; - - if constexpr (enable_prefetch) - __builtin_prefetch(aggregate_data); - } - } - places[j - agg_process_info.start_row] = aggregate_data; + if unlikely (!emplace_result_holder.has_value()) + { + LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); + break; + } + + auto emplace_result = emplace_result_holder.value(); + if constexpr (only_lookup) + { + if (emplace_result.isFound()) + { + aggregate_data = emplace_result.getMapped(); + } + else + { + agg_process_info.not_found_rows.push_back(j); + } + } + else + { + if (emplace_result.isInserted()) + { + emplace_result.setMapped(nullptr); + + aggregate_data + = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + + emplace_result.setMapped(aggregate_data); + } + else + { + aggregate_data = emplace_result.getMapped(); + + if constexpr (collect_hit_rate) + ++agg_process_info.hit_row_cnt; + + if constexpr (enable_prefetch) + __builtin_prefetch(aggregate_data); + } + } + places[j - agg_process_info.start_row] = aggregate_data; processed_rows = j; } From bdb21d367dbf32824e9f81c14dbd56e2641cc7e9 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 27 Dec 2024 16:05:36 +0800 Subject: [PATCH 49/61] refine comment Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 15c7ef78a9e..a03136bbed8 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -52,17 +52,14 @@ struct HashMethodOneNumber static constexpr bool is_serialized_key = false; const FieldType * vec; - const size_t total_rows; /// If the keys of a fixed length then key_sizes contains their lengths, empty otherwise. HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) - : total_rows(key_columns[0]->size()) { vec = &static_cast *>(key_columns[0])->getData()[0]; } explicit HashMethodOneNumber(const IColumn * column) - : total_rows(column->size()) { vec = &static_cast *>(column)->getData()[0]; } @@ -105,13 +102,11 @@ struct HashMethodString const IColumn::Offset * offsets; const UInt8 * chars; TiDB::TiDBCollatorPtr collator = nullptr; - const size_t total_rows; HashMethodString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators & collators) - : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); @@ -151,10 +146,8 @@ struct HashMethodStringBin const IColumn::Offset * offsets; const UInt8 * chars; - const size_t total_rows; HashMethodStringBin(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators &) - : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); @@ -353,12 +346,10 @@ struct HashMethodFastPathTwoKeysSerialized Key1Desc key_1_desc; Key2Desc key_2_desc; - const size_t total_rows; HashMethodFastPathTwoKeysSerialized(const ColumnRawPtrs & key_columns, const Sizes &, const TiDB::TiDBCollators &) : key_1_desc(key_columns[0]) , key_2_desc(key_columns[1]) - , total_rows(key_columns[0]->size()) {} ALWAYS_INLINE inline KeyHolderType getKeyHolder(ssize_t row, Arena * pool, std::vector &) const @@ -393,13 +384,11 @@ struct HashMethodFixedString size_t n; const ColumnFixedString::Chars_t * chars; TiDB::TiDBCollatorPtr collator = nullptr; - const size_t total_rows; HashMethodFixedString( const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators & collators) - : total_rows(key_columns[0]->size()) { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); @@ -443,7 +432,6 @@ struct HashMethodKeysFixed Sizes key_sizes; size_t keys_size; - const size_t total_rows; /// SSSE3 shuffle method can be used. Shuffle masks will be calculated and stored here. #if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) @@ -469,7 +457,6 @@ struct HashMethodKeysFixed : Base(key_columns) , key_sizes(std::move(key_sizes_)) , keys_size(key_columns.size()) - , total_rows(key_columns[0]->size()) { if (usePreparedKeys(key_sizes)) { @@ -602,7 +589,6 @@ struct HashMethodSerialized ColumnRawPtrs key_columns; size_t keys_size; TiDB::TiDBCollators collators; - const size_t total_rows; HashMethodSerialized( const ColumnRawPtrs & key_columns_, @@ -611,7 +597,6 @@ struct HashMethodSerialized : key_columns(key_columns_) , keys_size(key_columns_.size()) , collators(collators_) - , total_rows(key_columns_[0]->size()) {} ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena * pool, std::vector & sort_key_containers) @@ -640,12 +625,10 @@ struct HashMethodHashed ColumnRawPtrs key_columns; TiDB::TiDBCollators collators; - const size_t total_rows; HashMethodHashed(ColumnRawPtrs key_columns_, const Sizes &, const TiDB::TiDBCollators & collators_) : key_columns(std::move(key_columns_)) , collators(collators_) - , total_rows(key_columns[0]->size()) {} ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena *, std::vector & sort_key_containers) From 4153592b66d737ef8ca96a8b8acf627ebf3cc666 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 27 Dec 2024 16:18:43 +0800 Subject: [PATCH 50/61] refine Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashingImpl.h | 3 --- dbms/src/Interpreters/Aggregator.cpp | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index 4797cd07f4c..bf130d2bd29 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -130,7 +129,6 @@ class HashMethodBase using Cache = LastElementCache; using Derived = TDerived; - // Emplace key without hashval, and this method doesn't support prefetch. template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, @@ -194,7 +192,6 @@ class HashMethodBase } } - // This method is performance critical, so there are two emplaceImpl to make sure caller can use the one they need. template ALWAYS_INLINE inline EmplaceResult emplaceImpl(KeyHolder & key_holder, Data & data, size_t hashval) { diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index ebde1cbc7d3..becb64c132b 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -879,7 +879,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; -#undef HANDLE_AGG_EMPLACE_RESULT return; } @@ -1026,8 +1025,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if likely (processed_rows) agg_process_info.start_row = *processed_rows + 1; - -#undef HANDLE_AGG_EMPLACE_RESULT } void NO_INLINE @@ -1877,7 +1874,7 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( ++data_index; }); - size_t prefetch_idx = 16; + size_t prefetch_idx = agg_prefetch_step; for (size_t i = 0; i < rows; ++i) { if (prefetch_idx < rows) From bbd5c45c1f5672f9f680258726e2c1dce2ef1ec5 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 29 Dec 2024 17:28:15 +0800 Subject: [PATCH 51/61] vec 16 Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index becb64c132b..b145df5691c 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -748,8 +748,8 @@ std::optional::Res template ALWAYS_INLINE inline std::pair getCurrentHashAndDoPrefetch( - size_t cur_row_idx, - size_t start_row_idx, + size_t hashvals_idx, + size_t prefetch_row_idx, size_t end_row_idx, Method & method, typename Method::State & state, @@ -760,9 +760,6 @@ ALWAYS_INLINE inline std::pair Date: Tue, 7 Jan 2025 11:06:37 +0800 Subject: [PATCH 52/61] refine Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index b145df5691c..5f6e69ced0d 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -830,7 +830,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if constexpr (enable_prefetch) { auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( - (i - agg_process_info.start_row) % 15, + (i - agg_process_info.start_row) % agg_prefetch_step, i + agg_prefetch_step, end, method, @@ -932,11 +932,11 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( for (size_t j = i; j < cur_batch_end; ++j) { AggregateDataPtr aggregate_data = nullptr; - const size_t relative_idx = j - agg_process_info.start_row; + const size_t index_relative_to_start_row = j - agg_process_info.start_row; if constexpr (enable_prefetch) { auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( - relative_idx & 15, + index_relative_to_start_row % agg_prefetch_step, j + agg_prefetch_step, end, method, @@ -995,7 +995,7 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( __builtin_prefetch(aggregate_data); } } - places[relative_idx] = aggregate_data; + places[index_relative_to_start_row] = aggregate_data; processed_rows = j; } From 530bb27cdee732ce2340be2c938b16b52ee57477 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 7 Jan 2025 11:26:12 +0800 Subject: [PATCH 53/61] compute hashvals before each mini batch Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 107 +++++++++++++++++++-------- 1 file changed, 78 insertions(+), 29 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 5f6e69ced0d..8c5b6ae31aa 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -746,6 +746,23 @@ std::optional::Res } } +template +ALWAYS_INLINE inline void prepareBatch(size_t row_idx, size_t end_row, std::vector & hashvals, std::vector & key_holders, + Arena * aggregates_pool, std::vector & sort_key_containers, Method & method, typename Method::State & state) +{ + assert(hashvals.size() == key_holders.size()); + + size_t j = 0; + for (size_t i = row_idx; i < row_idx + hashvals.size() && i < end_row; ++i, ++j) + { + key_holders[j] = static_cast(&state)->getKeyHolder( + i, + aggregates_pool, + sort_key_containers); + hashvals[j] = method.data.hash(keyHolderGetKey(key_holders[j])); + } +} + template ALWAYS_INLINE inline std::pair getCurrentHashAndDoPrefetch( size_t hashvals_idx, @@ -799,22 +816,22 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( std::vector hashvals; std::vector key_holders; - if constexpr (enable_prefetch) - { - hashvals.resize(agg_prefetch_step); - key_holders.resize(agg_prefetch_step); - for (size_t i = 0; i < agg_prefetch_step && i + agg_process_info.start_row < agg_process_info.end_row; ++i) - { - auto key_holder = static_cast(&state)->getKeyHolder( - i + agg_process_info.start_row, - aggregates_pool, - sort_key_containers); - const size_t hashval = method.data.hash(keyHolderGetKey(key_holder)); - - key_holders[i] = std::move(key_holder); - hashvals[i] = hashval; - } - } + // if constexpr (enable_prefetch) + // { + // hashvals.resize(agg_prefetch_step); + // key_holders.resize(agg_prefetch_step); + // for (size_t i = 0; i < agg_prefetch_step && i + agg_process_info.start_row < agg_process_info.end_row; ++i) + // { + // auto key_holder = static_cast(&state)->getKeyHolder( + // i + agg_process_info.start_row, + // aggregates_pool, + // sort_key_containers); + // const size_t hashval = method.data.hash(keyHolderGetKey(key_holder)); + + // key_holders[i] = std::move(key_holder); + // hashvals[i] = hashval; + // } + // } /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) @@ -825,6 +842,23 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( std::optional::ResultType> emplace_result_holder; const auto end = agg_process_info.start_row + rows; + if constexpr (enable_prefetch) + { + hashvals.resize(agg_prefetch_step); + key_holders.resize(agg_prefetch_step); + for (size_t i = 0; i < agg_prefetch_step && i + agg_process_info.start_row < agg_process_info.end_row; ++i) + { + auto key_holder = static_cast(&state)->getKeyHolder( + i + agg_process_info.start_row, + aggregates_pool, + sort_key_containers); + const size_t hashval = method.data.hash(keyHolderGetKey(key_holder)); + + key_holders[i] = std::move(key_holder); + hashvals[i] = hashval; + } + } + for (size_t i = agg_process_info.start_row; i < end; ++i) { if constexpr (enable_prefetch) @@ -920,6 +954,12 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( size_t i = agg_process_info.start_row; const size_t end = agg_process_info.start_row + rows; const size_t mini_batch = 256; + + // std::vector hashvals; + hashvals.resize(mini_batch); + // std::vector key_holders; + key_holders.resize(mini_batch); + // i is the begin row index of each mini batch. while (i < end) { @@ -927,26 +967,35 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( if unlikely (i + batch_size > end) batch_size = end - i; + if constexpr (enable_prefetch) + prepareBatch(i, end, hashvals, key_holders, aggregates_pool, sort_key_containers, method, state); + const auto cur_batch_end = i + batch_size; // j is the row index inside each mini batch. - for (size_t j = i; j < cur_batch_end; ++j) + size_t k = 0; + for (size_t j = i; j < cur_batch_end; ++j, ++k) { AggregateDataPtr aggregate_data = nullptr; const size_t index_relative_to_start_row = j - agg_process_info.start_row; if constexpr (enable_prefetch) { - auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( - index_relative_to_start_row % agg_prefetch_step, - j + agg_prefetch_step, - end, - method, - state, - aggregates_pool, - sort_key_containers, - hashvals, - key_holders); - - emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); + // auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( + // index_relative_to_start_row % agg_prefetch_step, + // j + agg_prefetch_step, + // end, + // method, + // state, + // aggregates_pool, + // sort_key_containers, + // hashvals, + // key_holders); + + // emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); + + if unlikely (k + agg_prefetch_step < hashvals.size()) + method.data.prefetch(hashvals[k + agg_prefetch_step]); + + emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); } else { From 358e8518d01f99ed7efaceb453b3ca9629d4ca88 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 7 Jan 2025 16:35:06 +0800 Subject: [PATCH 54/61] handleMiniBatchImpl Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 247 ++++++++++----------------- dbms/src/Interpreters/Aggregator.h | 9 +- 2 files changed, 94 insertions(+), 162 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 8c5b6ae31aa..5eeba25b5c3 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -47,6 +47,7 @@ extern const char force_agg_prefetch[]; } // namespace FailPoints static constexpr size_t agg_prefetch_step = 16; +static constexpr size_t agg_mini_batch = 256; #define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME #define AggregationMethodNameTwoLevel(NAME) AggregatedDataVariants::AggregationMethod_##NAME##_two_level @@ -684,24 +685,19 @@ void NO_INLINE Aggregator::executeImpl( // 2. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. if constexpr (Method::State::is_serialized_key) { - // TODO: batch serialize method for Columns is still under development. - // if (!disable_prefetch) - // executeImplSerializedKeyByCol(); - // else - // executeImplByRow(method, state, aggregates_pool, agg_process_info); - executeImplByRow(method, state, aggregates_pool, agg_process_info); + executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); } else if constexpr (Method::Data::is_string_hash_map) { // StringHashMap doesn't support prefetch. - executeImplByRow(method, state, aggregates_pool, agg_process_info); + executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); } else { if (disable_prefetch) - executeImplByRow(method, state, aggregates_pool, agg_process_info); + executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); else - executeImplByRow(method, state, aggregates_pool, agg_process_info); + executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); } } @@ -797,7 +793,7 @@ ALWAYS_INLINE inline std::pair -ALWAYS_INLINE void Aggregator::executeImplByRow( +ALWAYS_INLINE void Aggregator::executeImplMiniBatch( Method & method, typename Method::State & state, Arena * aggregates_pool, @@ -806,116 +802,19 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( // collect_hit_rate and only_lookup cannot be true at the same time. static_assert(!(collect_hit_rate && only_lookup)); - std::vector sort_key_containers; - sort_key_containers.resize(params.keys_size, ""); - size_t rows = agg_process_info.end_row - agg_process_info.start_row; - fiu_do_on(FailPoints::force_agg_on_partial_block, { - if (rows > 0 && agg_process_info.start_row == 0) - rows = std::max(rows / 2, 1); - }); - - std::vector hashvals; - std::vector key_holders; - // if constexpr (enable_prefetch) - // { - // hashvals.resize(agg_prefetch_step); - // key_holders.resize(agg_prefetch_step); - // for (size_t i = 0; i < agg_prefetch_step && i + agg_process_info.start_row < agg_process_info.end_row; ++i) - // { - // auto key_holder = static_cast(&state)->getKeyHolder( - // i + agg_process_info.start_row, - // aggregates_pool, - // sort_key_containers); - // const size_t hashval = method.data.hash(keyHolderGetKey(key_holder)); - - // key_holders[i] = std::move(key_holder); - // hashvals[i] = hashval; - // } - // } - /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) - { - /// For all rows. - AggregateDataPtr place = aggregates_pool->alloc(0); - std::optional processed_rows; - std::optional::ResultType> emplace_result_holder; - const auto end = agg_process_info.start_row + rows; - - if constexpr (enable_prefetch) - { - hashvals.resize(agg_prefetch_step); - key_holders.resize(agg_prefetch_step); - for (size_t i = 0; i < agg_prefetch_step && i + agg_process_info.start_row < agg_process_info.end_row; ++i) - { - auto key_holder = static_cast(&state)->getKeyHolder( - i + agg_process_info.start_row, - aggregates_pool, - sort_key_containers); - const size_t hashval = method.data.hash(keyHolderGetKey(key_holder)); - - key_holders[i] = std::move(key_holder); - hashvals[i] = hashval; - } - } - - for (size_t i = agg_process_info.start_row; i < end; ++i) - { - if constexpr (enable_prefetch) - { - auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( - (i - agg_process_info.start_row) % agg_prefetch_step, - i + agg_prefetch_step, - end, - method, - state, - aggregates_pool, - sort_key_containers, - hashvals, - key_holders); - - emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); - } - else - { - emplace_result_holder - = emplaceOrFindKey(method, state, i, *aggregates_pool, sort_key_containers); - } - - if likely (emplace_result_holder.has_value()) - { - if constexpr (collect_hit_rate) - { - ++agg_process_info.hit_row_cnt; - } - - if constexpr (only_lookup) - { - if (!emplace_result_holder.value().isFound()) - agg_process_info.not_found_rows.push_back(i); - } - else - { - emplace_result_holder.value().setMapped(place); - } - processed_rows = i; - } - else - { - LOG_INFO(log, "HashTable resize throw ResizeException since the data is already marked for spill"); - break; - } - } - - if likely (processed_rows) - agg_process_info.start_row = *processed_rows + 1; - - return; - } + return handleMiniBatchImpl(method, state, agg_process_info, aggregates_pool); /// Optimization for special case when aggregating by 8bit key. if constexpr (std::is_same_v) { + size_t rows = agg_process_info.end_row - agg_process_info.start_row; + fiu_do_on(FailPoints::force_agg_on_partial_block, { + if (rows > 0 && agg_process_info.start_row == 0) + rows = std::max(rows / 2, 1); + }); + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; ++inst) { @@ -947,23 +846,42 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( } /// Generic case. - std::unique_ptr places(new AggregateDataPtr[rows]); + return handleMiniBatchImpl(method, state, agg_process_info, aggregates_pool); +} + +template +void Aggregator::handleMiniBatchImpl( + Method & method, + typename Method::State & state, + AggProcessInfo & agg_process_info, + Arena * aggregates_pool) const +{ + std::vector sort_key_containers; + sort_key_containers.resize(params.keys_size, ""); + size_t rows = agg_process_info.end_row - agg_process_info.start_row; + fiu_do_on(FailPoints::force_agg_on_partial_block, { + if (rows > 0 && agg_process_info.start_row == 0) + rows = std::max(rows / 2, 1); + }); + std::optional processed_rows; std::optional::ResultType> emplace_result_holder; + std::unique_ptr places{}; + auto * place = reinterpret_cast(0x1); + if constexpr (compute_agg_data) + places = std::unique_ptr(new AggregateDataPtr[rows]); + size_t i = agg_process_info.start_row; const size_t end = agg_process_info.start_row + rows; - const size_t mini_batch = 256; - // std::vector hashvals; - hashvals.resize(mini_batch); - // std::vector key_holders; - key_holders.resize(mini_batch); + std::vector hashvals(agg_mini_batch); + std::vector key_holders(agg_mini_batch); // i is the begin row index of each mini batch. while (i < end) { - size_t batch_size = mini_batch; + size_t batch_size = agg_mini_batch; if unlikely (i + batch_size > end) batch_size = end - i; @@ -971,7 +889,8 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( prepareBatch(i, end, hashvals, key_holders, aggregates_pool, sort_key_containers, method, state); const auto cur_batch_end = i + batch_size; - // j is the row index inside each mini batch. + // j is the row index of Column. + // k is the index of hashvals/key_holders. size_t k = 0; for (size_t j = i; j < cur_batch_end; ++j, ++k) { @@ -979,19 +898,6 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( const size_t index_relative_to_start_row = j - agg_process_info.start_row; if constexpr (enable_prefetch) { - // auto [key_holder, hashval] = getCurrentHashAndDoPrefetch( - // index_relative_to_start_row % agg_prefetch_step, - // j + agg_prefetch_step, - // end, - // method, - // state, - // aggregates_pool, - // sort_key_containers, - // hashvals, - // key_holders); - - // emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holder), hashval); - if unlikely (k + agg_prefetch_step < hashvals.size()) method.data.prefetch(hashvals[k + agg_prefetch_step]); @@ -1012,39 +918,55 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( auto emplace_result = emplace_result_holder.value(); if constexpr (only_lookup) { - if (emplace_result.isFound()) + if constexpr (compute_agg_data) { - aggregate_data = emplace_result.getMapped(); + if (emplace_result.isFound()) + { + aggregate_data = emplace_result.getMapped(); + } + else + { + agg_process_info.not_found_rows.push_back(j); + } } else { - agg_process_info.not_found_rows.push_back(j); + if (!emplace_result_holder.value().isFound()) + agg_process_info.not_found_rows.push_back(i); } } else { - if (emplace_result.isInserted()) + if constexpr (compute_agg_data) { - emplace_result.setMapped(nullptr); + if (emplace_result.isInserted()) + { + emplace_result.setMapped(nullptr); - aggregate_data - = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); + aggregate_data + = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + + emplace_result.setMapped(aggregate_data); + } + else + { + aggregate_data = emplace_result.getMapped(); + + if constexpr (collect_hit_rate) + ++agg_process_info.hit_row_cnt; - emplace_result.setMapped(aggregate_data); + if constexpr (enable_prefetch) + __builtin_prefetch(aggregate_data); + } } else { - aggregate_data = emplace_result.getMapped(); - - if constexpr (collect_hit_rate) - ++agg_process_info.hit_row_cnt; - - if constexpr (enable_prefetch) - __builtin_prefetch(aggregate_data); + emplace_result_holder.value().setMapped(place); } } - places[index_relative_to_start_row] = aggregate_data; + if constexpr (compute_agg_data) + places[index_relative_to_start_row] = aggregate_data; processed_rows = j; } @@ -1052,16 +974,19 @@ ALWAYS_INLINE void Aggregator::executeImplByRow( break; const size_t processed_size = *processed_rows - i + 1; - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; - ++inst) + if constexpr (compute_agg_data) { - inst->batch_that->addBatch( - i, - processed_size, - places.get() + i - agg_process_info.start_row, - inst->state_offset, - inst->batch_arguments, - aggregates_pool); + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + ++inst) + { + inst->batch_that->addBatch( + i, + processed_size, + places.get() + i - agg_process_info.start_row, + inst->state_offset, + inst->batch_arguments, + aggregates_pool); + } } if unlikely (processed_size != batch_size) diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index c55edcbd85c..7ab58d01818 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1454,12 +1454,19 @@ class Aggregator TiDB::TiDBCollators & collators) const; template - void executeImplByRow( + void executeImplMiniBatch( Method & method, typename Method::State & state, Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; + template + void handleMiniBatchImpl( + Method & method, + typename Method::State & state, + AggProcessInfo & agg_process_info, + Arena * aggregates_pool) const; + template std::optional::ResultType> emplaceOrFindKey( Method & method, From 3d79a209d197f4b34a3da3765c1f32030034214d Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 7 Jan 2025 17:15:03 +0800 Subject: [PATCH 55/61] fmt Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 43 +++++++++++++++++++++------- dbms/src/Interpreters/Aggregator.h | 15 ++++++---- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 5eeba25b5c3..4e9fba4a988 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -695,7 +695,11 @@ void NO_INLINE Aggregator::executeImpl( else { if (disable_prefetch) - executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); + executeImplMiniBatch( + method, + state, + aggregates_pool, + agg_process_info); else executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); } @@ -743,8 +747,15 @@ std::optional::Res } template -ALWAYS_INLINE inline void prepareBatch(size_t row_idx, size_t end_row, std::vector & hashvals, std::vector & key_holders, - Arena * aggregates_pool, std::vector & sort_key_containers, Method & method, typename Method::State & state) +ALWAYS_INLINE inline void prepareBatch( + size_t row_idx, + size_t end_row, + std::vector & hashvals, + std::vector & key_holders, + Arena * aggregates_pool, + std::vector & sort_key_containers, + Method & method, + typename Method::State & state) { assert(hashvals.size() == key_holders.size()); @@ -804,7 +815,11 @@ ALWAYS_INLINE void Aggregator::executeImplMiniBatch( /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) - return handleMiniBatchImpl(method, state, agg_process_info, aggregates_pool); + return handleMiniBatchImpl( + method, + state, + agg_process_info, + aggregates_pool); /// Optimization for special case when aggregating by 8bit key. if constexpr (std::is_same_v) @@ -846,15 +861,19 @@ ALWAYS_INLINE void Aggregator::executeImplMiniBatch( } /// Generic case. - return handleMiniBatchImpl(method, state, agg_process_info, aggregates_pool); + return handleMiniBatchImpl( + method, + state, + agg_process_info, + aggregates_pool); } template void Aggregator::handleMiniBatchImpl( - Method & method, - typename Method::State & state, - AggProcessInfo & agg_process_info, - Arena * aggregates_pool) const + Method & method, + typename Method::State & state, + AggProcessInfo & agg_process_info, + Arena * aggregates_pool) const { std::vector sort_key_containers; sort_key_containers.resize(params.keys_size, ""); @@ -901,7 +920,8 @@ void Aggregator::handleMiniBatchImpl( if unlikely (k + agg_prefetch_step < hashvals.size()) method.data.prefetch(hashvals[k + agg_prefetch_step]); - emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); + emplace_result_holder + = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); } else { @@ -976,7 +996,8 @@ void Aggregator::handleMiniBatchImpl( const size_t processed_size = *processed_rows - i + 1; if constexpr (compute_agg_data) { - for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); inst->that; + for (AggregateFunctionInstruction * inst = agg_process_info.aggregate_functions_instructions.data(); + inst->that; ++inst) { inst->batch_that->addBatch( diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 7ab58d01818..cb8c2623b91 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1460,12 +1460,17 @@ class Aggregator Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template + template < + bool collect_hit_rate, + bool only_lookup, + bool enable_prefetch, + bool need_compute_agg_func, + typename Method> void handleMiniBatchImpl( - Method & method, - typename Method::State & state, - AggProcessInfo & agg_process_info, - Arena * aggregates_pool) const; + Method & method, + typename Method::State & state, + AggProcessInfo & agg_process_info, + Arena * aggregates_pool) const; template std::optional::ResultType> emplaceOrFindKey( From 51a90b5baf65a4b2b374ff769aad74c7db03b955 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 7 Jan 2025 17:16:40 +0800 Subject: [PATCH 56/61] remove useless code Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 51 +++------------------------- dbms/src/Interpreters/Aggregator.h | 2 +- 2 files changed, 6 insertions(+), 47 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 4e9fba4a988..2162c10dda9 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -678,11 +678,6 @@ void NO_INLINE Aggregator::executeImpl( const bool disable_prefetch = (method.data.getBufferSizeInBytes() < prefetch_threshold); #endif - // key_serialized needs column-wise handling for prefetch. - // Because: - // 1. getKeyHolder of key_serialized have to copy real data into Arena. - // It means we better getKeyHolder for all Columns once and then use it both for getHash() and emplaceKey(). - // 2. For other group by key(key_int8/16/32/...), it's ok to use row-wise handling even prefetch is enabled. if constexpr (Method::State::is_serialized_key) { executeImplMiniBatch(method, state, aggregates_pool, agg_process_info); @@ -759,8 +754,7 @@ ALWAYS_INLINE inline void prepareBatch( { assert(hashvals.size() == key_holders.size()); - size_t j = 0; - for (size_t i = row_idx; i < row_idx + hashvals.size() && i < end_row; ++i, ++j) + for (size_t i = row_idx, j = 0; i < row_idx + hashvals.size() && i < end_row; ++i, ++j) { key_holders[j] = static_cast(&state)->getKeyHolder( i, @@ -770,39 +764,6 @@ ALWAYS_INLINE inline void prepareBatch( } } -template -ALWAYS_INLINE inline std::pair getCurrentHashAndDoPrefetch( - size_t hashvals_idx, - size_t prefetch_row_idx, - size_t end_row_idx, - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - std::vector & sort_key_containers, - std::vector & hashvals, - std::vector & key_holders) -{ - assert(hashvals.size() == agg_prefetch_step); - - const size_t cur_hashval = hashvals[hashvals_idx]; - auto cur_key_holder = std::move(key_holders[hashvals_idx]); - - if likely (prefetch_row_idx < end_row_idx) - { - auto key_holder = static_cast(&state)->getKeyHolder( - prefetch_row_idx, - aggregates_pool, - sort_key_containers); - const size_t new_hashval = method.data.hash(keyHolderGetKey(key_holder)); - - key_holders[hashvals_idx] = std::move(key_holder); - hashvals[hashvals_idx] = new_hashval; - - method.data.prefetch(new_hashval); - } - return {cur_key_holder, cur_hashval}; -} - template ALWAYS_INLINE void Aggregator::executeImplMiniBatch( Method & method, @@ -815,7 +776,7 @@ ALWAYS_INLINE void Aggregator::executeImplMiniBatch( /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) - return handleMiniBatchImpl( + return handleMiniBatchImpl( method, state, agg_process_info, @@ -910,8 +871,7 @@ void Aggregator::handleMiniBatchImpl( const auto cur_batch_end = i + batch_size; // j is the row index of Column. // k is the index of hashvals/key_holders. - size_t k = 0; - for (size_t j = i; j < cur_batch_end; ++j, ++k) + for (size_t j = i, k = 0; j < cur_batch_end; ++j, ++k) { AggregateDataPtr aggregate_data = nullptr; const size_t index_relative_to_start_row = j - agg_process_info.start_row; @@ -1867,11 +1827,10 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( ++data_index; }); - size_t prefetch_idx = agg_prefetch_step; - for (size_t i = 0; i < rows; ++i) + for (size_t i = 0, prefetch_idx = agg_prefetch_step; i < rows; ++i, ++prefetch_idx) { if (prefetch_idx < rows) - __builtin_prefetch(places[prefetch_idx++]); + __builtin_prefetch(places[prefetch_idx]); size_t key_columns_vec_index = i / params.max_block_size; insertAggregatesIntoColumns(places[i], final_aggregate_columns_vec[key_columns_vec_index], arena); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index cb8c2623b91..4aa06ce7af9 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1464,7 +1464,7 @@ class Aggregator bool collect_hit_rate, bool only_lookup, bool enable_prefetch, - bool need_compute_agg_func, + bool compute_agg_data, typename Method> void handleMiniBatchImpl( Method & method, From e108584a6b32f23159cff1434b71033e64914082 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 7 Jan 2025 17:42:53 +0800 Subject: [PATCH 57/61] minor refine Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 2162c10dda9..160be7e63b5 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -895,7 +895,7 @@ void Aggregator::handleMiniBatchImpl( break; } - auto emplace_result = emplace_result_holder.value(); + auto & emplace_result = emplace_result_holder.value(); if constexpr (only_lookup) { if constexpr (compute_agg_data) @@ -911,7 +911,7 @@ void Aggregator::handleMiniBatchImpl( } else { - if (!emplace_result_holder.value().isFound()) + if (!emplace_result.isFound()) agg_process_info.not_found_rows.push_back(i); } } @@ -942,7 +942,7 @@ void Aggregator::handleMiniBatchImpl( } else { - emplace_result_holder.value().setMapped(place); + emplace_result.setMapped(place); } } if constexpr (compute_agg_data) From 761b34ddb8e0d9185282994eb96a0efbb6f1ebc0 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 7 Jan 2025 17:58:36 +0800 Subject: [PATCH 58/61] fmt Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 4aa06ce7af9..97731afb24f 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1460,12 +1460,7 @@ class Aggregator Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template < - bool collect_hit_rate, - bool only_lookup, - bool enable_prefetch, - bool compute_agg_data, - typename Method> + template void handleMiniBatchImpl( Method & method, typename Method::State & state, From 538bad441eeda0b310c7cd2c6c3d051943112dd3 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 9 Jan 2025 16:42:42 +0800 Subject: [PATCH 59/61] only enable mini batch when prefetch Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 160be7e63b5..16bd2a1748c 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -855,20 +855,30 @@ void Aggregator::handleMiniBatchImpl( size_t i = agg_process_info.start_row; const size_t end = agg_process_info.start_row + rows; - std::vector hashvals(agg_mini_batch); - std::vector key_holders(agg_mini_batch); + size_t mini_batch_size = rows; + std::vector hashvals; + std::vector key_holders; + if constexpr (enable_prefetch) + { + // mini batch will only be used when HashTable is big(a.k.a enable_prefetch is true), + // which can reduce cache miss of agg data. + mini_batch_size = agg_mini_batch; + hashvals.resize(agg_mini_batch); + key_holders.resize(agg_mini_batch); + } // i is the begin row index of each mini batch. while (i < end) { - size_t batch_size = agg_mini_batch; - if unlikely (i + batch_size > end) - batch_size = end - i; - if constexpr (enable_prefetch) + { + if unlikely (i + mini_batch_size > end) + mini_batch_size = end - i; + prepareBatch(i, end, hashvals, key_holders, aggregates_pool, sort_key_containers, method, state); + } - const auto cur_batch_end = i + batch_size; + const auto cur_batch_end = i + mini_batch_size; // j is the row index of Column. // k is the index of hashvals/key_holders. for (size_t j = i, k = 0; j < cur_batch_end; ++j, ++k) @@ -921,6 +931,7 @@ void Aggregator::handleMiniBatchImpl( { if (emplace_result.isInserted()) { + // exception-safety - if you can not allocate memory or create states, then destructors will not be called. emplace_result.setMapped(nullptr); aggregate_data @@ -970,7 +981,7 @@ void Aggregator::handleMiniBatchImpl( } } - if unlikely (processed_size != batch_size) + if unlikely (processed_size != mini_batch_size) break; i = cur_batch_end; From 534301449c878c8a47ddf02954bb40ded567ef23 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 12 Jan 2025 12:16:20 +0800 Subject: [PATCH 60/61] fix Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 16bd2a1748c..a4795b98fc0 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -887,7 +887,7 @@ void Aggregator::handleMiniBatchImpl( const size_t index_relative_to_start_row = j - agg_process_info.start_row; if constexpr (enable_prefetch) { - if unlikely (k + agg_prefetch_step < hashvals.size()) + if likely (k + agg_prefetch_step < hashvals.size()) method.data.prefetch(hashvals[k + agg_prefetch_step]); emplace_result_holder From 9c9ff5e6943f664be3ac5e02d3c60e85898ae109 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 13 Jan 2025 16:26:41 +0800 Subject: [PATCH 61/61] refine convertToBlocksImplFinal Signed-off-by: guo-shaoge --- dbms/src/Interpreters/Aggregator.cpp | 29 ++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index a4795b98fc0..caa86212aaf 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -1827,8 +1827,10 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( const auto rows = data.size(); std::unique_ptr places(new AggregateDataPtr[rows]); + size_t current_bound = params.max_block_size; + size_t key_columns_vec_index = 0; + data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { - size_t key_columns_vec_index = data_index / params.max_block_size; if constexpr (!skip_convert_key) { agg_keys_helpers[key_columns_vec_index] @@ -1836,15 +1838,30 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( } places[data_index] = mapped; ++data_index; + + if unlikely (data_index == current_bound) + { + ++key_columns_vec_index; + current_bound += params.max_block_size; + } }); - for (size_t i = 0, prefetch_idx = agg_prefetch_step; i < rows; ++i, ++prefetch_idx) + data_index = 0; + current_bound = params.max_block_size; + key_columns_vec_index = 0; + while (data_index < rows) { - if (prefetch_idx < rows) - __builtin_prefetch(places[prefetch_idx]); + if (data_index + agg_prefetch_step < rows) + __builtin_prefetch(places[data_index + agg_prefetch_step]); - size_t key_columns_vec_index = i / params.max_block_size; - insertAggregatesIntoColumns(places[i], final_aggregate_columns_vec[key_columns_vec_index], arena); + insertAggregatesIntoColumns(places[data_index], final_aggregate_columns_vec[key_columns_vec_index], arena); + ++data_index; + + if unlikely (data_index == current_bound) + { + ++key_columns_vec_index; + current_bound += params.max_block_size; + } } }