Repository: kuzudb/kuzu
Pull request #5031: Distinct Aggregate vectorization
Author: login: benjaminwinger
Created: 2025-03-11T14:50:57Z | Merged: 2025-03-22T01:05:59Z

This involved refactoring the AggregateHashTable a bit, since the distinct tables end up having mixed flat/unflat vectors when you add the distinct key (since it always is the last column, but may be flat when there are unflat vectors, so the existing split of flat vectors first, then unflat vectors didn't really work).

The performance difference isn't huge for most queries, but it makes a much bigger difference for aggregation over complex data types where there is more overhead when calculating the hashes (the list and struct hashing functions allocate a temporary ValueVector, which is probably the main difference in the performance).

Performance (first is from Clickbench, the others were just constructed to see how similar queries perform with lists/structs as the keys):
| Query | Before | After |
| --- | --- | --- |
| `MATCH (h:hits) RETURN h.RegionID, COUNT(DISTINCT h.UserID) AS u ORDER BY u DESC LIMIT 10;` | 2.25s | 2.1s |
| `MATCH (h:hits) RETURN h.RegionID, COUNT(DISTINCT [h.UserID]) AS u ORDER BY u DESC LIMIT 10;` | 17.5s | 2.7s |
| `MATCH (h:hits) RETURN h.RegionID, COUNT(DISTINCT {"id": h.UserID}) AS u ORDER BY u DESC LIMIT 10;` | 3.7s | 2.3s |


Diff:
diff --git a/src/include/planner/operator/factorization/flatten_resolver.h b/src/include/planner/operator/factorization/flatten_resolver.h
index 62bafbcd081..b99124468e0 100644
--- a/src/include/planner/operator/factorization/flatten_resolver.h
+++ b/src/include/planner/operator/factorization/flatten_resolver.h
@@ -8,8 +8,8 @@ namespace planner {
 class GroupDependencyAnalyzer;
 
 struct FlattenAllButOne {
-    static f_group_pos_set getGroupsPosToFlatten(const binder::expression_vector& exprs,
-        const Schema& schema);
+    static std::pair<f_group_pos, f_group_pos_set> getGroupsPosToFlatten(
+        const binder::expression_vector& exprs, const Schema& schema);
     static f_group_pos_set getGroupsPosToFlatten(std::shared_ptr<binder::Expression> expr,
         const Schema& schema);
     // Assume no requiredFlatGroups
diff --git a/src/include/planner/operator/logical_aggregate.h b/src/include/planner/operator/logical_aggregate.h
index ce494972cbf..352ea38ac4f 100644
--- a/src/include/planner/operator/logical_aggregate.h
+++ b/src/include/planner/operator/logical_aggregate.h
@@ -40,8 +40,7 @@ class LogicalAggregate final : public LogicalOperator {
     void computeFactorizedSchema() override;
     void computeFlatSchema() override;
 
-    f_group_pos_set getGroupsPosToFlattenForGroupBy();
-    f_group_pos_set getGroupsPosToFlattenForAggregate();
+    f_group_pos_set getGroupsPosToFlatten();
 
     std::string getExpressionsForPrinting() const override;
 
@@ -66,7 +65,6 @@ class LogicalAggregate final : public LogicalOperator {
     }
 
 private:
-    bool hasDistinctAggregate();
     void insertAllExpressionsToGroupAndScope(f_group_pos groupPos);
 
 private:
diff --git a/src/include/processor/operator/aggregate/aggregate_hash_table.h b/src/include/processor/operator/aggregate/aggregate_hash_table.h
index bbe477520c6..712ef45590e 100644
--- a/src/include/processor/operator/aggregate/aggregate_hash_table.h
+++ b/src/include/processor/operator/aggregate/aggregate_hash_table.h
@@ -4,6 +4,7 @@
 
 #include "aggregate_input.h"
 #include "common/copy_constructors.h"
+#include "common/data_chunk/data_chunk_state.h"
 #include "common/types/types.h"
 #include "common/vector/value_vector.h"
 #include "function/aggregate_function.h"
@@ -67,17 +68,11 @@ class AggregateHashTable : public BaseHashTable {
         const std::vector<common::LogicalType>& distinctAggKeyTypes, uint64_t numEntriesToAllocate,
         FactorizedTableSchema tableSchema);
 
-    // Returns true if the value was distinct and was inserted
-    // otherwise if the value already existed, returns false and the hash table is unchanged
-    virtual bool insertAggregateValueIfDistinctForGroupByKeys(
-        const std::vector<common::ValueVector*>& groupByKeyVectors,
-        common::ValueVector* aggregateVector);
-
     //! merge aggregate hash table by combining aggregate states under the same key
     void merge(FactorizedTable&& other);
     void merge(AggregateHashTable&& other) { merge(std::move(*other.factorizedTable)); }
-    // Must be called after merging hash tables with distinct functions, but only when the merged
-    // distinct tuples match the merged non-distinct tuples
+    // Must be called after merging hash tables with distinct functions, but only when the
+    // merged distinct tuples match the merged non-distinct tuples
     void mergeDistinctAggregateInfo();
 
     void finalizeAggregateStates();
@@ -93,16 +88,29 @@ class AggregateHashTable : public BaseHashTable {
         return distinctHashTables[aggregateFunctionIdx].get();
     }
 
+    void appendDistinct(const std::vector<common::ValueVector*>& keyVectors,
+        common::ValueVector* aggregateVector, const common::DataChunkState* leadingState);
+
 protected:
-    virtual uint64_t matchFTEntries(std::span<const common::ValueVector*> flatKeyVectors,
-        std::span<const common::ValueVector*> unFlatKeyVectors, uint64_t numMayMatches,
-        uint64_t numNoMatches);
+    virtual uint64_t append(const std::vector<common::ValueVector*>& keyVectors,
+        const common::DataChunkState* leadingState,
+        const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity) {
+        return append(keyVectors, std::vector<common::ValueVector*>{} /*dependentKeyVectors*/,
+            leadingState, aggregateInputs, resultSetMultiplicity);
+    }
+
+    virtual uint64_t append(const std::vector<common::ValueVector*>& keyVectors,
+        const std::vector<common::ValueVector*>& dependentKeyVectors,
+        const common::DataChunkState* leadingState,
+        const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity);
+
+    virtual uint64_t matchFTEntries(std::span<const common::ValueVector*> keyVectors,
+        uint64_t numMayMatches, uint64_t numNoMatches);
 
     uint64_t matchFTEntries(const FactorizedTable& srcTable, uint64_t startOffset,
         uint64_t numMayMatches, uint64_t numNoMatches);
 
-    void initializeFTEntries(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    void initializeFTEntries(const std::vector<common::ValueVector*>& keyVectors,
         const std::vector<common::ValueVector*>& dependentKeyVectors,
         uint64_t numFTEntriesToInitialize);
     void initializeFTEntries(const FactorizedTable& sourceTable, uint64_t sourceStartOffset,
@@ -114,10 +122,9 @@ class AggregateHashTable : public BaseHashTable {
     uint64_t matchFlatVecWithFTColumn(const common::ValueVector* vector, uint64_t numMayMatches,
         uint64_t& numNoMatches, uint32_t colIdx);
 
-    void findHashSlots(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    void findHashSlots(const std::vector<common::ValueVector*>& keyVectors,
         const std::vector<common::ValueVector*>& dependentKeyVectors,
-        common::DataChunkState* leadingState);
+        const common::DataChunkState* leadingState);
 
     void findHashSlots(const FactorizedTable& data, uint64_t startOffset, uint64_t numTuples);
 
@@ -129,8 +136,8 @@ class AggregateHashTable : public BaseHashTable {
 
     void initializeTmpVectors();
 
-    // ! This function will only be used by distinct aggregate, which assumes that all groupByKeys
-    // are flat.
+    // ! This function will only be used by distinct aggregate, which assumes that all
+    // groupByKeys are flat.
     uint8_t* findEntryInDistinctHT(const std::vector<common::ValueVector*>& groupByKeyVectors,
         common::hash_t hash);
 
@@ -151,18 +158,19 @@ class AggregateHashTable : public BaseHashTable {
 
     void increaseHashSlotIdxes(uint64_t numNoMatches);
 
-    void updateAggState(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    void updateAggState(const std::vector<common::ValueVector*>& keyVectors,
         function::AggregateFunction& aggregateFunction, common::ValueVector* aggVector,
-        uint64_t multiplicity, uint32_t colIdx, uint32_t aggStateOffset);
+        uint64_t multiplicity, uint32_t aggStateOffset,
+        const common::DataChunkState* firstUnFlatState);
 
-    void updateAggStates(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
-        const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity);
+    void updateAggStates(const std::vector<common::ValueVector*>& keyVectors,
+        const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity,
+        const common::DataChunkState* firstUnFlatState);
 
     void fillEntryWithInitialNullAggregateState(FactorizedTable& table, uint8_t* entry);
 
-    //! find an uninitialized hash slot for given hash and fill hash slot with block id and offset
+    //! find an uninitialized hash slot for given hash and fill hash slot with block id and
+    //! offset
     void fillHashSlot(common::hash_t hash, uint8_t* groupByKeysAndAggregateStateBuffer);
 
     inline HashSlot* getHashSlot(uint64_t slotIdx) {
@@ -177,18 +185,14 @@ class AggregateHashTable : public BaseHashTable {
 
     void addDataBlocksIfNecessary(uint64_t maxNumHashSlots);
 
-    void updateNullAggVectorState(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    void updateNullAggVectorState(const common::DataChunkState& keyState,
         function::AggregateFunction& aggregateFunction, uint64_t multiplicity,
         uint32_t aggStateOffset);
 
-    void updateBothFlatAggVectorState(const std::vector<common::ValueVector*>& flatKeyVectors,
-        function::AggregateFunction& aggregateFunction, common::ValueVector* aggVector,
-        uint64_t multiplicity, uint32_t aggStateOffset);
+    void updateBothFlatAggVectorState(function::AggregateFunction& aggregateFunction,
+        common::ValueVector* aggVector, uint64_t multiplicity, uint32_t aggStateOffset);
 
-    void updateFlatUnFlatKeyFlatAggVectorState(
-        const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    void updateFlatUnFlatKeyFlatAggVectorState(const common::DataChunkState& unFlatKeyState,
         function::AggregateFunction& aggregateFunction, common::ValueVector* aggVector,
         uint64_t multiplicity, uint32_t aggStateOffset);
 
@@ -196,15 +200,10 @@ class AggregateHashTable : public BaseHashTable {
         function::AggregateFunction& aggregateFunction, common::ValueVector* aggVector,
         uint64_t multiplicity, uint32_t aggStateOffset);
 
-    void updateBothUnFlatSameDCAggVectorState(
-        const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
-        function::AggregateFunction& aggregateFunction, common::ValueVector* aggVector,
-        uint64_t multiplicity, uint32_t aggStateOffset);
+    void updateBothUnFlatSameDCAggVectorState(function::AggregateFunction& aggregateFunction,
+        common::ValueVector* aggVector, uint64_t multiplicity, uint32_t aggStateOffset);
 
-    void updateBothUnFlatDifferentDCAggVectorState(
-        const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    void updateBothUnFlatDifferentDCAggVectorState(const common::DataChunkState& unFlatKeyState,
         function::AggregateFunction& aggregateFunction, common::ValueVector* aggVector,
         uint64_t multiplicity, uint32_t aggStateOffset);
 
@@ -306,18 +305,14 @@ class PartitioningAggregateHashTable final : public AggregateHashTable {
               common::DEFAULT_VECTOR_CAPACITY /*minimum size*/, tableSchema.copy()),
           tableSchema{std::move(tableSchema)}, partitioningData{partitioningData} {}
 
-    uint64_t append(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors,
+    uint64_t append(const std::vector<common::ValueVector*>& keyVectors,
         const std::vector<common::ValueVector*>& dependentKeyVectors,
-        common::DataChunkState* leadingState, const std::vector<AggregateInput>& aggregateInputs,
-        uint64_t resultSetMultiplicity);
+        const common::DataChunkState* leadingState,
+        const std::vector<AggregateInput>& aggregateInputs,
+        uint64_t resultSetMultiplicity) override;
 
     void mergeIfFull(uint64_t tuplesToAdd, bool mergeAll = false);
 
-    bool insertAggregateValueIfDistinctForGroupByKeys(
-        const std::vector<common::ValueVector*>& groupByKeyVectors,
-        common::ValueVector* aggregateVector) override;
-
 private:
     FactorizedTableSchema tableSchema;
     AggregatePartitioningData* partitioningData;
diff --git a/src/include/processor/operator/aggregate/hash_aggregate.h b/src/include/processor/operator/aggregate/hash_aggregate.h
index ed394919065..5c4289f3256 100644
--- a/src/include/processor/operator/aggregate/hash_aggregate.h
+++ b/src/include/processor/operator/aggregate/hash_aggregate.h
@@ -9,6 +9,7 @@
 #include "aggregate_hash_table.h"
 #include "common/cast.h"
 #include "common/copy_constructors.h"
+#include "common/data_chunk/data_chunk_state.h"
 #include "common/in_mem_overflow_buffer.h"
 #include "common/mpsc_queue.h"
 #include "common/types/types.h"
@@ -114,8 +115,7 @@ class HashAggregateSharedState final : public BaseAggregateSharedState,
 };
 
 struct HashAggregateLocalState {
-    std::vector<common::ValueVector*> flatKeyVectors;
-    std::vector<common::ValueVector*> unFlatKeyVectors;
+    std::vector<common::ValueVector*> keyVectors;
     std::vector<common::ValueVector*> dependentKeyVectors;
     common::DataChunkState* leadingState = nullptr;
     std::unique_ptr<PartitioningAggregateHashTable> aggregateHashTable;
diff --git a/src/include/processor/result/base_hash_table.h b/src/include/processor/result/base_hash_table.h
index e282511931b..bb4188922a0 100644
--- a/src/include/processor/result/base_hash_table.h
+++ b/src/include/processor/result/base_hash_table.h
@@ -40,15 +40,11 @@ class BaseHashTable {
 
     uint64_t getSlotIdxForHash(common::hash_t hash) const { return hash & bitmask; }
     void setMaxNumHashSlots(uint64_t newSize);
-    void computeAndCombineVecHash(std::span<const common::ValueVector*> unFlatKeyVectors,
-        uint32_t startVecIdx);
 
-    void computeVectorHashes(const std::vector<common::ValueVector*>& flatKeyVectors,
-        const std::vector<common::ValueVector*>& unFlatKeyVectors) {
-        computeVectorHashes(constSpan(flatKeyVectors), constSpan(unFlatKeyVectors));
+    void computeVectorHashes(const std::vector<common::ValueVector*>& flatKeyVectors) {
+        computeVectorHashes(constSpan(flatKeyVectors));
     }
-    void computeVectorHashes(std::span<const common::ValueVector*> flatKeyVectors,
-        std::span<const common::ValueVector*> unFlatKeyVectors);
+    void computeVectorHashes(std::span<const common::ValueVector*> flatKeyVectors);
     void initSlotConstant(uint64_t numSlotsPerBlock);
     bool matchFlatVecWithEntry(const std::vector<common::ValueVector*>& keyVectors,
         const uint8_t* entry);
diff --git a/src/include/processor/result/pattern_creation_info_table.h b/src/include/processor/result/pattern_creation_info_table.h
index 65a7d77c788..5012896ef20 100644
--- a/src/include/processor/result/pattern_creation_info_table.h
+++ b/src/include/processor/result/pattern_creation_info_table.h
@@ -25,9 +25,8 @@ class PatternCreationInfoTable : public AggregateHashTable {
 
     PatternCreationInfo getPatternCreationInfo(const std::vector<common::ValueVector*>& keyVectors);
 
-    uint64_t matchFTEntries(std::span<const common::ValueVector*> flatKeyVectors,
-        std::span<const common::ValueVector*> unFlatKeyVectors, uint64_t numMayMatches,
-        uint64_t numNoMatches) override;
+    uint64_t matchFTEntries(std::span<const common::ValueVector*> keyVectors,
+        uint64_t numMayMatches, uint64_t numNoMatches) override;
 
 private:
     uint8_t* tuple;
diff --git a/src/optimizer/factorization_rewriter.cpp b/src/optimizer/factorization_rewriter.cpp
index 1fed2889a3f..927831234a7 100644
--- a/src/optimizer/factorization_rewriter.cpp
+++ b/src/optimizer/factorization_rewriter.cpp
@@ -92,10 +92,8 @@ void FactorizationRewriter::visitAccumulate(planner::LogicalOperator* op) {
 
 void FactorizationRewriter::visitAggregate(planner::LogicalOperator* op) {
     auto& aggregate = op->cast<LogicalAggregate>();
-    auto groupsPosToFlattenForGroupBy = aggregate.getGroupsPosToFlattenForGroupBy();
-    aggregate.setChild(0, appendFlattens(aggregate.getChild(0), groupsPosToFlattenForGroupBy));
-    auto groupsPosToFlattenForAggregate = aggregate.getGroupsPosToFlattenForAggregate();
-    aggregate.setChild(0, appendFlattens(aggregate.getChild(0), groupsPosToFlattenForAggregate));
+    auto groupsPosToFlatten = aggregate.getGroupsPosToFlatten();
+    aggregate.setChild(0, appendFlattens(aggregate.getChild(0), groupsPosToFlatten));
 }
 
 void FactorizationRewriter::visitOrderBy(planner::LogicalOperator* op) {
diff --git a/src/planner/operator/factorization/flatten_resolver.cpp b/src/planner/operator/factorization/flatten_resolver.cpp
index 4cccfaf6eb6..0731a00988c 100644
--- a/src/planner/operator/factorization/flatten_resolver.cpp
+++ b/src/planner/operator/factorization/flatten_resolver.cpp
@@ -7,6 +7,7 @@
 #include "binder/expression/scalar_function_expression.h"
 #include "binder/expression/subquery_expression.h"
 #include "common/exception/not_implemented.h"
+#include "planner/operator/schema.h"
 
 using namespace kuzu::common;
 using namespace kuzu::binder;
@@ -14,8 +15,8 @@ using namespace kuzu::binder;
 namespace kuzu {
 namespace planner {
 
-f_group_pos_set FlattenAllButOne::getGroupsPosToFlatten(const expression_vector& exprs,
-    const Schema& schema) {
+std::pair<f_group_pos, f_group_pos_set> FlattenAllButOne::getGroupsPosToFlatten(
+    const expression_vector& exprs, const Schema& schema) {
     f_group_pos_set result;
     f_group_pos_set dependentGroups;
     for (auto expr : exprs) {
@@ -37,7 +38,11 @@ f_group_pos_set FlattenAllButOne::getGroupsPosToFlatten(const expression_vector&
     for (auto i = 1u; i < candidates.size(); ++i) {
         result.insert(candidates[i]);
     }
-    return result;
+    if (candidates.empty()) {
+        return std::make_pair(INVALID_F_GROUP_POS, result);
+    } else {
+        return std::make_pair(candidates[0], result);
+    }
 }
 
 f_group_pos_set FlattenAllButOne::getGroupsPosToFlatten(std::shared_ptr<Expression> expr,
diff --git a/src/planner/operator/logical_aggregate.cpp b/src/planner/operator/logical_aggregate.cpp
index a5041afa369..cc80f658d75 100644
--- a/src/planner/operator/logical_aggregate.cpp
+++ b/src/planner/operator/logical_aggregate.cpp
@@ -3,6 +3,7 @@
 #include "binder/expression/aggregate_function_expression.h"
 #include "binder/expression/expression_util.h"
 #include "planner/operator/factorization/flatten_resolver.h"
+#include "planner/operator/schema.h"
 
 namespace kuzu {
 namespace planner {
@@ -28,19 +29,31 @@ void LogicalAggregate::computeFlatSchema() {
     insertAllExpressionsToGroupAndScope(0 /* groupPos */);
 }
 
-f_group_pos_set LogicalAggregate::getGroupsPosToFlattenForGroupBy() {
-    if (hasDistinctAggregate()) {
-        return FlattenAll::getGroupsPosToFlatten(getAllKeys(), *children[0]->getSchema());
-    } else {
-        return FlattenAllButOne::getGroupsPosToFlatten(getAllKeys(), *children[0]->getSchema());
-    }
-}
-
-f_group_pos_set LogicalAggregate::getGroupsPosToFlattenForAggregate() {
-    if (hasDistinctAggregate()) {
-        return FlattenAll::getGroupsPosToFlatten(aggregates, *children[0]->getSchema());
+f_group_pos_set LogicalAggregate::getGroupsPosToFlatten() {
+    auto [unflatGroup, flattenedGroups] =
+        FlattenAllButOne::getGroupsPosToFlatten(getAllKeys(), *children[0]->getSchema());
+    // Flatten distinct aggregates if they are from a different group than the unflat key group
+    // Regular aggregates can be processed when unflat, but distinct aggregates get added to their
+    // own AggregateHashTable and have the same input limitations as the aggregate groups
+    if (unflatGroup != INVALID_F_GROUP_POS) {
+        for (const auto& aggregate : aggregates) {
+            auto funcExpr = aggregate->constPtrCast<binder::AggregateFunctionExpression>();
+            auto analyzer = GroupDependencyAnalyzer(false /* collectDependentExpr */,
+                *children[0]->getSchema());
+            analyzer.visit(aggregate);
+            for (const auto& group : analyzer.getRequiredFlatGroups()) {
+                flattenedGroups.insert(group);
+            }
+            if (funcExpr->isDistinct()) {
+                for (const auto& group : analyzer.getDependentGroups()) {
+                    if (group != unflatGroup) {
+                        flattenedGroups.insert(group);
+                    }
+                }
+            }
+        }
     }
-    return f_group_pos_set{};
+    return flattenedGroups;
 }
 
 std::string LogicalAggregate::getExpressionsForPrinting() const {
@@ -59,16 +72,6 @@ std::string LogicalAggregate::getExpressionsForPrinting() const {
     return result;
 }
 
-bool LogicalAggregate::hasDistinctAggregate() {
-    for (auto& expression : aggregates) {
-        auto funcExpr = expression->constPtrCast<binder::AggregateFunctionExpression>();
-        if (funcExpr->isDistinct()) {
-            return true;
-        }
-    }
-    return false;
-}
-
 void LogicalAggregate::insertAllExpressionsToGroupAndScope(f_group_pos groupPos) {
     for (auto& expression : keys) {
         schema->insertToGroupAndScopeMayRepeat(expression, groupPos);
diff --git a/src/planner/plan/append_aggregate.cpp b/src/planner/plan/append_aggregate.cpp
index 35dd9323361..ee6a2338749 100644
--- a/src/planner/plan/append_aggregate.cpp
+++ b/src/planner/plan/append_aggregate.cpp
@@ -10,9 +10,7 @@ void Planner::appendAggregate(const expression_vector& expressionsToGroupBy,
     const expression_vector& expressionsToAggregate, LogicalPlan& plan) {
     auto aggregate = make_shared<LogicalAggregate>(expressionsToGroupBy, expressionsToAggregate,
         plan.getLastOperator());
-    appendFlattens(aggregate->getGroupsPosToFlattenForGroupBy(), plan);
-    aggregate->setChild(0, plan.getLastOperator());
-    appendFlattens(aggregate->getGroupsPosToFlattenForAggregate(), plan);
+    appendFlattens(aggregate->getGroupsPosToFlatten(), plan);
     aggregate->setChild(0, plan.getLastOperator());
     aggregate->computeFactorizedSchema();
     aggregate->setCardinality(cardinalityEstimator.estimateAggregate(*aggregate));
diff --git a/src/processor/operator/aggregate/aggregate_hash_table.cpp b/src/processor/operator/aggregate/aggregate_hash_table.cpp
index fdd521edbb3..4219986c936 100644
--- a/src/processor/operator/aggregate/aggregate_hash_table.cpp
+++ b/src/processor/operator/aggregate/aggregate_hash_table.cpp
@@ -6,11 +6,13 @@
 #include "common/assert.h"
 #include "common/constants.h"
 #include "common/data_chunk/data_chunk_state.h"
+#include "common/data_chunk/sel_vector.h"
 #include "common/in_mem_overflow_buffer.h"
 #include "common/type_utils.h"
 #include "common/types/types.h"
 #include "common/utils.h"
 #include "common/vector/value_vector.h"
+#include "processor/operator/aggregate/aggregate_input.h"
 #include "processor/result/factorized_table.h"
 #include "processor/result/factorized_table_schema.h"
 
@@ -45,33 +47,15 @@ AggregateHashTable::AggregateHashTable(MemoryManager& memoryManager,
     initializeTmpVectors();
 }
 
-bool PartitioningAggregateHashTable::insertAggregateValueIfDistinctForGroupByKeys(
-    const std::vector<ValueVector*>& groupByFlatKeyVectors, ValueVector* aggregateVector) {
-    mergeIfFull(1);
-    return AggregateHashTable::insertAggregateValueIfDistinctForGroupByKeys(groupByFlatKeyVectors,
-        aggregateVector);
-}
-
-bool AggregateHashTable::insertAggregateValueIfDistinctForGroupByKeys(
-    const std::vector<ValueVector*>& groupByFlatKeyVectors, ValueVector* aggregateVector) {
-    auto pos = aggregateVector->state->getSelVector()[0];
-    if (aggregateVector->isNull(pos)) {
-        return false;
-    }
-    std::vector<ValueVector*> distinctKeyVectors(groupByFlatKeyVectors.size() + 1);
-    for (auto i = 0u; i < groupByFlatKeyVectors.size(); i++) {
-        distinctKeyVectors[i] = groupByFlatKeyVectors[i];
-    }
-    distinctKeyVectors[groupByFlatKeyVectors.size()] = aggregateVector;
-    computeVectorHashes(distinctKeyVectors, std::vector<ValueVector*>() /* unFlatKeyVectors */);
-    auto hash = hashVector->getValue<hash_t>(hashVector->state->getSelVector()[0]);
-    auto distinctHTEntry = findEntryInDistinctHT(distinctKeyVectors, hash);
-    if (distinctHTEntry == nullptr) {
-        resizeHashTableIfNecessary(1);
-        createEntryInDistinctHT(distinctKeyVectors, hash);
-        return true;
-    }
-    return false;
+uint64_t AggregateHashTable::append(const std::vector<ValueVector*>& keyVectors,
+    const std::vector<ValueVector*>& dependentKeyVectors, const DataChunkState* leadingState,
+    const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity) {
+    const auto numFlatTuples = leadingState->getSelVector().getSelSize();
+    resizeHashTableIfNecessary(numFlatTuples);
+    computeVectorHashes(keyVectors);
+    findHashSlots(keyVectors, dependentKeyVectors, leadingState);
+    updateAggStates(keyVectors, aggregateInputs, resultSetMultiplicity, leadingState);
+    return numFlatTuples;
 }
 
 hash_t getHash(const FactorizedTable& table, ft_tuple_idx_t tupleIdx) {
@@ -143,15 +127,16 @@ void AggregateHashTable::mergeDistinctAggregateInfo() {
                     distinctHashEntriesProcessed[distinctIdx], numTuplesToScan, colIdxToScan);
                 // Compute hashes for the scanned keys but not the distinct key (i.e. the groups in
                 // the main hash table)
-                computeVectorHashes(std::span<const ValueVector*>{},
-                    std::span(const_cast<const ValueVector**>(vectorPtrs.data()),
-                        vectorPtrs.size() - 1));
+                computeVectorHashes(std::span(const_cast<const ValueVector**>(vectorPtrs.data()),
+                    vectorPtrs.size() - 1));
                 for (size_t i = 0; i < numTuplesToScan; i++) {
                     // Find slot in current hash table corresponding to the entry in the distinct
                     // hash table
                     // updatePosState may not check nulls, and we avoid inserting any nulls into
                     // distinct hash tables
-                    KU_ASSERT(!vectors.back()->isNull(i));
+                    if (vectors.back()->isNull(i)) {
+                        continue;
+                    }
                     auto hash = hashVector->getValue<hash_t>(i);
                     // matchFTEntries expects that the index is the same as the index in the vector
                     // Doesn't need to be done each time, but maybe this can be vectorized so that
@@ -160,7 +145,7 @@ void AggregateHashTable::mergeDistinctAggregateInfo() {
                     auto entry = findEntry(hash, [&](auto entry) {
                         HashSlot slot{hash, entry};
                         hashSlotsToUpdateAggState[i] = &slot;
-                        return matchFTEntries(std::span<const ValueVector*>{},
+                        return matchFTEntries(
                                    std::span(const_cast<const ValueVector**>(vectorPtrs.data()),
                                        vectorPtrs.size() - 1),
                                    1, 0) == 0;
@@ -250,16 +235,17 @@ void AggregateHashTable::resize(uint64_t newSize) {
         [&](auto tuple) { fillHashSlot(*(hash_t*)(tuple + hashColOffsetInFT), tuple); });
 }
 
-uint64_t AggregateHashTable::matchFTEntries(std::span<const ValueVector*> flatKeyVectors,
-    std::span<const ValueVector*> unFlatKeyVectors, uint64_t numMayMatches, uint64_t numNoMatches) {
+uint64_t AggregateHashTable::matchFTEntries(std::span<const ValueVector*> keyVectors,
+    uint64_t numMayMatches, uint64_t numNoMatches) {
     auto colIdx = 0u;
-    for (const auto& flatKeyVector : flatKeyVectors) {
-        numMayMatches =
-            matchFlatVecWithFTColumn(flatKeyVector, numMayMatches, numNoMatches, colIdx++);
-    }
-    for (auto& unFlatKeyVector : unFlatKeyVectors) {
-        numMayMatches =
-            matchUnFlatVecWithFTColumn(unFlatKeyVector, numMayMatches, numNoMatches, colIdx++);
+    for (const auto& keyVector : keyVectors) {
+        if (keyVector->state->isFlat()) {
+            numMayMatches =
+                matchFlatVecWithFTColumn(keyVector, numMayMatches, numNoMatches, colIdx++);
+        } else {
+            numMayMatches =
+                matchUnFlatVecWithFTColumn(keyVector, numMayMatches, numNoMatches, colIdx++);
+        }
     }
     return numNoMatches;
 }
@@ -294,15 +280,15 @@ uint64_t AggregateHashTable::matchFTEntries(const FactorizedTable& srcTable, uin
     return numNoMatches;
 }
 
-void AggregateHashTable::initializeFTEntries(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors,
+void AggregateHashTable::initializeFTEntries(const std::vector<ValueVector*>& keyVectors,
     const std::vector<ValueVector*>& dependentKeyVectors, uint64_t numFTEntriesToInitialize) {
     auto colIdx = 0u;
-    for (auto flatKeyVector : flatKeyVectors) {
-        initializeFTEntryWithFlatVec(flatKeyVector, numFTEntriesToInitialize, colIdx++);
-    }
-    for (auto unFlatKeyVector : unFlatKeyVectors) {
-        initializeFTEntryWithUnFlatVec(unFlatKeyVector, numFTEntriesToInitialize, colIdx++);
+    for (auto keyVector : keyVectors) {
+        if (keyVector->state->isFlat()) {
+            initializeFTEntryWithFlatVec(keyVector, numFTEntriesToInitialize, colIdx++);
+        } else {
+            initializeFTEntryWithUnFlatVec(keyVector, numFTEntriesToInitialize, colIdx++);
+        }
     }
     for (auto dependentKeyVector : dependentKeyVectors) {
         if (dependentKeyVector->state->isFlat()) {
@@ -513,11 +499,10 @@ void AggregateHashTable::increaseHashSlotIdxes(uint64_t numNoMatches) {
     }
 }
 
-void AggregateHashTable::findHashSlots(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors,
-    const std::vector<ValueVector*>& dependentKeyVectors, DataChunkState* leadingState) {
+void AggregateHashTable::findHashSlots(const std::vector<ValueVector*>& keyVectors,
+    const std::vector<ValueVector*>& dependentKeyVectors, const DataChunkState* leadingState) {
     initTmpHashSlotsAndIdxes();
-    auto numEntriesToFindHashSlots = leadingState->getSelVector().getSelSize();
+    auto numEntriesToFindHashSlots = leadingState->getSelSize();
     KU_ASSERT(getNumEntries() + numEntriesToFindHashSlots < maxNumHashSlots);
     while (numEntriesToFindHashSlots > 0) {
         uint64_t numFTEntriesToUpdate = 0;
@@ -537,10 +522,8 @@ void AggregateHashTable::findHashSlots(const std::vector<ValueVector*>& flatKeyV
                 noMatchIdxes[numNoMatches++] = idx;
             }
         }
-        initializeFTEntries(flatKeyVectors, unFlatKeyVectors, dependentKeyVectors,
-            numFTEntriesToUpdate);
-        numNoMatches = matchFTEntries(constSpan(flatKeyVectors), constSpan(unFlatKeyVectors),
-            numMayMatches, numNoMatches);
+        initializeFTEntries(keyVectors, dependentKeyVectors, numFTEntriesToUpdate);
+        numNoMatches = matchFTEntries(constSpan(keyVectors), numMayMatches, numNoMatches);
         increaseHashSlotIdxes(numNoMatches);
         KU_ASSERT(numNoMatches <= numEntriesToFindHashSlots);
         numEntriesToFindHashSlots = numNoMatches;
@@ -579,33 +562,46 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector, const DataChunkState* leadingState) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    // The aggregateVector's state is either the same as the leading state (doesn't matter which we
+    // use), flat (where we must pass the original leading state), or unflat while the leadingState
+    // is flat (where we must pass the aggregateVector's state)
+    auto distinctLeadingState =
+        aggregateVector->state->isFlat() ? leadingState : aggregateVector->state.get();
+    append(distinctKeyVectors, distinctLeadingState, std::vector<AggregateInput>{},
+        1 /*multiplicity*/);
+}
+
+void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& keyVectors,
+    AggregateFunction& aggregateFunction, ValueVector* aggVector, uint64_t multiplicity,
+    uint32_t aggStateOffset, const DataChunkState* leadingState) {
+    //  There may be a mix of flat and unflat states, but any unflat states will be the same
+    bool allFlat = leadingState->isFlat();
     if (!aggVector) {
-        updateNullAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction, multiplicity,
-            aggStateOffset);
-    } else if (aggVector->state->isFlat() && unFlatKeyVectors.empty()) {
-        updateBothFlatAggVectorState(flatKeyVectors, aggregateFunction, aggVector, multiplicity,
-            aggStateOffset);
+        updateNullAggVectorState(*leadingState, aggregateFunction, multiplicity, aggStateOffset);
+    } else if (aggVector->state->isFlat() && allFlat) {
+        updateBothFlatAggVectorState(aggregateFunction, aggVector, multiplicity, aggStateOffset);
     } else if (aggVector->state->isFlat()) {
-        updateFlatUnFlatKeyFlatAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction,
-            aggVector, multiplicity, aggStateOffset);
-    } else if (unFlatKeyVectors.empty()) {
-        updateFlatKeyUnFlatAggVectorState(flatKeyVectors, aggregateFunction, aggVector,
+        updateFlatUnFlatKeyFlatAggVectorState(*leadingState, aggregateFunction, aggVector,
             multiplicity, aggStateOffset);
-    } else if (!unFlatKeyVectors.empty() && (aggVector->state == unFlatKeyVectors[0]->state)) {
-        updateBothUnFlatSameDCAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction,
-            aggVector, multiplicity, aggStateOffset);
+    } else if (allFlat) {
+        updateFlatKeyUnFlatAggVectorState(keyVectors, aggregateFunction, aggVector, multiplicity,
+            aggStateOffset);
+    } else if (aggVector->state.get() == leadingState) {
+        updateBothUnFlatSameDCAggVectorState(aggregateFunction, aggVector, multiplicity,
+            aggStateOffset);
     } else {
-        updateBothUnFlatDifferentDCAggVectorState(flatKeyVectors, unFlatKeyVectors,
-            aggregateFunction, aggVector, multiplicity, aggStateOffset);
+        updateBothUnFlatDifferentDCAggVectorState(*leadingState, aggregateFunction, aggVector,
+            multiplicity, aggStateOffset);
     }
 }
 
-void AggregateHashTable::updateAggStates(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors,
-    const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity) {
+void AggregateHashTable::updateAggStates(const std::vector<ValueVector*>& keyVectors,
+    const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity,
+    const DataChunkState* leadingState) {
     auto aggregateStateOffset = aggStateColOffsetInFT;
     for (auto i = 0u; i < aggregateFunctions.size(); i++) {
         if (!aggregateFunctions[i].isDistinct) {
@@ -613,14 +609,14 @@ void AggregateHashTable::updateAggStates(const std::vector<ValueVector*>& flatKe
             for (auto& dataChunk : aggregateInputs[i].multiplicityChunks) {
                 multiplicity *= dataChunk->state->getSelVector().getSelSize();
             }
-            updateAggState(flatKeyVectors, unFlatKeyVectors, aggregateFunctions[i],
-                aggregateInputs[i].aggregateVector, multiplicity, i, aggregateStateOffset);
+            updateAggState(keyVectors, aggregateFunctions[i], aggregateInputs[i].aggregateVector,
+                multiplicity, aggregateStateOffset, leadingState);
             aggregateStateOffset += aggregateFunctions[i].getAggregateStateSize();
         } else {
-            // If a function is distinct we still need to insert the value into the distinct hash
-            // table
-            distinctHashTables[i]->insertAggregateValueIfDistinctForGroupByKeys(flatKeyVectors,
-                aggregateInputs[i].aggregateVector);
+            // If a function is distinct we still need to insert the value into the distinct
+            // hash table
+            distinctHashTables[i]->appendDistinct(keyVectors, aggregateInputs[i].aggregateVector,
+                leadingState);
         }
     }
 }
@@ -666,23 +662,21 @@ void AggregateHashTable::resizeHashTableIfNecessary(uint32_t maxNumDistinctHashK
     }
 }
 
-void AggregateHashTable::updateNullAggVectorState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    uint64_t multiplicity, uint32_t aggStateOffset) {
-    if (unFlatKeyVectors.empty()) {
-        auto pos = flatKeyVectors[0]->state->getSelVector()[0];
+void AggregateHashTable::updateNullAggVectorState(const DataChunkState& keyState,
+    AggregateFunction& aggregateFunction, uint64_t multiplicity, uint32_t aggStateOffset) {
+    if (keyState.isFlat()) {
+        auto pos = keyState.getSelVector()[0];
         aggregateFunction.updatePosState(hashSlotsToUpdateAggState[pos]->entry + aggStateOffset,
             nullptr, multiplicity, 0 /* dummy pos */, memoryManager);
     } else {
-        unFlatKeyVectors[0]->state->getSelVector().forEach([&](auto pos) {
+        keyState.getSelVector().forEach([&](auto pos) {
             aggregateFunction.updatePosState(hashSlotsToUpdateAggState[pos]->entry + aggStateOffset,
                 nullptr, multiplicity, 0 /* dummy pos */, memoryManager);
         });
     }
 }
 
-void AggregateHashTable::updateBothFlatAggVectorState(
-    const std::vector<ValueVector*>& /*flatKeyVectors*/, AggregateFunction& aggregateFunction,
+void AggregateHashTable::updateBothFlatAggVectorState(AggregateFunction& aggregateFunction,
     ValueVector* aggVector, uint64_t multiplicity, uint32_t aggStateOffset) {
     auto aggPos = aggVector->state->getSelVector()[0];
     if (!aggVector->isNull(aggPos)) {
@@ -692,13 +686,12 @@ void AggregateHashTable::updateBothFlatAggVectorState(
     }
 }
 
-void AggregateHashTable::updateFlatUnFlatKeyFlatAggVectorState(
-    const std::vector<ValueVector*>& /*flatKeyVectors*/,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t aggStateOffset) {
+void AggregateHashTable::updateFlatUnFlatKeyFlatAggVectorState(const DataChunkState& unFlatKeyState,
+    AggregateFunction& aggregateFunction, ValueVector* aggVector, uint64_t multiplicity,
+    uint32_t aggStateOffset) {
     auto aggPos = aggVector->state->getSelVector()[0];
     if (!aggVector->isNull(aggPos)) {
-        unFlatKeyVectors[0]->state->getSelVector().forEach([&](auto pos) {
+        unFlatKeyState.getSelVector().forEach([&](auto pos) {
             aggregateFunction.updatePosState(hashSlotsToUpdateAggState[pos]->entry + aggStateOffset,
                 aggVector, multiplicity, aggPos, memoryManager);
         });
@@ -716,9 +709,7 @@ void AggregateHashTable::updateFlatKeyUnFlatAggVectorState(
     });
 }
 
-void AggregateHashTable::updateBothUnFlatSameDCAggVectorState(
-    const std::vector<ValueVector*>& /*flatKeyVectors*/,
-    const std::vector<ValueVector*>& /*unFlatKeyVectors*/, AggregateFunction& aggregateFunction,
+void AggregateHashTable::updateBothUnFlatSameDCAggVectorState(AggregateFunction& aggregateFunction,
     ValueVector* aggVector, uint64_t multiplicity, uint32_t aggStateOffset) {
     aggVector->forEachNonNull([&](auto pos) {
         aggregateFunction.updatePosState(hashSlotsToUpdateAggState[pos]->entry + aggStateOffset,
@@ -727,10 +718,9 @@ void AggregateHashTable::updateBothUnFlatSameDCAggVectorState(
 }
 
 void AggregateHashTable::updateBothUnFlatDifferentDCAggVectorState(
-    const std::vector<ValueVector*>& /*flatKeyVectors*/,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
+    const DataChunkState& unFlatKeyState, AggregateFunction& aggregateFunction,
     ValueVector* aggVector, uint64_t multiplicity, uint32_t aggStateOffset) {
-    unFlatKeyVectors[0]->state->getSelVector().forEach([&](auto pos) {
+    unFlatKeyState.getSelVector().forEach([&](auto pos) {
         aggregateFunction.updateAllState(hashSlotsToUpdateAggState[pos]->entry + aggStateOffset,
             aggVector, multiplicity, memoryManager);
     });
@@ -775,20 +765,22 @@ std::unique_ptr<AggregateHashTable> AggregateHashTableUtils::createDistinctHashT
         getTableSchemaForKeys(groupByKeyTypes, distinctKeyType));
 }
 
-uint64_t PartitioningAggregateHashTable::append(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors,
-    const std::vector<ValueVector*>& dependentKeyVectors, DataChunkState* leadingState,
-    const std::vector<AggregateInput>& aggregateInputs, uint64_t resultSetMultiplicity) {
+uint64_t PartitioningAggregateHashTable::append(const std::vector<ValueVector*>& keyVectors,
+    const std::vector<ValueVector*>& dependentKeyVectors,
+    const common::DataChunkState* leadingState, const std::vector<AggregateInput>& aggregateInputs,
+    uint64_t resultSetMultiplicity) {
     const auto numFlatTuples = leadingState->getSelVector().getSelSize();
 
     mergeIfFull(numFlatTuples);
 
     // mergeAll makes use of the hashVector, so it needs to be called before computeVectorHashes
-    computeVectorHashes(flatKeyVectors, unFlatKeyVectors);
-    findHashSlots(flatKeyVectors, unFlatKeyVectors, dependentKeyVectors, leadingState);
-    // Don't update distinct states since they can't be merged into the global hash tables. Instead
-    // we'll calculate them from scratch when merging.
-    updateAggStates(flatKeyVectors, unFlatKeyVectors, aggregateInputs, resultSetMultiplicity);
+    computeVectorHashes(keyVectors);
+    KU_ASSERT(
+        hashVector->getSelVectorPtr()->getSelSize() == leadingState->getSelVector().getSelSize());
+    findHashSlots(keyVectors, dependentKeyVectors, leadingState);
+    // Don't update distinct states since they can't be merged into the global hash tables.
+    // Instead we'll calculate them from scratch when merging.
+    updateAggStates(keyVectors, aggregateInputs, resultSetMultiplicity, leadingState);
     return numFlatTuples;
 }
 
@@ -802,8 +794,8 @@ void PartitioningAggregateHashTable::mergeIfFull(uint64_t tuplesToAdd, bool merg
     if (mergeAll || outOfSpace(*this, tuplesToAdd)) {
         partitioningData->appendTuples(*factorizedTable,
             tableSchema.getColOffset(tableSchema.getNumColumns() - 1));
-        // Move overflow data into the shared state so that it isn't obliterated when we clear the
-        // factorized table
+        // Move overflow data into the shared state so that it isn't obliterated when we clear
+        // the factorized table
         partitioningData->appendOverflow(std::move(*factorizedTable->getInMemOverflowBuffer()));
         clear();
     }
@@ -825,9 +817,9 @@ void PartitioningAggregateHashTable::mergeIfFull(uint64_t tuplesToAdd, bool merg
     std::vector<ft_col_idx_t> colIdxToScan;
     if (distinctHashTables.size() > 0) {
         // We need the hashes of the key columns to partition them appropriately.
-        // These will be the same for each of the distinct hash tables since we exclude the distinct
-        // aggregate key Reserve inside here so that we don't unnecessarily allocate memory if there
-        // are no distinct hash tables
+        // These will be the same for each of the distinct hash tables since we exclude the
+        // distinct aggregate key Reserve inside here so that we don't unnecessarily allocate
+        // memory if there are no distinct hash tables
         colIdxToScan.resize(keyTypes.size());
         std::iota(colIdxToScan.begin(), colIdxToScan.end(), 0);
         vectors.reserve(keyTypes.size());
@@ -851,12 +843,12 @@ void PartitioningAggregateHashTable::mergeIfFull(uint64_t tuplesToAdd, bool merg
             while (startTupleIdx < distinctFactorizedTable->getNumTuples()) {
                 distinctFactorizedTable->scan(vectors, startTupleIdx, numTuplesToScan,
                     colIdxToScan);
-                computeVectorHashes(std::vector<ValueVector*>{}, vectors);
+                computeVectorHashes(vectors);
                 for (uint64_t tupleIdx = 0; tupleIdx < numTuplesToScan; tupleIdx++) {
                     auto* tuple = distinctFactorizedTable->getTuple(startTupleIdx + tupleIdx);
-                    // The distinct value needs to be partitioned according to the group that stores
-                    // its aggregate state So we need to ignore the aggregate key when calculating
-                    // the hash for partitioning
+                    // The distinct value needs to be partitioned according to the group that
+                    // stores its aggregate state So we need to ignore the aggregate key when
+                    // calculating the hash for partitioning
                     const auto hash = hashVector->getValue<hash_t>(tupleIdx);
                     partitioningData->appendDistinctTuple(distinctIdx,
                         std::span(tuple, distinctTableSchema->getNumBytesPerTuple()), hash);
diff --git a/src/processor/operator/aggregate/hash_aggregate.cpp b/src/processor/operator/aggregate/hash_aggregate.cpp
index c690aad19bc..db5da31702a 100644
--- a/src/processor/operator/aggregate/hash_aggregate.cpp
+++ b/src/processor/operator/aggregate/hash_aggregate.cpp
@@ -206,13 +206,18 @@ void HashAggregateLocalState::init(HashAggregateSharedState* sharedState, Result
     std::vector<LogicalType> keyDataTypes;
     for (auto& pos : info.flatKeysPos) {
         auto vector = resultSet.getValueVector(pos).get();
-        flatKeyVectors.push_back(vector);
+        keyVectors.push_back(vector);
         keyDataTypes.push_back(vector->dataType.copy());
     }
     for (auto& pos : info.unFlatKeysPos) {
         auto vector = resultSet.getValueVector(pos).get();
-        unFlatKeyVectors.push_back(vector);
+        keyVectors.push_back(vector);
         keyDataTypes.push_back(vector->dataType.copy());
+        leadingState = vector->state.get();
+    }
+    if (leadingState == nullptr) {
+        // All vectors are flat, so any can be the leading state
+        leadingState = keyVectors.front()->state.get();
     }
     std::vector<LogicalType> payloadDataTypes;
     for (auto& pos : info.dependentKeysPos) {
@@ -220,8 +225,6 @@ void HashAggregateLocalState::init(HashAggregateSharedState* sharedState, Result
         dependentKeyVectors.push_back(vector);
         payloadDataTypes.push_back(vector->dataType.copy());
     }
-    leadingState = unFlatKeyVectors.empty() ? flatKeyVectors[0]->state.get() :
-                                              unFlatKeyVectors[0]->state.get();
 
     aggregateHashTable = std::make_unique<PartitioningAggregateHashTable>(sharedState,
         *context->getMemoryManager(), std::move(keyDataTypes), std::move(payloadDataTypes),
@@ -230,8 +233,8 @@ void HashAggregateLocalState::init(HashAggregateSharedState* sharedState, Result
 
 uint64_t HashAggregateLocalState::append(const std::vector<AggregateInput>& aggregateInputs,
     uint64_t multiplicity) const {
-    return aggregateHashTable->append(flatKeyVectors, unFlatKeyVectors, dependentKeyVectors,
-        leadingState, aggregateInputs, multiplicity);
+    return aggregateHashTable->append(keyVectors, dependentKeyVectors, leadingState,
+        aggregateInputs, multiplicity);
 }
 
 void HashAggregate::initLocalStateInternal(ResultSet* resultSet, ExecutionContext* context) {
diff --git a/src/processor/operator/aggregate/simple_aggregate.cpp b/src/processor/operator/aggregate/simple_aggregate.cpp
index 35ee8e75be9..003da35a46b 100644
--- a/src/processor/operator/aggregate/simple_aggregate.cpp
+++ b/src/processor/operator/aggregate/simple_aggregate.cpp
@@ -207,10 +207,10 @@ void SimpleAggregate::executeInternal(ExecutionContext* context) {
         for (auto i = 0u; i < aggregateFunctions.size(); i++) {
             auto aggregateFunction = &aggregateFunctions[i];
             if (aggregateFunction->isFunctionDistinct()) {
-                // Just add distinct value to the hash table. We'll calculate the final hash table
-                // once it's been merged into the global state
-                distinctHashTables[i].get()->insertAggregateValueIfDistinctForGroupByKeys(
-                    std::vector<ValueVector*>{}, aggInputs[i].aggregateVector);
+                // Just add distinct value to the hash table. We'll calculate the aggregate state
+                // once it's been merged into the shared state
+                distinctHashTables[i]->appendDistinct(std::vector<ValueVector*>{},
+                    aggInputs[i].aggregateVector, aggInputs[i].aggregateVector->state.get());
             } else {
                 computeAggregate(aggregateFunction, &aggInputs[i], localAggregateStates[i].get(),
                     memoryManager);
diff --git a/src/processor/operator/hash_join/join_hash_table.cpp b/src/processor/operator/hash_join/join_hash_table.cpp
index ef2b92bade8..08d8ddfed2d 100644
--- a/src/processor/operator/hash_join/join_hash_table.cpp
+++ b/src/processor/operator/hash_join/join_hash_table.cpp
@@ -201,8 +201,7 @@ uint8_t* JoinHashTable::insertEntry(uint8_t* tuple) const {
 }
 
 void JoinHashTable::computeVectorHashes(std::vector<common::ValueVector*> keyVectors) {
-    std::vector<ValueVector*> dummyUnFlatKeyVectors;
-    BaseHashTable::computeVectorHashes(keyVectors, dummyUnFlatKeyVectors);
+    BaseHashTable::computeVectorHashes(keyVectors);
 }
 
 offset_t JoinHashTable::getHashValueColOffset() const {
diff --git a/src/processor/result/base_hash_table.cpp b/src/processor/result/base_hash_table.cpp
index 8760f55b93d..9ecb3cd630e 100644
--- a/src/processor/result/base_hash_table.cpp
+++ b/src/processor/result/base_hash_table.cpp
@@ -29,10 +29,13 @@ void BaseHashTable::setMaxNumHashSlots(uint64_t newSize) {
     bitmask = maxNumHashSlots - 1;
 }
 
-void BaseHashTable::computeAndCombineVecHash(std::span<const ValueVector*> unFlatKeyVectors,
-    uint32_t startVecIdx) {
-    for (; startVecIdx < unFlatKeyVectors.size(); startVecIdx++) {
-        auto keyVector = unFlatKeyVectors[startVecIdx];
+void BaseHashTable::computeVectorHashes(std::span<const ValueVector*> keyVectors) {
+    hashVector->state = keyVectors[0]->state;
+    VectorHashFunction::computeHash(*keyVectors[0], keyVectors[0]->state->getSelVector(),
+        *hashVector.get(), hashVector->state->getSelVector());
+
+    for (size_t startVecIdx = 1; startVecIdx < keyVectors.size(); startVecIdx++) {
+        auto keyVector = keyVectors[startVecIdx];
         tmpHashResultVector->state = keyVector->state;
         tmpHashCombineResultVector->state = keyVector->state;
         VectorHashFunction::computeHash(*keyVector, keyVector->state->getSelVector(),
@@ -46,24 +49,6 @@ void BaseHashTable::computeAndCombineVecHash(std::span<const ValueVector*> unFla
     }
 }
 
-void BaseHashTable::computeVectorHashes(std::span<const ValueVector*> flatKeyVectors,
-    std::span<const ValueVector*> unFlatKeyVectors) {
-    if (!flatKeyVectors.empty()) {
-        hashVector->state = flatKeyVectors[0]->state;
-        VectorHashFunction::computeHash(*flatKeyVectors[0],
-            flatKeyVectors[0]->state->getSelVector(), *hashVector.get(),
-            hashVector->state->getSelVector());
-        computeAndCombineVecHash(flatKeyVectors, 1 /* startVecIdx */);
-        computeAndCombineVecHash(unFlatKeyVectors, 0 /* startVecIdx */);
-    } else {
-        hashVector->state = unFlatKeyVectors[0]->state;
-        VectorHashFunction::computeHash(*unFlatKeyVectors[0],
-            unFlatKeyVectors[0]->state->getSelVector(), *hashVector.get(),
-            hashVector->state->getSelVector());
-        computeAndCombineVecHash(unFlatKeyVectors, 1 /* startVecIdx */);
-    }
-}
-
 template<typename T>
 static bool compareEntry(const common::ValueVector* vector, uint32_t vectorPos,
     const uint8_t* entry) {
diff --git a/src/processor/result/pattern_creation_info_table.cpp b/src/processor/result/pattern_creation_info_table.cpp
index 0650e19e9ce..7b8ca592174 100644
--- a/src/processor/result/pattern_creation_info_table.cpp
+++ b/src/processor/result/pattern_creation_info_table.cpp
@@ -33,9 +33,8 @@ PatternCreationInfo PatternCreationInfoTable::getPatternCreationInfo(
         return PatternCreationInfo{tuple, hasCreated};
     } else {
         resizeHashTableIfNecessary(1);
-        computeVectorHashes(keyVectors, std::vector<common::ValueVector*>{});
-        findHashSlots(keyVectors, std::vector<common::ValueVector*>{},
-            std::vector<common::ValueVector*>{}, keyVectors[0]->state.get());
+        computeVectorHashes(keyVectors);
+        findHashSlots(keyVectors, std::vector<common::ValueVector*>{}, keyVectors[0]->state.get());
         hasCreated = tuple != nullptr;
         auto idTuple = tuple == nullptr ?
                            factorizedTable->getTuple(factorizedTable->getNumTuples() - 1) :
@@ -44,13 +43,9 @@ PatternCreationInfo PatternCreationInfoTable::getPatternCreationInfo(
     }
 }
 
-uint64_t PatternCreationInfoTable::matchFTEntries(
-    std::span<const common::ValueVector*> flatKeyVectors,
-    std::span<const common::ValueVector*> unFlatKeyVectors, uint64_t numMayMatches,
-    uint64_t numNoMatches) {
-    KU_ASSERT(unFlatKeyVectors.empty());
-    numNoMatches = AggregateHashTable::matchFTEntries(flatKeyVectors, unFlatKeyVectors,
-        numMayMatches, numNoMatches);
+uint64_t PatternCreationInfoTable::matchFTEntries(std::span<const common::ValueVector*> keyVectors,
+    uint64_t numMayMatches, uint64_t numNoMatches) {
+    numNoMatches = AggregateHashTable::matchFTEntries(keyVectors, numMayMatches, numNoMatches);
     KU_ASSERT(numMayMatches <= 1);
     // If we found the entry for the target key, we set tuple to the key tuple. Otherwise, simply
     // set tuple to nullptr.


Reviews:
user: login: acquamarin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T19:16:42Z
user: login: ray6080
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T02:17:14Z
user: login: benjaminwinger
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T14:23:45Z
user: login: benjaminwinger
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T14:23:47Z
user: login: benjaminwinger
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:32:25Z
user: login: acquamarin
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T00:37:27Z
user: login: benjaminwinger
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T21:33:18Z

Review comments:
diff_hunk: @@ -45,33 +47,30 @@ AggregateHashTable::AggregateHashTable(MemoryManager& memoryManager,
     initializeTmpVectors();
 }
 
-bool PartitioningAggregateHashTable::insertAggregateValueIfDistinctForGroupByKeys(
-    const std::vector<ValueVector*>& groupByFlatKeyVectors, ValueVector* aggregateVector) {
-    mergeIfFull(1);
-    return AggregateHashTable::insertAggregateValueIfDistinctForGroupByKeys(groupByFlatKeyVectors,
-        aggregateVector);
+static DataChunkState* getFirstUnflatState(const std::vector<ValueVector*>& keyVectors) {
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: acquamarin
body: Instead of computing the first unflat state on each `AggregateHashTable::append`, I think we should consider using the leading state.
created_at: 2025-03-11T19:11:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -114,10 +114,8 @@ class HashAggregateSharedState final : public BaseAggregateSharedState,
 };
 
 struct HashAggregateLocalState {
-    std::vector<common::ValueVector*> flatKeyVectors;
-    std::vector<common::ValueVector*> unFlatKeyVectors;
+    std::vector<common::ValueVector*> keyVectors;
path: src/include/processor/operator/aggregate/hash_aggregate.h
user: login: acquamarin
body: Why remove the leading state? leading state is a static information which can be computed at compile time, so we don't have to compute it over again during execution.
created_at: 2025-03-11T19:12:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -579,48 +577,77 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    if (!keyVectors.empty() && !aggregateVector->state->isFlat() &&
+        getFirstUnflatState(keyVectors) != aggregateVector->state.get()) {
+        // If the aggregateVector is unFlat, with a different state from the unflat vectors in the
+        // keys, flatten the aggregate vector and append values one by one since there can be at
+        // most one unflat group in the keyVectors.
+        auto oldState = std::move(aggregateVector->state);
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: acquamarin
body: Instead of doing the crossproduct there, i suggest we flat the aggreagate vectors when aggregateVector and keyVector are both unflat and from different chunks.
created_at: 2025-03-11T19:16:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -579,48 +577,77 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    if (!keyVectors.empty() && !aggregateVector->state->isFlat() &&
+        getFirstUnflatState(keyVectors) != aggregateVector->state.get()) {
+        // If the aggregateVector is unFlat, with a different state from the unflat vectors in the
+        // keys, flatten the aggregate vector and append values one by one since there can be at
+        // most one unflat group in the keyVectors.
+        auto oldState = std::move(aggregateVector->state);
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: ray6080
body: I'm also curious if we do flatten all but one, how much performance difference did you observe? 
created_at: 2025-03-12T02:17:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -579,48 +577,77 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    if (!keyVectors.empty() && !aggregateVector->state->isFlat() &&
+        getFirstUnflatState(keyVectors) != aggregateVector->state.get()) {
+        // If the aggregateVector is unFlat, with a different state from the unflat vectors in the
+        // keys, flatten the aggregate vector and append values one by one since there can be at
+        // most one unflat group in the keyVectors.
+        auto oldState = std::move(aggregateVector->state);
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: benjaminwinger
body: I didn't know how to do that, but I think I've figured it out.
created_at: 2025-03-12T14:23:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -114,10 +114,8 @@ class HashAggregateSharedState final : public BaseAggregateSharedState,
 };
 
 struct HashAggregateLocalState {
-    std::vector<common::ValueVector*> flatKeyVectors;
-    std::vector<common::ValueVector*> unFlatKeyVectors;
+    std::vector<common::ValueVector*> keyVectors;
path: src/include/processor/operator/aggregate/hash_aggregate.h
user: login: benjaminwinger
body: That's a good point. I guess I removed it since it wasn't being used, but we could calculate it ahead of time. The only concern I have is that the leading state wouldn't be the same if all the key vectors are flat for the main hash table, but the aggregate vector is unflat, though I guess in that case we can pass the distinct aggregate vector's state as the leading state.
created_at: 2025-03-12T14:23:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -579,48 +577,77 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    if (!keyVectors.empty() && !aggregateVector->state->isFlat() &&
+        getFirstUnflatState(keyVectors) != aggregateVector->state.get()) {
+        // If the aggregateVector is unFlat, with a different state from the unflat vectors in the
+        // keys, flatten the aggregate vector and append values one by one since there can be at
+        // most one unflat group in the keyVectors.
+        auto oldState = std::move(aggregateVector->state);
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: benjaminwinger
body: > I'm also curious if we do flatten all but one, how much performance difference did you observe?

The performance was more or less identical after that change.
created_at: 2025-03-13T14:32:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -579,48 +562,61 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector, const DataChunkState* leadingState) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    // The aggregateVector's state is either the same as the leading state (doesn't matter which we
+    // use), flat (where we must pass the original leading state), or unflat while the leadingState
+    // is flat (where we must pass the aggregateVector's state)
+    auto distinctLeadingState =
+        aggregateVector->state->isFlat() ? leadingState : aggregateVector->state.get();
+    append(distinctKeyVectors, distinctLeadingState, std::vector<AggregateInput>{},
+        1 /*multiplicity*/);
+}
+
+void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& keyVectors,
+    AggregateFunction& aggregateFunction, ValueVector* aggVector, uint64_t multiplicity,
+    uint32_t aggStateOffset, const DataChunkState* leadingState) {
+    //  There may be a mix of flat and unflat states, but any unflat states will be the same
+    bool allFlat = leadingState->isFlat();
     if (!aggVector) {
-        updateNullAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction, multiplicity,
-            aggStateOffset);
-    } else if (aggVector->state->isFlat() && unFlatKeyVectors.empty()) {
-        updateBothFlatAggVectorState(flatKeyVectors, aggregateFunction, aggVector, multiplicity,
-            aggStateOffset);
+        updateNullAggVectorState(*leadingState, aggregateFunction, multiplicity, aggStateOffset);
+    } else if (aggVector->state->isFlat() && allFlat) {
+        updateBothFlatAggVectorState(aggregateFunction, aggVector, multiplicity, aggStateOffset);
     } else if (aggVector->state->isFlat()) {
-        updateFlatUnFlatKeyFlatAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction,
-            aggVector, multiplicity, aggStateOffset);
-    } else if (unFlatKeyVectors.empty()) {
-        updateFlatKeyUnFlatAggVectorState(flatKeyVectors, aggregateFunction, aggVector,
+        updateFlatUnFlatKeyFlatAggVectorState(*leadingState, aggregateFunction, aggVector,
             multiplicity, aggStateOffset);
-    } else if (!unFlatKeyVectors.empty() && (aggVector->state == unFlatKeyVectors[0]->state)) {
-        updateBothUnFlatSameDCAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction,
-            aggVector, multiplicity, aggStateOffset);
+    } else if (allFlat) {
+        updateFlatKeyUnFlatAggVectorState(keyVectors, aggregateFunction, aggVector, multiplicity,
+            aggStateOffset);
+    } else if (aggVector->state.get() == leadingState) {
+        updateBothUnFlatSameDCAggVectorState(aggregateFunction, aggVector, multiplicity,
+            aggStateOffset);
     } else {
-        updateBothUnFlatDifferentDCAggVectorState(flatKeyVectors, unFlatKeyVectors,
-            aggregateFunction, aggVector, multiplicity, aggStateOffset);
+        updateBothUnFlatDifferentDCAggVectorState(*leadingState, aggregateFunction, aggVector,
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: acquamarin
body: Can you add a test case for this line?
created_at: 2025-03-18T00:37:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -579,48 +562,61 @@ void AggregateHashTable::findHashSlots(const FactorizedTable& srcTable, uint64_t
     }
 }
 
-void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& flatKeyVectors,
-    const std::vector<ValueVector*>& unFlatKeyVectors, AggregateFunction& aggregateFunction,
-    ValueVector* aggVector, uint64_t multiplicity, uint32_t /*colIdx*/, uint32_t aggStateOffset) {
+void AggregateHashTable::appendDistinct(const std::vector<ValueVector*>& keyVectors,
+    ValueVector* aggregateVector, const DataChunkState* leadingState) {
+    std::vector<ValueVector*> distinctKeyVectors(keyVectors);
+    distinctKeyVectors.push_back(aggregateVector);
+    // The aggregateVector's state is either the same as the leading state (doesn't matter which we
+    // use), flat (where we must pass the original leading state), or unflat while the leadingState
+    // is flat (where we must pass the aggregateVector's state)
+    auto distinctLeadingState =
+        aggregateVector->state->isFlat() ? leadingState : aggregateVector->state.get();
+    append(distinctKeyVectors, distinctLeadingState, std::vector<AggregateInput>{},
+        1 /*multiplicity*/);
+}
+
+void AggregateHashTable::updateAggState(const std::vector<ValueVector*>& keyVectors,
+    AggregateFunction& aggregateFunction, ValueVector* aggVector, uint64_t multiplicity,
+    uint32_t aggStateOffset, const DataChunkState* leadingState) {
+    //  There may be a mix of flat and unflat states, but any unflat states will be the same
+    bool allFlat = leadingState->isFlat();
     if (!aggVector) {
-        updateNullAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction, multiplicity,
-            aggStateOffset);
-    } else if (aggVector->state->isFlat() && unFlatKeyVectors.empty()) {
-        updateBothFlatAggVectorState(flatKeyVectors, aggregateFunction, aggVector, multiplicity,
-            aggStateOffset);
+        updateNullAggVectorState(*leadingState, aggregateFunction, multiplicity, aggStateOffset);
+    } else if (aggVector->state->isFlat() && allFlat) {
+        updateBothFlatAggVectorState(aggregateFunction, aggVector, multiplicity, aggStateOffset);
     } else if (aggVector->state->isFlat()) {
-        updateFlatUnFlatKeyFlatAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction,
-            aggVector, multiplicity, aggStateOffset);
-    } else if (unFlatKeyVectors.empty()) {
-        updateFlatKeyUnFlatAggVectorState(flatKeyVectors, aggregateFunction, aggVector,
+        updateFlatUnFlatKeyFlatAggVectorState(*leadingState, aggregateFunction, aggVector,
             multiplicity, aggStateOffset);
-    } else if (!unFlatKeyVectors.empty() && (aggVector->state == unFlatKeyVectors[0]->state)) {
-        updateBothUnFlatSameDCAggVectorState(flatKeyVectors, unFlatKeyVectors, aggregateFunction,
-            aggVector, multiplicity, aggStateOffset);
+    } else if (allFlat) {
+        updateFlatKeyUnFlatAggVectorState(keyVectors, aggregateFunction, aggVector, multiplicity,
+            aggStateOffset);
+    } else if (aggVector->state.get() == leadingState) {
+        updateBothUnFlatSameDCAggVectorState(aggregateFunction, aggVector, multiplicity,
+            aggStateOffset);
     } else {
-        updateBothUnFlatDifferentDCAggVectorState(flatKeyVectors, unFlatKeyVectors,
-            aggregateFunction, aggVector, multiplicity, aggStateOffset);
+        updateBothUnFlatDifferentDCAggVectorState(*leadingState, aggregateFunction, aggVector,
path: src/processor/operator/aggregate/aggregate_hash_table.cpp
user: login: benjaminwinger
body: It's covered now. I noticed that I'd forgotten to make the flattening of aggregates with a different group from the leadingState's group only apply to distinct aggregates. That was making this case impossible to hit.
created_at: 2025-03-21T21:33:18Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-11T15:21:02Z
body: # Benchmark Result

Master commit hash: `77dff7ffabc02f94646c7ee55b8c210be120051a` 
Branch commit hash: `9e8e66439707baa2751f167d279037f8f1ae977f` 

| Query Group | Query Name | Mean Time - Commit (ms) | Mean Time - Master (ms) | Diff |
| --- | --- | --- | --- | --- |
| aggregation | q24 | 723.64 | 716.93 | 6.71 (0.94%) |
| aggregation | q28 | 6561.65 | 6573.93 | -12.27 (-0.19%) |
| filter | q14 | 126.07 | 116.15 | 9.92 (8.54%) |
| filter | q15 | 124.14 | 114.69 | 9.45 (8.24%) |
| filter | q16 | 338.19 | 337.98 | 0.21 (0.06%) |
| filter | q17 | 446.68 | 437.15 | 9.53 (2.18%) |
| filter | q18 | 2011.96 | 2008.40 | 3.56 (0.18%) |
| filter | zonemap-node | 90.00 | 82.03 | 7.98 (9.72%) |
| filter | zonemap-node-lhs-cast | 89.81 | 81.48 | 8.33 (10.22%) |
| filter | zonemap-node-null | 89.43 | 81.17 | 8.26 (10.18%) |
| filter | zonemap-rel | 5607.31 | 5757.51 | -150.20 (-2.61%) |
| fixed_size_expr_evaluator | q07 | 690.01 | 672.70 | 17.31 (2.57%) |
| fixed_size_expr_evaluator | q08 | 972.10 | 952.21 | 19.89 (2.09%) |
| fixed_size_expr_evaluator | q09 | 977.23 | 954.90 | 22.33 (2.34%) |
| fixed_size_expr_evaluator | q10 | 264.86 | 247.01 | 17.85 (7.23%) |
| fixed_size_expr_evaluator | q11 | 262.95 | 247.58 | 15.36 (6.21%) |
| fixed_size_expr_evaluator | q12 | 240.89 | 223.37 | 17.53 (7.85%) |
| fixed_size_expr_evaluator | q13 | 1554.67 | 1559.95 | -5.27 (-0.34%) |
| **fixed_size_seq_scan** | **q23** | **120.05** | **98.54** | **21.51 (21.83%)** |
| join | q29 | 757.07 | 774.35 | -17.29 (-2.23%) |
| join | q30 | 1747.80 | 1716.75 | 31.06 (1.81%) |
| join | q31 | 5.87 | 6.10 | -0.22 (-3.67%) |
| **join** | **SelectiveTwoHopJoin** | **52.31** | **40.39** | **11.92 (29.52%)** |
| ldbc_snb_ic | q35 | 9.54 | 7.99 | 1.55 (19.36%) |
| ldbc_snb_ic | q36 | 95.70 | 87.87 | 7.83 (8.91%) |
| **ldbc_snb_is** | **q32** | **6.94** | **4.14** | **2.80 (67.61%)** |
| ldbc_snb_is | q33 | 13.04 | 13.37 | -0.33 (-2.49%) |
| ldbc_snb_is | q34 | 1.26 | 1.20 | 0.06 (5.20%) |
| multi-rel | multi-rel-large-scan | 1438.38 | 1410.06 | 28.32 (2.01%) |
| multi-rel | multi-rel-lookup | 10.33 | 11.45 | -1.12 (-9.77%) |
| multi-rel | multi-rel-small-scan | 192.21 | 203.55 | -11.34 (-5.57%) |
| order_by | q25 | 131.24 | 123.52 | 7.73 (6.25%) |
| order_by | q26 | 445.66 | 443.66 | 2.00 (0.45%) |
| order_by | q27 | 1374.90 | 1412.85 | -37.95 (-2.69%) |
| recursive_join | recursive-join-bidirection | 275.52 | 289.80 | -14.28 (-4.93%) |
| recursive_join | recursive-join-dense | 7139.68 | 7134.59 | 5.09 (0.07%) |
| recursive_join | recursive-join-path | 23120.71 | 23136.60 | -15.89 (-0.07%) |
| recursive_join | recursive-join-sparse | 627.36 | 630.20 | -2.83 (-0.45%) |
| recursive_join | recursive-join-trail | 7072.10 | 7052.34 | 19.76 (0.28%) |
| scan_after_filter | q01 | 169.28 | 159.70 | 9.58 (6.00%) |
| scan_after_filter | q02 | 156.32 | 146.50 | 9.82 (6.70%) |
| **shortest_path_ldbc100** | **q37** | **102.98** | **84.15** | **18.83 (22.37%)** |
| shortest_path_ldbc100 | q38 | 325.47 | 316.59 | 8.88 (2.80%) |
| **shortest_path_ldbc100** | **q39** | **45.11** | **57.72** | **-12.61 (-21.85%)** |
| shortest_path_ldbc100 | q40 | 356.69 | 388.62 | -31.93 (-8.22%) |
| var_size_expr_evaluator | q03 | 2106.37 | 2074.39 | 31.97 (1.54%) |
| var_size_expr_evaluator | q04 | 2319.19 | 2302.18 | 17.01 (0.74%) |
| var_size_expr_evaluator | q05 | 2692.62 | 2671.23 | 21.39 (0.80%) |
| var_size_expr_evaluator | q06 | 1341.06 | 1331.90 | 9.16 (0.69%) |
| var_size_seq_scan | q19 | 1430.43 | 1424.41 | 6.01 (0.42%) |
| var_size_seq_scan | q20 | 2599.66 | 2696.99 | -97.33 (-3.61%) |
| var_size_seq_scan | q21 | 2250.66 | 2260.95 | -10.30 (-0.46%) |
| var_size_seq_scan | q22 | 126.91 | 124.81 | 2.10 (1.68%) |

author_association: NONE
user: login: codecov[bot]
created_at: 2025-03-11T15:33:22Z
body: ## [Codecov](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb) Report
Attention: Patch coverage is `97.89474%` with `2 lines` in your changes missing coverage. Please review.
> Project coverage is 87.50%. Comparing base [(`377df44`)](https://app.codecov.io/gh/kuzudb/kuzu/commit/377df44d91db444ab06ca1190d32a34dfafbce11?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb) to head [(`ed9ce29`)](https://app.codecov.io/gh/kuzudb/kuzu/commit/ed9ce299deeab047c1bcba7ea487b9b738c402f6?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb).
> Report is 1 commits behind head on master.

| [Files with missing lines](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb) | Patch % | Lines |
|---|---|---|
| [...cessor/operator/aggregate/aggregate\_hash\_table.cpp](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?src=pr&el=tree&filepath=src%2Fprocessor%2Foperator%2Faggregate%2Faggregate_hash_table.cpp&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb#diff-c3JjL3Byb2Nlc3Nvci9vcGVyYXRvci9hZ2dyZWdhdGUvYWdncmVnYXRlX2hhc2hfdGFibGUuY3Bw) | 98.21% | [1 Missing :warning: ](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb) |
| [...rc/processor/operator/aggregate/hash\_aggregate.cpp](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?src=pr&el=tree&filepath=src%2Fprocessor%2Foperator%2Faggregate%2Fhash_aggregate.cpp&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb#diff-c3JjL3Byb2Nlc3Nvci9vcGVyYXRvci9hZ2dyZWdhdGUvaGFzaF9hZ2dyZWdhdGUuY3Bw) | 85.71% | [1 Missing :warning: ](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #5031      +/-   ##
==========================================
- Coverage   87.52%   87.50%   -0.03%     
==========================================
  Files        1403     1403              
  Lines       63557    63538      -19     
  Branches     7509     7509              
==========================================
- Hits        55631    55601      -30     
- Misses       7897     7908      +11     
  Partials       29       29              
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/kuzudb/kuzu/pull/5031?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kuzudb).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-12T16:33:57Z
body: # Benchmark Result

Master commit hash: `617f170d64d37390f503e6998862f10fabec5603` 
Branch commit hash: `0fe9723be3cdd35bab92d6fb17dc9c6ee462163e` 

| Query Group | Query Name | Mean Time - Commit (ms) | Mean Time - Master (ms) | Diff |
| --- | --- | --- | --- | --- |
| aggregation | q24 | 873.95 | 735.21 | 138.74 (18.87%) |
| aggregation | q28 | 6577.35 | 6583.11 | -5.76 (-0.09%) |
| filter | q14 | 118.69 | 133.87 | -15.18 (-11.34%) |
| filter | q15 | 117.20 | 135.71 | -18.50 (-13.63%) |
| filter | q16 | 333.27 | 348.16 | -14.89 (-4.28%) |
| filter | q17 | 442.62 | 451.78 | -9.17 (-2.03%) |
| filter | q18 | 1992.34 | 2026.24 | -33.90 (-1.67%) |
| filter | zonemap-node | 81.00 | 97.66 | -16.66 (-17.06%) |
| filter | zonemap-node-lhs-cast | 82.14 | 97.02 | -14.87 (-15.33%) |
| filter | zonemap-node-null | 81.19 | 96.70 | -15.51 (-16.04%) |
| filter | zonemap-rel | 5692.37 | 5821.31 | -128.95 (-2.22%) |
| fixed_size_expr_evaluator | q07 | 672.99 | 692.57 | -19.58 (-2.83%) |
| fixed_size_expr_evaluator | q08 | 952.91 | 969.84 | -16.93 (-1.75%) |
| fixed_size_expr_evaluator | q09 | 958.40 | 975.72 | -17.33 (-1.78%) |
| fixed_size_expr_evaluator | q10 | 247.46 | 262.80 | -15.34 (-5.84%) |
| fixed_size_expr_evaluator | q11 | 245.73 | 262.33 | -16.60 (-6.33%) |
| fixed_size_expr_evaluator | q12 | 223.33 | 242.59 | -19.27 (-7.94%) |
| fixed_size_expr_evaluator | q13 | 1558.01 | 1568.76 | -10.75 (-0.69%) |
| fixed_size_seq_scan | q23 | 110.55 | 124.57 | -14.02 (-11.25%) |
| join | q29 | 723.50 | 767.23 | -43.73 (-5.70%) |
| join | q30 | 1687.48 | 1626.44 | 61.04 (3.75%) |
| **join** | **q31** | **4.15** | **6.00** | **-1.84 (-30.75%)** |
| **join** | **SelectiveTwoHopJoin** | **36.74** | **48.38** | **-11.64 (-24.06%)** |
| ldbc_snb_ic | q35 | 10.46 | 10.79 | -0.33 (-3.05%) |
| ldbc_snb_ic | q36 | 94.92 | 97.16 | -2.23 (-2.30%) |
| **ldbc_snb_is** | **q32** | **6.61** | **5.35** | **1.26 (23.54%)** |
| ldbc_snb_is | q33 | 12.75 | 11.98 | 0.76 (6.36%) |
| ldbc_snb_is | q34 | 1.12 | 1.24 | -0.13 (-10.12%) |
| multi-rel | multi-rel-large-scan | 1928.56 | 1683.93 | 244.64 (14.53%) |
| multi-rel | multi-rel-lookup | 10.55 | 10.45 | 0.11 (1.00%) |
| multi-rel | multi-rel-small-scan | 192.74 | 196.05 | -3.31 (-1.69%) |
| order_by | q25 | 119.43 | 139.39 | -19.95 (-14.31%) |
| order_by | q26 | 439.95 | 469.97 | -30.02 (-6.39%) |
| order_by | q27 | 1369.30 | 1401.62 | -32.32 (-2.31%) |
| recursive_join | recursive-join-bidirection | 311.96 | 300.31 | 11.65 (3.88%) |
| recursive_join | recursive-join-dense | 7157.37 | 7121.11 | 36.25 (0.51%) |
| recursive_join | recursive-join-path | 23091.98 | 24371.03 | -1279.05 (-5.25%) |
| recursive_join | recursive-join-sparse | 630.69 | 633.37 | -2.69 (-0.42%) |
| recursive_join | recursive-join-trail | 7062.37 | 7015.02 | 47.35 (0.68%) |
| scan_after_filter | q01 | 162.52 | 175.06 | -12.54 (-7.16%) |
| scan_after_filter | q02 | 148.59 | 162.02 | -13.43 (-8.29%) |
| shortest_path_ldbc100 | q37 | 94.65 | 93.98 | 0.67 (0.71%) |
| shortest_path_ldbc100 | q38 | 319.40 | 351.78 | -32.38 (-9.20%) |
| shortest_path_ldbc100 | q39 | 55.46 | 49.65 | 5.81 (11.70%) |
| shortest_path_ldbc100 | q40 | 359.96 | 340.69 | 19.28 (5.66%) |
| var_size_expr_evaluator | q03 | 2075.00 | 2120.29 | -45.29 (-2.14%) |
| var_size_expr_evaluator | q04 | 2279.27 | 2388.37 | -109.10 (-4.57%) |
| var_size_expr_evaluator | q05 | 2695.46 | 2683.39 | 12.07 (0.45%) |
| var_size_expr_evaluator | q06 | 1347.18 | 1359.73 | -12.55 (-0.92%) |
| var_size_seq_scan | q19 | 1417.68 | 1459.12 | -41.44 (-2.84%) |
| var_size_seq_scan | q20 | 2687.11 | 2729.13 | -42.03 (-1.54%) |
| var_size_seq_scan | q21 | 2213.78 | 2290.74 | -76.96 (-3.36%) |
| var_size_seq_scan | q22 | 124.70 | 127.38 | -2.68 (-2.10%) |

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-21T19:06:06Z
body: # Benchmark Result

Master commit hash: `655399ef12cc1968d53393ce00d8074ac9956fee` 
Branch commit hash: `f479567ab3f5d13d932e5d9a22ef47bddecafde2` 

| Query Group | Query Name | Mean Time - Commit (ms) | Mean Time - Master (ms) | Diff |
| --- | --- | --- | --- | --- |
| aggregation | q24 | 719.90 | 742.63 | -22.72 (-3.06%) |
| aggregation | q28 | 6542.72 | 6593.21 | -50.49 (-0.77%) |
| filter | q14 | 119.09 | 127.16 | -8.07 (-6.35%) |
| filter | q15 | 118.30 | 129.58 | -11.29 (-8.71%) |
| filter | q16 | 335.62 | 352.56 | -16.94 (-4.80%) |
| filter | q17 | 437.95 | 452.27 | -14.32 (-3.17%) |
| filter | q18 | 1906.83 | 2018.85 | -112.01 (-5.55%) |
| filter | zonemap-node | 80.77 | 90.52 | -9.75 (-10.77%) |
| filter | zonemap-node-lhs-cast | 81.40 | 90.77 | -9.37 (-10.32%) |
| filter | zonemap-node-null | 81.15 | 90.46 | -9.31 (-10.29%) |
| filter | zonemap-rel | 5729.03 | 5745.43 | -16.41 (-0.29%) |
| fixed_size_expr_evaluator | q07 | 673.11 | 690.43 | -17.32 (-2.51%) |
| fixed_size_expr_evaluator | q08 | 953.03 | 981.27 | -28.23 (-2.88%) |
| fixed_size_expr_evaluator | q09 | 955.45 | 984.91 | -29.45 (-2.99%) |
| fixed_size_expr_evaluator | q10 | 248.94 | 258.38 | -9.44 (-3.65%) |
| fixed_size_expr_evaluator | q11 | 247.89 | 260.39 | -12.50 (-4.80%) |
| fixed_size_expr_evaluator | q12 | 226.32 | 237.33 | -11.01 (-4.64%) |
| fixed_size_expr_evaluator | q13 | 1546.83 | 1587.42 | -40.58 (-2.56%) |
| fixed_size_seq_scan | q23 | 105.46 | 119.88 | -14.42 (-12.03%) |
| join | q29 | 718.26 | 755.31 | -37.06 (-4.91%) |
| join | q30 | 1644.93 | 1648.16 | -3.23 (-0.20%) |
| join | q31 | 6.89 | 6.49 | 0.40 (6.12%) |
| join | SelectiveTwoHopJoin | 45.01 | 54.35 | -9.34 (-17.19%) |
| ldbc_snb_ic | q35 | 10.60 | 9.65 | 0.96 (9.94%) |
| ldbc_snb_ic | q36 | 107.08 | 103.32 | 3.77 (3.65%) |
| ldbc_snb_is | q32 | 3.85 | 3.45 | 0.40 (11.63%) |
| ldbc_snb_is | q33 | 13.03 | 15.19 | -2.16 (-14.22%) |
| ldbc_snb_is | q34 | 0.97 | 1.20 | -0.24 (-19.62%) |
| multi-rel | multi-rel-large-scan | 1744.44 | 1869.37 | -124.93 (-6.68%) |
| **multi-rel** | **multi-rel-lookup** | **11.79** | **5.61** | **6.18 (110.14%)** |
| multi-rel | multi-rel-small-scan | 212.68 | 189.02 | 23.65 (12.51%) |
| order_by | q25 | 122.16 | 130.57 | -8.42 (-6.45%) |
| order_by | q26 | 433.24 | 458.68 | -25.44 (-5.55%) |
| order_by | q27 | 1366.24 | 1392.76 | -26.52 (-1.90%) |
| recursive_join | recursive-join-bidirection | 281.87 | 290.47 | -8.60 (-2.96%) |
| recursive_join | recursive-join-dense | 7145.24 | 7225.56 | -80.32 (-1.11%) |
| recursive_join | recursive-join-path | 23033.10 | 23636.75 | -603.64 (-2.55%) |
| recursive_join | recursive-join-sparse | 626.39 | 640.43 | -14.04 (-2.19%) |
| recursive_join | recursive-join-trail | 7067.65 | 7153.91 | -86.26 (-1.21%) |
| scan_after_filter | q01 | 159.78 | 174.90 | -15.12 (-8.64%) |
| scan_after_filter | q02 | 146.40 | 158.38 | -11.98 (-7.57%) |
| shortest_path_ldbc100 | q37 | 92.70 | 86.73 | 5.97 (6.88%) |
| **shortest_path_ldbc100** | **q38** | **326.20** | **172.43** | **153.77 (89.18%)** |
| shortest_path_ldbc100 | q39 | 57.01 | 56.68 | 0.33 (0.58%) |
| **shortest_path_ldbc100** | **q40** | **410.35** | **255.14** | **155.20 (60.83%)** |
| var_size_expr_evaluator | q03 | 2107.80 | 2136.78 | -28.97 (-1.36%) |
| var_size_expr_evaluator | q04 | 2203.44 | 2306.51 | -103.07 (-4.47%) |
| var_size_expr_evaluator | q05 | 2609.76 | 2641.51 | -31.75 (-1.20%) |
| var_size_expr_evaluator | q06 | 1336.66 | 1377.96 | -41.30 (-3.00%) |
| var_size_seq_scan | q19 | 1423.77 | 1442.54 | -18.77 (-1.30%) |
| var_size_seq_scan | q20 | 2723.61 | 2704.85 | 18.76 (0.69%) |
| var_size_seq_scan | q21 | 2272.37 | 2282.56 | -10.19 (-0.45%) |
| var_size_seq_scan | q22 | 125.95 | 125.98 | -0.03 (-0.02%) |

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-21T22:22:36Z
body: # Benchmark Result

Master commit hash: `655399ef12cc1968d53393ce00d8074ac9956fee` 
Branch commit hash: `060846c1ffb410d8bd958957c5e74b67f4fd96bb` 

| Query Group | Query Name | Mean Time - Commit (ms) | Mean Time - Master (ms) | Diff |
| --- | --- | --- | --- | --- |
| aggregation | q24 | 714.86 | 742.63 | -27.77 (-3.74%) |
| aggregation | q28 | 6566.39 | 6593.21 | -26.82 (-0.41%) |
| filter | q14 | 116.34 | 127.16 | -10.82 (-8.51%) |
| filter | q15 | 119.46 | 129.58 | -10.12 (-7.81%) |
| filter | q16 | 334.82 | 352.56 | -17.75 (-5.03%) |
| filter | q17 | 436.91 | 452.27 | -15.36 (-3.40%) |
| filter | q18 | 1906.32 | 2018.85 | -112.53 (-5.57%) |
| filter | zonemap-node | 80.71 | 90.52 | -9.81 (-10.84%) |
| filter | zonemap-node-lhs-cast | 80.28 | 90.77 | -10.48 (-11.55%) |
| filter | zonemap-node-null | 81.01 | 90.46 | -9.45 (-10.45%) |
| filter | zonemap-rel | 5724.94 | 5745.43 | -20.49 (-0.36%) |
| fixed_size_expr_evaluator | q07 | 678.10 | 690.43 | -12.34 (-1.79%) |
| fixed_size_expr_evaluator | q08 | 956.75 | 981.27 | -24.52 (-2.50%) |
| fixed_size_expr_evaluator | q09 | 960.30 | 984.91 | -24.61 (-2.50%) |
| fixed_size_expr_evaluator | q10 | 248.12 | 258.38 | -10.25 (-3.97%) |
| fixed_size_expr_evaluator | q11 | 251.40 | 260.39 | -8.99 (-3.45%) |
| fixed_size_expr_evaluator | q12 | 225.89 | 237.33 | -11.44 (-4.82%) |
| fixed_size_expr_evaluator | q13 | 1548.69 | 1587.42 | -38.72 (-2.44%) |
| fixed_size_seq_scan | q23 | 103.31 | 119.88 | -16.56 (-13.82%) |
| join | q29 | 667.40 | 755.31 | -87.92 (-11.64%) |
| join | q30 | 1635.02 | 1648.16 | -13.14 (-0.80%) |
| join | q31 | 5.64 | 6.49 | -0.85 (-13.11%) |
| **join** | **SelectiveTwoHopJoin** | **42.28** | **54.35** | **-12.08 (-22.22%)** |
| ldbc_snb_ic | q35 | 10.92 | 9.65 | 1.27 (13.21%) |
| ldbc_snb_ic | q36 | 104.22 | 103.32 | 0.91 (0.88%) |
| **ldbc_snb_is** | **q32** | **6.35** | **3.45** | **2.90 (83.97%)** |
| ldbc_snb_is | q33 | 13.87 | 15.19 | -1.32 (-8.69%) |
| ldbc_snb_is | q34 | 1.13 | 1.20 | -0.07 (-5.79%) |
| multi-rel | multi-rel-large-scan | 1781.72 | 1869.37 | -87.65 (-4.69%) |
| **multi-rel** | **multi-rel-lookup** | **11.90** | **5.61** | **6.29 (112.17%)** |
| multi-rel | multi-rel-small-scan | 205.08 | 189.02 | 16.06 (8.50%) |
| order_by | q25 | 121.95 | 130.57 | -8.62 (-6.60%) |
| order_by | q26 | 432.82 | 458.68 | -25.86 (-5.64%) |
| order_by | q27 | 1372.52 | 1392.76 | -20.24 (-1.45%) |
| recursive_join | recursive-join-bidirection | 301.91 | 290.47 | 11.44 (3.94%) |
| recursive_join | recursive-join-dense | 7157.92 | 7225.56 | -67.64 (-0.94%) |
| recursive_join | recursive-join-path | 23161.53 | 23636.75 | -475.22 (-2.01%) |
| recursive_join | recursive-join-sparse | 633.85 | 640.43 | -6.58 (-1.03%) |
| recursive_join | recursive-join-trail | 7057.36 | 7153.91 | -96.54 (-1.35%) |
| scan_after_filter | q01 | 161.37 | 174.90 | -13.53 (-7.74%) |
| scan_after_filter | q02 | 148.66 | 158.38 | -9.73 (-6.14%) |
| shortest_path_ldbc100 | q37 | 80.88 | 86.73 | -5.85 (-6.75%) |
| **shortest_path_ldbc100** | **q38** | **322.07** | **172.43** | **149.64 (86.79%)** |
| shortest_path_ldbc100 | q39 | 46.94 | 56.68 | -9.74 (-17.19%) |
| **shortest_path_ldbc100** | **q40** | **312.38** | **255.14** | **57.24 (22.43%)** |
| var_size_expr_evaluator | q03 | 2105.85 | 2136.78 | -30.93 (-1.45%) |
| var_size_expr_evaluator | q04 | 2191.12 | 2306.51 | -115.39 (-5.00%) |
| var_size_expr_evaluator | q05 | 2606.27 | 2641.51 | -35.25 (-1.33%) |
| var_size_expr_evaluator | q06 | 1331.64 | 1377.96 | -46.32 (-3.36%) |
| var_size_seq_scan | q19 | 1418.07 | 1442.54 | -24.46 (-1.70%) |
| var_size_seq_scan | q20 | 2702.97 | 2704.85 | -1.88 (-0.07%) |
| var_size_seq_scan | q21 | 2265.07 | 2282.56 | -17.49 (-0.77%) |
| var_size_seq_scan | q22 | 125.13 | 125.98 | -0.85 (-0.68%) |

author_association: NONE