Repository: mabel-dev/opteryx

Issue:
🪲 Performance Stats illogical
time_evaluate_dataset: 0.00137
time_evaluating: 0.00057
time_exiting: 0.00002
time_grouping: 0.068
time_inner_join: 27.145
time_planning: 8.723
time_reading_blobs: 17.179

--------
time_total: 36.3028

The numbers don't make sense, 27 + 8 + 17 = 52, which is a lot more than 36


Discussion:
the inner join contained the time for the blob reading

10 + 8 +17 = 35 (rounded etc, looks about right)

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 321
+__build__ = 322
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/operators/cross_join_node.py b/opteryx/operators/cross_join_node.py
--- a/opteryx/operators/cross_join_node.py
+++ b/opteryx/operators/cross_join_node.py
@@ -18,6 +18,7 @@
 This performs a CROSS JOIN - CROSS JOIN is not natively supported by PyArrow so this is written
 here rather than calling the join() functions
 """
+import time
 from typing import Generator
 from typing import Tuple
 
@@ -208,15 +209,27 @@ def execute(self) -> Generator:
         )  # type:ignore
 
         if self._unnest_column is None:
-            yield from _cross_join(left_node, right_table)
+            start = time.monotonic_ns()
+            for morsel in _cross_join(left_node, right_table):
+                self.statistics.time_cross_join += time.monotonic_ns() - start
+                yield morsel
+                start = time.monotonic_ns()
 
         elif isinstance(self._unnest_column.value, tuple):
-            yield from _cross_join_unnest_literal(
+            start = time.monotonic_ns()
+            for morsel in _cross_join_unnest_literal(
                 morsels=left_node,
                 source=self._unnest_column.value,
                 target_column=self._unnest_target,
-            )
+            ):
+                self.statistics.time_cross_join_literal += time.monotonic_ns() - start
+                yield morsel
+                start = time.monotonic_ns()
         else:
-            yield from _cross_join_unnest_column(
+            start = time.monotonic_ns()
+            for morsel in _cross_join_unnest_column(
                 morsels=left_node, source=self._unnest_column, target_column=self._unnest_target
-            )
+            ):
+                self.statistics.time_cross_join_unnest += time.monotonic_ns() - start
+                yield morsel
+                start = time.monotonic_ns()
diff --git a/opteryx/operators/inner_join_node.py b/opteryx/operators/inner_join_node.py
--- a/opteryx/operators/inner_join_node.py
+++ b/opteryx/operators/inner_join_node.py
@@ -113,16 +113,16 @@ def execute(self) -> Generator:
         left_node = self._producers[0]  # type:ignore
         right_node = self._producers[1]  # type:ignore
 
-        start = time.monotonic_ns()
-
         left_relation = pyarrow.concat_tables(left_node.execute(), promote_options="none")
         # in place until #1295 resolved
         if not self._left_columns[0] in left_relation.column_names:
             self._right_columns, self._left_columns = self._left_columns, self._right_columns
 
+        start = time.monotonic_ns()
         left_hash = preprocess_left(left_relation, self._left_columns)
-
+        self.statistics.time_inner_join += time.monotonic_ns() - start
         for morsel in right_node.execute():
+            start = time.monotonic_ns()
             # do the join
             new_morsel = inner_join_with_preprocessed_left_side(
                 left_relation=left_relation,
@@ -132,4 +132,3 @@ def execute(self) -> Generator:
             )
             self.statistics.time_inner_join += time.monotonic_ns() - start
             yield new_morsel
-            start = time.monotonic_ns()


Test patch: