Repository: mabel-dev/opteryx

Issue:
✨ Short Cut AND processing
Not all conjunctions are split and shortcutted by the planner, for example, if below an OR, implement short-cutting to reduce rows processed by subsequent AND predicate evaluations.


Discussion:

Resolving patch:
diff --git a/build_counter.py b/build_counter.py
--- a/build_counter.py
+++ b/build_counter.py
@@ -14,4 +14,8 @@
         f.write(f"__build__ = {build_number}\n")
         f.write("\n".join(contents))
 
-    print(f"Build Number: {build_number}")
+__version__ = "notset"
+with open(f"opteryx/__version__.py", mode="r") as v:
+    vers = v.read()
+exec(vers)  # nosec
+print(__version__)
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 172
+__build__ = 176
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
diff --git a/opteryx/managers/expression/__init__.py b/opteryx/managers/expression/__init__.py
--- a/opteryx/managers/expression/__init__.py
+++ b/opteryx/managers/expression/__init__.py
@@ -123,6 +123,33 @@ def identities(self):
         return list(self.results.keys())
 
 
+def short_cut_and(root, table, context):
+    # Convert to NumPy arrays
+    true_indices = numpy.arange(table.num_rows)
+
+    # Evaluate left expression
+    left_result = numpy.array(evaluate(root.left, table, context))
+
+    # If all values in left_result are False, no need to evaluate the right expression
+    if not left_result.any():
+        return left_result
+
+    # Filter out indices where left_result is FALSE
+    subset_indices = true_indices[left_result]
+
+    # Create a subset table for evaluating the right expression
+    subset_table = table.take(subset_indices)
+
+    # Evaluate right expression on the subset table
+    right_result = numpy.array(evaluate(root.right, subset_table, context))
+
+    # Combine results
+    # Iterate over subset_indices and update left_result at those positions
+    left_result[subset_indices] = right_result
+
+    return left_result
+
+
 def short_cut_or(root, table, context):
     # Assuming table.num_rows returns the number of rows in the table
     false_indices = numpy.arange(table.num_rows)
@@ -200,6 +227,8 @@ def _inner_evaluate(root: Node, table: Table, context: ExecutionContext):
     if node_type & LOGICAL_TYPE == LOGICAL_TYPE:
         if node_type == NodeType.OR:
             return short_cut_or(root, table, context)
+        if node_type == NodeType.AND:
+            return short_cut_and(root, table, context)
 
         if node_type in LOGICAL_OPERATIONS:
             left = _inner_evaluate(root.left, table, context) if root.left else None


Test patch:
diff --git a/tests/sql_battery/test_shapes_and_errors_battery.py b/tests/sql_battery/test_shapes_and_errors_battery.py
--- a/tests/sql_battery/test_shapes_and_errors_battery.py
+++ b/tests/sql_battery/test_shapes_and_errors_battery.py
@@ -369,6 +369,26 @@
         ("SELECT * FROM $satellites WHERE (id = 5 AND (2>1)) OR name = 'Europa';", 1, 8, None),
         ("SELECT * FROM $satellites WHERE id = 5 OR (name = 'Europa' AND ('a'='a'));", 1, 8, None),
         ("SELECT * FROM $satellites WHERE (id = 5 OR (3=4)) OR name = 'Europa';", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 AND (1=1)) OR (name = 'Europa' AND (TRUE OR FALSE)));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR (name = 'Europa' AND (TRUE AND (2>1))));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 OR FALSE) AND TRUE) OR (name = 'Europa' AND ('x'='x'));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 OR (4=5)) AND (6!=7)) OR (name = 'Europa');", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 AND (NOT FALSE)) OR (name = 'Europa' OR (7=8));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 AND ((8<>8) OR TRUE)) OR name = 'Europa');", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR (name = 'Europa' AND (FALSE AND TRUE)));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 OR (9>10)) AND ('a'!='b')) OR name = 'Europa';", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR (name = 'Europa' AND ('x'='x' OR (10=11))));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR ((11<12) AND name = 'Europa'));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 AND (TRUE AND (1=1))) OR (name = 'Europa' AND (FALSE OR (2!=2))));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR ((name = 'Europa' OR (TRUE AND FALSE)) AND (3>3)));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (((id = 5 OR FALSE) AND (TRUE OR (4<4))) OR (name = 'Europa' AND ('x'='y')));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 AND ((5=6) OR TRUE)) OR (name = 'Europa' AND (FALSE OR (6=7))));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR (name = 'Europa' AND (TRUE OR (7!=7) AND FALSE)));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 AND (NOT (8=8))) OR (name = 'Europa' AND (9>10 OR TRUE)));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (((id = 5 OR (10<11)) AND ('a'='b')) OR (name = 'Europa' AND (TRUE AND (11=11))));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (id = 5 OR ((name = 'Europa' OR (FALSE AND TRUE)) AND (12>13)));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE (((id = 5 AND (14=14)) OR FALSE) AND ((name = 'Europa' OR (14<15)) AND ('x'='x')));", 1, 8, None),
+        ("SELECT * FROM $satellites WHERE ((id = 5 OR (name = 'Europa' AND ((15=16) OR FALSE))) AND (TRUE AND (16!=17)));", 1, 8, None),
 
         ("SELECT COUNT(*) FROM $satellites", 1, 1, None),
         ("SELECT count(*) FROM $satellites", 1, 1, None),
@@ -1431,6 +1451,8 @@ def test_sql_battery(statement, rows, columns, exception):
 
     nl = "\n"
 
+    failures = []
+
     print(f"RUNNING BATTERY OF {len(STATEMENTS)} SHAPE TESTS")
     for index, (statement, rows, cols, err) in enumerate(STATEMENTS):
         start = time.monotonic_ns()
@@ -1458,6 +1480,7 @@ def test_sql_battery(statement, rows, columns, exception):
             print(f"\033[0;31m{str(int((time.monotonic_ns() - start)/1e6)).rjust(4)}ms ❌ *\033[0m")
             print(">", err)
             failed += 1
+            failures.append((statement, err))
 
     print("--- ✅ \033[0;32mdone\033[0m")
     print(
@@ -1465,3 +1488,7 @@ def test_sql_battery(statement, rows, columns, exception):
         f"  \033[38;2;26;185;67m{passed} passed ({(passed * 100) // (passed + failed)}%)\033[0m\n"
         f"  \033[38;2;255;121;198m{failed} failed\033[0m"
     )
+    if failed > 0:
+        print("\n\033[38;2;139;233;253m\033[3mFAILURES\033[0m")
+        for statement, err in failures:
+            print(err)