Repository: mabel-dev/opteryx

Issue:
🪲 complex predicates not pushing correctly
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Describe the bug** _A clear and specific description of what the bug is. What the error, incorrect or unexpected behaviour was._


**Expected behaviour** _A clear and concise description of what you expected to happen._


**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql

~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 333
+__build__ = 336
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/components/cost_based_optimizer/strategies/predicate_pushdown.py b/opteryx/components/cost_based_optimizer/strategies/predicate_pushdown.py
--- a/opteryx/components/cost_based_optimizer/strategies/predicate_pushdown.py
+++ b/opteryx/components/cost_based_optimizer/strategies/predicate_pushdown.py
@@ -25,6 +25,9 @@
 from .optimization_strategy import OptimizationStrategy
 from .optimization_strategy import OptimizerContext
 
+IN_REWRITES = {"InList": "Eq", "NotInList": "NotEq"}
+LIKE_REWRITES = {"Like": "Eq", "NotList": "NotEq"}
+
 
 def _add_condition(existing_condition, new_condition):
     if not existing_condition:
@@ -35,10 +38,6 @@ def _add_condition(existing_condition, new_condition):
     return _and
 
 
-IN_REWRITES = {"InList": "Eq", "NotInList": "NotEq"}
-LIKE_REWRITES = {"Like": "Eq", "NotList": "NotEq"}
-
-
 def _rewrite_predicate(predicate):
     """
     Rewrite individual predicates
@@ -62,6 +61,18 @@ def _rewrite_predicate(predicate):
     return predicate
 
 
+def _tag_predicate(predicate):
+    """
+    Add flags, tags, labels, and notes to predicates
+    """
+
+    # predicate.relations = set()
+    # add label for if a predicate is a filter or a join
+    # add in nominal per-row cost information / time to execute 1 million times
+
+    return predicate
+
+
 class PredicatePushdownStrategy(OptimizationStrategy):
     def visit(self, node: LogicalPlanNode, context: OptimizerContext) -> OptimizerContext:
         if not context.optimized_plan:
@@ -81,8 +92,10 @@ def visit(self, node: LogicalPlanNode, context: OptimizerContext) -> OptimizerCo
 
         elif node.node_type == LogicalPlanStepType.Filter:
             # collect predicates we can probably push
-            if len(node.relations) > 0 and not get_all_nodes_of_type(
-                node.condition, (NodeType.AGGREGATOR,)
+            if (
+                len(node.relations) > 0
+                and not get_all_nodes_of_type(node.condition, (NodeType.AGGREGATOR,))
+                and len(get_all_nodes_of_type(node.condition, (NodeType.IDENTIFIER,))) == 1
             ):
                 # record where the node was, so we can put it back
                 node.nid = context.node_id
diff --git a/opteryx/connectors/sql_connector.py b/opteryx/connectors/sql_connector.py
--- a/opteryx/connectors/sql_connector.py
+++ b/opteryx/connectors/sql_connector.py
@@ -14,6 +14,7 @@
 The SQL Connector downloads data from remote servers and converts them
 to pyarrow tables so they can be processed as per any other data source.
 """
+import time
 from decimal import Decimal
 from typing import Any
 from typing import Dict
@@ -167,6 +168,8 @@ def read_dataset(  # type:ignore
         row_factory = Row.create_class(result_schema, tuples_only=True)
         at_least_once = False
 
+        convert_time = 0.0
+
         with self._engine.connect() as conn:
             # DEBUG: log ("READ DATASET\n", str(query_builder))
             # DEBUG: log ("PARAMETERS\n", parameters)
@@ -181,9 +184,12 @@ def read_dataset(  # type:ignore
                     break
 
                 # convert each SqlAlchemy Row to an orso Row
+                b = time.monotonic_ns()
                 for row in batch_rows:
                     morsel._rows.append(row_factory(row))
-                yield morsel.arrow()
+                arr = morsel.arrow()
+                convert_time += time.monotonic_ns() - b
+                yield arr
                 at_least_once = True
 
                 # Dynamically adjust chunk size based on the data size, we start by downloading
@@ -204,6 +210,8 @@ def read_dataset(  # type:ignore
         if not at_least_once:
             yield morsel.arrow()
 
+        # DEBUG: log (f"time spent converting: {convert_time/1e9}s")
+
     def get_dataset_schema(self) -> RelationSchema:
         from sqlalchemy import Table
 


Test patch:
diff --git a/tests/plan_optimization/test_predicate_pushdown_parquet.py b/tests/plan_optimization/test_predicate_pushdown_parquet.py
--- a/tests/plan_optimization/test_predicate_pushdown_parquet.py
+++ b/tests/plan_optimization/test_predicate_pushdown_parquet.py
@@ -42,7 +42,7 @@ def test_predicate_pushdowns_blobs_parquet():
         "SELECT * FROM 'testdata/flat/planets/parquet/planets.parquet' WHERE rotationPeriod = lengthOfDay;"
     )
     assert cur.rowcount == 3, cur.rowcount
-    assert cur.stats.get("rows_read", 0) == 3, cur.stats
+    assert cur.stats.get("rows_read", 0) == 9, cur.stats
 
     conn.close()
 
diff --git a/tests/plan_optimization/test_predicate_pushdown_postgres.py b/tests/plan_optimization/test_predicate_pushdown_postgres.py
--- a/tests/plan_optimization/test_predicate_pushdown_postgres.py
+++ b/tests/plan_optimization/test_predicate_pushdown_postgres.py
@@ -34,8 +34,9 @@
     ("SELECT * FROM pg.planets WHERE gravity > 3.7", 6, 6),
     ("SELECT * FROM pg.planets WHERE gravity >= 3.7", 8, 8),
     ("SELECT * FROM pg.planets WHERE name LIKE '%a%'", 4, 4),
-    ("SELECT * FROM pg.planets WHERE id > gravity", 2, 2),
+    ("SELECT * FROM pg.planets WHERE id > gravity", 2, 9),
     ("SELECT * FROM pg.planets WHERE surface_pressure IS NULL", 4, 4),
+    ("SELECT * FROM pg.planets WHERE gravity - mass > 10", 0, 9),
 ]
 
 
diff --git a/tests/plan_optimization/test_predicate_pushdown_sqlite.py b/tests/plan_optimization/test_predicate_pushdown_sqlite.py
--- a/tests/plan_optimization/test_predicate_pushdown_sqlite.py
+++ b/tests/plan_optimization/test_predicate_pushdown_sqlite.py
@@ -105,7 +105,7 @@ def test_predicate_pushdown_sqlite_other():
 
     res = opteryx.query("SELECT * FROM sqlite.planets WHERE id > gravity")
     assert res.rowcount == 2, res.rowcount
-    assert res.stats.get("rows_read", 0) == 2, res.stats
+    assert res.stats.get("rows_read", 0) == 9, res.stats
 
 
 if __name__ == "__main__":  # pragma: no cover
diff --git a/tests/plan_optimization/test_predicate_pushdown_zstd.py b/tests/plan_optimization/test_predicate_pushdown_zstd.py
--- a/tests/plan_optimization/test_predicate_pushdown_zstd.py
+++ b/tests/plan_optimization/test_predicate_pushdown_zstd.py
@@ -42,7 +42,7 @@ def test_predicate_pushdowns_blobs_zstd():
     cur.execute("SELECT * FROM testdata.flat.formats.zstd WHERE following < followers;")
     # test two identifiers
     assert cur.rowcount == 37319, cur.rowcount
-    assert cur.stats.get("rows_read", 0) == 37319, cur.stats
+    assert cur.stats.get("rows_read", 0) == 100000, cur.stats
 
     conn.close()
 
diff --git a/tests/storage/test_sql_cockroach.py b/tests/storage/test_sql_cockroach.py
--- a/tests/storage/test_sql_cockroach.py
+++ b/tests/storage/test_sql_cockroach.py
@@ -56,7 +56,7 @@ def test_cockroach_storage():
 
     results = opteryx.query("SELECT * FROM cockroach.planets WHERE distancefromsun < lengthofday")
     assert results.rowcount == 2, results.rowcount
-    assert results.stats.get("rows_read", 0) == 2, results.stats
+    assert results.stats.get("rows_read", 0) == 9, results.stats
 
 
 if __name__ == "__main__":  # pragma: no cover
diff --git a/tests/storage/test_sql_duckdb.py b/tests/storage/test_sql_duckdb.py
--- a/tests/storage/test_sql_duckdb.py
+++ b/tests/storage/test_sql_duckdb.py
@@ -53,7 +53,7 @@ def test_duckdb_storage():
 
     results = opteryx.query("SELECT * FROM duckdb.planets WHERE id > gravity")
     assert results.rowcount == 2, results.rowcount
-    assert results.stats.get("rows_read", 0) == 2, results.stats
+    assert results.stats.get("rows_read", 0) == 9, results.stats
 
 
 if __name__ == "__main__":  # pragma: no cover
diff --git a/tests/storage/test_sql_mysql.py b/tests/storage/test_sql_mysql.py
--- a/tests/storage/test_sql_mysql.py
+++ b/tests/storage/test_sql_mysql.py
@@ -48,7 +48,7 @@ def test_mysql_storage():
 
     results = opteryx.query("SELECT * FROM mysql.planets WHERE id > gravity")
     assert results.rowcount == 2, results.rowcount
-    assert results.stats.get("rows_read", 0) == 2, results.stats
+    assert results.stats.get("rows_read", 0) == 9, results.stats
 
 
 def test_database_paging():
diff --git a/tests/storage/test_sql_postgres.py b/tests/storage/test_sql_postgres.py
--- a/tests/storage/test_sql_postgres.py
+++ b/tests/storage/test_sql_postgres.py
@@ -55,7 +55,7 @@ def test_postgres_storage():
 
     results = opteryx.query("SELECT * FROM pg.planets WHERE id > gravity")
     assert results.rowcount == 2, results.rowcount
-    assert results.stats.get("rows_read", 0) == 2, results.stats
+    assert results.stats.get("rows_read", 0) == 9, results.stats
 
 
 if __name__ == "__main__":  # pragma: no cover
diff --git a/tests/storage/test_sql_sqlite.py b/tests/storage/test_sql_sqlite.py
--- a/tests/storage/test_sql_sqlite.py
+++ b/tests/storage/test_sql_sqlite.py
@@ -49,7 +49,7 @@ def test_sqlite_storage():
 
     results = opteryx.query("SELECT * FROM sqlite.planets WHERE id > gravity")
     assert results.rowcount == 2, results.rowcount
-    assert results.stats.get("rows_read", 0) == 2, results.stats
+    assert results.stats.get("rows_read", 0) == 9, results.stats
 
 
 if __name__ == "__main__":  # pragma: no cover