Repository: mabel-dev/opteryx

Issue:
🪲 ANY on a literal always returns no matches
~~~sql
SELECT $missions.* FROM $missions WHERE Mission = ANY(@@user_memberships)
~~~


Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 538
+__build__ = 540
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/connectors/cql_connector.py b/opteryx/connectors/cql_connector.py
--- a/opteryx/connectors/cql_connector.py
+++ b/opteryx/connectors/cql_connector.py
@@ -56,7 +56,7 @@ class CqlConnector(BaseConnector, PredicatePushable):
 
     PUSHABLE_OPS: Dict[str, bool] = {
         "Eq": True,
-        "NotEq": True,
+        # "NotEq": True,
         "Gt": True,
         "GtEq": True,
         "Lt": True,
@@ -65,7 +65,7 @@ class CqlConnector(BaseConnector, PredicatePushable):
 
     OPS_XLAT: Dict[str, str] = {
         "Eq": ":left = :right",
-        "NotEq": ":left != :right",
+        # "NotEq": ":left not in (:right)",
         "Gt": ":left > :right",
         "GtEq": ":left >= :right",
         "Lt": ":left < :right",
diff --git a/opteryx/planner/cost_based_optimizer/strategies/predicate_pushdown.py b/opteryx/planner/cost_based_optimizer/strategies/predicate_pushdown.py
--- a/opteryx/planner/cost_based_optimizer/strategies/predicate_pushdown.py
+++ b/opteryx/planner/cost_based_optimizer/strategies/predicate_pushdown.py
@@ -16,6 +16,8 @@
 
 """
 import re
+from typing import Callable
+from typing import Dict
 
 from orso.tools import random_string
 from orso.types import OrsoTypes
@@ -34,7 +36,7 @@
 from .optimization_strategy import OptimizerContext
 
 IN_REWRITES = {"InList": "Eq", "NotInList": "NotEq"}
-LIKE_REWRITES = {"Like": "Eq", "NotList": "NotEq"}
+LIKE_REWRITES = {"Like": "Eq", "NotLike": "NotEq"}
 
 
 def _add_condition(existing_condition, new_condition):
@@ -46,101 +48,130 @@ def _add_condition(existing_condition, new_condition):
     return _and
 
 
-def _rewrite_predicate(predicate):
+def remove_adjacent_wildcards(predicate):
     """
-    Rewrite individual predicates to forms able to push to more places or that
-    are just faster.
+    Remove adjacent wildcards from LIKE/ILIKE/NotLike/NotILike conditions.
+
+    This optimization removes redundant wildcards from LIKE patterns to help
+    improve matching with subsequent optimizations
     """
-    # remove adjacent wildcards
-    if (
-        predicate.value in {"Like", "ILike", "NotLike", "NotILike"}
-        and "%%" in predicate.right.value
-    ):
+    if "%%" in predicate.right.value:
         predicate.right.value = re.sub(r"%+", "%", predicate.right.value)
+    return predicate
+
+
+def rewrite_to_starts_with(predicate):
+    """
+    Rewrite LIKE/ILIKE conditions with a single trailing wildcard to STARTS_WITH function.
+
+    This optimization converts patterns like 'abc%' to a STARTS_WITH function, which can be
+    more efficiently processed by the underlying engine compared to a generic LIKE pattern.
+    """
+    ignore_case = predicate.value == "ILike"
+    predicate.right.value = predicate.right.value[:-1]
+    predicate.node_type = NodeType.FUNCTION
+    predicate.value = "STARTS_WITH"
+    predicate.parameters = [
+        predicate.left,
+        predicate.right,
+        Node(node_type=NodeType.LITERAL, type=OrsoTypes.BOOLEAN, value=ignore_case),
+    ]
+    return predicate
+
+
+def rewrite_to_ends_with(predicate):
+    """
+    Rewrite LIKE/ILIKE conditions with a single leading wildcard to ENDS_WITH function.
+
+    This optimization converts patterns like '%abc' to an ENDS_WITH function, which can be
+    more efficiently processed by the underlying engine compared to a generic LIKE pattern.
+    """
+    ignore_case = predicate.value == "ILike"
+    predicate.right.value = predicate.right.value[1:]
+    predicate.node_type = NodeType.FUNCTION
+    predicate.value = "ENDS_WITH"
+    predicate.parameters = [
+        predicate.left,
+        predicate.right,
+        Node(node_type=NodeType.LITERAL, type=OrsoTypes.BOOLEAN, value=ignore_case),
+    ]
+    return predicate
+
+
+def rewrite_to_search(predicate):
+    """
+    Rewrite LIKE/ILIKE conditions with leading and trailing wildcards to SEARCH function.
+
+    This optimization converts patterns like '%abc%' to a SEARCH function, which can be
+    more efficiently processed by the underlying engine compared to a generic LIKE pattern.
+    """
+    ignore_case = predicate.value == "ILike"
+    predicate.right.value = predicate.right.value[1:-1]
+    predicate.node_type = NodeType.FUNCTION
+    predicate.value = "SEARCH"
+    predicate.parameters = [
+        predicate.left,
+        predicate.right,
+        Node(node_type=NodeType.LITERAL, type=OrsoTypes.BOOLEAN, value=ignore_case),
+    ]
+    return predicate
+
 
-    if predicate.value in LIKE_REWRITES:
-        # LIKE conditions with no wildcards => Eq
-        if (
-            predicate.right.node_type == NodeType.LITERAL
-            and "%" not in predicate.right.value
-            and "_" not in predicate.right.value
-        ):
+def rewrite_in_to_eq(predicate):
+    """
+    Rewrite IN conditions with a single value to equality conditions.
+
+    If the IN condition contains only one value, it is equivalent to an equality check.
+    This optimization replaces the IN condition with a faster equality check.
+    """
+    predicate.value = IN_REWRITES[predicate.value]
+    predicate.right.value = predicate.right.value.pop()
+    predicate.right.type = predicate.right.sub_type or OrsoTypes.VARCHAR
+    predicate.right.sub_type = None
+    return predicate
+
+
+# Define dispatcher conditions and actions
+dispatcher: Dict[str, Callable] = {
+    "remove_adjacent_wildcards": remove_adjacent_wildcards,
+    "rewrite_to_starts_with": rewrite_to_starts_with,
+    "rewrite_to_ends_with": rewrite_to_ends_with,
+    "rewrite_to_search": rewrite_to_search,
+    "rewrite_in_to_eq": rewrite_in_to_eq,
+}
+
+
+# Dispatcher conditions
+def _rewrite_predicate(predicate):
+    if predicate.value in {"Like", "ILike", "NotLike", "NotILike"}:
+        if "%%" in predicate.right.value:
+            predicate = dispatcher["remove_adjacent_wildcards"](predicate)
+
+    if predicate.value in {"Like", "NotLike"}:
+        if "%" not in predicate.right.value and "_" not in predicate.right.value:
             predicate.value = LIKE_REWRITES[predicate.value]
-            return predicate
-    if (
-        predicate.value in {"Like", "ILike"}
-        and predicate.right.value
-        and predicate.left.source_connector.isdisjoint({"Sql", "Cql"})
-    ):
-        ignore_case = predicate.value == "ILike"
-        # Rewrite LIKEs as STARTS_WITH
-        if (
-            predicate.right.node_type == NodeType.LITERAL
-            and predicate.right.value[-1] == "%"
-            and predicate.right.value.count("%") == 1
-            and "_" not in predicate.right.value
-        ):
-            predicate.right.value = predicate.right.value[:-1]
-            predicate.node_type = NodeType.FUNCTION
-            predicate.value = "STARTS_WITH"
-            predicate.parameters = [
-                predicate.left,
-                predicate.right,
-                Node(
-                    node_type=NodeType.LITERAL,
-                    type=OrsoTypes.BOOLEAN,
-                    value=ignore_case,
-                ),
-            ]
-            return predicate
-        # Rewrite LIKEs as ENDS_WITH
-        if (
-            predicate.right.node_type == NodeType.LITERAL
-            and predicate.right.value[0] == "%"
-            and predicate.right.value.count("%") == 1
-            and "_" not in predicate.right.value
-        ):
-            predicate.right.value = predicate.right.value[1:]
-            predicate.node_type = NodeType.FUNCTION
-            predicate.value = "ENDS_WITH"
-            predicate.parameters = [
-                predicate.left,
-                predicate.right,
-                Node(
-                    node_type=NodeType.LITERAL,
-                    type=OrsoTypes.BOOLEAN,
-                    value=ignore_case,
-                ),
-            ]
-            return predicate
-        if (
-            predicate.right.node_type == NodeType.LITERAL
-            and predicate.right.value[0] == "%"
-            and predicate.right.value[-1] == "%"
-            and predicate.right.value.count("%") == 2
-            and "_" not in predicate.right.value
-        ):
-            predicate.right.value = predicate.right.value[1:-1]
-            predicate.node_type = NodeType.FUNCTION
-            predicate.value = "SEARCH"
-            predicate.parameters = [
-                predicate.left,
-                predicate.right,
-                Node(
-                    node_type=NodeType.LITERAL,
-                    type=OrsoTypes.BOOLEAN,
-                    value=ignore_case,
-                ),
-            ]
-            return predicate
+
+    if predicate.value in {"Like", "ILike"}:
+        if predicate.left.source_connector.isdisjoint({"Sql", "Cql"}):
+            if predicate.right.node_type == NodeType.LITERAL:
+                if predicate.right.value[-1] == "%" and predicate.right.value.count("%") == 1:
+                    return dispatcher["rewrite_to_starts_with"](predicate)
+                if predicate.right.value[0] == "%" and predicate.right.value.count("%") == 1:
+                    return dispatcher["rewrite_to_ends_with"](predicate)
+                if (
+                    predicate.right.value[0] == "%"
+                    and predicate.right.value[-1] == "%"
+                    and predicate.right.value.count("%") == 2
+                ):
+                    return dispatcher["rewrite_to_search"](predicate)
+
+    if predicate.value == "AnyOpEq":
+        if predicate.right.node_type == NodeType.LITERAL:
+            predicate.value = "InList"
+
     if predicate.value in IN_REWRITES:
-        # IN conditions on single values => Eq
         if predicate.right.node_type == NodeType.LITERAL and len(predicate.right.value) == 1:
-            predicate.value = IN_REWRITES[predicate.value]
-            predicate.right.value = predicate.right.value.pop()
-            predicate.right.type = predicate.right.sub_type
-            predicate.right.sub_type = None
-            return predicate
+            return dispatcher["rewrite_in_to_eq"](predicate)
 
     return predicate
 


Test patch:
diff --git a/tests/sql_battery/test_shapes_and_errors_battery.py b/tests/sql_battery/test_shapes_and_errors_battery.py
--- a/tests/sql_battery/test_shapes_and_errors_battery.py
+++ b/tests/sql_battery/test_shapes_and_errors_battery.py
@@ -1345,6 +1345,8 @@
         ("SELECT name[id] as m FROM $planets", None, None, UnsupportedSyntaxError),
         ("SELECT * FROM $astronauts WHERE LIST_CONTAINS_ANY(missions, @@user_memberships)", 3, 19, None),
         ("SELECT $missions.* FROM $missions INNER JOIN $user ON Mission = value WHERE attribute = 'membership'", 1, 8, None),
+        ("SELECT * FROM $planets WHERE name = any(@@user_memberships)", 0, 20, None),
+        ("SELECT name FROM sqlite.planets WHERE name = ANY(('Earth', 'Mars'))", 2, 1, None),
 
         # TEST FUNCTIONS
         ("EXECUTE PLANETS_BY_ID (id=1)", 1, 20, None),  # simple case
@@ -1616,6 +1618,7 @@ def test_sql_battery(statement, rows, columns, exception):
         raise Exception(err) from err
     except Exception as err:
         if type(err) != exception:
+            quit()
             raise Exception(
                 f"{format_sql(statement)}\nQuery failed with error {type(err)} but error {exception} was expected"
             ) from err
diff --git a/tests/storage/test_cql_datastax.py b/tests/storage/test_cql_datastax.py
--- a/tests/storage/test_cql_datastax.py
+++ b/tests/storage/test_cql_datastax.py
@@ -8,8 +8,6 @@
 import sys
 import pytest
 
-# os.environ["OPTERYX_DEBUG"] = "1"
-
 sys.path.insert(1, os.path.join(sys.path[0], "../.."))
 
 import opteryx
diff --git a/tests/storage/test_sql_sqlite.py b/tests/storage/test_sql_sqlite.py
--- a/tests/storage/test_sql_sqlite.py
+++ b/tests/storage/test_sql_sqlite.py
@@ -71,6 +71,25 @@
     ("SELECT name FROM sqlite.planets WHERE surfacePressure IS NOT NULL", 5, 1, None),
     ("SELECT name FROM sqlite.planets WHERE numberOfMoons IS NOT TRUE", 2, 1, None),
     ("SELECT name FROM sqlite.planets WHERE numberOfMoons IS TRUE", 7, 1, None),
+    ("SELECT name FROM sqlite.planets WHERE name LIKE 'M%';", 2, 1, None),  # Mars, Mercury
+    ("SELECT name FROM sqlite.planets WHERE name NOT LIKE 'M%';", 7, 1, None),  # All except Mars and Mercury
+    ("SELECT name FROM sqlite.planets WHERE name LIKE '%e%';", 5, 1, None),  # Earth, Jupiter, Neptune, Mercury, Venus
+    ("SELECT name FROM sqlite.planets WHERE name NOT LIKE '%e%';", 4, 1, None),  # Mars, Saturn, Uranus, Pluto
+    ("SELECT name FROM sqlite.planets WHERE name ILIKE 'p%';", 1, 1, None),  # Pluto
+    ("SELECT name FROM sqlite.planets WHERE name NOT ILIKE 'p%';", 8, 1, None),  # All except Pluto
+    ("SELECT name FROM sqlite.planets WHERE name ILIKE '%U%';", 7, 1, None),
+    ("SELECT name FROM sqlite.planets WHERE name NOT ILIKE '%U%';", 2, 1, None),
+    ("SELECT name FROM sqlite.planets WHERE name LIKE '__r%';", 3, 1, None),  # Earth, Uranus
+    ("SELECT name FROM sqlite.planets WHERE name NOT LIKE '__r%';", 6, 1, None), 
+    ("SELECT name FROM sqlite.planets WHERE name LIKE '%t';", 0, 1, None), 
+    ("SELECT name FROM sqlite.planets WHERE name NOT LIKE '%t';", 9, 1, None), 
+    ("SELECT name FROM sqlite.planets WHERE name ILIKE '_a%';", 3, 1, None),  # Mars, Saturn, Uranus
+    ("SELECT name FROM sqlite.planets WHERE name NOT ILIKE '_a%';", 6, 1, None),  # All except Mars, Saturn, Uranus
+    ("SELECT name FROM sqlite.planets WHERE name LIKE '____';", 1, 1, None), 
+    ("SELECT name FROM sqlite.planets WHERE name NOT LIKE '____';", 8, 1, None),  # All except Mars, Earth
+    ("SELECT name FROM sqlite.planets WHERE name ILIKE '%o';", 1, 1, None),  # Pluto
+    ("SELECT name FROM sqlite.planets WHERE name NOT ILIKE '%o';", 8, 1, None)  # All except Pluto
+
 ]
 # fmt: on
 
@@ -169,7 +188,7 @@ def test_sql_battery(statement, rows, columns, exception):
             failed += 1
             failures.append((statement, err))
 
-    print("--- ✅ \033[0;32mdone\033[0m")
+    print("#- ✅ \033[0;32mdone\033[0m")
 
     if failed > 0:
         print("\n\033[38;2;139;233;253m\033[3mFAILURES\033[0m")