Repository: mabel-dev/opteryx

Issue:
🪲 IP Containment testing fails after bitwise OR implemented
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Describe the bug** _A clear and specific description of what the bug is. What the error, incorrect or unexpected behaviour was._


**Expected behaviour** _A clear and concise description of what you expected to happen._


**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql

~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/functions/binary_operators.py b/opteryx/functions/binary_operators.py
--- a/opteryx/functions/binary_operators.py
+++ b/opteryx/functions/binary_operators.py
@@ -14,6 +14,8 @@
 import datetime
 from typing import Any
 from typing import Dict
+from typing import List
+from typing import Optional
 from typing import Tuple
 from typing import Union
 
@@ -44,6 +46,7 @@
 
 INTERVALS = (pyarrow.lib.MonthDayNano, pyarrow.lib.MonthDayNanoIntervalArray)
 LISTS = (pyarrow.Array, numpy.ndarray, list, array.ArrayType)
+STRINGS = (str, numpy.str_)
 
 # Also supported by the AST but not implemented
 
@@ -102,15 +105,36 @@ def _date_minus_interval(left, right):
     return result
 
 
-def _has_intervals(left, right):
+def _ip_containment(left: List[Optional[str]], right: List[str]) -> List[Optional[bool]]:
+    """
+    Check if each IP address in 'left' is contained within the network specified in 'right'.
+
+    Parameters:
+        left: List[Optional[str]]
+            List of IP addresses as strings.
+        right: List[str]
+            List containing the network as a string.
+
+    Returns:
+        List[Optional[bool]]:
+            A list of boolean values indicating if each corresponding IP in 'left' is in 'right'.
+    """
+    from ipaddress import IPv4Address
+    from ipaddress import IPv4Network
+
+    network = IPv4Network(right[0], strict=False)
+    return [(IPv4Address(ip) in network) if ip else None for ip in left]
+
+
+def _either_side_is_type(left, right, types):
     def _check_type(obj, types: Union[type, Tuple[type, ...]]) -> bool:
         return any(isinstance(obj, t) for t in types)
 
     return (
-        _check_type(left, INTERVALS)
-        or _check_type(right, INTERVALS)
-        or (_check_type(left, LISTS) and _check_type(left[0], INTERVALS))
-        or (_check_type(right, LISTS) and _check_type(right[0], INTERVALS))
+        _check_type(left, types)
+        or _check_type(right, types)
+        or (_check_type(left, LISTS) and _check_type(left[0], types))
+        or (_check_type(right, LISTS) and _check_type(right[0], types))
     )
 
 
@@ -134,15 +158,19 @@ def binary_operations(left, operator: str, right) -> Union[numpy.ndarray, pyarro
     if operation is None:
         raise NotImplementedError(f"Operator `{operator}` is not implemented!")
 
-    if operator == "Minus" or operator == "Plus":
-        if _has_intervals(left, right):
+    if operator in ["Minus", "Plus"]:
+        if _either_side_is_type(left, right, INTERVALS):
             return (
                 _date_minus_interval(left, right)
                 if operator == "Minus"
                 else _date_plus_interval(left, right)
             )
 
-    if operator == "StringConcat":
+    elif operator == "BitwiseOr":
+        if _either_side_is_type(left, right, STRINGS):
+            return _ip_containment(left, right)
+
+    elif operator == "StringConcat":
         empty = numpy.full(len(left), "")
         joined = compute.binary_join_element_wise(left, right, empty)
         return joined
diff --git a/opteryx/functions/string_functions.py b/opteryx/functions/string_functions.py
--- a/opteryx/functions/string_functions.py
+++ b/opteryx/functions/string_functions.py
@@ -11,6 +11,7 @@
 # limitations under the License.
 
 from typing import List
+from typing import Union
 
 import numpy
 from pyarrow import compute
@@ -241,24 +242,34 @@ def ends_w(arr, test):
     return compute.ends_with(arr, test[0])
 
 
-def substring(arr, from_pos, count):
+def substring(arr: List[str], from_pos: List[int], count: List[Union[int, float]]) -> List[str]:
+    """
+    Extracts substrings from each string in the 'arr' list.
+
+    Parameters:
+        arr: List[str]
+            List of strings from which substrings will be extracted.
+        from_pos: List[int]
+            List of starting positions for each substring.
+        count: List[Union[int, float]]
+            List of lengths for each substring. Can be a float (NaN) to signify until the end.
+
+    Returns:
+        List[str]
+            List of extracted substrings.
+    """
     if len(arr) == 0:
         return [[]]
 
     def _inner(val, _from, _for):
-        _from = int(_from)
         if _from > 0:
             _from -= 1
-        _for = int(_for) if _for == _for else None  # nosec
+        _for = int(_for) if _for and _for == _for else None  # nosec
         if _for is None:
             return val[_from:]
         return val[_from : _for + _from]
 
-    def _outer():
-        for index, value in enumerate(arr):
-            yield _inner(value, from_pos[index], count[index])
-
-    return list(_outer())
+    return [_inner(val, _from, _for) for val, _from, _for in zip(arr, from_pos, count)]
 
 
 def position(sub, string):
diff --git a/opteryx/managers/expression/formatter.py b/opteryx/managers/expression/formatter.py
--- a/opteryx/managers/expression/formatter.py
+++ b/opteryx/managers/expression/formatter.py
@@ -36,7 +36,7 @@ def format_expression(root, qualify: bool = False):
         if literal_type == OrsoTypes.VARCHAR:
             return "'" + root.value.replace("'", "'") + "'"
         if literal_type == OrsoTypes.TIMESTAMP:
-            return "'" + str(root) + "'"
+            return "'" + str(root.value) + "'"
         if literal_type == OrsoTypes.INTERVAL:
             return "<INTERVAL>"
         return str(root.value)


Test patch:
diff --git a/tests/sql_battery/test_shapes_and_errors_battery.py b/tests/sql_battery/test_shapes_and_errors_battery.py
--- a/tests/sql_battery/test_shapes_and_errors_battery.py
+++ b/tests/sql_battery/test_shapes_and_errors_battery.py
@@ -942,8 +942,7 @@
         ("SET @variable = 44; SET @var = 'name'; SHOW VARIABLES LIKE '@%ri%';", 1, 4, UnsupportedSyntaxError),
         ("SHOW PARAMETER disable_optimizer", 1, 2, None),
         ("SET disable_optimizer = true; SHOW PARAMETER disable_optimizer;", 1, 2, None),
-]
-A = [
+
         ("SELECT id FROM $planets WHERE NOT NOT id > 3", 6, 1, None),
         ("SELECT id FROM $planets WHERE NOT NOT id < 3", 2, 1, None),
         ("SELECT id FROM $planets WHERE NOT id > 3", 3, 1, None),
@@ -958,10 +957,10 @@
         ("SELECT * FROM $planets WHERE 1 = 1", 9, 20, None),
         ("SELECT * FROM $planets WHERE NOT 1 = 2", 9, 20, None),
 
-        ("SHOW CREATE TABLE $planets", 1, 1, None),
-        ("SHOW CREATE TABLE $satellites", 1, 1, None),
-        ("SHOW CREATE TABLE $astronauts", 1, 1, None),
-        ("SHOW CREATE TABLE testdata.partitioned.framed FOR '2021-03-28'", 1, 1, None),
+        ("SHOW CREATE TABLE $planets", 1, 1, UnsupportedSyntaxError),
+        ("SHOW CREATE TABLE $satellites", 1, 1, UnsupportedSyntaxError),
+        ("SHOW CREATE TABLE $astronauts", 1, 1, UnsupportedSyntaxError),
+        ("SHOW CREATE TABLE testdata.partitioned.framed FOR '2021-03-28'", 1, 1, UnsupportedSyntaxError),
         ("SET disable_optimizer = true;\nSET disable_morsel_defragmentation = false;\nSELECT COUNT(*) FROM $planets WHERE id > 3 AND name ILIKE '%e%'", 1, 1, None),
         ("SET disable_optimizer = false;\nSET disable_morsel_defragmentation = false;\nSELECT COUNT(*) FROM $planets WHERE id > 3 AND name ILIKE '%e%'", 1, 1, None),
         ("SET disable_optimizer = false;\nSET disable_morsel_defragmentation = true;\nSELECT COUNT(*) FROM $planets WHERE id > 3 AND name ILIKE '%e%'", 1, 1, None),
@@ -969,8 +968,8 @@
         ("SELECT COUNT(*) FROM $planets WHERE id > 3 AND name ILIKE '%e%' AND id > 1 AND id > 0 AND id > 2 AND name ILIKE '%e%'", 1, 1, None),
 
         ("SELECT planets.* FROM $planets AS planets LEFT JOIN $planets FOR '1600-01-01' AS older ON planets.id = older.id WHERE older.name IS NULL", 3, 20, None),
-        ("SELECT * FROM generate_series(1,10) LEFT JOIN $planets FOR '1600-01-01' ON id = generate_series", 10, 21, None),
-        ("SELECT DISTINCT name FROM generate_series(1,10) LEFT JOIN $planets FOR '1600-01-01' ON id = generate_series", 7, 1, None),
+        ("SELECT * FROM generate_series(1,10) AS GS LEFT JOIN $planets FOR '1600-01-01' ON id = GS", 10, 21, None),
+        ("SELECT DISTINCT name FROM generate_series(1,10) AS GS LEFT JOIN $planets FOR '1600-01-01' ON id = GS", 7, 1, None),
         ("SELECT 1 WHERE ' a  b ' \t = \n\n ' ' || 'a' || ' ' || \n ' b '", 1, 1, None),
         ("SELECT name FROM $planets WHERE SUBSTRING ( name, 1, 1 ) = 'M'", 2, 1, None),
         ("SELECT name FROM $planets WHERE SUBSTRING ( name, 2, 1 ) = 'a'", 3, 1, None),
@@ -980,10 +979,10 @@
         ("SELECT * FROM $satellites WHERE NULLIF(planetId, 5) IS NULL", 67, 8, None),
         ("SELECT * FROM $satellites WHERE NULLIF(planetId, 5) IS NOT NULL", 110, 8, None),
 
-        ("SHOW STORES LIKE 'apple'", None, None, SqlError),
+        ("SHOW STORES LIKE 'apple'", None, None, UnsupportedSyntaxError),
         ("SELECT name FROM $astronauts WHERE LEFT(name, POSITION(' ' IN name) - 1) = 'Andrew'", 3, 1, None),
         ("SELECT name FROM $astronauts WHERE LEFT(name, POSITION(' ' IN name)) = 'Andrew '", 3, 1, None),
-        
+
         ("SELECT ARRAY_AGG(name) from $satellites GROUP BY planetId", 7, 1, None),
         ("SELECT ARRAY_AGG(DISTINCT name) from $satellites GROUP BY planetId", 7, 1, None),
         ("SELECT ARRAY_AGG(name ORDER BY name) from $satellites GROUP BY TRUE", 1, 1, None),
@@ -995,8 +994,9 @@
 
         ("SELECT name FROM $satellites WHERE '192.168.0.1' | '192.168.0.0/24'", 177, 1, None),
         ("SELECT name FROM $satellites WHERE '192.168.0.1' | '192.167.0.0/24'", 0, 1, None),
-        ("SELECT name FROM $satellites WHERE 12 | 22", None, None, NotImplementedError),
-
+        ("SELECT name FROM $satellites WHERE 12 | 22", 177, 1, None),
+]
+A = [
         ("SELECT COUNT(*), place FROM (SELECT CASE id WHEN 3 THEN 'Earth' WHEN 1 THEN 'Mercury' ELSE 'Elsewhere' END as place FROM $planets) AS SQ GROUP BY place;", 3, 2, None),
         ("SELECT COUNT(*), place FROM (SELECT CASE id WHEN 3 THEN 'Earth' WHEN 1 THEN 'Mercury' END as place FROM $planets) AS SQ GROUP BY place HAVING place IS NULL;", 1, 2, None),
         ("SELECT COUNT(*), place FROM (SELECT CASE id WHEN 3 THEN 'Earth' WHEN 1 THEN 'Mercury' ELSE 'Elsewhere' END as place FROM $planets) AS SQ GROUP BY place HAVING place IS NULL;", 0, 2, None),