internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mabel-dev/opteryx

Issue:
🧹 resync SQLoxide
AST to SQL and AST visitor appear to have been added 


Discussion:

Resolving patch:
diff --git a/opteryx/third_party/sqloxide/__init__.py b/opteryx/third_party/sqloxide/__init__.py
--- a/opteryx/third_party/sqloxide/__init__.py
+++ b/opteryx/third_party/sqloxide/__init__.py
@@ -1,8 +1,16 @@
-try:
-    from .sqloxide import parse_sql
-except ImportError as e:  # pragma: no cover
-    print(e)
-    if str(e) != "PyO3 modules may only be initialized once per interpreter process":
-        raise e
-
-__all__ = ["parse_sql"]
+"""
+This module provides an interface to the sqloxide library, which is responsible for parsing SQL,
+restoring the Abstract Syntax Tree (AST), and performing various mutations on expressions and relations.
+
+For more information about sqloxide: https://github.com/wseaton/sqloxide
+
+This module is not from sqloxide, it is written for Opteryx.
+"""
+
+from .sqloxide import mutate_expressions
+from .sqloxide import mutate_relations
+from .sqloxide import parse_sql
+from .sqloxide import restore_ast
+
+# Explicitly define the API of this module for external consumers
+__all__ = ["parse_sql", "restore_ast", "mutate_expressions", "mutate_relations"]


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲 sqlparser-rs breaking change
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Describe the bug** _A clear and specific description of what the bug is. What the error, incorrect or unexpected behaviour was._


**Expected behaviour** _A clear and concise description of what you expected to happen._


**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql

~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/components/logical_planner_builders.py b/opteryx/components/logical_planner_builders.py
--- a/opteryx/components/logical_planner_builders.py
+++ b/opteryx/components/logical_planner_builders.py
@@ -144,6 +144,10 @@ def wildcard_filter(branch, alias=None, key=None):
     return Node(NodeType.WILDCARD)
 
 
+def expressions(branch, alias=None, key=None):
+    return [build(part) for part in branch]
+
+
 def expression_with_alias(branch, alias=None, key=None):
     """an alias"""
     return build(branch["expr"], alias=branch["alias"]["value"])
@@ -664,6 +668,7 @@ def build(value, alias=None, key=None):
     "CompoundIdentifier": compound_identifier,
     "DoubleQuotedString": literal_string,
     "Expr": build,
+    "Expressions": expressions,
     "ExprWithAlias": expression_with_alias,
     "Extract": extract,
     "Floor": floor,


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲 `null` rendered as `none`
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Describe the bug** _A clear and specific description of what the bug is. What the error, incorrect or unexpected behaviour was._


**Expected behaviour** _A clear and concise description of what you expected to happen._


**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql

~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/managers/expression/__init__.py b/opteryx/managers/expression/__init__.py
--- a/opteryx/managers/expression/__init__.py
+++ b/opteryx/managers/expression/__init__.py
@@ -179,6 +179,7 @@ def _inner_evaluate(root: Node, table: Table, context: ExecutionContext):
 
         if node_type == NodeType.NOT:
             centre = _inner_evaluate(root.centre, table, context) if root.centre else None
+            centre = pyarrow.array(centre, type=pyarrow.bool_())
             return pyarrow.compute.invert(centre)
 
     # INTERAL IDENTIFIERS
diff --git a/opteryx/managers/expression/formatter.py b/opteryx/managers/expression/formatter.py
--- a/opteryx/managers/expression/formatter.py
+++ b/opteryx/managers/expression/formatter.py
@@ -39,6 +39,8 @@ def format_expression(root, qualify: bool = False):
             return "'" + str(root.value) + "'"
         if literal_type == OrsoTypes.INTERVAL:
             return "<INTERVAL>"
+        if literal_type == OrsoTypes.NULL:
+            return "null"
         return str(root.value)
     # INTERAL IDENTIFIERS
     if node_type & INTERNAL_TYPE == INTERNAL_TYPE:


Test patch:
diff --git a/tests/sql_battery/tests/results/null_handling_03.results_tests b/tests/sql_battery/tests/results/null_handling_03.results_tests
--- a/tests/sql_battery/tests/results/null_handling_03.results_tests
+++ b/tests/sql_battery/tests/results/null_handling_03.results_tests
@@ -1,5 +1,5 @@
 {
     "summary": "Not of null is also null",
     "statement": "SELECT NOT 1 = NULL AS V",
-    "result": {"NOT 1.0 = null": [null]}
+    "result": {"NOT 1 = null": [null]}
 }
\ No newline at end of file
Repository: mabel-dev/opteryx

Issue:
🪲 aliases not handled correctly
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql
SELECT pl.orbital_period FROM $planets as pl
~~~

returns `('orbitalPeriod',)` not the requested alias.

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/components/binder/binder_visitor.py b/opteryx/components/binder/binder_visitor.py
--- a/opteryx/components/binder/binder_visitor.py
+++ b/opteryx/components/binder/binder_visitor.py
@@ -338,8 +338,19 @@ def name_column(qualifier, column):
                 ):
                     if projection_column.alias:
                         return projection_column.alias
-            if len(context.schemas) > 1 or needs_qualifier:
-                return f"{qualifier}.{column.name}"
+
+                    if len(context.relations) > 1 or needs_qualifier:
+                        if isinstance(projection_column, LogicalColumn):
+                            if qualifier:
+                                projection_column.sourcce = qualifier
+                            return projection_column.qualified_name
+                        return f"{qualifier}.{column.name}"
+
+                    if projection_column.query_column:
+                        return str(projection_column.query_column)
+                    if projection_column.current_name:
+                        return projection_column.current_name
+
             return column.name
 
         def keep_column(column, identities):


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲 column not in SELECT order
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql
SELECT
    p1.name AS planet1_name,
    p2.name AS planet2_name,
    p1.diameter AS planet1_diameter,
    p2.gravity AS planet2_gravity
FROM
    $planets p1
JOIN $planets p2 ON p1.id = p2.id
WHERE
    p1.diameter > 10000
ORDER BY
    p1.name, p2.name
~~~

returns in this order `('planet2_name', 'planet2_gravity', 'planet1_name', 'planet1_diameter')`



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -17,4 +17,4 @@
 """
 
 # __version__ = "0.4.0-alpha.6"
-__version__ = "0.12.0-alpha.8"
+__version__ = "0.12.0-alpha.9"
diff --git a/opteryx/components/binder/binder_visitor.py b/opteryx/components/binder/binder_visitor.py
--- a/opteryx/components/binder/binder_visitor.py
+++ b/opteryx/components/binder/binder_visitor.py
@@ -358,7 +358,12 @@ def keep_column(column, identities):
                 if node.columns[0].value:
                     if isinstance(column.origin, str):
                         column.origin = [column.origin]
-                    return node.columns[0].value[0] in column.origin
+                    if node.columns[0].value[0] in column.origin:
+                        identities.append(column.identity)
+                        return True
+                    else:
+                        return False
+                identities.append(column.identity)
                 return True
             return column.identity in identities
 
@@ -381,7 +386,9 @@ def keep_column(column, identities):
                     )
                     columns.append(column_reference)
 
-        node.columns = columns
+        # we bound as we came across items in schemas, not the order the user wants them
+        desired_order = {id: index for index, id in enumerate(identities)}
+        node.columns = sorted(columns, key=lambda item: desired_order[item.schema_column.identity])
 
         context.schemas["$derived"] = derived.schema()
 


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲

**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql
SELECT * FROM $planets INNER JOIN (SELECT * FROM UNNEST((1, 2, 3)) AS id) AS PID USING(id)
~~~

> NoOpNode object has no attribute 'columns'

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:
~~~sql
SELECT * FROM $planets INNER JOIN UNNEST((1, 2, 3)) AS id USING(id)
~~~

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 201
+__build__ = 203
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/components/logical_planner.py b/opteryx/components/logical_planner.py
--- a/opteryx/components/logical_planner.py
+++ b/opteryx/components/logical_planner.py
@@ -646,7 +646,7 @@ def create_node_relation(relation):
             node_type=LogicalPlanStepType.FunctionDataset, function=function_name
         )
         if function_name == "UNNEST":
-            function_step.alias = f"$unnest-{random_string()}"
+            function_step.alias = f"$unnest-{random_string(4)}"
             function_step.unnest_target = function["alias"]["name"]["value"]
         else:
             function_step.alias = function["alias"]["name"]["value"]
diff --git a/opteryx/operators/function_dataset_node.py b/opteryx/operators/function_dataset_node.py
--- a/opteryx/operators/function_dataset_node.py
+++ b/opteryx/operators/function_dataset_node.py
@@ -78,6 +78,7 @@ def __init__(self, properties: QueryProperties, **config):
         self.alias = config.get("alias")
         self.function = config["function"]
         self.parameters = config
+        self.columns = config.get("columns", [])
 
     @property
     def config(self):  # pragma: no cover


Test patch:
diff --git a/tests/sql_battery/test_shapes_and_errors_battery.py b/tests/sql_battery/test_shapes_and_errors_battery.py
--- a/tests/sql_battery/test_shapes_and_errors_battery.py
+++ b/tests/sql_battery/test_shapes_and_errors_battery.py
@@ -1437,6 +1437,10 @@
         ("SELECT TIMESTAMP '2023-01-01' + INTERVAL '1' MONTH FROM $planets", 9, 1, None),
         ("SELECT * FROM $planets WHERE DATE '2023-01-01' + INTERVAL '1' MONTH < current_time", 9, 20, None),
         ("SELECT * FROM $planets WHERE TIMESTAMP '2023-01-01' + INTERVAL '1' MONTH < current_time", 9, 20, None),
+        # 1380
+        ("SELECT * FROM $planets INNER JOIN (SELECT * FROM UNNEST((1, 2, 3)) AS id) AS PID USING(id)", 3, 20, None),
+        ("SELECT * FROM $planets INNER JOIN UNNEST((1, 2, 3)) AS id USING(id)", 3, 20, None),
+        ("SELECT * FROM UNNEST((1, 2, 3)) AS id INNER JOIN $planets USING(id)", 3, 20, None),
 ]
 # fmt:on
Repository: mabel-dev/opteryx

Issue:
🪲 ARM test fails 

~~~
ValueError: 'orso/bitarray/cbitarray.pyx' doesn't match any files
~~~

https://github.com/mabel-dev/opteryx/actions/runs/7535073365/job/20510453555


Discussion:
confirmed app runs on raspberry pi - it's about 8x slower but runs without error.

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 244
+__build__ = 248
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.


Test patch:
Repository: mabel-dev/opteryx

Issue:
✨ improve CROSS JOIN UNNEST performance 
Usage logs show queries which use CROSS JOIN UNNEST are often amongst the slowest running queries.

Whilst CROSS JOIN and UNNEST are relatively slow operations, opportunities to improve the performance should be investigated.


Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 271
+__build__ = 276
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,9 +27,9 @@ class VersionStatus(Enum):
 
 
 _major = 0
-_minor = 13
-_revision = 2
-_status = VersionStatus.RELEASE
+_minor = 14
+_revision = 0
+_status = VersionStatus.ALPHA
 
 __author__ = "@joocer"
 __version__ = f"{_major}.{_minor}.{_revision}" + (
diff --git a/opteryx/operators/cross_join_node.py b/opteryx/operators/cross_join_node.py
--- a/opteryx/operators/cross_join_node.py
+++ b/opteryx/operators/cross_join_node.py
@@ -48,6 +48,7 @@ def _cross_join_unnest_column(
     Returns:
         A generator that yields the resulting `pyarrow.Table` objects.
     """
+    from opteryx.compiled import build_rows_indices_and_column
 
     # Check if the source node type is an identifier, raise error otherwise
     if source.node_type != NodeType.IDENTIFIER:
@@ -62,20 +63,26 @@ def _cross_join_unnest_column(
         for left_block in left_morsel.to_batches(max_chunksize=batch_size):
             # Fetch the data of the column to be unnested
             column_data = left_block[source.schema_column.identity]
-
+            # we need the offsets before we drop the rows
+            valid_offsets = column_data.is_valid()
+            column_data = column_data.drop_null()
+            # if there's no valid records, continue to the next record
+            if len(column_data) == 0:
+                continue
+            # drop the rows here, wait until we know we need to
+            left_block = left_block.filter(valid_offsets)
             # Set column_type if it hasn't been determined already
             if column_type is None:
                 column_type = column_data.type.value_type
 
-            from opteryx.compiled import build_rows_indices_and_column
-
-            indices, new_column_data = build_rows_indices_and_column(column_data)
+            indices, new_column_data = build_rows_indices_and_column(column_data.to_numpy(False))
 
             # If no new data was generated, skip to next iteration
-            if not new_column_data:
+            if len(new_column_data) == 0:
                 continue
 
-            new_block = left_morsel.take(indices)
+            new_block = left_block.take(indices)
+            new_block = pyarrow.Table.from_batches([new_block], schema=left_morsel.schema)
             new_block = new_block.append_column(target_column.identity, [new_column_data])
             yield new_block
 


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲stats for distinct incorrect
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Describe the bug** _A clear and specific description of what the bug is. What the error, incorrect or unexpected behaviour was._


**Expected behaviour** _A clear and concise description of what you expected to happen._


**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql

~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 296
+__build__ = 298
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/operators/distinct_node.py b/opteryx/operators/distinct_node.py
--- a/opteryx/operators/distinct_node.py
+++ b/opteryx/operators/distinct_node.py
@@ -29,10 +29,13 @@
 
 class DistinctNode(BasePlanNode):
     def __init__(self, properties: QueryProperties, **config):
+        from opteryx.compiled.functions import HashSet
+
         super().__init__(properties=properties)
         self._distinct_on = config.get("on")
         if self._distinct_on:
             self._distinct_on = [col.schema_column.identity for col in self._distinct_on]
+        self.hash_set = HashSet()
 
     @property
     def config(self):  # pragma: no cover
@@ -48,7 +51,6 @@ def name(self):  # pragma: no cover
 
     def execute(self) -> Generator[pyarrow.Table, None, None]:
 
-        from opteryx.compiled.functions import HashSet
         from opteryx.compiled.functions import distinct
 
         # We create a HashSet outside the distinct call, this allows us to pass
@@ -59,16 +61,17 @@ def execute(self) -> Generator[pyarrow.Table, None, None]:
         # dataset with 7 distinct entries.
         # Being able to run morsel-by-morsel means if we have a LIMIT clause, we can
         # limit processing
-        hash_set = HashSet()
 
         morsels = self._producers[0]  # type:ignore
 
-        start = time.monotonic_ns()
         for morsel in morsels.execute():
-            deduped, hash_set = distinct(
-                morsel, columns=self._distinct_on, seen_hashes=hash_set, return_seen_hashes=True
+            start = time.monotonic_ns()
+            deduped, self.hash_set = distinct(
+                morsel,
+                columns=self._distinct_on,
+                seen_hashes=self.hash_set,
+                return_seen_hashes=True,
             )
+            self.statistics.time_distincting += time.monotonic_ns() - start
             if deduped.num_rows > 0:
-                self.statistics.time_distincting += time.monotonic_ns() - start
                 yield deduped
-                start = time.monotonic_ns()


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲 Column Names not Aliased

**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

Example from user
~~~sql
SELECT *
  FROM $planets AS P
 INNER JOIN $satellites AS S
      ON P.id = S.id
~~~

Simplified example
~~~sql
SELECT *
  FROM $planets
 INNER JOIN $satellites
      ON $planets.id = $satellites.id
~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/operators/exit_node.py b/opteryx/operators/exit_node.py
--- a/opteryx/operators/exit_node.py
+++ b/opteryx/operators/exit_node.py
@@ -64,6 +64,14 @@ def execute(self) -> Generator:
                 message=f"Query result contains multiple instances of the same column(s) - `{'`, `'.join(matches)}`"
             )
 
+        if len(set(final_names)) != len(final_names):  # we have duplicate names
+            final_names = []
+            for column in self.columns:
+                if column.schema_column.origin:
+                    final_names.append(f"{column.schema_column.origin[0]}.{column.current_name}")
+                else:
+                    final_names.append(column.qualified_name)
+
         self.statistics.time_exiting += time.monotonic_ns() - start
         for morsel in morsels.execute():
             start = time.monotonic_ns()


Test patch:
Repository: mabel-dev/opteryx

Issue:
🪲 Unhelpful Error message
**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql
SELECT id, * FROM $planets
~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 320
+__build__ = 321
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/components/logical_planner/logical_planner.py b/opteryx/components/logical_planner/logical_planner.py
--- a/opteryx/components/logical_planner/logical_planner.py
+++ b/opteryx/components/logical_planner/logical_planner.py
@@ -328,6 +328,10 @@ def inner_query_planner(ast_branch):
 
     # groups
     _projection = logical_planner_builders.build(ast_branch["Select"].get("projection")) or []
+    if len(_projection) > 1 and any(p.node_type == NodeType.WILDCARD for p in _projection):
+        from opteryx.exceptions import SqlError
+
+        raise SqlError("SELECT * cannot coexist with additional columns.")
     _aggregates = get_all_nodes_of_type(_projection, select_nodes=(NodeType.AGGREGATOR,))
     _groups = logical_planner_builders.build(ast_branch["Select"].get("group_by"))
     if _groups is not None and _groups != []:


Test patch:
diff --git a/tests/sql_battery/test_shapes_and_errors_battery.py b/tests/sql_battery/test_shapes_and_errors_battery.py
--- a/tests/sql_battery/test_shapes_and_errors_battery.py
+++ b/tests/sql_battery/test_shapes_and_errors_battery.py
@@ -1480,6 +1480,9 @@
         ("SELECT COUNT(*), planetId FROM $satellites", None, None, SqlError),
         # 1462 - unhandled exception when working with empty datasets
         ("SELECT * FROM testdata.partitioned.empty FOR '2000-01-01'", None, None, EmptyDatasetError),
+        # 1474
+        ("SELECT *, id FROM $planets;", None, None, SqlError),
+        ("SELECT id, * FROM $planets;", None, None, SqlError),
 ]
 # fmt:on