Repository: mabel-dev/opteryx

Issue:
✨ support pyarrow's IPC format
### Thanks for stopping by to let us know something could be better!

**Is your feature request related to a problem? Please describe.** _A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]_

**Describe the solution you'd like** _A clear and concise description of what you want to happen._

**Describe alternatives you've considered** _A clear and concise description of any alternative solutions or features you've considered._

**Additional context** _Add any other context or screenshots about the feature request here._



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 312
+__build__ = 313
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/utils/file_decoders.py b/opteryx/utils/file_decoders.py
--- a/opteryx/utils/file_decoders.py
+++ b/opteryx/utils/file_decoders.py
@@ -273,6 +273,32 @@ def avro_decoder(buffer, projection: List = None, selection=None, just_schema: b
     return table
 
 
+def ipc_decoder(buffer, projection: List = None, selection=None, just_schema: bool = False):
+    from itertools import chain
+
+    from pyarrow import ipc
+
+    stream = io.BytesIO(buffer)
+    reader = ipc.open_stream(stream)
+
+    batch_one = next(reader, None)
+    if batch_one is None:
+        return None
+
+    schema = batch_one.schema
+
+    if just_schema:
+        return convert_arrow_schema_to_orso_schema(schema)
+
+    table = pyarrow.Table.from_batches([batch for batch in chain([batch_one], reader)])
+    if selection:
+        table = filter_records(selection, table)
+    if projection:
+        table = post_read_projector(table, projection)
+
+    return table
+
+
 # for types we know about, set up how we handle them
 KNOWN_EXTENSIONS: Dict[str, Tuple[Callable, str]] = {
     "avro": (avro_decoder, ExtentionType.DATA),
@@ -280,6 +306,7 @@ def avro_decoder(buffer, projection: List = None, selection=None, just_schema: b
     "ignore": (do_nothing, ExtentionType.CONTROL),
     "arrow": (arrow_decoder, ExtentionType.DATA),  # feather
     "csv": (csv_decoder, ExtentionType.DATA),
+    "ipc": (ipc_decoder, ExtentionType.DATA),
     "jsonl": (jsonl_decoder, ExtentionType.DATA),
     "orc": (orc_decoder, ExtentionType.DATA),
     "parquet": (parquet_decoder, ExtentionType.DATA),


Test patch:
diff --git a/tests/sql_battery/test_battery_formats.py b/tests/sql_battery/test_battery_formats.py
--- a/tests/sql_battery/test_battery_formats.py
+++ b/tests/sql_battery/test_battery_formats.py
@@ -53,6 +53,10 @@
         # .json.parquet - appears to be handled incorrectly
         ("SELECT * FROM testdata.flat.formats.misnamed_parquet WITH (NO_PARTITION)", 100000, 13, False),
         ("SELECT user_name, user_verified FROM testdata.flat.formats.misnamed_parquet WITH(NO_PARTITION) WHERE user_name ILIKE '%news%'", 122, 2, False),
+
+        # PyArrow IPC streams
+        ("SELECT * FROM testdata.flat.formats.ipc", 100000, 13, False),
+        ("SELECT user_name, user_verified FROM testdata.flat.formats.ipc WHERE user_name ILIKE '%news%'", 122, 2, False),
     ]
 # fmt:on