Repository: ibis-project/ibis

Issue:
BUG: TopK used as filter is not working on OmniSciDB backend
TopK is not working on OmniSciDB/MapD backend. eg.:

```
>>> alltypes[alltypes.string_col.topk(3)].count().compile()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-39-95fa1f84c573> in <module>
----> 1 alltypes[alltypes.string_col.topk(3)].count().compile()

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/expr/types.py in compile(self, limit, params)
    213         from ibis.client import compile
    214 
--> 215         return compile(self, limit=limit, params=params)
    216 
    217     def verify(self):

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/client.py in compile(expr, limit, params, **kwargs)
    288 def compile(expr, limit=None, params=None, **kwargs):
    289     backend, = validate_backends(list(find_backends(expr)))
--> 290     return backend.compile(expr, limit=limit, params=params, **kwargs)
    291 
    292 

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/client.py in compile(self, expr, params, limit)
    199         """
    200         query_ast = self._build_ast_ensure_limit(expr, limit, params=params)
--> 201         return query_ast.compile()
    202 
    203     def _build_ast_ensure_limit(self, expr, limit, params=None):

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/sql/compiler.py in compile(self)
     52     def compile(self):
     53         compiled_setup_queries = [q.compile() for q in self.setup_queries]
---> 54         compiled_queries = [q.compile() for q in self.queries]
     55         compiled_teardown_queries = [
     56             q.compile() for q in self.teardown_queries

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/sql/compiler.py in <listcomp>(.0)
     52     def compile(self):
     53         compiled_setup_queries = [q.compile() for q in self.setup_queries]
---> 54         compiled_queries = [q.compile() for q in self.queries]
     55         compiled_teardown_queries = [
     56             q.compile() for q in self.teardown_queries

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/sql/compiler.py in compile(self)
   1565 
   1566         # FROM, JOIN, UNION
-> 1567         from_frag = self.format_table_set()
   1568 
   1569         # WHERE

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/sql/compiler.py in format_table_set(self)
   1682 
   1683         helper = self.table_set_formatter(self, self.table_set)
-> 1684         fragment += helper.get_result()
   1685 
   1686         return fragment

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/mapd/compiler.py in get_result(self)
    143 
    144         if isinstance(op, ops.Join):
--> 145             self._walk_join_tree(op)
    146         else:
    147             self.join_tables.append(self._format_table(self.expr))

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/sql/compiler.py in _walk_join_tree(self, op)
   1793         self._validate_join_predicates(op.predicates)
   1794 
-> 1795         jname = self._get_join_type(op)
   1796 
   1797         # Read off tables and join predicates left-to-right in

/mnt/sda1/dev/quansight/ibis-project/ibis/ibis/sql/compiler.py in _get_join_type(self, op)
   1834 
   1835     def _get_join_type(self, op):
-> 1836         return self._join_names[type(op)]
   1837 
   1838     def _quote_identifier(self, name):

KeyError: <class 'ibis.expr.operations.LeftSemiJoin'>

```

[Parquet] ParquetClient.table "path" parameter default value
Checking the method ParquetClient.table

```
    def table(self, name, path):
        if name not in self.list_tables(path):
            raise AttributeError(name)

        if path is None:
            path = self.root

        # get the schema
        f = path / "{}.parquet".format(name)

        parquet_file = pq.ParquetFile(str(f))
        schema = sch.infer(parquet_file.schema)

        table = self.table_class(name, schema, self).to_expr()
        self.dictionary[name] = f

        return table
```

it seems `path` should have a default value `None`. Similar approach used by `CSVClient.table`: https://github.com/ibis-project/ibis/blob/master/ibis/file/csv.py#L57


Discussion:

Resolving patch:
diff --git a/ibis/file/parquet.py b/ibis/file/parquet.py
--- a/ibis/file/parquet.py
+++ b/ibis/file/parquet.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
 import pyarrow as pa
 import pyarrow.parquet as pq
 import regex as re
@@ -6,6 +8,7 @@
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
 import ibis.expr.schema as sch
+import ibis.expr.types as ir
 from ibis.file.client import FileClient
 from ibis.pandas.api import PandasDialect
 from ibis.pandas.core import execute, execute_node
@@ -75,7 +78,7 @@ def insert(self, path, expr, **kwargs):
         table = pa.Table.from_pandas(df)
         pq.write_table(table, str(path))
 
-    def table(self, name, path):
+    def table(self, name: str, path: Optional[str] = None) -> ir.TableExpr:
         if name not in self.list_tables(path):
             raise AttributeError(name)
 
diff --git a/ibis/omniscidb/compiler.py b/ibis/omniscidb/compiler.py
--- a/ibis/omniscidb/compiler.py
+++ b/ibis/omniscidb/compiler.py
@@ -180,6 +180,7 @@ class OmniSciDBTableSetFormatter(compiles.TableSetFormatter):
     _join_names = {
         ops.InnerJoin: 'JOIN',
         ops.LeftJoin: 'LEFT JOIN',
+        ops.LeftSemiJoin: 'JOIN',  # needed by topk as filter
         ops.CrossJoin: 'JOIN',
     }
 


Test patch:
diff --git a/ibis/tests/all/test_aggregation.py b/ibis/tests/all/test_aggregation.py
--- a/ibis/tests/all/test_aggregation.py
+++ b/ibis/tests/all/test_aggregation.py
@@ -2,7 +2,14 @@
 import pytest
 from pytest import param
 
-from ibis.tests.backends import Clickhouse, MySQL, OmniSciDB, SQLite
+from ibis.tests.backends import (
+    Clickhouse,
+    MySQL,
+    OmniSciDB,
+    PostgreSQL,
+    PySpark,
+    SQLite,
+)
 
 
 @pytest.mark.parametrize(
@@ -156,3 +163,57 @@ def test_group_concat(backend, alltypes, df, result_fn, expected_fn):
     expected = expected_fn(df)
 
     assert set(result.iloc[:, 1]) == set(expected.iloc[:, 1])
+
+
+@pytest.mark.parametrize(
+    ('result_fn', 'expected_fn'),
+    [
+        param(
+            lambda t: t.string_col.topk(3),
+            lambda t: t.groupby('string_col')['string_col'].count().head(3),
+            id='string_col_top3',
+        )
+    ],
+)
+@pytest.mark.xfail_unsupported
+@pytest.mark.xfail_backends([PySpark])  # Issue #2130
+def test_topk_op(backend, alltypes, df, result_fn, expected_fn):
+    # TopK expression will order rows by "count" but each backend
+    # can have different result for that.
+    # Note: Maybe would be good if TopK could order by "count"
+    # and the field used by TopK
+    t = alltypes.sort_by(alltypes.string_col)
+    df = df.sort_values('string_col')
+    result = result_fn(t).execute()
+    expected = expected_fn(df)
+    assert all(result['count'].values == expected.values)
+
+
+@pytest.mark.parametrize(
+    ('result_fn', 'expected_fn'),
+    [
+        param(
+            lambda t: t[t.string_col.topk(3)],
+            lambda t: t[
+                t.string_col.isin(
+                    t.groupby('string_col')['string_col'].count().head(3).index
+                )
+            ],
+            id='string_col_filter_top3',
+        )
+    ],
+)
+@pytest.mark.xfail_unsupported
+# Issues #2133 #2132# #2133
+@pytest.mark.xfail_backends([Clickhouse, MySQL, PostgreSQL])
+@pytest.mark.skip_backends([SQLite], reason='Issue #2128')
+def test_topk_filter_op(backend, alltypes, df, result_fn, expected_fn):
+    # TopK expression will order rows by "count" but each backend
+    # can have different result for that.
+    # Note: Maybe would be good if TopK could order by "count"
+    # and the field used by TopK
+    t = alltypes.sort_by(alltypes.string_col)
+    df = df.sort_values('string_col')
+    result = result_fn(t).execute()
+    expected = expected_fn(df)
+    assert result.shape[0] == expected.shape[0]