Repository: ibis-project/ibis

Issue:
Partition by failure on postgres backend
I am working with Ibis version: '1.0.0+5.g80db366'

Trying to execute the following code:

```
seller_dimensions_table = monster.seller_dimension
last_snapshot_gb = seller_dimensions_table.groupby(['merchant_customer_id', 'snapshot_date'])
sorted_snapshot = last_snapshot_gb.order_by(seller_dimensions_table.snapshot_date).mutate(rank=ibis.row_number())
sorted_snapshot.execute(limit=10)
```

Gives this long traceback:

```
-----------------------------------------------------------------------
ProgrammingError                      Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1181                         parameters,
-> 1182                         context)
   1183         except BaseException as e:

~/anaconda3/lib/python3.6/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    469     def do_execute(self, cursor, statement, parameters, context=None):
--> 470         cursor.execute(statement, parameters)
    471 

ProgrammingError: Frame clause should not be specified for ranking window functions
```

The generated SQL looks like this:

```
SELECT ..., row_number() OVER (PARTITION BY t0.merchant_customer_id, t0.snapshot_date ORDER BY t0.snapshot_date ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - %(param_1)s AS rank 
FROM monster.seller_dimension AS t0
```


Discussion:
@FedericoV Thanks for the report. I'm going to release 1.1.0 in the next couple of weeks which will contain a fix for this issue.
@FedericoV What version of postgres are you using?
I'm using the postgres backend to work with Redshift, which usually works
very well.

On Fri, Jun 7, 2019, 10:55 AM Phillip Cloud <notifications@github.com>
wrote:

> @FedericoV <https://github.com/FedericoV> What version of postgres are
> you using?
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/ibis-project/ibis/issues/1757?email_source=notifications&email_token=AAEZ25U2BNMBPVZ2CRDNNXLPZKOKJA5CNFSM4HFSGM4KYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGODXGRQII#issuecomment-499980321>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAEZ25XFN35LHIZYJLENJHDPZKOKJANCNFSM4HFSGM4A>
> .
>

Interesting, using postgres proper (not redshift) doesn't raise an error when specifying a frame clause for ranking functions. We should be able to remove them with no effect except to allow redshift to work.

Resolving patch:
diff --git a/ibis/impala/compiler.py b/ibis/impala/compiler.py
--- a/ibis/impala/compiler.py
+++ b/ibis/impala/compiler.py
@@ -378,6 +378,7 @@ def _foll(f: Optional[int]) -> str:
         ops.Lead,
         ops.DenseRank,
         ops.MinRank,
+        ops.NTile,
         ops.PercentRank,
         ops.RowNumber,
     )
diff --git a/ibis/sql/alchemy.py b/ibis/sql/alchemy.py
--- a/ibis/sql/alchemy.py
+++ b/ibis/sql/alchemy.py
@@ -1,6 +1,5 @@
 import contextlib
 import functools
-import numbers
 import operator
 
 import pandas as pd
@@ -11,8 +10,6 @@
 from sqlalchemy.dialects.postgresql.base import PGDialect as PostgreSQLDialect
 from sqlalchemy.dialects.sqlite.base import SQLiteDialect
 from sqlalchemy.engine.interfaces import Dialect as SQLAlchemyDialect
-from sqlalchemy.ext.compiler import compiles as sa_compiles
-from sqlalchemy.sql.elements import Over as _Over
 
 import ibis
 import ibis.common as com
@@ -539,12 +536,30 @@ def _window(t, expr):
 
     partition_by = list(map(t.translate, window._group_by))
 
-    result = Over(
-        reduction,
-        partition_by=partition_by,
-        order_by=order_by,
-        preceding=window.preceding,
-        following=window.following,
+    frame_clause_not_allowed = (
+        ops.Lag,
+        ops.Lead,
+        ops.DenseRank,
+        ops.MinRank,
+        ops.NTile,
+        ops.PercentRank,
+        ops.RowNumber,
+    )
+
+    how = {'range': 'range_'}.get(window.how, window.how)
+    preceding = window.preceding
+    additional_params = (
+        {}
+        if isinstance(window_op, frame_clause_not_allowed)
+        else {
+            how: (
+                -preceding if preceding is not None else preceding,
+                window.following,
+            )
+        }
+    )
+    result = reduction.over(
+        partition_by=partition_by, order_by=order_by, **additional_params
     )
 
     if isinstance(
@@ -1138,7 +1153,7 @@ def _add_groupby(self, fragment):
 
         if len(self.having) > 0:
             having_args = [self._translate(arg) for arg in self.having]
-            having_clause = _and_all(having_args)
+            having_clause = functools.reduce(sql.and_, having_args)
             fragment = fragment.having(having_clause)
 
         return fragment
@@ -1150,7 +1165,7 @@ def _add_where(self, fragment):
         args = [
             self._translate(pred, permit_subquery=True) for pred in self.where
         ]
-        clause = _and_all(args)
+        clause = functools.reduce(sql.and_, args)
         return fragment.where(clause)
 
     def _add_order_by(self, fragment):
@@ -1216,7 +1231,7 @@ def get_result(self):
         ):
             if len(preds):
                 sqla_preds = [self._translate(pred) for pred in preds]
-                onclause = _and_all(sqla_preds)
+                onclause = functools.reduce(sql.and_, sqla_preds)
             else:
                 onclause = None
 
@@ -1316,13 +1331,6 @@ def _can_lower_sort_column(table_set, expr):
         return False
 
 
-def _and_all(clauses):
-    result = clauses[0]
-    for clause in clauses[1:]:
-        result = sql.and_(result, clause)
-    return result
-
-
 class AlchemyProxy:
     """
     Wraps a SQLAlchemy ResultProxy and ensures that .close() is called on
@@ -1357,11 +1365,10 @@ def _nullifzero(expr):
 @compiles(ops.Divide)
 def _true_divide(t, expr):
     op = expr.op()
-    left, right = op.args
+    left, right = args = op.args
 
-    if util.all_of(op.args, ir.IntegerValue):
-        new_expr = left.div(right.cast('double'))
-        return t.translate(new_expr)
+    if util.all_of(args, ir.IntegerValue):
+        return t.translate(left.div(right.cast('double')))
 
     return fixed_arity(lambda x, y: x / y, 2)(t, expr)
 
@@ -1372,49 +1379,3 @@ def _sort_key(t, expr):
     by, ascending = expr.op().args
     sort_direction = sa.asc if ascending else sa.desc
     return sort_direction(t.translate(by))
-
-
-_valid_frame_types = numbers.Integral, str, type(None)
-
-
-class Over(_Over):
-    def __init__(
-        self,
-        element,
-        order_by=None,
-        partition_by=None,
-        preceding=None,
-        following=None,
-    ):
-        super().__init__(element, order_by=order_by, partition_by=partition_by)
-        if not isinstance(preceding, _valid_frame_types):
-            raise TypeError(
-                'preceding must be a string, integer or None, got %r'
-                % (type(preceding).__name__)
-            )
-        if not isinstance(following, _valid_frame_types):
-            raise TypeError(
-                'following must be a string, integer or None, got %r'
-                % (type(following).__name__)
-            )
-        self.preceding = preceding if preceding is not None else 'UNBOUNDED'
-        self.following = following if following is not None else 'UNBOUNDED'
-
-
-@sa_compiles(Over)
-def compile_over_with_frame(element, compiler, **kw):
-    clauses = ' '.join(
-        '%s BY %s' % (word, compiler.process(clause, **kw))
-        for word, clause in (
-            ('PARTITION', element.partition_by),
-            ('ORDER', element.order_by),
-        )
-        if clause is not None and len(clause)
-    )
-    return '%s OVER (%s%sROWS BETWEEN %s PRECEDING AND %s FOLLOWING)' % (
-        compiler.process(getattr(element, 'element', element.element), **kw),
-        clauses,
-        ' ' if clauses else '',  # only add a space if we order by or group by
-        str(element.preceding).upper(),
-        str(element.following).upper(),
-    )
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -15,13 +15,13 @@
 
 VERSION = sys.version_info.major, sys.version_info.minor
 
-impala_requires = ['hdfs>=2.0.16', 'sqlalchemy', 'requests']
+impala_requires = ['hdfs>=2.0.16', 'sqlalchemy>=1.1', 'requests']
 if VERSION == (3, 5):
     impala_requires.append('impyla<0.14.2')
 else:
     impala_requires.append('impyla>=0.15.0')
 
-sqlite_requires = ['sqlalchemy']
+sqlite_requires = ['sqlalchemy>=1.1']
 postgres_requires = sqlite_requires + ['psycopg2']
 mysql_requires = sqlite_requires + ['pymysql']
 


Test patch:
diff --git a/ibis/sql/postgres/tests/test_functions.py b/ibis/sql/postgres/tests/test_functions.py
--- a/ibis/sql/postgres/tests/test_functions.py
+++ b/ibis/sql/postgres/tests/test_functions.py
@@ -1242,8 +1242,7 @@ def test_rank(con):
     sqla_expr = expr.compile()
     result = str(sqla_expr.compile(compile_kwargs=dict(literal_binds=True)))
     expected = (
-        "SELECT rank() OVER (ORDER BY t0.double_col ROWS BETWEEN "
-        "UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS tmp \n"
+        "SELECT rank() OVER (ORDER BY t0.double_col) - 1 AS tmp \n"
         "FROM functional_alltypes AS t0"
     )
     assert result == expected
@@ -1255,8 +1254,7 @@ def test_percent_rank(con):
     sqla_expr = expr.compile()
     result = str(sqla_expr.compile(compile_kwargs=dict(literal_binds=True)))
     expected = (
-        "SELECT percent_rank() OVER (ORDER BY t0.double_col ROWS "
-        "BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS "
+        "SELECT percent_rank() OVER (ORDER BY t0.double_col) AS "
         "tmp \nFROM functional_alltypes AS t0"
     )
     assert result == expected
@@ -1268,8 +1266,7 @@ def test_ntile(con):
     sqla_expr = expr.compile()
     result = str(sqla_expr.compile(compile_kwargs=dict(literal_binds=True)))
     expected = (
-        "SELECT ntile(7) OVER (ORDER BY t0.double_col ROWS BETWEEN "
-        "UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - 1 AS tmp \n"
+        "SELECT ntile(7) OVER (ORDER BY t0.double_col) - 1 AS tmp \n"
         "FROM functional_alltypes AS t0"
     )
     assert result == expected