Repository: blaze/blaze

Issue:
Use char_length instead of length
String length is currently using `length` which in a few databases returns the number of bytes in a string. This is problematic for multibyte characters, such as `é`. The proper function to use is `char_length`.



Discussion:

Resolving patch:
diff --git a/blaze/compute/sql.py b/blaze/compute/sql.py
--- a/blaze/compute/sql.py
+++ b/blaze/compute/sql.py
@@ -18,7 +18,7 @@
 
 import sqlalchemy as sa
 import sqlalchemy
-from sqlalchemy import sql, Table, MetaData, Column
+from sqlalchemy import sql, Table, MetaData
 from sqlalchemy.sql import Selectable, Select
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.elements import ClauseElement, ColumnElement
@@ -33,10 +33,9 @@
 from ..dispatch import dispatch
 from ..expr import Projection, Selection, Field, Broadcast, Expr
 from ..expr import (BinOp, UnaryOp, USub, Join, mean, var, std, Reduction,
-                    count, FloorDiv, UnaryStringFunction, DateTime,
-                    Millisecond, Microsecond)
+                    count, FloorDiv, UnaryStringFunction, strlen, DateTime)
 from ..expr import nunique, Distinct, By, Sort, Head, Label, ReLabel, Merge
-from ..expr import common_subexpression, Summary, Like, nelements, DateTime
+from ..expr import common_subexpression, Summary, Like, nelements
 from ..compatibility import reduce
 from .core import compute_up, compute, base
 from ..utils import listpack
@@ -568,11 +567,27 @@ def compute_up(t, s, **kwargs):
 
 
 string_func_names = {
-    'strlen': 'length'
+    # <blaze function name>: <SQL function name>
 }
 
 
-@dispatch(UnaryStringFunction, Column)
+# TODO: remove if the alternative fix goes into PyHive
+@compiles(sa.sql.functions.Function, 'hive')
+def compile_char_length_on_hive(element, compiler, **kwargs):
+    assert len(element.clauses) == 1, \
+        'char_length must have a single clause, got %s' % list(element.clauses)
+    if element.name == 'char_length':
+        return compiler.visit_function(sa.func.length(*element.clauses),
+                                       **kwargs)
+    return compiler.visit_function(element, **kwargs)
+
+
+@dispatch(strlen, ColumnElement)
+def compute_up(expr, data, **kwargs):
+    return sa.sql.functions.char_length(data).label(expr._name)
+
+
+@dispatch(UnaryStringFunction, ColumnElement)
 def compute_up(expr, data, **kwargs):
     func_name = type(expr).__name__
     func_name = string_func_names.get(func_name, func_name)


Test patch:
diff --git a/blaze/compute/tests/test_sql_compute.py b/blaze/compute/tests/test_sql_compute.py
--- a/blaze/compute/tests/test_sql_compute.py
+++ b/blaze/compute/tests/test_sql_compute.py
@@ -24,22 +24,22 @@
 s = sa.Table('accounts', metadata,
              sa.Column('name', sa.String),
              sa.Column('amount', sa.Integer),
-             sa.Column('id', sa.Integer, primary_key=True),
-             )
+             sa.Column('id', sa.Integer, primary_key=True))
 
-tbig = symbol('tbig', 'var * {name: string, sex: string[1], amount: int, id: int}')
+tbig = symbol('tbig',
+              'var * {name: string, sex: string[1], amount: int, id: int}')
 
 sbig = sa.Table('accountsbig', metadata,
-             sa.Column('name', sa.String),
-             sa.Column('sex', sa.String),
-             sa.Column('amount', sa.Integer),
-             sa.Column('id', sa.Integer, primary_key=True),
-             )
+                sa.Column('name', sa.String),
+                sa.Column('sex', sa.String),
+                sa.Column('amount', sa.Integer),
+                sa.Column('id', sa.Integer, primary_key=True))
+
 
 def normalize(s):
-    s2 = ' '.join(s.strip().split()).lower().replace('_', '')
-    s3 = re.sub('alias\d*', 'alias', s2)
-    return s3
+    s = ' '.join(s.strip().split()).lower()
+    s = re.sub(r'(alias)_?\d*', r'\1', s)
+    return re.sub(r'__([A-Za-z_][A-Za-z_0-9]*)', r'\1', s)
 
 
 def test_table():
@@ -633,7 +633,7 @@ def test_like():
 def test_strlen():
     expr = t.name.strlen()
     result = str(compute(expr, s))
-    expected = "SELECT length(accounts.name) as name FROM accounts"
+    expected = "SELECT char_length(accounts.name) as name FROM accounts"
     assert normalize(result) == normalize(expected)
 
 
@@ -1118,7 +1118,7 @@ def test_join_count():
           WHERE t1.x > ?) as alias
           """
     expected2 = """
-    SELECT count(alias2.x) AS __count
+    SELECT count(alias2.x) AS count
     FROM (SELECT alias1.x AS x, alias1.y AS y, t2.b AS b
           FROM (SELECT t1.x AS x, t1.y AS y
                 FROM t1
@@ -1217,5 +1217,5 @@ def test_math():
 
     result = compute(t.amount // 2, s)
     assert normalize(str(result)) == normalize("""
-            SELECT floor(accounts.amount / :amount1) AS amount
+            SELECT floor(accounts.amount / :amount_1) AS amount
             FROM accounts""")