Repository: ibis-project/ibis

Issue:
partition insert fails when partition key is a string
For partition keys that are string, table.insert() does not appropriately wrap the static partition value of the string in quotes, in particular where the static partition value is contained within a Python variable. The documentation example uses integer partition values and thus does not address this case.

Example:

tblSchema = ibis.schema(
    [
        ('a': int32),
        ('b': 'string')
    ]
)
db.create_table('test_table', schema=tblSchema, partition=['b'])
tbl = db.table('test_table')
the_partition='the_partition_value'
tbl.insert(some_table_reference, partition={'b': the_partition})

--> impala.error.HiveServer2Error: AnalysisException: Non-constant expressions are not supported as static partition-key values in 'b=the_partition_value'.



Discussion:
Thanks, will fix soon

Thanks Wes -

also found this morning that in interactive mode, table.info() does not
print a newline after the final line of the output, so you get the line
cursor on the same line as the last row of the table description.

Thanks!

Matt

On Wednesday, January 6, 2016, Wes McKinney notifications@github.com
wrote:

> Thanks, will fix soon
> 
> —
> Reply to this email directly or view it on GitHub.<
> https://ci6.googleusercontent.com/proxy/G2jFhI71Jda-iMFvdGSSJLIAfBGHBVR5Fdo36kwiXG5P9cw1ALp9sxoUIpqqnY_NUPYLsUMSSH90TRnfcPPfd804pVr6gj0P1z0-pBb0NtPZzeqvGWi1OtJNG0sCBbFy17OiLQCL7jPUm_nesq-ltnCJw156Sg=s0-d-e1-ft#https://github.com/notifications/beacon/AE4pkT_KQewwNKeuvbYlN1cKp3aRAqYuks5pXTi5gaJpZM4G_Km0.gif

## 

Matt Anthony
statmatt@gmail.com
"Achieving the impossible is merely stringing together a series of the
possible." - GOLF54 teaching mantra
"The best way to predict the future is to create it." - Peter Drucker


Resolving patch:
diff --git a/ibis/impala/ddl.py b/ibis/impala/ddl.py
--- a/ibis/impala/ddl.py
+++ b/ibis/impala/ddl.py
@@ -20,6 +20,7 @@
 
 from ibis.expr.datatypes import validate_type
 from ibis.compat import py_string
+import ibis.expr.datatypes as dt
 import ibis.expr.rules as rules
 
 
@@ -417,19 +418,29 @@ def _format_partition(partition, partition_schema):
     if isinstance(partition, dict):
         for name in partition_schema:
             if name in partition:
-                tok = '{0}={1}'.format(name, partition[name])
+                tok = _format_partition_kv(name, partition[name],
+                                           partition_schema[name])
             else:
                 # dynamic partitioning
                 tok = name
             tokens.append(tok)
     else:
         for name, value in zip(partition_schema, partition):
-            tok = '{0}={1}'.format(name, value)
+            tok = _format_partition_kv(name, value, partition_schema[name])
             tokens.append(tok)
 
     return 'PARTITION ({0})'.format(', '.join(tokens))
 
 
+def _format_partition_kv(k, v, type):
+    if type == dt.string:
+        value_formatted = '"{0}"'.format(v)
+    else:
+        value_formatted = str(v)
+
+    return '{0}={1}'.format(k, value_formatted)
+
+
 class LoadData(ImpalaDDL):
 
     """


Test patch:
diff --git a/ibis/impala/tests/test_ddl.py b/ibis/impala/tests/test_ddl.py
--- a/ibis/impala/tests/test_ddl.py
+++ b/ibis/impala/tests/test_ddl.py
@@ -149,6 +149,17 @@ def test_add_partition(self):
         expected = 'ALTER TABLE tbl ADD PARTITION (year=2007, month=4)'
         assert result == expected
 
+    def test_add_partition_string_key(self):
+        part_schema = ibis.schema([('foo', 'int32'),
+                                   ('bar', 'string')])
+        table_name = 'tbl'
+
+        stmt = ddl.AddPartition('tbl', {'foo': 5, 'bar': 'qux'}, part_schema)
+
+        result = stmt.compile()
+        expected = 'ALTER TABLE tbl ADD PARTITION (foo=5, bar="qux")'
+        assert result == expected
+
     def test_drop_partition(self):
         stmt = ddl.DropPartition(self.table_name,
                                  {'year': 2007, 'month': 4},
diff --git a/ibis/impala/tests/test_partition.py b/ibis/impala/tests/test_partition.py
--- a/ibis/impala/tests/test_partition.py
+++ b/ibis/impala/tests/test_partition.py
@@ -37,7 +37,7 @@ def setUpClass(cls):
         ImpalaE2E.setup_e2e(cls, ENV)
 
         df = pd.DataFrame({'year': [2009, 2009, 2009, 2010, 2010, 2010],
-                           'month': [1, 2, 3, 1, 2, 3],
+                           'month': ['1', '2', '3', '1', '2', '3'],
                            'value': [1, 2, 3, 4, 5, 6]})
         df = pd.concat([df] * 10, ignore_index=True)
         df['id'] = df.index.values
@@ -50,7 +50,7 @@ def setUpClass(cls):
     def test_is_partitioned(self):
         schema = ibis.schema([('foo', 'string'),
                               ('year', 'int32'),
-                              ('month', 'int16')])
+                              ('month', 'string')])
         name = _tmp_name()
         self.db.create_table(name, schema=schema,
                              partition=['year', 'month'])
@@ -58,7 +58,7 @@ def test_is_partitioned(self):
 
     def test_create_table_with_partition_column(self):
         schema = ibis.schema([('year', 'int32'),
-                              ('month', 'int8'),
+                              ('month', 'string'),
                               ('day', 'int8'),
                               ('value', 'double')])
 
@@ -72,21 +72,21 @@ def test_create_table_with_partition_column(self):
         ex_schema = ibis.schema([('day', 'int8'),
                                  ('value', 'double'),
                                  ('year', 'int32'),
-                                 ('month', 'int8')])
+                                 ('month', 'string')])
         table_schema = self.con.get_schema(name, database=self.tmp_db)
         assert_equal(table_schema, ex_schema)
 
         partition_schema = self.db.table(name).partition_schema()
 
         expected = ibis.schema([('year', 'int32'),
-                                ('month', 'int8')])
+                                ('month', 'string')])
         assert_equal(partition_schema, expected)
 
     def test_create_partitioned_separate_schema(self):
         schema = ibis.schema([('day', 'int8'),
                               ('value', 'double')])
         part_schema = ibis.schema([('year', 'int32'),
-                                   ('month', 'int8')])
+                                   ('month', 'string')])
 
         name = _tmp_name()
         self.con.create_table(name, schema=schema, partition=part_schema)
@@ -96,7 +96,7 @@ def test_create_partitioned_separate_schema(self):
         ex_schema = ibis.schema([('day', 'int8'),
                                  ('value', 'double'),
                                  ('year', 'int32'),
-                                 ('month', 'int8')])
+                                 ('month', 'string')])
         table_schema = self.con.get_schema(name)
         assert_equal(table_schema, ex_schema)
 
@@ -171,7 +171,7 @@ def test_load_data_partition(self):
                                                 part_keys)
 
         # trim the runtime of this test
-        df = df[df.month == 1].reset_index(drop=True)
+        df = df[df.month == '1'].reset_index(drop=True)
 
         unique_keys = df[part_keys].drop_duplicates()