Repository: CartoDB/cartoframes
Pull request #1142 (2019-10-31 11:52:11)

Fix encoding in data upload
Fixes https://github.com/CartoDB/cartoframes/issues/1133 and https://github.com/CartoDB/support/issues/2219


Patch:
diff --git a/cartoframes/data/dataset/registry/dataframe_dataset.py b/cartoframes/data/dataset/registry/dataframe_dataset.py
index 5a9baa66..b2079f58 100644
--- a/cartoframes/data/dataset/registry/dataframe_dataset.py
+++ b/cartoframes/data/dataset/registry/dataframe_dataset.py
@@ -7,7 +7,7 @@ from tqdm import tqdm
 from ....utils.columns import DataframeColumnsInfo, _first_value
 from ....utils.geom_utils import (compute_geodataframe, decode_geometry,
                                   save_index_as_column)
-from ....utils.utils import is_geojson, load_geojson, map_geom_type
+from ....utils.utils import is_geojson, load_geojson, map_geom_type, encode_row
 from .base_dataset import BaseDataset
 
 # avoid _lock issue: https://github.com/tqdm/tqdm/issues/457
@@ -161,7 +161,7 @@ def _rows(df, dataframe_columns_info, with_lnglat):
                 else:
                     val = ''
 
-            row_data.append(_encoded(val))
+            row_data.append(encode_row(val))
 
         if with_lnglat:
             lng_val = row[with_lnglat[0]]
@@ -170,23 +170,14 @@ def _rows(df, dataframe_columns_info, with_lnglat):
                 val = 'SRID=4326;POINT ({lng} {lat})'.format(lng=lng_val, lat=lat_val)
             else:
                 val = ''
-            row_data.append(_encoded(val))
+            row_data.append(encode_row(val))
 
-        csv_row = _encoded('|').join(row_data)
-        csv_row += _encoded('\n')
+        csv_row = b'|'.join(row_data)
+        csv_row += b'\n'
 
         yield csv_row
 
 
-def _encoded(val):
-    if isinstance(val, type(u'')):
-        return val.encode('utf-8')
-    elif isinstance(val, type(b'')):
-        return val
-    else:
-        return u'{}'.format(val).encode('utf-8')
-
-
 def _is_null(val):
     vnull = pd.isnull(val)
     if isinstance(vnull, bool):
diff --git a/cartoframes/utils/utils.py b/cartoframes/utils/utils.py
index 424dc88f..b85df00c 100644
--- a/cartoframes/utils/utils.py
+++ b/cartoframes/utils/utils.py
@@ -16,6 +16,11 @@ import numpy as np
 from functools import wraps
 from warnings import catch_warnings, filterwarnings
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
 
 GEOM_TYPE_POINT = 'point'
 GEOM_TYPE_LINE = 'line'
@@ -309,3 +314,18 @@ def remove_column_from_dataframe(dataframe, name):
     if dataframe.index.name == name:
         dataframe.reset_index(inplace=True)
         del dataframe[name]
+
+
+def encode_row(row):
+    if isinstance(row, type(b'')):
+        # Decode the input if it's a bytestring
+        row = row.decode('utf-8')
+
+    special_keys = ['"', '|', '\n']
+    if isinstance(row, basestring) and any(key in row for key in special_keys):
+        # If the input contains any special key:
+        # - replace " by ""
+        # - cover the row with "..."
+        row = '"{}"'.format(row.replace('"', '""'))
+
+    return '{}'.format(row).encode('utf-8')


Test patch:
diff --git a/tests/e2e/data/dataset/test_dataset_strategy_changes.py b/tests/unit/data/dataset/test_dataset_strategy_changes.py
similarity index 100%
rename from tests/e2e/data/dataset/test_dataset_strategy_changes.py
rename to tests/unit/data/dataset/test_dataset_strategy_changes.py
diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py
index 0c32f441..aef81fb6 100644
--- a/tests/unit/utils/test_utils.py
+++ b/tests/unit/utils/test_utils.py
@@ -8,7 +8,7 @@ import requests
 
 from cartoframes.utils.utils import (camel_dictionary, cssify, debug_print,
                                      dict_items, importify_params,
-                                     snake_to_camel)
+                                     snake_to_camel, encode_row)
 
 
 class TestUtils(unittest.TestCase):
@@ -175,3 +175,15 @@ class TestUtils(unittest.TestCase):
         # verbose = False
         verbose = 0
         self.assertIsNone(debug_print(verbose, resp=test_str))
+
+    def test_encode_row(self):
+        assert encode_row('Hello') == b'Hello'
+        assert encode_row('Hello \'world\'') == b'Hello \'world\''
+        assert encode_row('Hello "world"') == b'"Hello ""world"""'
+        assert encode_row('Hello | world') == b'"Hello | world"'
+        assert encode_row('Hello \n world') == b'"Hello \n world"'
+        assert encode_row(b'Hello') == b'Hello'
+        assert encode_row(b'Hello \'world\'') == b'Hello \'world\''
+        assert encode_row(b'Hello "world"') == b'"Hello ""world"""'
+        assert encode_row(b'Hello | world') == b'"Hello | world"'
+        assert encode_row(b'Hello \n world') == b'"Hello \n world"'