Repository: weecology/retriever

Issue:
Reserved keyword 'order' not being properly replaced
`clean_column_name()` in `table.py` should replace the column name `order` with `sporder`. It does not appear to be doing this in #286.



Discussion:
Looks like `clean_column_name()` is only called in `auto_get_columns()`, so it only gets triggered when reading columns from a header row of a file. I suspect that it won't be uncommon that someone writing a script that requires providing column names will accidentally include either bad characters or reserved keywords, so we should probably always run `clean_column_name()`.

This should be as easy as just calling clean_column_name() every time we manually set columns, right? or you have something else in mind?

At first glance it looks like, in `tables.py`, maybe adding something like Line 39 in `auto_get_columns` to `get_insert_columns` might be the simplest and most robust solution.

I was thinking maybe we could do something like override `__setattr__` for table class, that way every time **column** is updated/assigned we just clean it up before adding to dictionary. But I guess as you said, cleaning them up in get_insert_columns, seems simpler.

Feel free to try it either way and we can discuss it more when we have a PR
to look at.
On Apr 1, 2015 9:34 PM, "Pankaj" notifications@github.com wrote:

> I was thinking maybe we could do something like override _setattr_ for
> table class, that way every time _column_ is updated/assigned we just
> clean it up before adding to dictionary. But I guess as you said, cleaning
> them up in get_insert_columns, seems simpler.
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/weecology/retriever/issues/292#issuecomment-88715459.


Resolving patch:
diff --git a/lib/table.py b/lib/table.py
--- a/lib/table.py
+++ b/lib/table.py
@@ -1,7 +1,8 @@
-from retriever.lib.cleanup import *
 import csv
 import StringIO
 
+from retriever.lib.cleanup import *
+
 
 class Table:
     """Information about a database table."""
@@ -19,6 +20,7 @@ def __init__(self, name, **kwargs):
         self.replace_columns = []
         self.escape_single_quotes=True
         self.escape_double_quotes=True
+        self.cleaned_columns = False
         for key, item in kwargs.items():
             setattr(self, key, item[0] if isinstance(item, tuple) else item)
 
@@ -39,36 +41,53 @@ def auto_get_columns(self, header):
 
         columns = map(lambda x: self.clean_column_name(x), column_names)
         column_values = {x:[] for x in columns if x}
-
+        self.cleaned_columns = True
         return [[x, None] for x in columns if x], column_values
 
     def clean_column_name(self, column_name):
-        '''Makes sure a column name is formatted correctly by removing reserved
-        words, symbols, numbers, etc.'''
-        column_name = column_name.lower()
+        """Clean column names using the expected sql guidelines
+
+        remove leading whitespaces, replace sql key words, etc..
+        """
+        column_name = column_name.lower().strip()
         replace_columns = {old.lower(): new.lower()
                            for old, new in self.replace_columns}
-        column_name = replace_columns.get(column_name, column_name)
+
+        column_name = replace_columns.get(column_name, column_name).strip()
         replace = [
-                   ("%", "percent"),
-                   ("&", "and"),
-                   ("\xb0", "degrees"),
-                   ("group", "grp"),
-                   ("order", "sporder"),
-                   ("check", "checked"),
-                   ("references", "refs"),
-                   ("long", "lon"),
-                   ("date", "record_date"),
-                   ("?", ""),
-                   ]
+            ("%", "percent"),
+            ("&", "and"),
+            ("\xb0", "degrees"),
+            ("?", ""),
+        ]
         replace += [(x, '') for x in (")", "\n", "\r", '"', "'")]
-        replace += [(x, '_') for x in (" ", "(", "/", ".", "-")]
+        replace += [(x, '_') for x in (" ", "(", "/", ".", "-", "*", ":")]
         column_name = reduce(lambda x, y: x.replace(*y), replace, column_name)
-
         while "__" in column_name:
             column_name = column_name.replace("__", "_")
         column_name = column_name.lstrip("0123456789_").rstrip("_")
-
+        replace_dict = {
+            "group": "grp",
+            "order": "ordered",
+            "check": "checked",
+            "references": "refs",
+            "long": "lon",
+            "column": "columns",
+            "cursor": "cursors",
+            "delete": "deleted",
+            "insert": "inserted",
+            "join": "joins",
+            "select": "selects",
+            "table": "tables",
+            "update": "updates",
+            "date": "record_date"
+        }
+        for x in (")", "\n", "\r", '"', "'"):
+            replace_dict[x] = ''
+        for x in (" ", "(", "/", ".", "-"):
+            replace_dict[x] = '_'
+        if column_name in replace_dict:
+            column_name = replace_dict[column_name]
         return column_name
 
     def split_on_delimiter(self, line):
@@ -129,6 +148,11 @@ def extract_values(self, line):
     def get_insert_columns(self, join=True):
         """Gets a set of column names for insert statements."""
         columns = ""
+        if not self.cleaned_columns:
+            column_names = list(self.columns)
+            self.columns[:] = []
+            self.columns = [(self.clean_column_name(name[0]), name[1]) for name in column_names]
+            self.cleaned_columns = True
         for item in self.columns:
             thistype = item[1][0]
             if ((thistype != "skip") and (thistype !="combine") and
@@ -148,4 +172,3 @@ def get_column_datatypes(self):
                 if item == column[0]:
                     columns.append(column[1][0])
         return columns
-


Test patch:
diff --git a/test/test_regression_cli.py b/test/test_regression_cli.py
--- a/test/test_regression_cli.py
+++ b/test/test_regression_cli.py
@@ -7,7 +7,7 @@
 known_md5s = {
 
             'sqlite': {'AvianBodySize' : '72256f681cdce96eba32d4ece270bcb2',
-                     'DelMoral2010' : '92b6cd535f8e6c82d6fc19802d6ba64f',
+                     'DelMoral2010' : '2f936afb990c8f818223b27e1e8d6212',
                      'MoM2003' : 'd3bdce86e0fc5888449884dfb0ef4611'},
             
             'download': {'AvianBodySize' : 'dce81ee0f040295cd14c857c18cc3f7e',
@@ -15,7 +15,7 @@
                      'MoM2003' : 'b54b80d0d1959bdea0bb8a59b70fa871'},                          
             
             'csv': {'AvianBodySize' : 'f42702a53e7d99d16e909676f30e5aa8',
-                  'DelMoral2010' : '606f97c3ddbfd6d63b474bc76d01646a',
+                  'DelMoral2010' : 'e79d55ac15f1a70a6c7d3ad4e678ec0e',
                   'MoM2003' : 'ef0a31c132cfe1c6594739c872f70f54'},
             
             'mysql': {'AvianBodySize' : 'f60ac93d9be4671dbef77da9d10676b8',
@@ -23,7 +23,7 @@
                     'MoM2003' : '9728728d72af4c21a2a6e29fec3edb48'},
             
             'postgres': {'AvianBodySize' : '60c252af74d914e3c15fa9af43edefca',
-                     'DelMoral2010' : '1e8de8fa3ddbd4ca3a7cab921926e70e',
+                     'DelMoral2010' : '63d91a972b5ca07bf90900a576286a8a',
                      'MoM2003' : 'a55c8308722c8e20950e0d1e6d9639e6'}
              }
 
@@ -137,4 +137,4 @@ def check_download_regression(self, dataset, known_md5):
         del(stub_test)
 
 if __name__ == '__main__':
-    nose.runmodule()
\ No newline at end of file
+    nose.runmodule()