Repository: wireservice/csvkit

Issue:
Parse non-US locale numbers
Sometimes numeric data contains thousands separators, typically ',', '_' or  in Europe ','.
Also Europeans sometimes use ',' as the decimal point.


Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 8f1617b..9dc61c3 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -3,8 +3,9 @@
 
 Improvements:
 
-* Add a :code:`--version` (:code:`-V`) flag.
+* Add a :code:`--version` flag.
 * Add a :code:`--skip-lines` option to skip initial lines (e.g. comments, copyright notices, empty rows).
+* Add a :code:`--locale` option to set the locale of any formatted numbers.
 * :code:`-I` is the short option for :code:`--no-inference`.
 * :doc:`/scripts/csvjoin` supports :code:`--snifflimit` and :code:`--no-inference`.
 * :doc:`/scripts/csvstat` adds a :code:`--freq-count` option to set the maximum number of frequent values to display.
diff --git a/csvkit/cli.py b/csvkit/cli.py
index cf52724..f69f961 100644
--- a/csvkit/cli.py
+++ b/csvkit/cli.py
@@ -159,7 +159,10 @@ class CSVKitUtility(object):
                                         help='Maximum length of a single field in the input CSV file.')
         if 'e' not in self.override_flags:
             self.argparser.add_argument('-e', '--encoding', dest='encoding', default='utf-8',
-                                        help='Specify the encoding the input CSV file.')
+                                        help='Specify the encoding of the input CSV file.')
+        if 'L' not in self.override_flags:
+            self.argparser.add_argument('-L', '--locale', dest='locale', default='en_US',
+                                        help='Specify the locale (en_US) of any formatted numbers.')
         if 'S' not in self.override_flags:
             self.argparser.add_argument('-S', '--skipinitialspace', dest='skipinitialspace', action='store_true',
                                         help='Ignore whitespace immediately following the delimiter.')
@@ -283,7 +286,7 @@ class CSVKitUtility(object):
         else:
             return agate.TypeTester(types=[
                 agate.Boolean(),
-                agate.Number(),
+                agate.Number(locale=self.args.locale),
                 agate.TimeDelta(),
                 agate.Date(),
                 agate.DateTime(),
diff --git a/csvkit/utilities/sql2csv.py b/csvkit/utilities/sql2csv.py
index 98bf911..1b56f52 100644
--- a/csvkit/utilities/sql2csv.py
+++ b/csvkit/utilities/sql2csv.py
@@ -10,7 +10,7 @@ from csvkit.cli import CSVKitUtility
 
 class SQL2CSV(CSVKitUtility):
     description = 'Execute an SQL query on a database and output the result to a CSV file.'
-    override_flags = 'f,b,d,e,H,K,p,q,S,t,u,z,zero'.split(',')
+    override_flags = 'f,b,d,e,H,K,L,p,q,S,t,u,z,zero'.split(',')
 
     def add_arguments(self):
         self.argparser.add_argument('--db', dest='connection_string', default='sqlite://',
@@ -20,7 +20,7 @@ class SQL2CSV(CSVKitUtility):
         self.argparser.add_argument('--query', default=None,
                                     help="The SQL query to execute. If specified, it overrides FILE and STDIN.")
         self.argparser.add_argument('-e', '--encoding', dest='encoding', default='utf-8',
-                                    help='Specify the encoding the input query file.')
+                                    help='Specify the encoding of the input query file.')
         self.argparser.add_argument('-H', '--no-header-row', dest='no_header_row', action='store_true',
                                     help='Do not output column names.')
 
diff --git a/docs/common_arguments.rst b/docs/common_arguments.rst
index b4af001..337ac0a 100644
--- a/docs/common_arguments.rst
+++ b/docs/common_arguments.rst
@@ -24,7 +24,9 @@ All tools which accept CSV as input share a set of common command-line arguments
                         Maximum length of a single field in the input CSV
                         file.
   -e ENCODING, --encoding ENCODING
-                        Specify the encoding the input CSV file.
+                        Specify the encoding of the input CSV file.
+  -L LOCALE, --locale LOCALE
+                        Specify the locale (en_US) of any formatted numbers.
   -S, --skipinitialspace
                         Ignore whitespace immediately following the delimiter.
   -H, --no-header-row   Specify that the input CSV file has no header row.
diff --git a/docs/scripts/sql2csv.rst b/docs/scripts/sql2csv.rst
index 1ea81e2..bfd3439 100644
--- a/docs/scripts/sql2csv.rst
+++ b/docs/scripts/sql2csv.rst
@@ -30,7 +30,7 @@ Executes arbitrary commands against a SQL database and outputs the results as a
       --query QUERY         The SQL query to execute. If specified, it overrides
                             FILE and STDIN.
       -e ENCODING, --encoding ENCODING
-                            Specify the encoding the input query file.
+                            Specify the encoding of the input query file.
       -H, --no-header-row   Do not output column names.
 
 Examples


Test patch:
diff --git a/examples/test_locale.csv b/examples/test_locale.csv
new file mode 100644
index 0000000..4924ddc
--- /dev/null
+++ b/examples/test_locale.csv
@@ -0,0 +1,2 @@
+a,b,c
+"1,7","200.000.000",
diff --git a/examples/test_locale_converted.csv b/examples/test_locale_converted.csv
new file mode 100644
index 0000000..3cd0f59
--- /dev/null
+++ b/examples/test_locale_converted.csv
@@ -0,0 +1,2 @@
+a,b,c
+1.7,200000000,
diff --git a/tests/test_utilities/test_in2csv.py b/tests/test_utilities/test_in2csv.py
index 5bedf05..ce9382d 100644
--- a/tests/test_utilities/test_in2csv.py
+++ b/tests/test_utilities/test_in2csv.py
@@ -34,6 +34,9 @@ class TestIn2CSV(CSVKitTestCase, EmptyFileTests):
 
         self.assertEqual(e.exception.code, 0)
 
+    def test_locale(self):
+        self.assertConverted('csv', 'examples/test_locale.csv', 'examples/test_locale_converted.csv', ['--locale', 'de_DE'])
+
     def test_convert_csv(self):
         self.assertConverted('csv', 'examples/testfixed_converted.csv', 'examples/testfixed_converted.csv')