Repository: pandas-dev/pandas

Issue:
read csv thousands separator
From [this SO question](http://stackoverflow.com/questions/17797384/date-and-time-in-pandas-read-csv), with input to include thousands:

```

In [1]: s = '06.02.2013;13:00;1.000,215;0,215;0,185;0,205;0,00'

In [2]: pd.read_csv(StringIO(s), sep=';', header=None, parse_dates={'Dates': [0, 1]}, index_col=0, decimal=',')
Out[2]:
                              2      3      4      5  6
Dates
2013-06-02 13:00:00  10.000,215  0.215  0.185  0.205  0

In [3]: pd.read_csv(StringIO(s), sep=';', header=None, parse_dates={'Dates': [0, 1]}, index_col=0, decimal=',', thousands='.')
Out[3]:
                        2      3      4      5  6
Dates
6022013 13:00   1.000,215  0.215  0.185  0.205  0
```

_Note: the Dates column (as well as the thousands not being converted._



Discussion:
Possibly this is a dupe of #2594

this work? http://pandas.pydata.org/pandas-docs/dev/io.html#thousand-separators

@jreback  not sure what you're asking?

is this a bug? e.g. the dtypes not working with thousands or a bug in thousands sep? or  a feature request (which is there already)?

@jreback this is a bug, thousand separator doesn't seem to work

in the docs you link to it says

> For large integers that have been written with a thousands separator, you can set the thousands keyword to True so that integers will be parsed correctly:

in the docstring for read_csv it asks for:

> thousands : str, default None
>     Thousands separator

Doesn't seem to be working with `'.'`, _as well as_ screwing up the dates.

ahh ok...i c now.....

closed by #4598

@jreback The date aspect of this pr is still not fixed. For some reason the thousands separator attacks the date and makes it a string.

Wow, so this is a very edge case... it's cos the date column is just `06.02.2013` which is read as a number `0602013`... it's possible dates are sometimes written this way on the continent (along with . thousands): http://en.wikipedia.org/wiki/Date_and_time_notation_in_Europe

Not sure what solution is.

but it should ignore dates columns entirely (for thousands parsing...).....hmmm...why don't you open a separate issue and can cross-link it


Resolving patch:
diff --git a/pandas/io/date_converters.py b/pandas/io/date_converters.py
--- a/pandas/io/date_converters.py
+++ b/pandas/io/date_converters.py
@@ -26,7 +26,7 @@ def parse_all_fields(year_col, month_col, day_col, hour_col, minute_col,
     minute_col = _maybe_cast(minute_col)
     second_col = _maybe_cast(second_col)
     return lib.try_parse_datetime_components(year_col, month_col, day_col,
-                                             hour_col, minute_col, second_col)
+                    hour_col, minute_col, second_col)
 
 
 def generic_parser(parse_func, *cols):
diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py
--- a/pandas/io/parsers.py
+++ b/pandas/io/parsers.py
@@ -1020,6 +1020,14 @@ def _set(x):
                 else:
                     _set(val)
 
+        elif isinstance(self.parse_dates, dict):
+            for val in self.parse_dates.values():
+                if isinstance(val, list):
+                    for k in val:
+                        _set(k)
+                else:
+                    _set(val)
+
     def set_error_bad_lines(self, status):
         self._reader.set_error_bad_lines(int(status))
 
@@ -1269,6 +1277,7 @@ def __init__(self, f, **kwds):
             self._make_reader(f)
         else:
             self.data = f
+
         self.columns = self._infer_columns()
 
         # we are processing a multi index column
@@ -1292,6 +1301,38 @@ def __init__(self, f, **kwds):
                 self.index_names = index_names
         self._first_chunk = True
 
+        if self.parse_dates:
+            self._no_thousands_columns = self._set_no_thousands_columns()
+        else:
+            self._no_thousands_columns = None
+
+    def _set_no_thousands_columns(self):
+        # Create a set of column ids that are not to be stripped of thousands operators.
+        noconvert_columns = set()
+
+        def _set(x):
+            if com.is_integer(x):
+                noconvert_columns.add(x)
+            else:
+                noconvert_columns.add(self.columns.index(x))
+
+        if isinstance(self.parse_dates, list):
+            for val in self.parse_dates:
+                if isinstance(val, list):
+                    for k in val:
+                        _set(k)
+                else:
+                    _set(val)
+
+        elif isinstance(self.parse_dates, dict):
+            for val in self.parse_dates.values():
+                if isinstance(val, list):
+                    for k in val:
+                        _set(k)
+                else:
+                    _set(val)
+        return noconvert_columns
+
     def _make_reader(self, f):
         sep = self.delimiter
 
@@ -1500,7 +1541,6 @@ def _next_line(self):
             line = next(self.data)
 
         line = self._check_comments([line])[0]
-        line = self._check_thousands([line])[0]
 
         self.pos += 1
         self.buf.append(line)
@@ -1532,9 +1572,10 @@ def _check_thousands(self, lines):
         ret = []
         for l in lines:
             rl = []
-            for x in l:
+            for i, x in enumerate(l):
                 if (not isinstance(x, compat.string_types) or
                     self.thousands not in x or
+                    (self._no_thousands_columns and i in self._no_thousands_columns) or
                         nonnum.search(x.strip())):
                     rl.append(x)
                 else:
@@ -1608,7 +1649,6 @@ def _rows_to_cols(self, content):
             raise AssertionError()
 
         if col_len != zip_len and self.index_col is not False:
-            row_num = -1
             i = 0
             for (i, l) in enumerate(content):
                 if len(l) != col_len:


Test patch:
diff --git a/pandas/io/tests/test_parsers.py b/pandas/io/tests/test_parsers.py
--- a/pandas/io/tests/test_parsers.py
+++ b/pandas/io/tests/test_parsers.py
@@ -233,6 +233,18 @@ def test_1000_sep_with_decimal(self):
         df = self.read_table(StringIO(data_with_odd_sep), sep='|', thousands='.', decimal=',')
         tm.assert_frame_equal(df, expected)
 
+    def test_separator_date_conflict(self):
+        # Regression test for issue #4678: make sure thousands separator and
+        # date parsing do not conflict.
+        data = '06-02-2013;13:00;1-000.215'
+        expected = DataFrame(
+            [[datetime(2013, 6, 2, 13, 0, 0), 1000.215]],
+            columns=['Date', 2]
+        )
+
+        df = self.read_csv(StringIO(data), sep=';', thousands='-', parse_dates={'Date': [0, 1]}, header=None)
+        tm.assert_frame_equal(df, expected)
+
     def test_squeeze(self):
         data = """\
 a,1