Repository: pandas-dev/pandas

Issue:
BUG: add support for writing datetime.date and datetime.time columns using to_sql
Hi-
the following commands throw a DataError --

``` python
con = sqlalchemy.create_engine("mssql+pyodbc://server?driver=SQL Server Native Client 11.0")
df = pd.DataFrame([datetime.time(7,10), datetime.time(7,20)], columns="a")
sql.to_sql(df, "TBL_TEMP", con, index=False)
```

throws the following error:

``` python
Traceback (most recent call last):

  File "<ipython-input-275-80a6d739629c>", line 1, in <module>
    sql.to_sql(df, "TBL_TEMP3", con, index=False)

  File "N:\Python\sql.py", line 399, in to_sql
    index_label=index_label)

  File "N:\Python\sql.py", line 774, in to_sql
    table.insert()

  File "N:\Python\sql.py", line 538, in insert
    self.pd_sql.execute(ins, data_list)

  File "N:\Python\sql.py", line 734, in execute
    return self.engine.execute(*args, **kwargs)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\base.py", line 1598, in execute
    return connection.execute(statement, *multiparams, **params)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\base.py", line 664, in execute
    return meth(self, multiparams, params)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\sql\elements.py", line 282, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\base.py", line 761, in _execute_clauseelement
    compiled_sql, distilled_params

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\base.py", line 874, in _execute_context
    context)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\base.py", line 1023, in _handle_dbapi_exception
    exc_info

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\util\compat.py", line 174, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=exc_value)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\util\compat.py", line 167, in reraise
    raise value.with_traceback(tb)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\base.py", line 856, in _execute_context
    context)

  File "C:\WinPython3.3.3.2\python-3.3.3.amd64\lib\site-packages\sqlalchemy\engine\default.py", line 385, in do_executemany
    cursor.executemany(statement, parameters)

DataError: (DataError) ('22018', '[22018] [Microsoft][SQL Server Native Client 11.0][SQL Server]Operand type clash: time is incompatible with text (206) (SQLExecDirectW)') 'INSERT INTO [TBL_TEMP3] (a) VALUES (?)' ((datetime.time(7, 10),), (datetime.time(7, 20),))
```

I have two columns, one with datetime.date and one with datetime.time, which both exhibited this problem. I force-converted the datetime.date column via pd.to_datetime into a datetimeindex, which to_sql/sqlalchemy correctly formats into an SQL-acceptable date format. However, to_datetime does not work on datetime.date, leaving the pandas datatype as "object" instead of datetime64ns.

Thanks,



Discussion:
`datetime.date` and `datetime.time` are always `object` formats; these are not force converted, unless you explicty pass a dtype (e.g. `datetime64[ns]`). This is sort of a compatibility issue as people want to use them, so they are represented as object dtype.

@jreback Is there any way to detect the 'real' type of a column when it has dtype `object`? (I suppose just looking at the type of eg the first value, but that is not really robust) 
Because, in principle, sqlalchemy has `datetime.date/time` types that map to sql types, and mysql has a `DATE` and `TIME` type, so they could actually be written to the database correctly.

BTW, this line now in the codebase: https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L655 I suppose is useless, as the dtype of a dataframe column can never be `datetime.date`?

yes that line would never be true and is useless

lib.infer_dtype(com._ensure_object(arr))

would return 'date' or 'time' if all elements are date or time
else would return 'mixed' 

I updated this issue to also remember that `datetime.date` should be added (there is a line of code for that, but it is not working, and it was never tested).

So adding `datetime.time` and `datetime.date`:
- so if dtype=object, check if it is date or time (https://github.com/pydata/pandas/issues/6932#issuecomment-41147038)
- return the appropriate SQLAlchemy type


Resolving patch:
diff --git a/pandas/io/sql.py b/pandas/io/sql.py
--- a/pandas/io/sql.py
+++ b/pandas/io/sql.py
@@ -11,6 +11,7 @@
 import re
 import numpy as np
 
+import pandas.lib as lib
 import pandas.core.common as com
 from pandas.compat import lzip, map, zip, raise_with_traceback, string_types
 from pandas.core.api import DataFrame, Series
@@ -684,13 +685,14 @@ def _get_column_names_and_types(self, dtype_mapper):
         if self.index is not None:
             for i, idx_label in enumerate(self.index):
                 idx_type = dtype_mapper(
-                    self.frame.index.get_level_values(i).dtype)
+                    self.frame.index.get_level_values(i))
                 column_names_and_types.append((idx_label, idx_type))
 
-        column_names_and_types += zip(
-            list(map(str, self.frame.columns)),
-            map(dtype_mapper, self.frame.dtypes)
-            )
+        column_names_and_types += [
+            (str(self.frame.columns[i]),
+             dtype_mapper(self.frame.iloc[:,i]))
+            for i in range(len(self.frame.columns))
+            ]
         return column_names_and_types
 
     def _create_table_statement(self):
@@ -756,30 +758,33 @@ def _harmonize_columns(self, parse_dates=None):
             except KeyError:
                 pass  # this column not in results
 
-    def _sqlalchemy_type(self, arr_or_dtype):
+    def _sqlalchemy_type(self, col):
         from sqlalchemy.types import (BigInteger, Float, Text, Boolean,
-            DateTime, Date, Interval)
+            DateTime, Date, Time, Interval)
 
-        if arr_or_dtype is date:
-            return Date
-        if com.is_datetime64_dtype(arr_or_dtype):
+        if com.is_datetime64_dtype(col):
             try:
-                tz = arr_or_dtype.tzinfo
+                tz = col.tzinfo
                 return DateTime(timezone=True)
             except:
                 return DateTime
-        if com.is_timedelta64_dtype(arr_or_dtype):
+        if com.is_timedelta64_dtype(col):
             warnings.warn("the 'timedelta' type is not supported, and will be "
                           "written as integer values (ns frequency) to the "
                           "database.", UserWarning)
             return BigInteger
-        elif com.is_float_dtype(arr_or_dtype):
+        elif com.is_float_dtype(col):
             return Float
-        elif com.is_integer_dtype(arr_or_dtype):
+        elif com.is_integer_dtype(col):
             # TODO: Refine integer size.
             return BigInteger
-        elif com.is_bool_dtype(arr_or_dtype):
+        elif com.is_bool_dtype(col):
             return Boolean
+        inferred = lib.infer_dtype(com._ensure_object(col))
+        if inferred == 'date':
+            return Date
+        if inferred == 'time':
+            return Time
         return Text
 
     def _numpy_type(self, sqltype):
@@ -908,7 +913,11 @@ def _create_sql_schema(self, frame, table_name):
     },
     'date': {
         'mysql': 'DATE',
-        'sqlite': 'TIMESTAMP',
+        'sqlite': 'DATE',
+    },
+    'time': {
+        'mysql': 'TIME',
+        'sqlite': 'TIME',
     },
     'bool': {
         'mysql': 'BOOLEAN',
@@ -1014,8 +1023,8 @@ def _create_table_statement(self):
         create_statement = template % {'name': self.name, 'columns': columns}
         return create_statement
 
-    def _sql_type_name(self, dtype):
-        pytype = dtype.type
+    def _sql_type_name(self, col):
+        pytype = col.dtype.type
         pytype_name = "text"
         if issubclass(pytype, np.floating):
             pytype_name = "float"
@@ -1029,10 +1038,14 @@ def _sql_type_name(self, dtype):
         elif issubclass(pytype, np.datetime64) or pytype is datetime:
             # Caution: np.datetime64 is also a subclass of np.number.
             pytype_name = "datetime"
-        elif pytype is datetime.date:
-            pytype_name = "date"
         elif issubclass(pytype, np.bool_):
             pytype_name = "bool"
+        elif issubclass(pytype, np.object):
+            pytype = lib.infer_dtype(com._ensure_object(col))
+            if pytype == "date":
+                pytype_name = "date"
+            elif pytype == "time":
+                pytype_name = "time"
 
         return _SQL_TYPES[pytype_name][self.pd_sql.flavor]
 


Test patch:
diff --git a/pandas/io/tests/test_sql.py b/pandas/io/tests/test_sql.py
--- a/pandas/io/tests/test_sql.py
+++ b/pandas/io/tests/test_sql.py
@@ -26,7 +26,7 @@
 import warnings
 import numpy as np
 
-from datetime import datetime
+from datetime import datetime, date, time
 
 from pandas import DataFrame, Series, Index, MultiIndex, isnull
 from pandas import date_range, to_datetime, to_timedelta
@@ -35,6 +35,7 @@
 from pandas.core.datetools import format as date_format
 
 import pandas.io.sql as sql
+from pandas.io.sql import read_sql_table, read_sql_query
 import pandas.util.testing as tm
 
 
@@ -976,6 +977,21 @@ def test_datetime_NaT(self):
         else:
             tm.assert_frame_equal(result, df)
 
+    def test_datetime_date(self):
+        # test support for datetime.date
+        df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"])
+        df.to_sql('test_date', self.conn, index=False)
+        res = read_sql_table('test_date', self.conn)
+        # comes back as datetime64
+        tm.assert_series_equal(res['a'], to_datetime(df['a']))
+
+    def test_datetime_time(self):
+        # test support for datetime.time
+        df = DataFrame([time(9, 0, 0), time(9, 1, 30)], columns=["a"])
+        df.to_sql('test_time', self.conn, index=False)
+        res = read_sql_table('test_time', self.conn)
+        tm.assert_frame_equal(res, df)
+
     def test_mixed_dtype_insert(self):
         # see GH6509
         s1 = Series(2**25 + 1,dtype=np.int32)
@@ -1269,6 +1285,21 @@ def test_roundtrip(self):
     def test_execute_sql(self):
         self._execute_sql()
 
+    def test_datetime_date(self):
+        # test support for datetime.date
+        df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"])
+        df.to_sql('test_date', self.conn, index=False, flavor=self.flavor)
+        res = read_sql_query('SELECT * FROM test_date', self.conn)
+        if self.flavor == 'sqlite':
+            # comes back as strings
+            tm.assert_frame_equal(res, df.astype(str))
+        elif self.flavor == 'mysql':
+            tm.assert_frame_equal(res, df)
+    
+    def test_datetime_time(self):
+        # test support for datetime.time
+        raise nose.SkipTest("datetime.time not supported for sqlite fallback")
+
 
 class TestMySQLLegacy(TestSQLiteLegacy):
     """