Repository: pandas-dev/pandas

Issue:
No way to specify higher precision (e.g. Double) when saving DataFrame with floating number to MySQL
When saving DataFrame to MySQL, Pandas will map Python float (by default double precision) to MySQL FLOAT (by default single precision). I couldn't find a way to either specify the use of MySQL DOUBLE, or MySQL DECIMAL.

When saving some financial data this will cause loss of precision. For example, 2-year treasury future price quotes are multiples of 1/4/32 = 0.0078125, and there are usually 3 digits before the decimal point. A real-world example is 109.8828125 (TUZ4 settlement price on 12/04/2014). Saving a DataFrame with this number to MySQL results in the rounded 109.883. The error itself is not huge (2.4% of a tick), but still undesirable, especially considering using MySQL DOUBLE or DECIMAL can easily improve the precision.

Currently a workaround is to modify pandas/io/sql.py to map Python float to MySQL FLOAT(53), which forces the use of DOUBLE. There are two places to change, one inside method definition SQLTable._sqlalchemy_type(self, col), the other inside dictionary definition _SQL_TYPES. This work around will waste storage when only single precision is needed. It will be better if some option is provided to utilize MySQL DOUBLE and DECIMAL only when needed.



Discussion:
@zhopan77 Thanks for the report!

There was recently introduced a new feature that allows you to override the default chosen type using the `dtype` kwarg in `to_sql` (will be released in 0.15.2 in one or two weeks). See #8926. 
So that should solve the immediate possibility to change the default (without changing the `_sqlalchemy_type` or `_SQL_TYPES`).

But regarding that default. For the mysql legacy `SQL_TYPES`, I am not going to change it anymore, as it is deprecated anyways. But for the sqlalchemy mode, I think we should use something by default that does not result in data loss. 
Can you give a small example? Because eg with posgresql, `FLOAT` does mean double precision (and if you write a table, it also use the postgresql `double precision` type)

Example with mysql:

```
In [57]: df = pd.DataFrame(np.random.randn(5, 5), columns=list('abcde'))

In [58]: df.to_sql('test_float', engine_pymysql, index=False, if_exists='replace')

In [59]: res = pd.read_sql_table('test_float', engine_pymysql)

In [60]: df.iloc[0,0]
Out[60]: -0.50432201678449218

In [61]: res.iloc[0,0]
Out[61]: -0.50432200000000005

In [62]: from sqlalchemy.dialects.mysql import DOUBLE

In [63]: df.to_sql('test_float', engine_pymysql, index=False, if_exists='replace', dtype={'a':DOUBLE})

In [64]: res = pd.read_sql_table('test_float', engine_pymysql)

In [65]: res.iloc[0,0]
Out[65]: -0.50432201679999999
```

It is a bit annoying that the `Float` sqlalchemy type means something different for differen flavors.


Resolving patch:
diff --git a/pandas/io/sql.py b/pandas/io/sql.py
--- a/pandas/io/sql.py
+++ b/pandas/io/sql.py
@@ -908,7 +908,7 @@ def _sqlalchemy_type(self, col):
 
         col_type = self._get_notnull_col_dtype(col)
 
-        from sqlalchemy.types import (BigInteger, Float, Text, Boolean,
+        from sqlalchemy.types import (BigInteger, Integer, Float, Text, Boolean,
             DateTime, Date, Time)
 
         if col_type == 'datetime64' or col_type == 'datetime':
@@ -923,10 +923,15 @@ def _sqlalchemy_type(self, col):
                           "database.", UserWarning)
             return BigInteger
         elif col_type == 'floating':
-            return Float
+            if col.dtype == 'float32':
+                return Float(precision=23)
+            else:
+                return Float(precision=53)
         elif col_type == 'integer':
-            # TODO: Refine integer size.
-            return BigInteger
+            if col.dtype == 'int32':
+                return Integer
+            else:
+                return BigInteger
         elif col_type == 'boolean':
             return Boolean
         elif col_type == 'date':
@@ -1187,9 +1192,17 @@ def has_table(self, name, schema=None):
     def get_table(self, table_name, schema=None):
         schema = schema or self.meta.schema
         if schema:
-            return self.meta.tables.get('.'.join([schema, table_name]))
+            tbl = self.meta.tables.get('.'.join([schema, table_name]))
         else:
-            return self.meta.tables.get(table_name)
+            tbl = self.meta.tables.get(table_name)
+
+        # Avoid casting double-precision floats into decimals
+        from sqlalchemy import Numeric
+        for column in tbl.columns:
+            if isinstance(column.type, Numeric):
+                column.type.asdecimal = False
+
+        return tbl
 
     def drop_table(self, table_name, schema=None):
         schema = schema or self.meta.schema
@@ -1198,8 +1211,9 @@ def drop_table(self, table_name, schema=None):
             self.get_table(table_name, schema).drop()
             self.meta.clear()
 
-    def _create_sql_schema(self, frame, table_name, keys=None):
-        table = SQLTable(table_name, self, frame=frame, index=False, keys=keys)
+    def _create_sql_schema(self, frame, table_name, keys=None, dtype=None):
+        table = SQLTable(table_name, self, frame=frame, index=False, keys=keys,
+                         dtype=dtype)
         return str(table.sql_schema())
 
 
@@ -1213,7 +1227,7 @@ def _create_sql_schema(self, frame, table_name, keys=None):
         'sqlite': 'TEXT',
     },
     'floating': {
-        'mysql': 'FLOAT',
+        'mysql': 'DOUBLE',
         'sqlite': 'REAL',
     },
     'integer': {
@@ -1520,13 +1534,13 @@ def drop_table(self, name, schema=None):
         drop_sql = "DROP TABLE %s" % name
         self.execute(drop_sql)
 
-    def _create_sql_schema(self, frame, table_name, keys=None):
+    def _create_sql_schema(self, frame, table_name, keys=None, dtype=None):
         table = SQLiteTable(table_name, self, frame=frame, index=False,
-                            keys=keys)
+                            keys=keys, dtype=dtype)
         return str(table.sql_schema())
 
 
-def get_schema(frame, name, flavor='sqlite', keys=None, con=None):
+def get_schema(frame, name, flavor='sqlite', keys=None, con=None, dtype=None):
     """
     Get the SQL db table schema for the given frame.
 
@@ -1545,11 +1559,14 @@ def get_schema(frame, name, flavor='sqlite', keys=None, con=None):
         Using SQLAlchemy makes it possible to use any DB supported by that
         library.
         If a DBAPI2 object, only sqlite3 is supported.
+    dtype : dict of column name to SQL type, default None
+        Optional specifying the datatype for columns. The SQL type should
+        be a SQLAlchemy type, or a string for sqlite3 fallback connection.
 
     """
 
     pandas_sql = pandasSQL_builder(con=con, flavor=flavor)
-    return pandas_sql._create_sql_schema(frame, name, keys=keys)
+    return pandas_sql._create_sql_schema(frame, name, keys=keys, dtype=dtype)
 
 
 # legacy names, with depreciation warnings and copied docs


Test patch:
diff --git a/pandas/io/tests/test_sql.py b/pandas/io/tests/test_sql.py
--- a/pandas/io/tests/test_sql.py
+++ b/pandas/io/tests/test_sql.py
@@ -651,6 +651,14 @@ def test_get_schema(self):
                                     con=self.conn)
         self.assertTrue('CREATE' in create_sql)
 
+    def test_get_schema_dtypes(self):
+        float_frame = DataFrame({'a':[1.1,1.2], 'b':[2.1,2.2]})
+        dtype = sqlalchemy.Integer if self.mode == 'sqlalchemy' else 'INTEGER'
+        create_sql = sql.get_schema(float_frame, 'test', 'sqlite',
+                                    con=self.conn, dtype={'b':dtype})
+        self.assertTrue('CREATE' in create_sql)
+        self.assertTrue('INTEGER' in create_sql)
+
     def test_chunksize_read(self):
         df = DataFrame(np.random.randn(22, 5), columns=list('abcde'))
         df.to_sql('test_chunksize', self.conn, index=False)
@@ -1233,7 +1241,6 @@ def test_dtype(self):
         df.to_sql('dtype_test3', self.conn, dtype={'B': sqlalchemy.String(10)})
         meta.reflect()
         sqltype = meta.tables['dtype_test3'].columns['B'].type
-        print(sqltype)
         self.assertTrue(isinstance(sqltype, sqlalchemy.String))
         self.assertEqual(sqltype.length, 10)
 
@@ -1262,6 +1269,36 @@ def test_notnull_dtype(self):
         self.assertTrue(isinstance(col_dict['Int'].type, sqltypes.Integer))
         self.assertTrue(isinstance(col_dict['Float'].type, sqltypes.Float))
 
+    def test_double_precision(self):
+        V = 1.23456789101112131415
+
+        df = DataFrame({'f32':Series([V,], dtype='float32'),
+                        'f64':Series([V,], dtype='float64'),
+                        'f64_as_f32':Series([V,], dtype='float64'),
+                        'i32':Series([5,], dtype='int32'),
+                        'i64':Series([5,], dtype='int64'),
+                        })
+
+        df.to_sql('test_dtypes', self.conn, index=False, if_exists='replace', 
+            dtype={'f64_as_f32':sqlalchemy.Float(precision=23)})
+        res = sql.read_sql_table('test_dtypes', self.conn)
+        
+        # check precision of float64
+        self.assertEqual(np.round(df['f64'].iloc[0],14), 
+                         np.round(res['f64'].iloc[0],14))
+
+        # check sql types
+        meta = sqlalchemy.schema.MetaData(bind=self.conn)
+        meta.reflect()
+        col_dict = meta.tables['test_dtypes'].columns
+        self.assertEqual(str(col_dict['f32'].type), 
+                         str(col_dict['f64_as_f32'].type))
+        self.assertTrue(isinstance(col_dict['f32'].type, sqltypes.Float))
+        self.assertTrue(isinstance(col_dict['f64'].type, sqltypes.Float))
+        self.assertTrue(isinstance(col_dict['i32'].type, sqltypes.Integer))
+        self.assertTrue(isinstance(col_dict['i64'].type, sqltypes.BigInteger))
+
+
 
 class TestSQLiteAlchemy(_TestSQLAlchemy):
     """