Repository: pandas-dev/pandas

Issue:
SQL: use of connection context manager not supported by mysql.connector
Coming from discussion in #8232, current master breaks with `mysql.connector` as the MySQL driver in the fallback modus (deprecated) due to the use of context managers.

This was introduced in #8062, to ensure that the different chunks would be inserted in one transaction (when using the `chunksize` argument).

The connection context manager is however nog DBAPI specified behaviour, and differs among drivers:
- sqlite3 returns a connection from the context manager, but has a connection.executemany, so that of course works (otherwise the tests would not pass at all)
- pymysql returns a cursor, but does not close that cursor automatically (I don't know if this is a problem)
- MySQLdb does now support such a context manager (from 1.2.5, so only the latest version), and does the same as pymysql
- mysql.connector does not support it, so the current implementation fails with that.

So problem is that with the current code in master, we break the fallback mode when using `mysql.connector`.

Solution: I think remove the context managers for now (which will make the code a bit more complex), until mysql support is fully dropped



Discussion:

Resolving patch:
diff --git a/pandas/io/sql.py b/pandas/io/sql.py
--- a/pandas/io/sql.py
+++ b/pandas/io/sql.py
@@ -19,6 +19,7 @@
 from pandas.core.base import PandasObject
 from pandas.tseries.tools import to_datetime
 
+from contextlib import contextmanager
 
 class SQLAlchemyRequired(ImportError):
     pass
@@ -637,13 +638,9 @@ def insert_data(self):
 
         return column_names, data_list
 
-    def get_session(self):
-        con = self.pd_sql.engine.connect()
-        return con.begin()
-
-    def _execute_insert(self, trans, keys, data_iter):
+    def _execute_insert(self, conn, keys, data_iter):
         data = [dict( (k, v) for k, v in zip(keys, row) ) for row in data_iter]
-        trans.connection.execute(self.insert_statement(), data)
+        conn.execute(self.insert_statement(), data)
 
     def insert(self, chunksize=None):
         keys, data_list = self.insert_data()
@@ -653,7 +650,7 @@ def insert(self, chunksize=None):
             chunksize = nrows
         chunks = int(nrows / chunksize) + 1
 
-        with self.get_session() as trans:
+        with self.pd_sql.run_transaction() as conn:
             for i in range(chunks):
                 start_i = i * chunksize
                 end_i = min((i + 1) * chunksize, nrows)
@@ -661,7 +658,7 @@ def insert(self, chunksize=None):
                     break
 
                 chunk_iter = zip(*[arr[start_i:end_i] for arr in data_list])
-                self._execute_insert(trans, keys, chunk_iter)
+                self._execute_insert(conn, keys, chunk_iter)
 
     def read(self, coerce_float=True, parse_dates=None, columns=None):
 
@@ -884,6 +881,9 @@ def __init__(self, engine, schema=None, meta=None):
 
         self.meta = meta
 
+    def run_transaction(self):
+       return self.engine.begin()
+
     def execute(self, *args, **kwargs):
         """Simple passthrough to SQLAlchemy engine"""
         return self.engine.execute(*args, **kwargs)
@@ -1017,9 +1017,9 @@ def sql_schema(self):
         return str(";\n".join(self.table))
 
     def _execute_create(self):
-        with self.get_session():
+        with self.pd_sql.run_transaction() as conn:
             for stmt in self.table:
-                self.pd_sql.execute(stmt)
+                conn.execute(stmt)
 
     def insert_statement(self):
         names = list(map(str, self.frame.columns))
@@ -1038,12 +1038,9 @@ def insert_statement(self):
             self.name, col_names, wildcards)
         return insert_statement
 
-    def get_session(self):
-        return self.pd_sql.con
-
-    def _execute_insert(self, trans, keys, data_iter):
+    def _execute_insert(self, conn, keys, data_iter):
         data_list = list(data_iter)
-        trans.executemany(self.insert_statement(), data_list)
+        conn.executemany(self.insert_statement(), data_list)
 
     def _create_table_setup(self):
         """Return a list of SQL statement that create a table reflecting the 
@@ -1125,6 +1122,17 @@ def __init__(self, con, flavor, is_cursor=False):
         else:
             self.flavor = flavor
 
+    @contextmanager
+    def run_transaction(self):
+        cur = self.con.cursor()
+        try:
+            yield cur
+            self.con.commit()
+        except:
+            self.con.rollback()
+        finally:
+            cur.close()
+
     def execute(self, *args, **kwargs):
         if self.is_cursor:
             cur = self.con


Test patch:
diff --git a/pandas/io/tests/test_sql.py b/pandas/io/tests/test_sql.py
--- a/pandas/io/tests/test_sql.py
+++ b/pandas/io/tests/test_sql.py
@@ -331,6 +331,28 @@ def _to_sql_save_index(self):
         ix_cols = self._get_index_columns('test_to_sql_saves_index')
         self.assertEqual(ix_cols, [['A',],])
 
+    def _transaction_test(self):
+        self.pandasSQL.execute("CREATE TABLE test_trans (A INT, B TEXT)")
+
+        ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')"
+        
+        # Make sure when transaction is rolled back, no rows get inserted
+        try:
+            with self.pandasSQL.run_transaction() as trans:
+                trans.execute(ins_sql)
+                raise Exception('error')
+        except:
+            # ignore raised exception
+            pass
+        res = self.pandasSQL.read_sql('SELECT * FROM test_trans')
+        self.assertEqual(len(res), 0)
+        
+        # Make sure when transaction is committed, rows do get inserted
+        with self.pandasSQL.run_transaction() as trans:
+            trans.execute(ins_sql)
+        res2 = self.pandasSQL.read_sql('SELECT * FROM test_trans')
+        self.assertEqual(len(res2), 1)
+
 
 #------------------------------------------------------------------------------
 #--- Testing the public API
@@ -1072,6 +1094,8 @@ def _get_index_columns(self, tbl_name):
     def test_to_sql_save_index(self):
         self._to_sql_save_index()
 
+    def test_transactions(self):
+        self._transaction_test()
 
 class TestSQLiteAlchemy(_TestSQLAlchemy):
     """
@@ -1380,6 +1404,8 @@ def _get_index_columns(self, tbl_name):
     def test_to_sql_save_index(self):
         self._to_sql_save_index()
 
+    def test_transactions(self):
+        self._transaction_test()
 
 class TestMySQLLegacy(TestSQLiteLegacy):
     """