Repository: ibis-project/ibis

Issue:
ImpalaCursor cursor attribute sensitive to how (quickly?) its accessed
``` python
In [2]: out = client.raw_sql('select 1', True); out.cursor.fetchone()
Out[2]: (1,)

In [3]: client.raw_sql('select 1', True).cursor.fetchone()
---------------------------------------------------------------------------
ProgrammingError                          Traceback (most recent call last)
<ipython-input-3-29dd48e2437a> in <module>()
----> 1 client.raw_sql('select 1', True).cursor.fetchone()

/u/dlovell/.conda/envs/ibis/lib/python3.4/site-packages/impala/hiveserver2.py in fetchone(self)
    306         # PEP 249
    307         if not self.has_result_set:
--> 308             raise ProgrammingError("Tried to fetch but no results.")
    309         log.info('Fetching a single row')
    310         try:

ProgrammingError: Tried to fetch but no results.
```



Discussion:
This is very strange. I'm investigating

@tsdlovell Using the `cursor` attribute of the result of `raw_sql` is not supported -- it is intended to be a managed cursor which is automatically closed on garbage collection of the parent `ImpalaCursor` class. Pull request coming shortly, and I'm making the `.cursor` a `._cursor` attribute to make thsi more clear


Resolving patch:
diff --git a/ibis/impala/client.py b/ibis/impala/client.py
--- a/ibis/impala/client.py
+++ b/ibis/impala/client.py
@@ -182,18 +182,21 @@ class ImpalaCursor(object):
 
     def __init__(self, cursor, con, impyla_con, database,
                  options):
-        self.cursor = cursor
+        self._cursor = cursor
         self.con = con
         self.impyla_con = impyla_con
         self.database = database
         self.options = options
+        self.released = False
 
     def __del__(self):
         self._close_cursor()
+        with self.con.lock:
+            self.con.connection_pool_size -= 1
 
     def _close_cursor(self):
         try:
-            self.cursor.close()
+            self._cursor.close()
         except HS2Error as e:
             # connection was closed elsewhere
             if 'invalid session' not in e.args[0].lower():
@@ -208,17 +211,19 @@ def __exit__(self, type, value, tb):
     def set_options(self):
         for k, v in self.options.items():
             query = 'SET {0}={1}'.format(k, v)
-            self.cursor.execute(query)
+            self._cursor.execute(query)
 
     @property
     def description(self):
-        return self.cursor.description
+        return self._cursor.description
 
     def release(self):
-        self.con.release(self)
+        if not self.released:
+            self.con.release(self)
+            self.released = True
 
     def execute(self, stmt, async=False):
-        self.cursor.execute_async(stmt)
+        self._cursor.execute_async(stmt)
         if async:
             return
         else:
@@ -241,11 +246,11 @@ def _sleep_interval(start_time):
                 return 0.5
             return 1.0
 
-        cur = self.cursor
+        cur = self._cursor
         try:
             while True:
                 state = cur.status()
-                if self.cursor._op_state_is_error(state):
+                if self._cursor._op_state_is_error(state):
                     raise OperationalError("Operation is in ERROR_STATE")
                 if not cur._op_state_is_executing(state):
                     break
@@ -259,16 +264,19 @@ def is_finished(self):
         return not self.is_executing()
 
     def is_executing(self):
-        return self.cursor.is_executing()
+        return self._cursor.is_executing()
 
     def cancel(self):
-        self.cursor.cancel_operation()
+        self._cursor.cancel_operation()
+
+    def fetchone(self):
+        return self._cursor.fetchone()
 
     def fetchall(self, columnar=False):
         if columnar:
-            return self.cursor.fetchcolumnar()
+            return self._cursor.fetchcolumnar()
         else:
-            return self.cursor.fetchall()
+            return self._cursor.fetchall()
 
 
 class ImpalaQuery(Query):


Test patch:
diff --git a/ibis/impala/tests/test_client.py b/ibis/impala/tests/test_client.py
--- a/ibis/impala/tests/test_client.py
+++ b/ibis/impala/tests/test_client.py
@@ -43,6 +43,11 @@ def test_execute_exprs_default_backend(self):
             result = expr.execute()
             assert result == expected
 
+    def test_cursor_garbage_collection(self):
+        for i in range(5):
+            self.con.raw_sql('select 1', True).fetchall()
+            self.con.raw_sql('select 1', True).fetchone()
+
     def test_raise_ibis_error_no_hdfs(self):
         # #299
         client = connect_test(ENV, with_hdfs=False)