Repository: aio-libs-abandoned/aioredis-py

Issue:
Transactions and not transaction commands in one connection
On a highly-loaded process with single connection I've got an `AssertionError from`MultiExec._do_execute`.

``` python
assert len(results) == len(waiters), (results, waiters)
# results, waiters looked like: 
([('OK', 0)], [])
```

It seems, that when I use multi like this:

``` python
with (yield from self.tr_lock):  # asyncio.Lock                            
     tr = self.redis.multi_exec()                             
     tr.hmset(self.data_key, *chunk_data)        
     tr.zadd(self.chunks_key, *items)            
     yield from tr.execute()
```

other green threads somehow can add other commands to this connection.
Is it possible for reconnecting redis? 

``` python
self.redis = yield from aioredis.create_reconnecting_redis(...)
```

Should I always use separate redis connection for transaction-only interactions?
By the way, the whole application freezes until Ctrl+C pressed.



Discussion:
Hmm...
This looks very strange, need to look into it...

Can you make a test to reproduce it?

I'll try to write an example script next week, because it's probably a race-condition effect.

Well, transaction and standalone commands itself don't break anything.
I wrote a simple script, that executes 120 transactions and 1200 standalone redis commands with single redis connection, and couldn't reproduce behavior described above.
Than I added "cancel monkey", than randomly cancels and re-launches green cycles, and in less than a minute script stucked with AssertionError. All other green cycles also stopped, except "cancel monkey".

Script itself is here: https://gist.github.com/anonymous/4ea757a6690ccfb27d99

Maybe `transaction.execute()` should be [shielded](https://docs.python.org/3/library/asyncio-task.html#asyncio.shield) from cancellation?

Thanks, I will take a look

``` python
yield from asyncio.shield(transaction.execute())
```

And no more freezes. Is PR without any tests appropriate for this bugfix?

Interesting, I've added `shield` to aiopg in December

Its a bit deeper problem, but I almost have a fix.


Resolving patch:
diff --git a/aioredis/commands/transaction.py b/aioredis/commands/transaction.py
--- a/aioredis/commands/transaction.py
+++ b/aioredis/commands/transaction.py
@@ -199,10 +199,7 @@ def _check_result(self, fut, waiter):
         elif fut.exception():
             waiter.set_exception(fut.exception())
         else:
-            self._set_result(fut, waiter)
-
-    def _set_result(self, fut, waiter):
-        waiter.set_result(fut.result())
+            waiter.set_result(fut.result())
 
 
 class MultiExec(Pipeline):
@@ -249,15 +246,19 @@ def _do_execute(self, *, return_exceptions=False):
         multi = conn.execute('MULTI')
         coros = list(self._send_pipeline(conn))
         exec_ = conn.execute('EXEC')
+        gather = asyncio.gather(multi, *coros, loop=self._loop,
+                                return_exceptions=True)
         try:
-            yield from asyncio.gather(multi, *coros,
-                                      loop=self._loop)
+            yield from asyncio.shield(gather, loop=self._loop)
         except asyncio.CancelledError:
-            pass
+            yield from gather
         finally:
-            if self._conn.closed:
+            if conn.closed:
                 for fut in waiters:
                     fut.cancel()
+                for fut in self._results:
+                    if not fut.done():
+                        fut.cancel()
             else:
                 try:
                     results = yield from exec_
@@ -265,10 +266,10 @@ def _do_execute(self, *, return_exceptions=False):
                     for fut in waiters:
                         fut.set_exception(err)
                 else:
-                    assert len(results) == len(waiters), (results, waiters)
+                    assert len(results) == len(waiters), (
+                        "Results does not match waiters", results, waiters)
                     self._resolve_waiters(results, return_exceptions)
-                    return (yield from self._gather_result(
-                        return_exceptions))
+            return (yield from self._gather_result(return_exceptions))
 
     def _resolve_waiters(self, results, return_exceptions):
         errors = []
@@ -281,15 +282,13 @@ def _resolve_waiters(self, results, return_exceptions):
         if errors and not return_exceptions:
             raise MultiExecError(errors)
 
-    def _set_result(self, fut, waiter):
-        # fut is done and must be 'QUEUED'
-        if fut in self._waiters:
-            self._waiters.remove(fut)
-            waiter.set_result(fut.result())
-        elif fut.result() not in {b'QUEUED', 'QUEUED'}:
-            waiter.set_result(fut.result())
-        else:
-            fut = asyncio.Future(loop=self._loop)
-            self._waiters.append(fut)
-            fut.add_done_callback(
-                functools.partial(self._check_result, waiter=waiter))
+    def _check_result(self, fut, waiter):
+        assert waiter not in self._waiters, (fut, waiter, self._waiters)
+        assert not waiter.done(), waiter
+        if fut.cancelled():     # yield from gather was cancelled
+            waiter.cancel()
+        elif fut.exception():   # server replied with error
+            waiter.set_exception(fut.exception())
+        elif fut.result() in {b'QUEUED', 'QUEUED'}:
+            # got result, it should be QUEUED
+            self._waiters.append(waiter)
diff --git a/aioredis/connection.py b/aioredis/connection.py
--- a/aioredis/connection.py
+++ b/aioredis/connection.py
@@ -7,7 +7,10 @@
 
 from .util import (
     encode_command,
-    wait_ok, _NOTSET,
+    wait_ok,
+    _NOTSET,
+    _set_result,
+    _set_exception,
     coerced_keys_dict,
     Channel,
     decode,
@@ -146,13 +149,8 @@ def _read_data(self):
     def _process_data(self, obj):
         """Processes command results."""
         waiter, encoding, cb = self._waiters.popleft()
-        if waiter.done():
-            logger.debug("Waiter future is already done %r", waiter)
-            assert waiter.cancelled(), (
-                "waiting future is in wrong state", waiter, obj)
-            return
         if isinstance(obj, RedisError):
-            waiter.set_exception(obj)
+            _set_exception(waiter, obj)
             if self._in_transaction is not None:
                 self._transaction_error = obj
         else:
@@ -160,15 +158,15 @@ def _process_data(self, obj):
                 try:
                     obj = decode(obj, encoding)
                 except Exception as exc:
-                    waiter.set_exception(exc)
+                    _set_exception(waiter, exc)
                     return
             if cb is not None:
                 try:
                     obj = cb(obj)
                 except Exception as exc:
-                    waiter.set_exception(exc)
+                    _set_exception(waiter, exc)
                     return
-            waiter.set_result(obj)
+            _set_result(waiter, obj)
             if self._in_transaction is not None:
                 self._in_transaction.append((encoding, cb))
 
@@ -330,24 +328,27 @@ def select(self, db):
         return wait_ok(fut)
 
     def _set_db(self, ok, args):
-        assert ok in {b'OK', 'OK'}, ok
+        assert ok in {b'OK', 'OK'}, ("Unexpected result of SELECT", ok)
         self._db = args[0]
         return ok
 
     def _start_transaction(self, ok):
-        assert self._in_transaction is None
+        assert self._in_transaction is None, (
+            "Connection is already in transaction", self._in_transaction)
         self._in_transaction = deque()
         self._transaction_error = None
         return ok
 
     def _end_transaction(self, obj, discard):
-        assert self._in_transaction is not None
+        assert self._in_transaction is not None, (
+            "Connection is not in transaction", obj)
         self._transaction_error = None
         recall, self._in_transaction = self._in_transaction, None
         recall.popleft()  # ignore first (its _start_transaction)
         if discard:
             return obj
-        assert len(obj) == len(recall), (obj, recall)
+        assert len(obj) == len(recall), (
+            "Wrong number of result items in mutli-exec", obj, recall)
         res = []
         for o, (encoding, cb) in zip(obj, recall):
             if not isinstance(o, RedisError):
diff --git a/aioredis/util.py b/aioredis/util.py
--- a/aioredis/util.py
+++ b/aioredis/util.py
@@ -3,11 +3,13 @@
 import sys
 
 from .errors import ChannelClosedError
+from .log import logger
 
 PY_35 = sys.version_info >= (3, 5)
 
 _NOTSET = object()
 
+
 # NOTE: never put here anything else;
 #       just this basic types
 _converters = {
@@ -283,6 +285,24 @@ def __anext__(self):
             return msg
 
 
+def _set_result(fut, result):
+    if fut.done():
+        logger.debug("Waiter future is already done %r", fut)
+        assert fut.cancelled(), (
+            "waiting future is in wrong state", fut, result)
+    else:
+        fut.set_result(result)
+
+
+def _set_exception(fut, exception):
+    if fut.done():
+        logger.debug("Waiter future is already done %r", fut)
+        assert fut.cancelled(), (
+            "waiting future is in wrong state", fut, exception)
+    else:
+        fut.set_exception(exception)
+
+
 if hasattr(asyncio, 'ensure_future'):
     async_task = asyncio.ensure_future
 else:


Test patch:
diff --git a/tests/transaction_commands_test.py b/tests/transaction_commands_test.py
--- a/tests/transaction_commands_test.py
+++ b/tests/transaction_commands_test.py
@@ -62,17 +62,22 @@ def test_connection_closed(self):
         fut1 = tr.quit()
         fut2 = tr.incrby('foo', 1.0)
         fut3 = tr.connection.execute('INCRBY', 'foo', '1.0')
-        res = yield from tr.execute()
-        self.assertIsNone(res)
+        with self.assertRaises(MultiExecError):
+            yield from tr.execute()
+
+        self.assertTrue(fut1.done())
+        self.assertTrue(fut2.done())
+        self.assertTrue(fut3.done())
+
         try:
             res = yield from fut1
             self.assertEqual(res, b'OK')
         except asyncio.CancelledError:
             pass
-        with self.assertRaises(TypeError):
-            yield from fut2
-        with self.assertRaises(asyncio.CancelledError):
-            yield from fut3
+        self.assertFalse(fut2.cancelled())
+        self.assertIsInstance(fut2.exception(), TypeError)
+
+        self.assertTrue(fut3.cancelled())
 
     @run_until_complete
     def test_discard(self):