Repository: aio-libs-abandoned/aioredis-py

Issue:
Pipelining is very slow
It seems each operation leads to separate `sendto` syscall. Correct me if I wrong but the whole idea behind pipelining is to pack multiple operations into one network send.


Discussion:
Test file

```import asyncio
import sys
import redis
import aioredis


N = 100


async def test0(loop):
    r = await aioredis.create_redis_pool('redis://localhost:6379', loop=loop)
    multi = r.pipeline()
    for i in range(N):
        multi.sismember('key', 'value')
    result = await multi.execute()
    print('Result0', result)


def test1():
    r = redis.Redis()
    multi = r.pipeline(False)
    for i in range(N):
        multi.sismember('key', 'value')
    result = multi.execute()
    print('Result1', result)


def main():
    if sys.argv[-1] == '1':
        print('Test1')
        test1()
    elif sys.argv[-1] == '0':
        print('Test0')
        loop = asyncio.get_event_loop()
        loop.run_until_complete(test0(loop))


if __name__ == '__main__':
    main()
```

Before patching:
```# strace -c python ../test.py 0 2>&1 | grep sendto
 16.49    0.003046          30       101           sendto
# strace -c python ../test.py 1 2>&1 | grep sendto
  0.17    0.000027          14         2           sendto
```
After patching:
```# strace -c python ../test.py 1 2>&1 | grep sendto
  0.67    0.000181          91         2           sendto
# strace -c python ../test.py 0 2>&1 | grep sendto
  0.10    0.000013           7         2           sendto
```
**The problem**
According to redis documentation: https://redis.io/topics/pipelining
when you use pipeline you send command in one step, so we can reduce the latency cost due to the round trip time.
As you can see in example above before patching it had 101 sendto calls and after patching just 2.

**Fix**
https://github.com/aio-libs/aioredis/pull/467 

Resolving patch:
diff --git a/aioredis/commands/transaction.py b/aioredis/commands/transaction.py
--- a/aioredis/commands/transaction.py
+++ b/aioredis/commands/transaction.py
@@ -41,6 +41,11 @@ def unwatch(self):
     def watch(self, key, *keys):
         """Watch the given keys to determine execution of the MULTI/EXEC block.
         """
+        # FIXME: we can send watch through one connection and then issue
+        #   'multi/exec' command through other.
+        # Possible fix:
+        #   "Remember" a connection that was used for 'watch' command
+        #   and then send 'multi / exec / discard' through it.
         fut = self._pool_or_conn.execute(b'WATCH', key, *keys)
         return wait_ok(fut)
 
@@ -197,15 +202,16 @@ async def _gather_result(self, return_exceptions):
         return results
 
     def _send_pipeline(self, conn):
-        for fut, cmd, args, kw in self._pipeline:
-            try:
-                result_fut = conn.execute(cmd, *args, **kw)
-                result_fut.add_done_callback(
-                    functools.partial(self._check_result, waiter=fut))
-            except Exception as exc:
-                fut.set_exception(exc)
-            else:
-                yield result_fut
+        with conn._buffered():
+            for fut, cmd, args, kw in self._pipeline:
+                try:
+                    result_fut = conn.execute(cmd, *args, **kw)
+                    result_fut.add_done_callback(
+                        functools.partial(self._check_result, waiter=fut))
+                except Exception as exc:
+                    fut.set_exception(exc)
+                else:
+                    yield result_fut
 
     def _check_result(self, fut, waiter):
         if fut.cancelled():
@@ -255,9 +261,10 @@ class MultiExec(Pipeline):
 
     async def _do_execute(self, conn, *, return_exceptions=False):
         self._waiters = waiters = []
-        multi = conn.execute('MULTI')
-        coros = list(self._send_pipeline(conn))
-        exec_ = conn.execute('EXEC')
+        with conn._buffered():
+            multi = conn.execute('MULTI')
+            coros = list(self._send_pipeline(conn))
+            exec_ = conn.execute('EXEC')
         gather = asyncio.gather(multi, *coros, loop=self._loop,
                                 return_exceptions=True)
         last_error = None
diff --git a/aioredis/connection.py b/aioredis/connection.py
--- a/aioredis/connection.py
+++ b/aioredis/connection.py
@@ -3,6 +3,7 @@
 import socket
 from functools import partial
 from collections import deque
+from contextlib import contextmanager
 
 from .util import (
     encode_command,
@@ -168,6 +169,7 @@ def __init__(self, reader, writer, *, address, encoding=None,
         self._pubsub_channels = coerced_keys_dict()
         self._pubsub_patterns = coerced_keys_dict()
         self._encoding = encoding
+        self._pipeline_buffer = None
 
     def __repr__(self):
         return '<RedisConnection [db:{}]>'.format(self._db)
@@ -273,14 +275,44 @@ def _process_pubsub(self, obj, *, process_waiters=True):
         else:
             logger.warning("Unknown pubsub message received %r", obj)
 
+    @contextmanager
+    def _buffered(self):
+        # XXX: we must ensure that no await happens
+        #   as long as we buffer commands.
+        #   Probably we can set some error-raising callback on enter
+        #   and remove it on exit
+        #   if some await happens in between -> throw an error.
+        #   This is creepy solution, 'cause some one might want to await
+        #   on some other source except redis.
+        #   So we must only raise error we someone tries to await
+        #   pending aioredis future
+        # One of solutions is to return coroutine instead of a future
+        # in `execute` method.
+        # In a coroutine we can check if buffering is enabled and raise error.
+
+        # TODO: describe in docs difference in pipeline mode for
+        #   conn.execute vs pipeline.execute()
+        if self._pipeline_buffer is None:
+            self._pipeline_buffer = bytearray()
+            try:
+                yield self
+                buf = self._pipeline_buffer
+                self._writer.write(buf)
+            finally:
+                self._pipeline_buffer = None
+        else:
+            yield self
+
     def execute(self, command, *args, encoding=_NOTSET):
         """Executes redis command and returns Future waiting for the answer.
 
         Raises:
         * TypeError if any of args can not be encoded as bytes.
-        * ReplyError on redis '-ERR' resonses.
+        * ReplyError on redis '-ERR' responses.
         * ProtocolError when response can not be decoded meaning connection
           is broken.
+        * ConnectionClosedError when either client or server has closed the
+          connection.
         """
         if self._reader is None or self._reader.at_eof():
             msg = self._close_msg or "Connection closed or corrupted"
@@ -311,7 +343,10 @@ def execute(self, command, *args, encoding=_NOTSET):
         if encoding is _NOTSET:
             encoding = self._encoding
         fut = self._loop.create_future()
-        self._writer.write(encode_command(command, *args))
+        if self._pipeline_buffer is None:
+            self._writer.write(encode_command(command, *args))
+        else:
+            encode_command(command, *args, buf=self._pipeline_buffer)
         self._waiters.append((fut, encoding, cb))
         return fut
 
@@ -344,7 +379,10 @@ def execute_pubsub(self, command, *channels):
             res.append(fut)
             cb = partial(self._update_pubsub, ch=ch)
             self._waiters.append((fut, None, cb))
-        self._writer.write(cmd)
+        if self._pipeline_buffer is None:
+            self._writer.write(cmd)
+        else:
+            self._pipeline_buffer.extend(cmd)
         return asyncio.gather(*res, loop=self._loop)
 
     def close(self):
@@ -361,6 +399,7 @@ def _do_close(self, exc):
         self._reader_task = None
         self._writer = None
         self._reader = None
+        self._pipeline_buffer = None
 
         if exc is not None:
             self._close_msg = str(exc)
diff --git a/aioredis/util.py b/aioredis/util.py
--- a/aioredis/util.py
+++ b/aioredis/util.py
@@ -16,19 +16,21 @@
 }
 
 
-def encode_command(*args):
+def encode_command(*args, buf=None):
     """Encodes arguments into redis bulk-strings array.
 
     Raises TypeError if any of args not of bytearray, bytes, float, int, or str
     type.
     """
-    buf = bytearray(b'*%d\r\n' % len(args))
+    if buf is None:
+        buf = bytearray()
+    buf.extend(b'*%d\r\n' % len(args))
 
     try:
         for arg in args:
             barg = _converters[type(arg)](arg)
             buf.extend(b'$%d\r\n%s\r\n' % (len(barg), barg))
-    except KeyError as exc:
+    except KeyError:
         raise TypeError("Argument {!r} expected to be of bytearray, bytes,"
                         " float, int, or str type".format(arg))
     return buf


Test patch:
diff --git a/tests/multi_exec_test.py b/tests/multi_exec_test.py
--- a/tests/multi_exec_test.py
+++ b/tests/multi_exec_test.py
@@ -1,13 +1,19 @@
 import asyncio
 from unittest import mock
+from contextlib import contextmanager
 
 from aioredis.commands import MultiExec
 from aioredis.commands import Redis
 
 
+@contextmanager
+def nullcontext(result):
+    yield result
+
+
 def test_global_loop():
     conn = mock.Mock(spec=(
-        'execute closed _transaction_error'
+        'execute closed _transaction_error _buffered'
         .split()))
     try:
         old_loop = asyncio.get_event_loop()
@@ -32,6 +38,7 @@ def make_fut(cmd, *args, **kw):
     conn.execute.side_effect = make_fut
     conn.closed = False
     conn._transaction_error = None
+    conn._buffered.side_effect = lambda: nullcontext(conn)
 
     async def go():
         tr.ping()
diff --git a/tests/transaction_commands_test.py b/tests/transaction_commands_test.py
--- a/tests/transaction_commands_test.py
+++ b/tests/transaction_commands_test.py
@@ -269,3 +269,15 @@ async def test_multi_exec_and_pool_release(redis):
     ret, = await tr.execute()
     assert ret is None
     assert (await fut1) is None
+
+
+@pytest.mark.run_loop
+async def test_multi_exec_db_select(redis):
+    await redis.set('foo', 'bar')
+
+    tr = redis.multi_exec()
+    f1 = tr.get('foo', encoding='utf-8')
+    f2 = tr.get('foo')
+    await tr.execute()
+    assert await f1 == 'bar'
+    assert await f2 == b'bar'