Repository: django/channels_redis

Issue:
PubSub: Sharding broken with multiple workers or worker restarts
The "classic" implementation selects shards like this:

```
    def consistent_hash(self, value):
        """
        Maps the value to a node value between 0 and 4095
        using CRC, then down to one of the ring nodes.
        """
        if isinstance(value, str):
            value = value.encode("utf8")
        bigval = binascii.crc32(value) & 0xFFF
        ring_divisor = 4096 / float(self.ring_size)
        return int(bigval / ring_divisor)
```

The PUBSUB implementation does this:

```
    def _get_shard(self, channel_or_group_name):
        """
        Return the shard that is used exclusively for this channel or group.
        """
        if len(self._shards) == 1:
            # Avoid the overhead of hashing and modulo when it is unnecessary.
            return self._shards[0]
        shard_index = abs(hash(channel_or_group_name)) % len(self._shards)
        return self._shards[shard_index]
```

I was surprised that there's a difference when I first looked at this, but now this started to really bite us in production after we introduced sharding: `hash()` is not guaranteed to return the same thing across multiple invocations of `python`, even on the same machine, unless `PYTHONHASHSEED` is set manually, which probably no-one ever does. See https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHASHSEED

I think we should just use the same hash function across implementations. I'll send a PR :)

Comments:

Patch:
diff --git a/channels_redis/core.py b/channels_redis/core.py
index af288bd..94531e9 100644
--- a/channels_redis/core.py
+++ b/channels_redis/core.py
@@ -1,6 +1,5 @@
 import asyncio
 import base64
-import binascii
 import collections
 import functools
 import hashlib
@@ -18,6 +17,8 @@ import msgpack
 from channels.exceptions import ChannelFull
 from channels.layers import BaseChannelLayer
 
+from .utils import _consistent_hash
+
 logger = logging.getLogger(__name__)
 
 AIOREDIS_VERSION = tuple(map(int, aioredis.__version__.split(".")))
@@ -858,15 +859,7 @@ class RedisChannelLayer(BaseChannelLayer):
     ### Internal functions ###
 
     def consistent_hash(self, value):
-        """
-        Maps the value to a node value between 0 and 4095
-        using CRC, then down to one of the ring nodes.
-        """
-        if isinstance(value, str):
-            value = value.encode("utf8")
-        bigval = binascii.crc32(value) & 0xFFF
-        ring_divisor = 4096 / float(self.ring_size)
-        return int(bigval / ring_divisor)
+        return _consistent_hash(value, self.ring_size)
 
     def make_fernet(self, key):
         """
diff --git a/channels_redis/pubsub.py b/channels_redis/pubsub.py
index 62907d5..b8d549e 100644
--- a/channels_redis/pubsub.py
+++ b/channels_redis/pubsub.py
@@ -8,6 +8,8 @@ import uuid
 import aioredis
 import msgpack
 
+from .utils import _consistent_hash
+
 logger = logging.getLogger(__name__)
 
 
@@ -106,11 +108,7 @@ class RedisPubSubLoopLayer:
         """
         Return the shard that is used exclusively for this channel or group.
         """
-        if len(self._shards) == 1:
-            # Avoid the overhead of hashing and modulo when it is unnecessary.
-            return self._shards[0]
-        shard_index = abs(hash(channel_or_group_name)) % len(self._shards)
-        return self._shards[shard_index]
+        return self._shards[_consistent_hash(channel_or_group_name, len(self._shards))]
 
     def _get_group_channel_name(self, group):
         """
diff --git a/channels_redis/utils.py b/channels_redis/utils.py
new file mode 100644
index 0000000..7b30fdc
--- /dev/null
+++ b/channels_redis/utils.py
@@ -0,0 +1,17 @@
+import binascii
+
+
+def _consistent_hash(value, ring_size):
+    """
+    Maps the value to a node value between 0 and 4095
+    using CRC, then down to one of the ring nodes.
+    """
+    if ring_size == 1:
+        # Avoid the overhead of hashing and modulo when it is unnecessary.
+        return 0
+
+    if isinstance(value, str):
+        value = value.encode("utf8")
+    bigval = binascii.crc32(value) & 0xFFF
+    ring_divisor = 4096 / float(ring_size)
+    return int(bigval / ring_divisor)


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 0000000..3a78330
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,20 @@
+import pytest
+
+from channels_redis.utils import _consistent_hash
+
+
+@pytest.mark.parametrize(
+    "value,ring_size,expected",
+    [
+        ("key_one", 1, 0),
+        ("key_two", 1, 0),
+        ("key_one", 2, 1),
+        ("key_two", 2, 0),
+        ("key_one", 10, 6),
+        ("key_two", 10, 4),
+        (b"key_one", 10, 6),
+        (b"key_two", 10, 4),
+    ],
+)
+def test_consistent_hash_result(value, ring_size, expected):
+    assert _consistent_hash(value, ring_size) == expected