Repository: celery/celery

Issue:
Make redis use of sorted sets for group results opt-out
Following up on #6245 - can we make the use of sorted set opt-out for 5.0 as discussed? In particular, this would simply be a change to the default value of the config value, and associated changes to tests and doco merged in #6245.


Discussion:
ping @thedrow - if you're on board with getting this into 5.0, would you please add it to the milestone? I'll get a PR up for you later today or tomorrow, won't be a complex one. Thanks!

Resolving patch:
diff --git a/celery/backends/redis.py b/celery/backends/redis.py
--- a/celery/backends/redis.py
+++ b/celery/backends/redis.py
@@ -405,7 +405,7 @@ def _chord_zset(self):
         transport_options = self.app.conf.get(
             'result_backend_transport_options', {}
         )
-        return transport_options.get('result_chord_ordered', False)
+        return transport_options.get('result_chord_ordered', True)
 
     def on_chord_part_return(self, request, state, result,
                              propagate=None, **kwargs):


Test patch:
diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py
--- a/t/integration/test_canvas.py
+++ b/t/integration/test_canvas.py
@@ -854,7 +854,7 @@ def test_chord_on_error(self, manager):
         redis_connection = get_redis_connection()
         # The redis key is either a list or zset depending on configuration
         if manager.app.conf.result_backend_transport_options.get(
-            'result_chord_ordered', False
+            'result_chord_ordered', True
         ):
             job_results = redis_connection.zrange(j_key, 0, 3)
         else:
diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py
--- a/t/unit/backends/test_redis.py
+++ b/t/unit/backends/test_redis.py
@@ -623,9 +623,9 @@ def test_on_chord_part_return(self, restore):
 
         for i in range(10):
             self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i)
-            assert self.b.client.rpush.call_count
-            self.b.client.rpush.reset_mock()
-        assert self.b.client.lrange.call_count
+            assert self.b.client.zadd.call_count
+            self.b.client.zadd.reset_mock()
+        assert self.b.client.zrangebyscore.call_count
         jkey = self.b.get_key_for_group('group_id', '.j')
         tkey = self.b.get_key_for_group('group_id', '.t')
         self.b.client.delete.assert_has_calls([call(jkey), call(tkey)])
@@ -683,9 +683,9 @@ def test_on_chord_part_return_no_expiry(self, restore):
 
         for i in range(10):
             self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i)
-            assert self.b.client.rpush.call_count
-            self.b.client.rpush.reset_mock()
-        assert self.b.client.lrange.call_count
+            assert self.b.client.zadd.call_count
+            self.b.client.zadd.reset_mock()
+        assert self.b.client.zrangebyscore.call_count
         jkey = self.b.get_key_for_group('group_id', '.j')
         tkey = self.b.get_key_for_group('group_id', '.t')
         self.b.client.delete.assert_has_calls([call(jkey), call(tkey)])
@@ -805,7 +805,7 @@ def test_on_chord_part_return__ChordError(self):
         with self.chord_context(1) as (_, request, callback):
             self.b.client.pipeline = ContextMock()
             raise_on_second_call(self.b.client.pipeline, ChordError())
-            self.b.client.pipeline.return_value.rpush().llen().get().expire(
+            self.b.client.pipeline.return_value.zadd().zcount().get().expire(
             ).expire().execute.return_value = (1, 1, 0, 4, 5)
             task = self.app._tasks['add'] = Mock(name='add_task')
             self.b.on_chord_part_return(request, states.SUCCESS, 10)
@@ -849,7 +849,7 @@ def test_on_chord_part_return__other_error(self):
         with self.chord_context(1) as (_, request, callback):
             self.b.client.pipeline = ContextMock()
             raise_on_second_call(self.b.client.pipeline, RuntimeError())
-            self.b.client.pipeline.return_value.rpush().llen().get().expire(
+            self.b.client.pipeline.return_value.zadd().zcount().get().expire(
             ).expire().execute.return_value = (1, 1, 0, 4, 5)
             task = self.app._tasks['add'] = Mock(name='add_task')
             self.b.on_chord_part_return(request, states.SUCCESS, 10)