Repository: redis/redis

Problem statement:
[BUG] delete keys operation not replicate to slave when 'remove all the keys from the slots we lost'
**Describe the bug**

we received an update message which removed ownership for certain slots we still have keys about, we need to remove all the keys from the slots we lost, **but these delete keys operation not replicate to replica**

**To reproduce**
https://github.com/redis/redis/blob/unstable/src/cluster.c#L1953
https://github.com/redis/redis/blob/unstable/src/cluster.c#L6981
```
/* Remove all the keys in the specified hash slot.
 * The number of removed items is returned. */
unsigned int delKeysInSlot(unsigned int hashslot) {
    unsigned int j = 0;
    dictEntry *de = (*server.db->slots_to_keys).by_slot[hashslot].head;
    while (de != NULL) {
        sds sdskey = dictGetKey(de);
        de = dictEntryNextInSlot(de);
        robj *key = createStringObject(sdskey, sdslen(sdskey));
        dbDelete(&server.db[0], key);
        decrRefCount(key);
        j++;
    }
    return j;
}

```
after exec 'dbDelete(&server.db[0], key)', I think we should replicate 'del key' to replica,  am i right? @oranagra @judeng 

**Expected behavior**


```
/* Remove all the keys in the specified hash slot.
 * The number of removed items is returned. */
unsigned int delKeysInSlot(unsigned int hashslot) {
    unsigned int j = 0;
    dictEntry *de = (*server.db->slots_to_keys).by_slot[hashslot].head;
    while (de != NULL) {
        sds sdskey = dictGetKey(de);
        de = dictEntryNextInSlot(de);
        robj *key = createStringObject(sdskey, sdslen(sdskey));
        dbDelete(&server.db[0], key);

        /******
        propagateDeletion(&server.db[0], key, server.lazyfree_lazy_server_del);
        ******/

        decrRefCount(key);
        j++;
    }
    return j;
}
```





Hints:
@weim0000 have you really reproduced this? can you share the reproduction scenario? or is it just a theoretical issue from reviewing the code.
@madolson please take a look.

Patch:
diff --git a/src/cluster.c b/src/cluster.c
index c739a389224..68807f33576 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -7061,8 +7061,13 @@ unsigned int delKeysInSlot(unsigned int hashslot) {
         de = dictEntryNextInSlot(de);
         robj *key = createStringObject(sdskey, sdslen(sdskey));
         dbDelete(&server.db[0], key);
+        propagateDeletion(&server.db[0], key, server.lazyfree_lazy_server_del);
+        propagatePendingCommands();
+        signalModifiedKey(NULL, &server.db[0], key);
+        moduleNotifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, server.db[0].id);
         decrRefCount(key);
         j++;
+        server.dirty++;
     }
     return j;
 }


Test patch:
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 5c951dda69b..274598947c9 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -99,6 +99,7 @@ set ::all_tests {
     unit/cluster/scripting
     unit/cluster/hostnames
     unit/cluster/multi-slot-operations
+    unit/cluster/slot-ownership
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
diff --git a/tests/unit/cluster/slot-ownership.tcl b/tests/unit/cluster/slot-ownership.tcl
new file mode 100644
index 00000000000..0f3e3cc4f77
--- /dev/null
+++ b/tests/unit/cluster/slot-ownership.tcl
@@ -0,0 +1,61 @@
+start_cluster 2 2 {tags {external:skip cluster}} {
+
+    test "Verify that slot ownership transfer through gossip propagates deletes to replicas" {
+        assert {[s -2 role] eq {slave}}
+        wait_for_condition 1000 50 {
+            [s -2 master_link_status] eq {up}
+        } else {
+            fail "Instance #2 master link status is not up"
+        }
+
+        assert {[s -3 role] eq {slave}}
+        wait_for_condition 1000 50 {
+            [s -3 master_link_status] eq {up}
+        } else {
+            fail "Instance #3 master link status is not up"
+        }
+
+        # Set a single key that will be used to test deletion
+        set key "FOO"
+        R 0 SET $key TEST
+        set key_slot [R 0 cluster keyslot $key]
+        set slot_keys_num [R 0 cluster countkeysinslot $key_slot]
+        assert {$slot_keys_num > 0}
+
+        # Wait for replica to have the key
+        R 2 readonly
+        wait_for_condition 1000 50 {
+            [R 2 exists $key] eq "1"
+        } else {
+            fail "Test key was not replicated"
+        }
+
+        assert_equal [R 2 cluster countkeysinslot $key_slot] $slot_keys_num
+
+        # Assert other shards in cluster doesn't have the key
+        assert_equal [R 1 cluster countkeysinslot $key_slot] "0"
+        assert_equal [R 3 cluster countkeysinslot $key_slot] "0"
+
+        set nodeid [R 1 cluster myid]
+
+        R 1 cluster bumpepoch
+        # Move $key_slot to node 1
+        assert_equal [R 1 cluster setslot $key_slot node $nodeid] "OK"
+
+        wait_for_cluster_propagation
+
+        # src master will delete keys in the slot
+        wait_for_condition 50 100 {
+            [R 0 cluster countkeysinslot $key_slot] eq 0
+        } else {
+            fail "master 'countkeysinslot $key_slot' did not eq 0"
+        }
+
+        # src replica will delete keys in the slot
+        wait_for_condition 50 100 {
+            [R 2 cluster countkeysinslot $key_slot] eq 0
+        } else {
+            fail "replica 'countkeysinslot $key_slot' did not eq 0"
+        }
+    }
+}