Repository: redis/redis (c)

Issue:
[CRASH] Redis 8.0.2 slave keep crashing when it startup.
Notice!
- If a Redis module was involved, please open an issue in the module's repo instead!
- If you're using docker on Apple M1, please make sure the image you're using was compiled for ARM!


**Crash report**

Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context.

```

=== REDIS BUG REPORT START: Cut & paste starting from here ===
1027239:S 12 Aug 2025 10:41:05.050 # Redis 8.0.2 crashed by signal: 11, si_code: 128
1027239:S 12 Aug 2025 10:41:05.050 # Accessing address: (nil)
1027239:S 12 Aug 2025 10:41:05.050 # Crashed running the instruction at: 0x466e68

------ STACK TRACE ------
EIP:
/usr/local/bin/redis-server 127.0.0.1:6379(dictSdsDestructor+0x8)[0x466e68]

1027240 bio_close_file
/lib64/libc.so.6(+0x8722a)[0x7fad0048722a]
/lib64/libc.so.6(pthread_cond_wait+0x132)[0x7fad00489782]
/usr/local/bin/redis-server 127.0.0.1:6379(bioProcessBackgroundJobs+0x1e1)[0x53a0f1]
/lib64/libc.so.6(+0x8a19a)[0x7fad0048a19a]
/lib64/libc.so.6(+0x10f210)[0x7fad0050f210]

1027243 io_thd_1
/lib64/libc.so.6(epoll_wait+0x5e)[0x7fad0050e84e]
/usr/local/bin/redis-server 127.0.0.1:6379[0x4631c7]
/usr/local/bin/redis-server 127.0.0.1:6379(aeMain+0xa4)[0x463864]
/usr/local/bin/redis-server 127.0.0.1:6379(IOThreadMain+0x67)[0x46c067]
/lib64/libc.so.6(+0x8a19a)[0x7fad0048a19a]
/lib64/libc.so.6(+0x10f210)[0x7fad0050f210]

1027244 io_thd_2
/lib64/libc.so.6(epoll_wait+0x5e)[0x7fad0050e84e]
/usr/local/bin/redis-server 127.0.0.1:6379[0x4631c7]
/usr/local/bin/redis-server 127.0.0.1:6379(aeMain+0xa4)[0x463864]
/usr/local/bin/redis-server 127.0.0.1:6379(IOThreadMain+0x67)[0x46c067]
/lib64/libc.so.6(+0x8a19a)[0x7fad0048a19a]
/lib64/libc.so.6(+0x10f210)[0x7fad0050f210]

1027241 bio_aof
/lib64/libc.so.6(+0x8722a)[0x7fad0048722a]
/lib64/libc.so.6(pthread_cond_wait+0x132)[0x7fad00489782]
/usr/local/bin/redis-server 127.0.0.1:6379(bioProcessBackgroundJobs+0x1e1)[0x53a0f1]
/lib64/libc.so.6(+0x8a19a)[0x7fad0048a19a]
/lib64/libc.so.6(+0x10f210)[0x7fad0050f210]

1027242 bio_lazy_free
/lib64/libc.so.6(+0x8722a)[0x7fad0048722a]
/lib64/libc.so.6(pthread_cond_wait+0x132)[0x7fad00489782]
/usr/local/bin/redis-server 127.0.0.1:6379(bioProcessBackgroundJobs+0x1e1)[0x53a0f1]
/lib64/libc.so.6(+0x8a19a)[0x7fad0048a19a]
/lib64/libc.so.6(+0x10f210)[0x7fad0050f210]

1027239 redis-server *
/lib64/libc.so.6(+0x3ebf0)[0x7fad0043ebf0]
/usr/local/bin/redis-server 127.0.0.1:6379(dictSdsDestructor+0x8)[0x466e68]
/usr/local/bin/redis-server 127.0.0.1:6379[0x5dc6fc]
/usr/local/bin/redis-server 127.0.0.1:6379(kvstoreEmpty+0xe9)[0x468aa9]
/usr/local/bin/redis-server 127.0.0.1:6379(emptyDbStructure+0x6d)[0x4ac46d]
/usr/local/bin/redis-server 127.0.0.1:6379[0x5def06]
/usr/local/bin/redis-server 127.0.0.1:6379(rdbLoadWithEmptyFunc+0xd3)[0x4cb203]
/usr/local/bin/redis-server 127.0.0.1:6379(readSyncBulkPayload+0xa8d)[0x4bc04d]
/usr/local/bin/redis-server 127.0.0.1:6379[0x5af4b1]
/usr/local/bin/redis-server 127.0.0.1:6379(aeMain+0xea)[0x4638aa]
/usr/local/bin/redis-server 127.0.0.1:6379(main+0x43f)[0x457b0f]
/lib64/libc.so.6(+0x295d0)[0x7fad004295d0]
/lib64/libc.so.6(__libc_start_main+0x80)[0x7fad00429680]
/usr/local/bin/redis-server 127.0.0.1:6379(_start+0x25)[0x4591d5]

6/6 expected stacktraces.

------ STACK TRACE DONE ------

------ REGISTERS ------
1027239:S 12 Aug 2025 10:41:05.053 #
RAX:00007facca831488 RBX:00007facca831488
RCX:0000000000466e60 RDX:00000000a7e532a0
RDI:5f726f7461726570 RSI:5f726f7461726570
RBP:00007ffdd12d1e00 RSP:00007ffdd12d1db8
R8 :00007fad00000f70 R9 :00007fad00bf3780
R10:00007fad00000f78 R11:0000000000000002
R12:8f4c00f40569736d R13:0000000000000000
R14:000000000054d541 R15:00007facca4d8000
RIP:0000000000466e68 EFL:0000000000010202
CSGSFS:002b000000000033
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc7) -> 000000000000003f
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc6) -> 000000000093d642
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc5) -> 0000000000000004
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc4) -> 00007fad000c6150
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc3) -> 00007fad0000c0f8
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc2) -> 0000000000468aa9
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc1) -> 00007ffdd12d1e40
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dc0) -> 0000000000000000
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dbf) -> 0000000000000000
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dbe) -> 00000000004b9150
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dbd) -> 00007fad00113400
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dbc) -> 00007facca4d8000
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dbb) -> 00000000000001b6
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1dba) -> 00000000004b9150
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1db9) -> 3832313138323730
1027239:S 12 Aug 2025 10:41:05.053 # (00007ffdd12d1db8) -> 00000000005dc6fc


------ CONFIG DEBUG OUTPUT ------
io-threads 3
repl-diskless-load disabled
repl-diskless-sync no
replica-read-only yes
lazyfree-lazy-expire no
slave-read-only yes
proto-max-bulk-len 512mb
list-compress-depth 0
lazyfree-lazy-eviction no
lazyfree-lazy-user-del no
activedefrag yes
lazyfree-lazy-server-del no
sanitize-dump-payload no
client-query-buffer-limit 512mb
lazyfree-lazy-user-flush no

------ FAST MEMORY TEST ------
1027239:S 12 Aug 2025 10:41:05.053 # Bio worker thread #0 terminated
1027239:S 12 Aug 2025 10:41:05.053 # Bio worker thread #1 terminated
1027239:S 12 Aug 2025 10:41:05.053 # Bio worker thread #2 terminated
1027239:S 12 Aug 2025 10:41:05.053 # IO thread(tid:140380968306240) terminated
1027239:S 12 Aug 2025 10:41:05.054 # IO thread(tid:140380959913536) terminated
*** Preparing to test memory region 792000 (487424 bytes)
*** Preparing to test memory region 1a34000 (135168 bytes)
*** Preparing to test memory region 7fabfa000000 (8388608 bytes)
*** Preparing to test memory region 7fabfa800000 (4185915392 bytes)
*** Preparing to test memory region 7facf4000000 (135168 bytes)
*** Preparing to test memory region 7facf82ff000 (22020096 bytes)
*** Preparing to test memory region 7facf9800000 (8388608 bytes)
*** Preparing to test memory region 7facfa000000 (8388608 bytes)
*** Preparing to test memory region 7facfa9f9000 (18874368 bytes)
*** Preparing to test memory region 7facfbbfa000 (8388608 bytes)
*** Preparing to test memory region 7facfc3fb000 (8388608 bytes)
*** Preparing to test memory region 7facfcbfc000 (8388608 bytes)
*** Preparing to test memory region 7facfd3fd000 (8388608 bytes)
*** Preparing to test memory region 7facfdbfe000 (8388608 bytes)
*** Preparing to test memory region 7facfe3ff000 (8388608 bytes)
*** Preparing to test memory region 7facff400000 (8388608 bytes)
*** Preparing to test memory region 7fad00000000 (4194304 bytes)
*** Preparing to test memory region 7fad005fb000 (53248 bytes)
*** Preparing to test memory region 7fad00a26000 (12288 bytes)
*** Preparing to test memory region 7fad00bf1000 (28672 bytes)
*** Preparing to test memory region 7fad00cf8000 (8192 bytes)
.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O
Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.

------ DUMPING CODE AROUND EIP ------
Symbol: dictSdsDestructor (base: 0x466e60)
Module: /usr/local/bin/redis-server 127.0.0.1:6379 (base 0x400000)
$ xxd -r -p /tmp/dump.hex /tmp/dump.bin
$ objdump --adjust-vma=0x466e60 -D -b binary -m i386:x86-64 /tmp/dump.bin
------
1027239:S 12 Aug 2025 10:41:13.295 # dump of function (hexdump of 136 bytes):
4889f74885f674300fb646ff83e0073c047e0de958b601000f1f8400000000000fb6c048630485c05e69004829c7e93db601000f1f440000c30f1f800000000049b873657479626465744889f8488b3d24a7320048b96172656e6567796c48be6d6f646e61726f6448330d01a7320048333502a7320048ba75657370656d6f734831c7483315e6a6

=== REDIS BUG REPORT END. Make sure to include from START to END. ===

```

**Additional information**

OS distribution and version: Rocky Linux release 9.6 (Blue Onyx)
Kernel: Linux 5.14.0-570.22.1.el9_6.x86_64

This is the slave bug report, master didn't crash. 
When I found it is crash, I try to restart it but it crash at the same place. 
Below is the last log before it crash. 

```
1027239:S 12 Aug 2025 10:40:39.502 * Full resync from master: 403f5eadaf40b46fa5a90ccde54b462606b52604:15123965342078
1027239:S 12 Aug 2025 10:40:51.718 * MASTER <-> REPLICA sync: receiving 1190754337 bytes from master to disk
1027239:S 12 Aug 2025 10:41:01.847 * MASTER <-> REPLICA sync: Loading DB in memory
1027239:S 12 Aug 2025 10:41:01.848 * MASTER <-> REPLICA sync: Flushing old data
```




Patch:
diff --git a/src/replication.c b/src/replication.c
index 486d94a5dc1..767b3473d2b 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1949,7 +1949,17 @@ static void rdbLoadEmptyDbFunc(void) {
     serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Flushing old data");
     int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC :
                                                         EMPTYDB_NO_FLAGS;
+
+    /* Temporarily disable active defragmentation during database flush.
+     * This prevents defrag from being triggered in replicationEmptyDbCallback()
+     * which could modify the database while it's being emptied. */
+    int orig_active_defrag = server.active_defrag_enabled;
+    server.active_defrag_enabled = 0;
+
     emptyData(-1, empty_db_flags, replicationEmptyDbCallback);
+
+    /* Restore the original active defragmentation. */
+    server.active_defrag_enabled = orig_active_defrag;
 }
 
 /* Once we have a link with the master and the synchronization was


Test patch:
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 68c44c6b484..062bff09233 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -67,6 +67,14 @@ run_solo {defrag} {
         }
     }
 
+    proc discard_replies_every {rd count frequency discard_num} {
+        if {$count % $frequency == 0} {
+            for {set k 0} {$k < $discard_num} {incr k} {
+                $rd read ; # Discard replies
+            }
+        }
+    }
+
     proc test_active_defrag {type} {
     if {[string match {*jemalloc*} [s mem_allocator]] && [r debug mallctl arenas.page] <= 8192} {
         test "Active defrag main dictionary: $type" {
@@ -339,11 +347,7 @@ run_solo {defrag} {
                     $rd hset bighash $j [concat "asdfasdfasdf" $j]
 
                     incr count
-                    if {$count % 10000 == 0} {
-                        for {set k 0} {$k < 10000} {incr k} {
-                            $rd read ; # Discard replies
-                        }
-                    }
+                    discard_replies_every $rd $count 10000 10000
                 }
                 # creating that big hash, increased used_memory, so the relative frag goes down
                 set expected_frag 1.3
@@ -355,11 +359,7 @@ run_solo {defrag} {
                 $rd setrange $j 150 a
 
                 incr count
-                if {$count % 10000 == 0} {
-                    for {set k 0} {$k < 10000} {incr k} {
-                        $rd read ; # Discard replies
-                    }
-                }
+                discard_replies_every $rd $count 10000 10000
             }
             assert_equal [r dbsize] 500016
 
@@ -369,11 +369,7 @@ run_solo {defrag} {
                 $rd del $j
 
                 incr count
-                if {$count % 10000 == 0} {
-                    for {set k 0} {$k < 10000} {incr k} {
-                        $rd read ; # Discard replies
-                    }
-                }
+                discard_replies_every $rd $count 10000 10000
             }
             assert_equal [r dbsize] 250016
 
@@ -769,12 +765,7 @@ run_solo {defrag} {
                 $rd lpush biglist2 $val
 
                 incr count
-                if {$count % 10000 == 0} {
-                    for {set k 0} {$k < 10000} {incr k} {
-                        $rd read ; # Discard replies
-                        $rd read ; # Discard replies
-                    }
-                }
+                discard_replies_every $rd $count 10000 20000
             }
 
             # create some fragmentation
@@ -884,11 +875,7 @@ run_solo {defrag} {
                     $rd setrange $j 600 x
 
                     incr count
-                    if {$count % 10000 == 0} {
-                        for {set k 0} {$k < 10000} {incr k} {
-                            $rd read ; # Discard replies
-                        }
-                    }
+                    discard_replies_every $rd $count 10000 10000
                 }
 
                 # create some fragmentation of 50%
@@ -898,11 +885,7 @@ run_solo {defrag} {
                     incr sent
                     incr j 1
 
-                    if {$sent % 10000 == 0} {
-                        for {set k 0} {$k < 10000} {incr k} {
-                            $rd read ; # Discard replies
-                        }
-                    }
+                    discard_replies_every $rd $sent 10000 10000
                 }
 
                 # create higher fragmentation in the first slab
@@ -959,6 +942,70 @@ run_solo {defrag} {
     }
     }
 
+    test "Active defrag can't be triggered during replicaof database flush. See issue #14267" {
+        start_server {tags {"repl"} overrides {save ""}} {
+            set master_host [srv 0 host]
+            set master_port [srv 0 port]
+
+            start_server {overrides {save ""}} {
+                set replica [srv 0 client]
+                set rd [redis_deferring_client 0]
+
+                $replica config set hz 100
+                $replica config set activedefrag no
+                $replica config set active-defrag-threshold-lower 5
+                $replica config set active-defrag-cycle-min 65
+                $replica config set active-defrag-cycle-max 75
+                $replica config set active-defrag-ignore-bytes 2mb
+
+                # add a mass of string keys
+                set count 0
+                for {set j 0} {$j < 500000} {incr j} {
+                    $rd setrange $j 150 a
+
+                    incr count
+                    discard_replies_every $rd $count 10000 10000
+                }
+                assert_equal [$replica dbsize] 500000
+
+                # create some fragmentation
+                set count 0
+                for {set j 0} {$j < 500000} {incr j 2} {
+                    $rd del $j
+
+                    incr count
+                    discard_replies_every $rd $count 10000 10000
+                }
+                $rd close
+                assert_equal [$replica dbsize] 250000
+
+                catch {$replica config set activedefrag yes} e
+                if {[$replica config get activedefrag] eq "activedefrag yes"} {
+                    # Start replication sync which will flush the replica's database,
+                    # then enable defrag to run concurrently with the database flush.
+                    $replica replicaof $master_host $master_port
+
+                    # wait for the active defrag to start working (decision once a second)
+                    wait_for_condition 50 100 {
+                        [s total_active_defrag_time] ne 0
+                    } else {
+                        after 120 ;# serverCron only updates the info once in 100ms
+                        puts [$replica info memory]
+                        puts [$replica info stats]
+                        puts [$replica memory malloc-stats]
+                        fail "defrag not started."
+                    }
+
+                    wait_for_sync $replica
+
+                    # wait for the active defrag to stop working (db has been emptied during replication sync)
+                    wait_for_defrag_stop 500 100
+                    assert_equal [$replica dbsize] 0
+                }
+            }
+        }
+    } {} {defrag external:skip tsan:skip cluster}
+
     start_cluster 1 0 {tags {"defrag external:skip tsan:skip cluster"} overrides {appendonly yes auto-aof-rewrite-percentage 0 save "" loglevel notice}} {
         test_active_defrag "cluster"
     }