Repository: juicedata/juicefs
Pull request #5572 (2025-01-17 06:20:01)

CI: increase code coverage of dump/load
pkg/meta/redis_bak.go: 81.3%
pkg/meta/sql_bak.go: 84.1%
pkg/meta/tkv_bak.go: 87.9%


Patch:
diff --git a/.github/scripts/command/dump_load.sh b/.github/scripts/command/dump_load.sh
new file mode 100755
index 00000000..524965a9
--- /dev/null
+++ b/.github/scripts/command/dump_load.sh
@@ -0,0 +1,262 @@
+#!/bin/bash -ex
+source .github/scripts/common/common.sh
+
+[[ -z "$META" ]] && META=sqlite3
+source .github/scripts/start_meta_engine.sh
+start_meta_engine $META
+META_URL=$(get_meta_url $META)
+META_URL2=$(get_meta_url2 $META)
+[[ -z "$SEED" ]] && SEED=$(date +%s)
+HEARTBEAT_INTERVAL=2
+DIR_QUOTA_FLUSH_INTERVAL=4
+# [[ -z "$SEED" ]] && SEED=1711594639
+[[ -z "$BINARY" ]] && BINARY=false
+[[ -z "$FAST" ]] && FAST=false
+
+trap "echo random seed is $SEED" EXIT
+
+if ! docker ps | grep -q minio; then
+    docker run -d -p 9000:9000 --name minio \
+            -e "MINIO_ACCESS_KEY=minioadmin" \
+            -e "MINIO_SECRET_KEY=minioadmin" \
+            -v /tmp/data:/data \
+            -v /tmp/config:/root/.minio \
+            minio/minio server /data
+fi
+[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
+sleep 3s
+mc alias set myminio http://localhost:9000 minioadmin minioadmin
+python3 -c "import xattr" || sudo pip install xattr
+
+test_dump_load_sustained_file(){
+    prepare_test
+    ./juicefs format $META_URL myjfs --trash-days 0
+    ./juicefs mount -d $META_URL /jfs
+    file_count=100
+    for i in $(seq 1 $file_count); do
+        touch /jfs/file$i
+        exec {fd}<>/jfs/file$i
+        echo fd is $fd
+        fds[$i]=$fd
+        rm /jfs/file$i
+    done
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    for i in $(seq 1 $file_count); do
+        fd=${fds[$i]}
+        exec {fd}>&-
+    done
+    if [[ "$BINARY" == "true" ]]; then
+        sustained=$(./juicefs load dump.json --binary --stat | grep sustained | awk -F"|" '{print $2}')
+    else
+        sustained=$(jq '.Sustained[].inodes | length' dump.json)
+    fi
+    echo "sustained file count: $sustained"
+    # TODO： uncomment this line 
+    # [[ "$sustained" -eq "$file_count" ]] || (echo "sustained file count($sustained) should be $file_count" && exit 1)
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs 
+}
+
+test_dump_load_with_copy_file_range(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    rm -rf /tmp/test
+    dd if=/dev/zero of=/tmp/test bs=1M count=1024
+    cp /tmp/test /jfs/test
+    node .github/scripts/copyFile.js /jfs/test /jfs/test1
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs
+    compare_md5sum /tmp/test /jfs/test1
+}
+
+test_dump_load_with_quota(){
+    prepare_test
+    ./juicefs format $META_URL myjfs 
+    ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
+    mkdir -p /jfs/d
+    ./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1
+    ./juicefs dump --log-level error $META_URL $(get_dump_option) > dump.json
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount $META_URL /jfs -d --heartbeat $HEARTBEAT_INTERVAL
+    ./juicefs quota get $META_URL --path /d
+    dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
+    sleep $DIR_QUOTA_FLUSH_INTERVAL
+    echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
+    grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+}
+
+test_dump_load_with_iflag(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs --enable-ioctl
+    echo "hello" > /jfs/hello.txt
+    chattr +i /jfs/hello.txt
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs --enable-ioctl
+    echo "hello" > /jfs/hello.txt && echo "write should fail" && exit 1 || true
+    chattr -i /jfs/hello.txt
+    echo "world" > /jfs/hello.txt
+    cat /jfs/hello.txt | grep world
+}
+
+test_dump_load_with_keep_secret_key()
+{
+    option=$@
+    prepare_test
+    ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
+    ./juicefs dump --keep-secret-key $META_URL dump.json $(get_dump_option)
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs
+    echo "hello" > /jfs/hello.txt
+    cat /jfs/hello.txt | grep hello
+
+    umount_jfs /jfs $META_URL
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true
+    ./juicefs config --secret-key minioadmin $META_URL
+    ./juicefs mount -d $META_URL /jfs
+    echo "hello" > /jfs/hello.txt
+    cat /jfs/hello.txt | grep hello
+}
+
+test_load_encrypted_meta_backup()
+{
+    prepare_test
+    [[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048
+    export JFS_RSA_PASSPHRASE=12345678
+    ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem
+    ./juicefs mount -d $META_URL /jfs
+    SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=50 STEP_COUNT=50 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true
+    umount /jfs
+    SKIP_BACKUP_META_CHECK=true ./juicefs mount -d --backup-meta 10s $META_URL /jfs
+    sleep 10s
+    backup_file=$(ls -l /var/jfs/myjfs/meta/ |tail -1 | awk '{print $NF}')
+    backup_path=/var/jfs/myjfs/meta/$backup_file
+    ls -l $backup_path
+
+    ./juicefs load sqlite3://test2.db $backup_path --encrypt-rsa-key my-priv-key.pem --encrypt-algo aes256gcm-rsa
+    ./juicefs mount -d sqlite3://test2.db /jfs2
+    diff -ur /jfs/test /jfs2/test --no-dereference
+    umount_jfs /jfs2 sqlite3://test2.db
+    rm test2.db -rf
+}
+
+test_dump_load_with_random_test()
+{
+    prepare_test
+    ./juicefs format $META_URL myjfs --enable-acl
+    ./juicefs mount -d $META_URL /jfs 
+    ./random-test runOp -baseDir /jfs/test -files 500000 -ops 5000000 -threads 50 -dirSize 10 -duration 30s -createOp 30,uniform -deleteOp 5,end --linkOp 10,uniform --removeLinkOp 1,end --symlinkOp 20,uniform --setXattrOp 10,uniform --truncateOp 10,uniform    
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    create_database $META_URL2
+    ./juicefs load $META_URL2 dump.json $(get_load_option)
+    ./juicefs dump $META_URL2 dump2.json $(get_dump_option)
+    ./juicefs mount -d $META_URL2 /jfs2
+    diff -ur /jfs/test /jfs2/test --no-dereference
+    diff -ur /jfs/.trash /jfs2/.trash --no-dereference
+    # compare_stat_acl_xattr /jfs/test /jfs2/test
+    umount_jfs /jfs2 $META_URL2
+    ./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '"' -f 4)
+    ./juicefs destroy --yes $META_URL2 $UUID
+}
+
+test_dump_load_with_fsrand()
+{
+    prepare_test
+    ./juicefs format $META_URL myjfs --trash-days 0 --enable-acl
+    ./juicefs mount -d $META_URL /jfs --enable-xattr
+    rm -rf /tmp/test
+    SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/test ROOT_DIR2=/tmp/test python3 .github/scripts/hypo/fs.py || true    
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    create_database $META_URL2
+    ./juicefs load $META_URL2 dump.json $(get_load_option)
+    ./juicefs dump $META_URL2 dump2.json $(get_dump_option)
+    # if [[ "$BINARY" == "false" ]]; then
+    #     compare_dump_json
+    # fi
+    ./juicefs mount -d $META_URL2 /jfs2
+    diff -ur /jfs/test /jfs2/test --no-dereference
+    compare_stat_acl_xattr /jfs/test /jfs2/test
+    umount_jfs /jfs2 $META_URL2
+    ./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '"' -f 4)
+    ./juicefs destroy --yes $META_URL2 $UUID
+}
+
+compare_dump_json(){
+    cp dump.json dump.json.bak
+    cp dump2.json dump2.json.bak
+    sed -i '/usedSpace/d' dump*.json.bak
+    sed -i '/usedInodes/d' dump*.json.bak
+    sed -i '/nextInodes/d' dump*.json.bak
+    sed -i '/nextChunk/d' dump*.json.bak
+    sed -i '/nextTrash/d' dump*.json.bak
+    sed -i '/nextSession/d' dump*.json.bak
+    sed -i 's/"inode":[0-9]\+/"inode":0/g' dump*.json.bak
+    diff -ur dump.json.bak dump2.json.bak
+}
+
+compare_stat_acl_xattr(){
+    dir1=$1
+    dir2=$2
+    files1=($(find "$dir1" -type f -o -type d -exec stat -c "%n" {} + | sort))
+    files2=($(find "$dir2" -type f -o -type d -exec stat -c "%n" {} + | sort))
+    [[ ${#files1[@]} -ne ${#files2[@]} ]] && echo "compare_stat_acl: number of files differs" && exit 1
+    for i in "${!files1[@]}"; do
+        stat1=$(stat -c "%F %a %s %h %U %G" "${files1[$i]}")
+        stat2=$(stat -c "%F %a %s %h %U %G" "${files2[$i]}")
+        acl1=$(getfacl -p "${files1[$i]}" | tail -n +2)
+        acl2=$(getfacl -p "${files2[$i]}" | tail -n +2)
+        xattr1=$(getfattr -d -m . -e hex "${files1[$i]}" 2>/dev/null | tail -n +2 | sort)
+        xattr2=$(getfattr -d -m . -e hex "${files2[$i]}" 2>/dev/null | tail -n +2 | sort)
+        [[ "$stat1" != "$stat2" ]] && echo "compare_stat_acl: stat for ${files1[$i]} and ${files2[$i]} differs" && echo $stat1 && echo $stat2 && exit 1
+        [[ "$acl1" != "$acl2" ]] && echo "compare_stat_acl: ACLs for ${files1[$i]} and ${files2[$i]} differs" && echo $acl1 && echo $acl2 && exit 1
+        [[ "$xattr1" != "$xattr2" ]] && echo "compare_stat_acl: xattrs for ${files1[$i]} and ${files2[$i]} differs" && echo $xattr1 && echo $xattr2 && exit 1
+
+    done
+    echo "compare_stat_acl: ACLs and stats are the same"
+}
+
+get_dump_option(){
+    if [[ "$BINARY" == "true" ]]; then 
+        option="--binary"
+    elif [[ "$FAST" == "true" ]]; then
+        option="--fast"
+    else
+        option=""
+    fi
+    echo $option
+}
+
+get_load_option(){
+    if [[ "$BINARY" == "true" ]]; then 
+        option="--binary"
+    else
+        option=""
+    fi
+    echo $option
+}
+
+prepare_test(){
+    umount_jfs /jfs $META_URL
+    umount_jfs /jfs2 sqlite3://test2.db
+    python3 .github/scripts/flush_meta.py $META_URL
+    rm test2.db -rf 
+    rm -rf /var/jfs/myjfs || true
+    mc rm --force --recursive myminio/test || true
+}
+
+source .github/scripts/common/run_test.sh && run_test $@
diff --git a/.github/scripts/command/load_dump_bench.sh b/.github/scripts/command/dump_load_bench.sh
similarity index 100%
rename from .github/scripts/command/load_dump_bench.sh
rename to .github/scripts/command/dump_load_bench.sh
diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh
deleted file mode 100755
index 3ea9cc5f..00000000
--- a/.github/scripts/command/load_dump.sh
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/bin/bash -e
-source .github/scripts/common/common.sh
-
-[[ -z "$META" ]] && META=sqlite3
-source .github/scripts/start_meta_engine.sh
-start_meta_engine $META
-META_URL=$(get_meta_url $META)
-[[ -z "$SEED" ]] && SEED=$(date +%s)
-# [[ -z "$SEED" ]] && SEED=1711594639
-
-trap "echo random seed is $SEED" EXIT
-
-if ! docker ps | grep -q minio; then
-    docker run -d -p 9000:9000 --name minio \
-            -e "MINIO_ACCESS_KEY=minioadmin" \
-            -e "MINIO_SECRET_KEY=minioadmin" \
-            -v /tmp/data:/data \
-            -v /tmp/config:/root/.minio \
-            minio/minio server /data
-fi
-[[ ! -f /usr/local/bin/mc ]] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
-sleep 3s
-mc alias set myminio http://localhost:9000 minioadmin minioadmin
-python3 -c "import xattr" || sudo pip install xattr
-
-test_dump_load_with_iflag(){
-    do_dump_load_with_iflag
-}
-
-test_dump_load_with_iflag_binary(){
-    do_dump_load_with_iflag --binary
-}
-
-do_dump_load_with_iflag(){
-    option=$@
-    prepare_test
-    ./juicefs format $META_URL myjfs
-    ./juicefs mount -d $META_URL /jfs --enable-ioctl
-    echo "hello" > /jfs/hello.txt
-    chattr +i /jfs/hello.txt
-    ./juicefs dump $META_URL dump.json $option
-    umount_jfs /jfs $META_URL
-    python3 .github/scripts/flush_meta.py $META_URL
-    ./juicefs load $META_URL dump.json $option
-    ./juicefs mount -d $META_URL /jfs --enable-ioctl
-    echo "hello" > /jfs/hello.txt && echo "write should fail" && exit 1 || true
-    chattr -i /jfs/hello.txt
-    echo "world" > /jfs/hello.txt
-    cat /jfs/hello.txt | grep world
-}
-
-test_dump_load_with_keep_secret_key(){
-    do_dump_load_with_keep_secret_key
-}
-
-test_dump_load_with_keep_secret_key_in_binary(){
-    do_dump_load_with_keep_secret_key --binary
-}
-
-do_dump_load_with_keep_secret_key()
-{
-    option=$@
-    prepare_test
-    ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
-    ./juicefs dump --keep-secret-key $META_URL dump.json $option
-    python3 .github/scripts/flush_meta.py $META_URL
-    ./juicefs load $META_URL dump.json $option
-    ./juicefs mount -d $META_URL /jfs
-    echo "hello" > /jfs/hello.txt
-    cat /jfs/hello.txt | grep hello
-
-    umount_jfs /jfs $META_URL
-    ./juicefs dump $META_URL dump.json $option
-    python3 .github/scripts/flush_meta.py $META_URL
-    ./juicefs load $META_URL dump.json $option
-    ./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true
-    ./juicefs config --secret-key minioadmin $META_URL
-    ./juicefs mount -d $META_URL /jfs
-    echo "hello" > /jfs/hello.txt
-    cat /jfs/hello.txt | grep hello
-}
-
-
-test_dump_load_with_trash_enable(){
-    do_dump_load_with_fsrand 1
-}
-test_dump_load_with_trash_disable(){
-    do_dump_load_with_fsrand 0
-}
-
-do_dump_load_with_fsrand(){
-    trash_days=$1
-    prepare_test
-    ./juicefs format $META_URL myjfs --trash-days $trash_days --enable-acl
-    ./juicefs mount -d $META_URL /jfs --enable-xattr
-    SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true
-    do_dump_load_and_compare 
-    do_dump_load_and_compare --binary
-    do_dump_load_and_compare --fast
-    do_dump_load_and_compare --skip-trash
-    do_dump_load_and_compare --fast --skip-trash
-}
-
-do_dump_load_and_compare()
-{
-    option=$@
-    echo option is $option
-    ./juicefs dump $META_URL dump.json $option
-    rm -rf test2.db 
-    if [[ "$option" == *"--binary"* ]]; then
-        ./juicefs load sqlite3://test2.db dump.json $option
-    else
-        ./juicefs load sqlite3://test2.db dump.json
-    fi
-    ./juicefs dump sqlite3://test2.db dump2.json $option
-    # if [[ "$option" != *"--binary"* ]]; then
-    #     compare_dump_json
-    # fi
-    ./juicefs mount -d sqlite3://test2.db /jfs2
-    diff -ur /jfs/fsrand /jfs2/fsrand --no-dereference
-    compare_stat_acl_xattr /jfs/fsrand /jfs2/fsrand
-    umount /jfs2
-}
-
-compare_dump_json(){
-    cp dump.json dump.json.bak
-    cp dump2.json dump2.json.bak
-    sed -i '/usedSpace/d' dump*.json.bak
-    sed -i '/usedInodes/d' dump*.json.bak
-    sed -i '/nextInodes/d' dump*.json.bak
-    sed -i '/nextChunk/d' dump*.json.bak
-    sed -i '/nextTrash/d' dump*.json.bak
-    sed -i '/nextSession/d' dump*.json.bak
-    sed -i 's/"inode":[0-9]\+/"inode":0/g' dump*.json.bak
-    diff -ur dump.json.bak dump2.json.bak
-}
-
-compare_stat_acl_xattr(){
-    dir1=$1
-    dir2=$2
-    files1=($(find "$dir1" -type f -o -type d -exec stat -c "%n" {} + | sort))
-    files2=($(find "$dir2" -type f -o -type d -exec stat -c "%n" {} + | sort))
-    [[ ${#files1[@]} -ne ${#files2[@]} ]] && echo "compare_stat_acl: number of files differs" && exit 1
-    for i in "${!files1[@]}"; do
-        stat1=$(stat -c "%F %a %s %h %U %G" "${files1[$i]}")
-        stat2=$(stat -c "%F %a %s %h %U %G" "${files2[$i]}")
-        acl1=$(getfacl -p "${files1[$i]}" | tail -n +2)
-        acl2=$(getfacl -p "${files2[$i]}" | tail -n +2)
-        xattr1=$(getfattr -d -m . -e hex "${files1[$i]}" 2>/dev/null | tail -n +2 | sort)
-        xattr2=$(getfattr -d -m . -e hex "${files2[$i]}" 2>/dev/null | tail -n +2 | sort)
-        [[ "$stat1" != "$stat2" ]] && echo "compare_stat_acl: stat for ${files1[$i]} and ${files2[$i]} differs" && echo $stat1 && echo $stat2 && exit 1
-        [[ "$acl1" != "$acl2" ]] && echo "compare_stat_acl: ACLs for ${files1[$i]} and ${files2[$i]} differs" && echo $acl1 && echo $acl2 && exit 1
-        [[ "$xattr1" != "$xattr2" ]] && echo "compare_stat_acl: xattrs for ${files1[$i]} and ${files2[$i]} differs" && echo $xattr1 && echo $xattr2 && exit 1
-
-    done
-    echo "compare_stat_acl: ACLs and stats are the same"
-}
-
-test_load_encrypted_meta_backup()
-{
-    prepare_test
-    [[ ! -f my-priv-key.pem ]] && openssl genrsa -out my-priv-key.pem -aes256 -passout pass:12345678 2048
-    export JFS_RSA_PASSPHRASE=12345678
-    ./juicefs format $META_URL myjfs --encrypt-rsa-key my-priv-key.pem
-    ./juicefs mount -d $META_URL /jfs
-    SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=50 STEP_COUNT=50 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true
-    umount /jfs
-    SKIP_BACKUP_META_CHECK=true ./juicefs mount -d --backup-meta 10s $META_URL /jfs
-    sleep 10s
-    backup_file=$(ls -l /var/jfs/myjfs/meta/ |tail -1 | awk '{print $NF}')
-    backup_path=/var/jfs/myjfs/meta/$backup_file
-    ls -l $backup_path
-
-    ./juicefs load sqlite3://test2.db $backup_path --encrypt-rsa-key my-priv-key.pem --encrypt-algo aes256gcm-rsa
-    ./juicefs mount -d sqlite3://test2.db /jfs2
-    diff -ur /jfs/fsrand /jfs2/fsrand --no-dereference
-    umount_jfs /jfs2 sqlite3://test2.db
-    rm test2.db -rf
-}
-
-prepare_test(){
-    umount_jfs /jfs $META_URL
-    umount_jfs /jfs2 sqlite3://test2.db
-    python3 .github/scripts/flush_meta.py $META_URL
-    rm test2.db -rf 
-    rm -rf /var/jfs/myjfs || true
-    mc rm --force --recursive myminio/test || true
-}
-
-source .github/scripts/common/run_test.sh && run_test $@
diff --git a/.github/scripts/copyFile.js b/.github/scripts/copyFile.js
new file mode 100644
index 00000000..d6d8934d
--- /dev/null
+++ b/.github/scripts/copyFile.js
@@ -0,0 +1,19 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+if (process.argv.length !== 4) {
+  console.error('Usage: node copyFile.js <sourceFile> <destinationFile>');
+  process.exit(1);
+}
+
+const sourceFile = path.resolve(process.argv[2]);
+const destinationFile = path.resolve(process.argv[3]);
+
+fs.copyFile(sourceFile, destinationFile, async (err) => {
+  if (err) {
+    console.error('Error copying file:', err);
+    process.exit(1);
+  }
+  console.log('File copied successfully.');
+});
\ No newline at end of file
diff --git a/.github/scripts/hypo/fs.py b/.github/scripts/hypo/fs.py
index 9394e50d..ee84dc6c 100644
--- a/.github/scripts/hypo/fs.py
+++ b/.github/scripts/hypo/fs.py
@@ -44,7 +44,7 @@ class JuicefsMachine(RuleBasedStateMachine):
     if os.getenv('EXCLUDE_RULES'):
         EXCLUDE_RULES = os.getenv('EXCLUDE_RULES').split(',')
     else:
-        EXCLUDE_RULES = ['clone_cp_file', 'clone_cp_dir', 'loop_symlink', 'symlink', 'readlines', 'readline']
+        EXCLUDE_RULES = ['readlines', 'readline']
         # EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'clone_cp_file', 'clone_cp_dir', 'loop_symlink', 'hardlink', 'rename_dir', 'chown']
     ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand')
     ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand')
diff --git a/.github/scripts/start_meta_engine.sh b/.github/scripts/start_meta_engine.sh
index 65d3dbb4..95c6698f 100755
--- a/.github/scripts/start_meta_engine.sh
+++ b/.github/scripts/start_meta_engine.sh
@@ -228,7 +228,7 @@ get_meta_url2(){
     elif [ "$meta" == "sqlite3" ]; then
         meta_url="sqlite3://test2.db"
     elif [ "$meta" == "tikv" ]; then
-        meta_url="tikv://127.0.0.1:2379/jfs"
+        meta_url="tikv://127.0.0.1:2379/jfs2"
     elif [ "$meta" == "badger" ]; then
         meta_url="badger:///tmp/test2"
     elif [ "$meta" == "mariadb" ]; then
@@ -238,7 +238,7 @@ get_meta_url2(){
     elif [ "$meta" == "etcd" ]; then
         meta_url="etcd://localhost:2379/test2"
     elif [ "$meta" == "fdb" ]; then
-        meta_url="fdb:///home/runner/fdb.cluster?prefix=kfs"
+        meta_url="fdb:///home/runner/fdb.cluster?prefix=jfs2"
     elif [ "$meta" == "ob" ]; then
         meta_url="mysql://root:@\\(127.0.0.1:2881\\)/test2"
     else
diff --git a/.github/workflows/command.yml b/.github/workflows/command.yml
index 6f65da3a..7e5ffcd5 100644
--- a/.github/workflows/command.yml
+++ b/.github/workflows/command.yml
@@ -136,50 +136,6 @@ jobs:
         uses: lhotari/action-upterm@v1
 
   command_test2:
-    needs: [build-matrix]
-    strategy:
-      fail-fast: false
-      matrix:
-        meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 1
-
-      - name: Remove unused software
-        run: |
-            echo "before remove unused software"
-            sudo df -h
-            sudo rm -rf /usr/share/dotnet
-            sudo rm -rf /usr/local/lib/android
-            sudo rm -rf /opt/ghc
-            echo "after remove unused software"
-            sudo df -h
-            
-      - name: Build 
-        uses: ./.github/actions/build
-
-      - name: Test Load & Dump
-        timeout-minutes: 30
-        run: |
-          sudo META=${{matrix.meta}} .github/scripts/command/load_dump.sh
-
-      - name: Log
-        if: always()
-        run: |
-          echo "juicefs log"
-          sudo tail -n 1000 /var/log/juicefs.log
-          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
-          
-      - name: Setup upterm session
-        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
-        # if: failure()
-        timeout-minutes: 60
-        uses: lhotari/action-upterm@v1
-
-  command_test3:
     needs: [build-matrix]
     strategy:
       fail-fast: false
@@ -223,7 +179,7 @@ jobs:
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
 
-  command_test4:
+  command_test3:
     needs: [build-matrix]
     strategy:
       fail-fast: false
@@ -239,13 +195,6 @@ jobs:
       - name: Build 
         uses: ./.github/actions/build
 
-      - name: Test Cache
-        timeout-minutes: 30
-        run: |
-          if [[ "${{matrix.meta}}" == "mysql" || "${{matrix.meta}}" == "tikv" ]]; then
-            sudo META=${{matrix.meta}} .github/scripts/cache.sh
-          fi
-
       - name: Test Graceful upgrade
         timeout-minutes: 30
         run: |
@@ -270,7 +219,7 @@ jobs:
 
   success-all-test:
     runs-on: ubuntu-latest
-    needs: [command_test1, command_test2, command_test3, command_test4]
+    needs: [command_test1, command_test2, command_test3]
     if: always()
     steps:
       - uses: technote-space/workflow-conclusion-action@v3
diff --git a/.github/workflows/dump_load.yml b/.github/workflows/dump_load.yml
new file mode 100644
index 00000000..5a11582a
--- /dev/null
+++ b/.github/workflows/dump_load.yml
@@ -0,0 +1,169 @@
+name: "dump_load"
+on:
+  push:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '**/dump_load.yml'
+      - '**/dump_load.sh'
+  pull_request:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '**/dump_load.yml'
+      - '**/dump_load.sh'
+  schedule:
+    - cron:  '0 19 * * *'
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
+
+jobs:
+  build-matrix:
+    runs-on: ubuntu-20.04
+    steps:
+      - id: set-matrix
+        run: |
+          echo "github.event_name is ${{github.event_name}}"
+          echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
+          if [ "${{github.event_name}}" == "schedule"  ] || [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
+            echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
+          else
+            echo 'meta_matrix=["redis", "sqlite3", "tikv"]' >> $GITHUB_OUTPUT
+          fi
+    outputs:
+      meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
+
+  dump_load:
+    timeout-minutes: 90
+    needs: [build-matrix]
+    strategy:
+      fail-fast: false
+      matrix:
+        meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
+
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
+      - name: Set Variable
+        id: vars
+        run: |
+          if [ "${{matrix.meta}}" == "fdb" ]; then
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
+          else
+            echo "target=juicefs" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build
+        timeout-minutes: 10
+        uses: ./.github/actions/build
+        with: 
+          target: ${{steps.vars.outputs.target}}
+
+      - name: Install nodejs
+        run: |
+          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+          export NVM_DIR="$HOME/.nvm"
+          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+          [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
+          nvm install 22
+          nvm use 22
+          node -v 
+          nvm current
+
+      - name: Download Random Test
+        run: |
+          wget https://juicefs-com-static.oss-cn-shanghai.aliyuncs.com/random-test/random-test
+          chmod +x random-test
+
+      - name: Test Load & Dump with Binary
+        timeout-minutes: 30
+        run: |
+          sudo BINARY=true GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/dump_load.sh
+
+      - name: Test Load & Dump with Json Fast Mode
+        timeout-minutes: 30
+        run: |
+          sudo FAST=true GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/dump_load.sh  
+
+      - name: Test Load & Dump with Json
+        timeout-minutes: 30
+        run: |
+          sudo GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/dump_load.sh     
+
+      - name: log
+        if: always()
+        run: | 
+          tail -500 /var/log/juicefs.log
+          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
+
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+
+      - name: Setup upterm session
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+        # if: failure()
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
+
+  success-all-test:
+    runs-on: ubuntu-latest
+    needs: [dump_load]
+    if: always()
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v3
+      - uses: actions/checkout@v3
+
+      - name: Check Failure
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: exit 1
+
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
+      - name: upload total coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-total-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+          
+      - name: Send Slack Notification
+        if: failure() && github.event_name != 'workflow_dispatch' 
+        uses: juicedata/slack-notify-action@main
+        with:
+          channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
+          slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
+
+      - name: Success
+        if: success()
+        run: echo "All Done"
+        
\ No newline at end of file
diff --git a/.github/workflows/load.yml b/.github/workflows/dump_load_bench.yml
similarity index 76%
rename from .github/workflows/load.yml
rename to .github/workflows/dump_load_bench.yml
index 601cb358..ddfc3b63 100644
--- a/.github/workflows/load.yml
+++ b/.github/workflows/dump_load_bench.yml
@@ -1,19 +1,19 @@
-name: "load"
+name: "dump_load_bench"
 on:
   push:
     branches:
       - 'main'
       - 'release-**'
     paths:
-      - '**/load.yml'
-      - '**/load_dump_bench.sh'
+      - '**/dump_load_bench.yml'
+      - '**/dump_load_bench.sh'
   pull_request:
     branches:
       - 'main'
       - 'release-**'
     paths:
-      - '**/load.yml'
-      - '**/load_dump_bench.sh'
+      - '**/dump_load_bench.yml'
+      - '**/dump_load_bench.sh'
   schedule:
     - cron:  '0 19 * * *'
   workflow_dispatch:
@@ -35,12 +35,12 @@ jobs:
           if [ "${{github.event_name}}" == "schedule"  ] || [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
             echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           else
-            echo 'meta_matrix=["redis", "tikv", "mysql"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["redis", "sqlite3", "tikv"]' >> $GITHUB_OUTPUT
           fi
     outputs:
       meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
 
-  load:
+  dump_load_bench:
     timeout-minutes: 90
     needs: [build-matrix]
     strategy:
@@ -80,10 +80,16 @@ jobs:
         with: 
           target: ${{steps.vars.outputs.target}}
 
-      - name: Test Load & Dump 
-        timeout-minutes: 30
+      - name: Install nodejs
         run: |
-          sudo GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/load_dump.sh          
+          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+          export NVM_DIR="$HOME/.nvm"
+          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+          [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
+          nvm install 22
+          nvm use 22
+          node -v 
+          nvm current
 
       - name: Benchmark dump load in binary format
         timeout-minutes: 60
@@ -94,7 +100,7 @@ jobs:
           META: ${{matrix.meta}}
           START_META: true
         run: |
-          sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_dump_load_in_binary
+          sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/dump_load_bench.sh test_dump_load_in_binary
       
       - name: Benchmark dump load fast
         timeout-minutes: 60
@@ -105,7 +111,7 @@ jobs:
           META: ${{matrix.meta}}
           START_META: false
         run: |
-          sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_dump_load_fast
+          sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/dump_load_bench.sh test_dump_load_fast
 
       - name: Benchmark dump load
         timeout-minutes: 60
@@ -116,20 +122,20 @@ jobs:
           META: ${{matrix.meta}}
           START_META: false
         run: |
-          sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_dump_load
+          sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/dump_load_bench.sh test_dump_load
   
       - name: log
         if: always()
         run: | 
           tail -500 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
-          
+
       - name: upload coverage report
         timeout-minutes: 5
         uses: ./.github/actions/upload-coverage
         with:
           UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
-          
+
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
@@ -138,7 +144,7 @@ jobs:
 
   success-all-test:
     runs-on: ubuntu-latest
-    needs: [load]
+    needs: [dump_load_bench]
     if: always()
     steps:
       - uses: technote-space/workflow-conclusion-action@v3
@@ -148,6 +154,21 @@ jobs:
         if: env.WORKFLOW_CONCLUSION == 'failure'
         run: exit 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
+      - name: upload total coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-total-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+          
       - name: Send Slack Notification
         if: failure() && github.event_name != 'workflow_dispatch' 
         uses: juicedata/slack-notify-action@main
diff --git a/cmd/stats.go b/cmd/stats.go
index 779ba219..9eba3989 100644
--- a/cmd/stats.go
+++ b/cmd/stats.go
@@ -50,7 +50,7 @@ Details: https://juicefs.com/docs/community/fault_diagnosis_and_analysis#stats`,
 			&cli.StringFlag{
 				Name:  "schema",
 				Value: "ufmco",
-				Usage: "schema string that controls the output sections (u: usage, f: fuse, m: meta, c: blockcache, o: object, g: go)",
+				Usage: "schema string of output sections (t:time, u: usage, f: fuse, m: meta, c: blockcache, o: object, g: go)",
 			},
 			&cli.UintFlag{
 				Name:  "interval",
@@ -62,6 +62,11 @@ Details: https://juicefs.com/docs/community/fault_diagnosis_and_analysis#stats`,
 				Aliases: []string{"l"},
 				Usage:   "verbosity level, 0 or 1 is enough for most cases",
 			},
+			&cli.UintFlag{
+				Name:    "count",
+				Aliases: []string{"c"},
+				Usage:   "number of updates to display before exiting",
+			},
 		},
 	}
 }
@@ -84,6 +89,7 @@ const (
 	COLOR_DARK_SEQ = "\033[0;" // %dm
 	UNDERLINE_SEQ  = "\033[4m"
 	CLEAR_SCREEM   = "\033[2J\033[1;1H"
+	UNIXTIME_FMT   = "01-02 15:04:05"
 	// BOLD_SEQ       = "\033[1m"
 )
 
@@ -119,6 +125,7 @@ const (
 	metricGauge
 	metricCounter
 	metricHist
+	metricUnixtime
 )
 
 type item struct {
@@ -136,6 +143,9 @@ func (w *statsWatcher) buildSchema(schema string, verbosity uint) {
 	for _, r := range schema {
 		var s section
 		switch r {
+		case 't':
+			s.name = "system"
+			s.items = append(s.items, &item{"time", "juicefs_timestamp", metricUnixtime})
 		case 'u':
 			s.name = "usage"
 			s.items = append(s.items, &item{"cpu", "juicefs_cpu_usage", metricCPU | metricCounter})
@@ -210,7 +220,11 @@ func (w *statsWatcher) formatHeader() {
 	for i, s := range w.sections {
 		subs := make([]string, 0, len(s.items))
 		for _, it := range s.items {
-			subs = append(subs, w.colorize(padding(it.nick, 5, ' '), BLUE, false, true))
+			if (it.typ & 0xF0) == metricUnixtime {
+				subs = append(subs, w.colorize(padding(it.nick, len(UNIXTIME_FMT), ' '), BLUE, false, true))
+			} else {
+				subs = append(subs, w.colorize(padding(it.nick, 5, ' '), BLUE, false, true))
+			}
 			if it.typ&metricHist != 0 {
 				if it.typ&metricTime != 0 {
 					subs = append(subs, w.colorize(" lat ", BLUE, false, true))
@@ -220,6 +234,9 @@ func (w *statsWatcher) formatHeader() {
 			}
 		}
 		width := 6*len(subs) - 1 // nick(5) + space(1)
+		if s.name == "system" {
+			width = len(UNIXTIME_FMT)
+		}
 		subHeaders[i] = strings.Join(subs, " ")
 		headers[i] = w.colorize(padding(s.name, width, '-'), BLUE, true, false)
 	}
@@ -301,6 +318,12 @@ func (w *statsWatcher) printDiff(left, right map[string]float64, dark bool) {
 		vals := make([]string, 0, len(s.items))
 		for _, it := range s.items {
 			switch it.typ & 0xF0 {
+			case metricUnixtime: // current timestamp
+				if dark {
+					vals = append(vals, w.colorize(time.Now().Format(UNIXTIME_FMT), BLACK, false, false))
+				} else {
+					vals = append(vals, w.colorize(time.Now().Format(UNIXTIME_FMT), WHITE, true, false))
+				}
 			case metricGauge: // currently must be metricByte
 				vals = append(vals, w.formatU64(right[it.name], dark, true))
 			case metricCounter:
@@ -388,6 +411,7 @@ func stats(ctx *cli.Context) error {
 	}
 	watcher.buildSchema(ctx.String("schema"), ctx.Uint("verbosity"))
 	watcher.formatHeader()
+	count := ctx.Uint("count")
 
 	var tick uint
 	var start, last, current map[string]float64
@@ -406,9 +430,13 @@ func stats(ctx *cli.Context) error {
 		} else {
 			watcher.printDiff(last, current, true)
 		}
+		if count > 0 && tick >= watcher.interval*(count-1) {
+			break
+		}
 		last = current
 		tick++
 		<-ticker.C
 		current = readStats(watcher.mp)
 	}
+	return nil
 }
diff --git a/cmd/webdav.go b/cmd/webdav.go
index d76467e4..e0fd4ac8 100644
--- a/cmd/webdav.go
+++ b/cmd/webdav.go
@@ -64,6 +64,12 @@ func cmdWebDav() *cli.Command {
 			Aliases: []string{"d"},
 			Usage:   "run in background",
 		},
+                &cli.IntFlag{
+                        Name:    "threads",
+                        Aliases: []string{"p"},
+                        Value:   50,
+                        Usage:   "number of threads for delete jobs (max 255)",
+                },
 	}
 
 	return &cli.Command{
@@ -95,6 +101,7 @@ func webdav(c *cli.Context) error {
 		CertFile:        c.String("cert-file"),
 		KeyFile:         c.String("key-file"),
 		EnableProppatch: c.Bool("enable-proppatch"),
+		MaxDeletes:      c.Int("threads"),
 	})
 	return jfs.Meta().CloseSession()
 }
diff --git a/pkg/chunk/cached_store.go b/pkg/chunk/cached_store.go
index ac05090b..4fc718fd 100644
--- a/pkg/chunk/cached_store.go
+++ b/pkg/chunk/cached_store.go
@@ -448,13 +448,18 @@ func (s *wSlice) upload(indx int) {
 		}
 		if s.store.conf.Writeback {
 			stagingPath := "unknown"
+			stageFailed := false
 			block.Acquire()
 			err := utils.WithTimeout(func() (err error) { // In case it hangs for more than 5 minutes(see fileWriter.flush), fallback to uploading directly to avoid `EIO`
 				defer block.Release()
 				stagingPath, err = s.store.bcache.stage(key, block.Data, s.store.shouldCache(blen))
+				if err == nil && stageFailed { // upload thread already marked me as failed because of timeout
+					_ = s.store.bcache.removeStage(key)
+				}
 				return err
 			}, s.store.conf.PutTimeout)
 			if err != nil {
+				stageFailed = true
 				if !errors.Is(err, errStageConcurrency) {
 					s.store.stageBlockErrors.Add(1)
 					logger.Warnf("write %s to disk: %s, upload it directly", stagingPath, err)
diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go
index 43ed5d24..c800a595 100644
--- a/pkg/fs/fs.go
+++ b/pkg/fs/fs.go
@@ -501,7 +501,7 @@ func (fs *FileSystem) Delete(ctx meta.Context, p string) (err syscall.Errno) {
 	return
 }
 
-func (fs *FileSystem) Rmr(ctx meta.Context, p string) (err syscall.Errno) {
+func (fs *FileSystem) Rmr(ctx meta.Context, p string, numthreads int) (err syscall.Errno) {
 	defer trace.StartRegion(context.TODO(), "fs.Rmr").End()
 	l := vfs.NewLogContext(ctx)
 	defer func() { fs.log(l, "Rmr (%s): %s", p, errstr(err)) }()
@@ -509,7 +509,7 @@ func (fs *FileSystem) Rmr(ctx meta.Context, p string) (err syscall.Errno) {
 	if err != 0 {
 		return
 	}
-	err = fs.m.Remove(ctx, parent.inode, path.Base(p), false, meta.RmrDefaultThreads, nil)
+	err = fs.m.Remove(ctx, parent.inode, path.Base(p), false, numthreads, nil)
 	fs.invalidateEntry(parent.inode, path.Base(p))
 	return
 }
diff --git a/pkg/fs/http.go b/pkg/fs/http.go
index b327b395..69cacea8 100644
--- a/pkg/fs/http.go
+++ b/pkg/fs/http.go
@@ -125,7 +125,7 @@ func (hfs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm o
 }
 
 func (hfs *webdavFS) RemoveAll(ctx context.Context, name string) error {
-	return econv(hfs.fs.Rmr(hfs.ctx, name))
+	return econv(hfs.fs.Rmr(hfs.ctx, name, hfs.config.MaxDeletes))
 }
 
 func (hfs *webdavFS) Rename(ctx context.Context, oldName, newName string) error {
@@ -252,6 +252,7 @@ type WebdavConfig struct {
 	Password        string
 	CertFile        string
 	KeyFile         string
+	MaxDeletes	int
 }
 
 type indexHandler struct {
diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go
index 7739c88f..5c6f7187 100644
--- a/pkg/gateway/gateway.go
+++ b/pkg/gateway/gateway.go
@@ -48,8 +48,9 @@ import (
 )
 
 const (
-	sep        = "/"
-	metaBucket = ".sys"
+	sep          = "/"
+	metaBucket   = ".sys"
+	subDirPrefix = 3 // 16^3=4096 slots
 )
 
 var mctx meta.Context
@@ -186,10 +187,14 @@ func (n *jfsObjects) tpath(p ...string) string {
 }
 
 func (n *jfsObjects) upath(bucket, uploadID string) string {
-	return n.tpath(bucket, "uploads", uploadID)
+	return n.tpath(bucket, "uploads", uploadID[:subDirPrefix], uploadID)
 }
 
 func (n *jfsObjects) ppath(bucket, uploadID, part string) string {
+	return n.tpath(bucket, "uploads", uploadID[:subDirPrefix], uploadID, part)
+}
+
+func (n *jfsObjects) ppathFlat(bucket, uploadID, part string) string { // compatible with tmp files uploaded by old versions(<1.2)
 	return n.tpath(bucket, "uploads", uploadID, part)
 }
 
@@ -539,7 +544,8 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	if minio.IsStringEqual(src, dst) {
 		return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})
 	}
-	tmp := n.tpath(dstBucket, "tmp", minio.MustGetUUID())
+	uuid := minio.MustGetUUID()
+	tmp := n.tpath(dstBucket, "tmp", uuid[:subDirPrefix], uuid)
 	f, eno := n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)
 	if eno == syscall.ENOENT {
 		_ = n.mkdirAll(ctx, path.Dir(tmp))
@@ -730,7 +736,8 @@ func (n *jfsObjects) mkdirAll(ctx context.Context, p string) error {
 }
 
 func (n *jfsObjects) putObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions, applyObjTaggingFunc func(tmpName string)) (err error) {
-	tmpname := n.tpath(bucket, "tmp", minio.MustGetUUID())
+	uuid := minio.MustGetUUID()
+	tmpname := n.tpath(bucket, "tmp", uuid[:subDirPrefix], uuid)
 	f, eno := n.fs.Create(mctx, tmpname, 0666, n.gConf.Umask)
 	if eno == syscall.ENOENT {
 		_ = n.mkdirAll(ctx, path.Dir(tmpname))
@@ -889,7 +896,7 @@ func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, pr
 		return // no found
 	}
 	defer f.Close(mctx)
-	entries, eno := f.ReaddirPlus(mctx, 0)
+	parents, eno := f.ReaddirPlus(mctx, 0)
 	if eno != 0 {
 		err = jfsToObjectErr(ctx, eno, bucket)
 		return
@@ -900,22 +907,38 @@ func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, pr
 	lmi.MaxUploads = maxUploads
 	lmi.Delimiter = delimiter
 	commPrefixSet := make(map[string]struct{})
-	for _, e := range entries {
-		uploadID := string(e.Name)
-		// todo: parallel
-		object_, eno := n.fs.GetXattr(mctx, n.upath(bucket, uploadID), uploadKeyName)
+	for _, p := range parents {
+		f, eno := n.fs.Open(mctx, n.tpath(bucket, "uploads", string(p.Name)), 0)
 		if eno != 0 {
-			logger.Warnf("get object xattr error %s: %s, ignore this item", n.upath(bucket, uploadID), eno)
-			continue
+			return
+		}
+		defer f.Close(mctx)
+		entries, eno := f.ReaddirPlus(mctx, 0)
+		if eno != 0 {
+			err = jfsToObjectErr(ctx, eno, bucket)
+			return
 		}
-		object := string(object_)
-		if strings.HasPrefix(object, prefix) {
-			if keyMarker != "" && object+uploadID > keyMarker+uploadIDMarker || keyMarker == "" {
-				lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
-					Object:    object,
-					UploadID:  uploadID,
-					Initiated: time.Unix(e.Attr.Atime, int64(e.Attr.Atimensec)),
-				})
+
+		for _, e := range entries {
+			if len(e.Name) != 36 {
+				continue // not an uuid
+			}
+			uploadID := string(e.Name)
+			// todo: parallel
+			object_, eno := n.fs.GetXattr(mctx, n.upath(bucket, uploadID), uploadKeyName)
+			if eno != 0 {
+				logger.Warnf("get object xattr error %s: %s, ignore this item", n.upath(bucket, uploadID), eno)
+				continue
+			}
+			object := string(object_)
+			if strings.HasPrefix(object, prefix) {
+				if keyMarker != "" && object+uploadID > keyMarker+uploadIDMarker || keyMarker == "" {
+					lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
+						Object:    object,
+						UploadID:  uploadID,
+						Initiated: time.Unix(e.Attr.Atime, int64(e.Attr.Atimensec)),
+					})
+				}
 			}
 		}
 	}
@@ -1080,6 +1103,10 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 	for _, part := range parts {
 		p := n.ppath(bucket, uploadID, strconv.Itoa(part.PartNumber))
 		copied, eno := n.fs.CopyFileRange(mctx, p, 0, tmp, total, 5<<30)
+		if eno == syscall.ENOENT { // try lookup from old path
+			p = n.ppathFlat(bucket, uploadID, strconv.Itoa(part.PartNumber))
+			copied, eno = n.fs.CopyFileRange(mctx, p, 0, tmp, total, 5<<30)
+		}
 		if eno != 0 {
 			err = jfsToObjectErr(ctx, eno, bucket, object, uploadID)
 			logger.Errorf("merge parts: %s", err)
@@ -1135,7 +1162,7 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 	}
 
 	// remove parts
-	_ = n.fs.Rmr(mctx, n.upath(bucket, uploadID))
+	_ = n.fs.Rmr(mctx, n.upath(bucket, uploadID), meta.RmrDefaultThreads)
 	return minio.ObjectInfo{
 		Bucket:   bucket,
 		Name:     object,
@@ -1152,12 +1179,12 @@ func (n *jfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 	if err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return
 	}
-	eno := n.fs.Rmr(mctx, n.upath(bucket, uploadID))
+	eno := n.fs.Rmr(mctx, n.upath(bucket, uploadID), meta.RmrDefaultThreads)
 	return jfsToObjectErr(ctx, eno, bucket, object, uploadID)
 }
 
 func (n *jfsObjects) cleanup() {
-	for t := range time.Tick(24 * time.Hour) {
+	for range time.Tick(24 * time.Hour) {
 		// default bucket tmp dirs
 		tmpDirs := []string{".sys/tmp/", ".sys/uploads/"}
 		if n.gConf.MultiBucket {
@@ -1172,27 +1199,40 @@ func (n *jfsObjects) cleanup() {
 			}
 		}
 		for _, dir := range tmpDirs {
-			f, errno := n.fs.Open(mctx, dir, 0)
-			if errno != 0 {
+			n.cleanupDir(dir)
+		}
+	}
+}
+
+func (n *jfsObjects) cleanupDir(dir string) bool {
+	f, errno := n.fs.Open(mctx, dir, 0)
+	if errno != 0 {
+		return false
+	}
+	defer f.Close(mctx)
+	entries, _ := f.ReaddirPlus(mctx, 0)
+	now := time.Now()
+	deleted := 0
+	for _, entry := range entries {
+		dirPath := n.path(dir, string(entry.Name))
+		if entry.Attr.Typ == meta.TypeDirectory && len(entry.Name) == subDirPrefix {
+			if !n.cleanupDir(strings.TrimPrefix(dirPath, "/")) {
 				continue
 			}
-			entries, _ := f.ReaddirPlus(mctx, 0)
-			for _, entry := range entries {
-				if _, err := uuid.Parse(string(entry.Name)); err != nil {
-					continue
-				}
-				if t.Sub(time.Unix(entry.Attr.Mtime, 0)) > 7*24*time.Hour {
-					p := n.path(dir, string(entry.Name))
-					if errno := n.fs.Rmr(mctx, p); errno != 0 {
-						logger.Errorf("failed to delete expired temporary files path: %s,", p)
-					} else {
-						logger.Infof("delete expired temporary files path: %s, mtime: %s", p, time.Unix(entry.Attr.Mtime, 0).Format(time.RFC3339))
-					}
-				}
+		} else if _, err := uuid.Parse(string(entry.Name)); err != nil {
+			logger.Warnf("unexpected file path: %s", dirPath)
+			continue
+		}
+		if now.Sub(time.Unix(entry.Attr.Mtime, 0)) > 7*24*time.Hour {
+			if errno = n.fs.Rmr(mctx, dirPath, meta.RmrDefaultThreads); errno != 0 {
+				logger.Errorf("failed to delete expired temporary files path: %s, err: %s", dirPath, errno)
+			} else {
+				deleted += 1
+				logger.Infof("delete expired temporary files path: %s, mtime: %s", dirPath, time.Unix(entry.Attr.Mtime, 0).Format(time.RFC3339))
 			}
-			_ = f.Close(mctx)
 		}
 	}
+	return deleted == len(entries)
 }
 
 type jfsFLock struct {
diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go
index 2a1267d0..9f46bf2f 100644
--- a/pkg/meta/sql.go
+++ b/pkg/meta/sql.go
@@ -290,8 +290,8 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) {
 
 	var vOpenConns int = 0
 	var vIdleConns int = runtime.GOMAXPROCS(-1) * 2
-	var vIdleTime  int = 300
-	var vLifeTime  int = 0
+	var vIdleTime int = 300
+	var vLifeTime int = 0
 
 	if optIndex != -1 {
 		baseurl := murl[:optIndex]
@@ -307,11 +307,11 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) {
 			}
 			if vals.Has("max_idle_time") {
 				vIdleTime, _ = strconv.Atoi(vals.Get("max_idle_time"))
-				vals.Del("max_idle_time");
+				vals.Del("max_idle_time")
 			}
 			if vals.Has("max_life_time") {
 				vLifeTime, _ = strconv.Atoi(vals.Get("max_life_time"))
-				vals.Del("max_life_time");
+				vals.Del("max_life_time")
 			}
 			optsurl = vals.Encode()
 		}
diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go
index 7587d7af..7d2682ca 100644
--- a/pkg/sync/sync.go
+++ b/pkg/sync/sync.go
@@ -30,7 +30,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/juicedata/juicefs/pkg/chunk"
 	"github.com/juicedata/juicefs/pkg/object"
 	"github.com/juicedata/juicefs/pkg/utils"
 	"github.com/juju/ratelimit"
@@ -553,15 +552,43 @@ func (w *withProgress) Read(b []byte) (int, error) {
 	return n, err
 }
 
+func dynAlloc(size int) []byte {
+	zeros := utils.PowerOf2(size)
+	b := *dynPools[zeros].Get().(*[]byte)
+	if cap(b) < size {
+		panic(fmt.Sprintf("%d < %d", cap(b), size))
+	}
+	return b[:size]
+}
+
+func dynFree(b []byte) {
+	dynPools[utils.PowerOf2(cap(b))].Put(&b)
+}
+
+var dynPools []*sync.Pool
+
+func init() {
+	dynPools = make([]*sync.Pool, 33) // 1 - 8G
+	for i := 0; i < 33; i++ {
+		func(bits int) {
+			dynPools[i] = &sync.Pool{
+				New: func() interface{} {
+					b := make([]byte, 1<<bits)
+					return &b
+				},
+			}
+		}(i)
+	}
+}
+
 func doUploadPart(src, dst object.ObjectStorage, srckey string, off, size int64, key, uploadID string, num int, calChksum bool) (*object.Part, uint32, error) {
 	if limiter != nil {
 		limiter.Wait(size)
 	}
 	start := time.Now()
 	sz := size
-	p := chunk.NewOffPage(int(size))
-	defer p.Release()
-	data := p.Data
+	data := dynAlloc(int(size))
+	defer dynFree(data)
 	var part *object.Part
 	var chksum uint32
 	err := try(3, func() error {
@@ -727,7 +754,14 @@ func doCopyMultiple(src, dst object.ObjectStorage, key string, size int64, uploa
 	return chksum, nil
 }
 
-func copyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {
+func InitForCopyData() {
+	concurrent = make(chan int, 10)
+	progress := utils.NewProgress(true)
+	copied = progress.AddCountSpinner("Copied objects")
+	copiedBytes = progress.AddByteSpinner("Copied bytes")
+}
+
+func CopyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {
 	start := time.Now()
 	var err error
 	var srcChksum uint32
@@ -825,7 +859,7 @@ func worker(tasks <-chan object.Object, src, dst object.ObjectStorage, config *C
 					logger.Errorf("copy link failed: %s", err)
 				}
 			} else {
-				srcChksum, err = copyData(src, dst, key, obj.Size(), config.CheckAll || config.CheckNew)
+				srcChksum, err = CopyData(src, dst, key, obj.Size(), config.CheckAll || config.CheckNew)
 			}
 
 			if err == nil && (config.CheckAll || config.CheckNew) {
diff --git a/pkg/utils/alloc.go b/pkg/utils/alloc.go
index 74a7f864..79988ea7 100644
--- a/pkg/utils/alloc.go
+++ b/pkg/utils/alloc.go
@@ -28,7 +28,7 @@ var used int64
 
 // Alloc returns size bytes memory from Go heap.
 func Alloc(size int) []byte {
-	zeros := powerOf2(size)
+	zeros := PowerOf2(size)
 	b := *pools[zeros].Get().(*[]byte)
 	if cap(b) < size {
 		panic(fmt.Sprintf("%d < %d", cap(b), size))
@@ -41,7 +41,7 @@ func Alloc(size int) []byte {
 func Free(b []byte) {
 	// buf could be zero length
 	atomic.AddInt64(&used, -int64(cap(b)))
-	pools[powerOf2(cap(b))].Put(&b)
+	pools[PowerOf2(cap(b))].Put(&b)
 }
 
 // AllocMemory returns the allocated memory
@@ -51,7 +51,7 @@ func AllocMemory() int64 {
 
 var pools []*sync.Pool
 
-func powerOf2(s int) int {
+func PowerOf2(s int) int {
 	var bits int
 	var p int = 1
 	for p < s {
diff --git a/pkg/vfs/backup.go b/pkg/vfs/backup.go
index 5bf28153..70382823 100644
--- a/pkg/vfs/backup.go
+++ b/pkg/vfs/backup.go
@@ -18,9 +18,13 @@ package vfs
 
 import (
 	"compress/gzip"
+	"errors"
 	"io"
 	"os"
+	"path/filepath"
 	"sort"
+	"strings"
+	"syscall"
 	"time"
 
 	"github.com/juicedata/juicefs/pkg/meta"
@@ -92,7 +96,17 @@ func Backup(m meta.Meta, blob object.ObjectStorage, interval time.Duration, skip
 
 func backup(m meta.Meta, blob object.ObjectStorage, now time.Time, fast, skipTrash bool) (string, error) {
 	name := "dump-" + now.UTC().Format("2006-01-02-150405") + ".json.gz"
-	fp, err := os.CreateTemp("", "juicefs-meta-*")
+	localDir := os.TempDir()
+	if !strings.HasSuffix(localDir, "/") {
+		localDir += "/"
+	}
+	fp, err := os.Create(filepath.Join(localDir, "meta", name))
+	if errors.Is(err, syscall.ENOENT) {
+		if err = os.MkdirAll(filepath.Join(localDir, "meta"), 0755); err != nil {
+			return "", err
+		}
+		fp, err = os.Create(filepath.Join(localDir, "meta", name))
+	}
 	if err != nil {
 		return "", err
 	}
@@ -108,11 +122,19 @@ func backup(m meta.Meta, blob object.ObjectStorage, now time.Time, fast, skipTra
 	if err != nil {
 		return "", err
 	}
-	if _, err = fp.Seek(0, io.SeekStart); err != nil {
+	size, err := fp.Seek(0, io.SeekCurrent)
+	if err != nil {
 		return "", err
 	}
+
 	fpath := "meta/" + name
-	return blob.String() + fpath, blob.Put(fpath, fp)
+	disk, err := object.CreateStorage("file", localDir, "", "", "")
+	if err != nil {
+		return "", err
+	}
+	osync.InitForCopyData()
+	_, err = osync.CopyData(disk, blob, fpath, size, true)
+	return blob.String() + fpath, err
 }
 
 func cleanupBackups(blob object.ObjectStorage, now time.Time) {
diff --git a/sdk/java/libjfs/main.go b/sdk/java/libjfs/main.go
index 08698e6a..e41e21ae 100644
--- a/sdk/java/libjfs/main.go
+++ b/sdk/java/libjfs/main.go
@@ -94,6 +94,17 @@ var (
 
 	userGroupCache = make(map[string]map[string][]string) // name -> (user -> groups)
 
+	MaxDeletes = meta.RmrDefaultThreads
+	caller     = CALLER_JAVA
+)
+
+const (
+	CALLER_JAVA = iota
+	CALLER_PYTHON
+)
+
+const (
+	BEHAVIOR_HADOOP = "Hadoop"
 )
 
 const (
@@ -210,7 +221,9 @@ func jfs_set_logger(cb unsafe.Pointer) {
 func (w *wrapper) withPid(pid int64) meta.Context {
 	// mapping Java Thread ID to global one
 	ctx := meta.NewContext(w.ctx.Pid()*1000+uint32(pid), w.ctx.Uid(), w.ctx.Gids())
-	ctx.WithValue(meta.CtxKey("behavior"), "Hadoop")
+	if caller == CALLER_JAVA {
+		ctx.WithValue(meta.CtxKey("behavior"), BEHAVIOR_HADOOP)
+	}
 	return ctx
 }
 
@@ -336,6 +349,7 @@ type javaConf struct {
 	PushAuth          string `json:"pushAuth"`
 	PushLabels        string `json:"pushLabels"`
 	PushGraphite      string `json:"pushGraphite"`
+	Caller            int    `json:"caller"`
 }
 
 func getOrCreate(name, user, group, superuser, supergroup string, f func() *fs.FileSystem) int64 {
@@ -467,6 +481,11 @@ func jfs_init(cname, jsonConf, user, group, superuser, supergroup *C.char) int64
 			utils.SetLogLevel(logrus.WarnLevel)
 		}
 
+		caller = jConf.Caller
+		if jConf.MaxDeletes > 0 {
+			MaxDeletes = jConf.MaxDeletes
+		}
+
 		metaConf := meta.DefaultConf()
 		metaConf.Retries = jConf.IORetries
 		metaConf.MaxDeletes = jConf.MaxDeletes
@@ -865,7 +884,7 @@ func jfs_rmr(pid int64, h int64, cpath *C.char) int32 {
 	if w == nil {
 		return EINVAL
 	}
-	return errno(w.Rmr(w.withPid(pid), C.GoString(cpath)))
+	return errno(w.Rmr(w.withPid(pid), C.GoString(cpath), MaxDeletes))
 }
 
 //export jfs_rename
diff --git a/sdk/python/juicefs/juicefs/juicefs.py b/sdk/python/juicefs/juicefs/juicefs.py
index beb6203e..b76bdf53 100644
--- a/sdk/python/juicefs/juicefs/juicefs.py
+++ b/sdk/python/juicefs/juicefs/juicefs.py
@@ -135,6 +135,7 @@ class Client(object):
         kwargs["pushAuth"] = push_auth
         kwargs["pushLabels"] = push_labels
         kwargs["pushGraphite"] = push_graphite
+        kwargs["caller"] = 1
 
         jsonConf = json.dumps(kwargs)
         self.umask = os.umask(0)


Test patch:
diff --git a/.github/scripts/hypo/fs_sdk_test.py b/.github/scripts/hypo/fs_sdk_test.py
index 7977905d..e2eb5604 100644
--- a/.github/scripts/hypo/fs_sdk_test.py
+++ b/.github/scripts/hypo/fs_sdk_test.py
@@ -269,5 +269,13 @@ class TestPySdk(unittest.TestCase):
         state.read(file=v2, mode='r', offset=0, user='root', whence=0, length=4)
         state.teardown()
 
+    def test_issue_f(self):
+        state = JuicefsMachine()
+        folders_0 = state.init_folders()
+        files_0 = state.create_file(content=b'', file_name='b', parent=folders_0, umask=18, user='root')
+        state.chown(entry=folders_0, owner='user1', user='root')
+        state.create_file(content=b'', file_name='a', parent=folders_0, umask=18, user='root')
+        state.teardown()
+
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()
diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go
index 47bede03..d1d91cbe 100644
--- a/pkg/fs/fs_test.go
+++ b/pkg/fs/fs_test.go
@@ -230,7 +230,7 @@ func TestFileSystem(t *testing.T) {
 	if err := fs.Delete(ctx, "/d/f"); err == 0 || !IsNotExist(err) {
 		t.Fatalf("delete /d/f: %s", err)
 	}
-	if e := fs.Rmr(ctx, "/d"); e != 0 {
+	if e := fs.Rmr(ctx, "/d", meta.RmrDefaultThreads); e != 0 {
 		t.Fatalf("delete /d -r: %s", e)
 	}
 
@@ -261,7 +261,7 @@ func TestFileSystem(t *testing.T) {
 	if err := fs.Rename(ctx, "/ddd/", "/ttt/", 0); err != 0 {
 		t.Fatalf("delete /ddd/: %s", err)
 	}
-	if err := fs.Rmr(ctx, "/ttt/"); err != 0 {
+	if err := fs.Rmr(ctx, "/ttt/", meta.RmrDefaultThreads); err != 0 {
 		t.Fatalf("rmr /ttt/: %s", err)
 	}
 	if _, err := fs.Stat(ctx, "/ttt/"); err != syscall.ENOENT {