Repository: juicedata/juicefs
Pull request #3395 (2023-03-28 08:46:54)

prepare for v1.0.4
Backported 26 commits (86888108, 57cef4d8, 5909126a, and 4ea2c848 have already been picked to release-1.0)
091a7bff java sdk: fix stats of symlink
7db673bc chunk: fix the issue that staging file may be uploaded for many times (#3157)
27ec92cf hadoop: add symlink test (#3165)
b40714cf fix reading large response from .control (#3170)
247e3df7 chunk: deleting an object should be idempotent (#3171)
1f63d3bf meta/sql: recreated session after some fault (#3190)
00863291 meta: recreate the session with info if it was cleaned before (#3197)
fcedbeec object/obs: fix obs encrypt check (#3199)
c8ea7449 disable server name check for sentinel (#3194)
9942d53b optionally turn off 'S3Disable100Continue' via url query parameter (#3228)
51fd6287 meta: move the control of max-deletes from chunk to meta (#3227)
e75256ae object: set encoding type when list object (#3244)
6225b48d remove password (#3256)
4a8b0911 dont set the default schema when remove password form titles (#3258)
bfcf95fa meta/redis: fix incorrect nil check on doRename with redis (#3266)
5a35d180 fix obs list encode (#3269)
ecf2c4d0 fix b2 get empty file info (#3274)
a8f520a4 meta/tikv: add background GC worker for TiKV (#3262)
585dcb32 object/obs&tos : fix range get (#3270)
67ce17ff update BadgerDB to v3.2103.5 (#3309)
40f6fce9 cmd/load: support load encrypted file (#3311)
75a092ec build(deps): bump golang.org/x/net (#3350)
60dca5f3 fix s3fs folder (#3378)
77ade77b object/OVH: support OVH region (#3133)
17589eef meta: deleteSlice synchronously for non-session clients (#3401)
a1e74882 meta/redis: fix transacion of copy_file_range (#3400)

More commits to fix the code:
9dc473c7 backport: 1. fix tkv not using interface in the txn function 2. delete the delimiter argument in blob.List function
4afbce8f fix minio set

SandyXSD: @p1u3o Hi, we are backporting this PR (#3266 ) to the release branch. Somehow the Github says that you haven't signed the CLA. Usually it should be requested before the PR merged, but I'm not sure now. Could you please sign it?
p1u3o: CLA has been signed.

Can I suggest back-porting https://github.com/juicedata/juicefs/pull/3133 too?

It's a simple enough patch and without it JuiceFS will not work on OVHs bucket storage
SandyXSD: > It's a simple enough patch and without it JuiceFS will not work on OVHs bucket storage

OK, picked it as well.

Patch:
diff --git a/.github/actions/version-compatible/action.yml b/.github/actions/version-compatible/action.yml
index 3607b58b..7c62d11f 100644
--- a/.github/actions/version-compatible/action.yml
+++ b/.github/actions/version-compatible/action.yml
@@ -114,7 +114,7 @@ runs:
                       "format") 
                           ./mc admin user remove minio juicedata || true
                           ./mc admin user add minio juicedata juicedata
-                          ./mc admin policy set minio consoleAdmin user=juicedata  
+                          ./mc admin policy attach minio consoleAdmin --user juicedata  
                           sudo ./$jfs_bin format --trash-days 0 --storage minio --bucket  ${{ inputs.bucket }} --access-key juicedata --secret-key juicedata  ${{ inputs.meta_url }} juicefs-version-compatible
                           echo "format finished" 
                           ;;
@@ -170,7 +170,7 @@ runs:
                           encrypt=$(echo $command | awk -F: '{print $1}'| xargs | awk -F_ '{print $2}')
                           ./mc admin user remove minio juicedata || true
                           ./mc admin user add minio juicedata juicedata1
-                          ./mc admin policy set minio consoleAdmin user=juicedata  
+                          ./mc admin policy attach minio consoleAdmin --user juicedata  
                           if [ -n "$encrypt" ]; then
                             sudo ./$jfs_bin config ${{inputs.meta_url}} --secret-key juicedata1 --encrypt-secret
                           else
diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml
index 943bbe72..86338450 100644
--- a/.github/workflows/config.yml
+++ b/.github/workflows/config.yml
@@ -67,13 +67,13 @@ jobs:
           mount_point=/tmp/sync-test/
           ./mc config host add minio http://127.0.0.1:9000 minioadmin minioadmin
           ./mc admin user add minio juicedata juicedata
-          ./mc admin policy set minio consoleAdmin user=juicedata
+          ./mc admin policy attach minio consoleAdmin --user juicedata
           ./juicefs format --storage minio --bucket http://localhost:9000/jfs-test --access-key juicedata --secret-key juicedata $meta_url myjfs
           ./juicefs mount $meta_url $mount_point -d --io-retries 1
 
           ./mc admin user remove minio juicedata
           ./mc admin user add minio juicedata1 juicedata1
-          ./mc admin policy set minio consoleAdmin user=juicedata1
+          ./mc admin policy attach minio consoleAdmin --user juicedata1
           ./juicefs config $meta_url --access-key juicedata1 --secret-key juicedata1
           sleep 60
           echo abc > abc.txt
diff --git a/cmd/format.go b/cmd/format.go
index dad56231..bf29c1eb 100644
--- a/cmd/format.go
+++ b/cmd/format.go
@@ -302,6 +302,17 @@ func test(store object.ObjectStorage) error {
 	return err
 }
 
+func loadEncrypt(keyPath string) string {
+	if keyPath == "" {
+		return ""
+	}
+	pem, err := os.ReadFile(keyPath)
+	if err != nil {
+		logger.Fatalf("load RSA key from %s: %s", keyPath, err)
+	}
+	return string(pem)
+}
+
 func format(c *cli.Context) error {
 	setup(c, 2)
 	removePassword(c.Args().Get(0))
@@ -320,16 +331,7 @@ func format(c *cli.Context) error {
 	if v := c.Int("shards"); v > 256 {
 		logger.Fatalf("too many shards: %d", v)
 	}
-	loadEncrypt := func(keyPath string) string {
-		if keyPath == "" {
-			return ""
-		}
-		pem, err := os.ReadFile(keyPath)
-		if err != nil {
-			logger.Fatalf("load RSA key from %s: %s", keyPath, err)
-		}
-		return string(pem)
-	}
+
 	var create, encrypted bool
 	format, err := m.Load(false)
 	if err == nil {
diff --git a/cmd/gc.go b/cmd/gc.go
index e06424cf..0656fabe 100644
--- a/cmd/gc.go
+++ b/cmd/gc.go
@@ -73,17 +73,13 @@ $ juicefs gc redis://localhost --delete`,
 	}
 }
 
-type dSlice struct {
-	id     uint64
-	length uint32
-}
-
 func gc(ctx *cli.Context) error {
 	setup(ctx, 1)
 	removePassword(ctx.Args().Get(0))
 	m := meta.NewClient(ctx.Args().Get(0), &meta.Config{
-		Retries: 10,
-		Strict:  true,
+		Retries:    10,
+		Strict:     true,
+		MaxDeletes: ctx.Int("threads"),
 	})
 	format, err := m.Load(true)
 	if err != nil {
@@ -96,7 +92,6 @@ func gc(ctx *cli.Context) error {
 		GetTimeout: time.Second * 60,
 		PutTimeout: time.Second * 60,
 		MaxUpload:  20,
-		MaxDeletes: ctx.Int("threads"),
 		BufferSize: 300 << 20,
 		CacheDir:   "memory",
 	}
@@ -150,14 +145,14 @@ func gc(ctx *cli.Context) error {
 		logger.Fatal("threads should be greater than 0 to delete objects")
 	}
 	var delSpin *utils.Bar
-	var sliceChan chan *dSlice // pending delete slices
+	var sliceChan chan meta.Slice // pending delete slices
 	var wg sync.WaitGroup
 	if delete {
 		delSpin = progress.AddCountSpinner("Deleted pending")
-		sliceChan = make(chan *dSlice, 10240)
+		sliceChan = make(chan meta.Slice, 10240)
 		m.OnMsg(meta.DeleteSlice, func(args ...interface{}) error {
 			delSpin.Increment()
-			sliceChan <- &dSlice{args[0].(uint64), args[1].(uint32)}
+			sliceChan <- meta.Slice{Id: args[0].(uint64), Size: args[1].(uint32)}
 			return nil
 		})
 		for i := 0; i < threads; i++ {
@@ -165,8 +160,8 @@ func gc(ctx *cli.Context) error {
 			go func() {
 				defer wg.Done()
 				for s := range sliceChan {
-					if err := store.Remove(s.id, int(s.length)); err != nil {
-						logger.Warnf("remove %d_%d: %s", s.id, s.length, err)
+					if err := store.Remove(s.Id, int(s.Size)); err != nil {
+						logger.Warnf("remove %d_%d: %s", s.Id, s.Size, err)
 					}
 				}
 			}()
diff --git a/cmd/load.go b/cmd/load.go
index e155a240..01d5bccc 100644
--- a/cmd/load.go
+++ b/cmd/load.go
@@ -17,9 +17,16 @@
 package cmd
 
 import (
+	"compress/gzip"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/juicedata/juicefs/pkg/object"
 
 	"github.com/juicedata/juicefs/pkg/meta"
 	"github.com/juicedata/juicefs/pkg/utils"
@@ -28,9 +35,15 @@ import (
 
 func cmdLoad() *cli.Command {
 	return &cli.Command{
-		Name:      "load",
-		Action:    load,
-		Category:  "ADMIN",
+		Name:     "load",
+		Action:   load,
+		Category: "ADMIN",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "encrypt-rsa-key",
+				Usage: "a path to RSA private key (PEM)",
+			},
+		},
 		Usage:     "Load metadata from a previously dumped JSON file",
 		ArgsUsage: "META-URL [FILE]",
 		Description: `
@@ -48,23 +61,71 @@ Details: https://juicefs.com/docs/community/metadata_dump_load`,
 
 func load(ctx *cli.Context) error {
 	setup(ctx, 1)
-	removePassword(ctx.Args().Get(0))
-	var fp io.ReadCloser
+	metaUri := ctx.Args().Get(0)
+	src := ctx.Args().Get(1)
+	removePassword(metaUri)
+	var r io.ReadCloser
 	if ctx.Args().Len() == 1 {
-		fp = os.Stdin
+		r = os.Stdin
+		src = "STDIN"
 	} else {
-		var err error
-		fp, err = os.Open(ctx.Args().Get(1))
-		if err != nil {
-			return err
+		var ioErr error
+		var fp io.ReadCloser
+		if ctx.String("encrypt-rsa-key") != "" {
+			passphrase := os.Getenv("JFS_RSA_PASSPHRASE")
+			encryptKey := loadEncrypt(ctx.String("encrypt-rsa-key"))
+			if passphrase == "" {
+				block, _ := pem.Decode([]byte(encryptKey))
+				// nolint:staticcheck
+				if block != nil && strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED") && x509.IsEncryptedPEMBlock(block) {
+					return fmt.Errorf("passphrase is required to private key, please try again after setting the 'JFS_RSA_PASSPHRASE' environment variable")
+				}
+			}
+			privKey, err := object.ParseRsaPrivateKeyFromPem([]byte(encryptKey), []byte(passphrase))
+			if err != nil {
+				return fmt.Errorf("parse rsa: %s", err)
+			}
+			encryptor := object.NewAESEncryptor(object.NewRSAEncryptor(privKey))
+			if err != nil {
+				return err
+			}
+			if _, err := os.Stat(src); err != nil {
+				return fmt.Errorf("failed to stat %s: %s", src, err)
+			}
+			var srcAbsPath string
+			srcAbsPath, err = filepath.Abs(src)
+			if err != nil {
+				return fmt.Errorf("failed to get absolute path of %s: %s", src, err)
+			}
+			fileBlob, err := object.CreateStorage("file", strings.TrimRight(src, filepath.Base(srcAbsPath)), "", "", "")
+			if err != nil {
+				return err
+			}
+			blob := object.NewEncrypted(fileBlob, encryptor)
+			fp, ioErr = blob.Get(filepath.Base(srcAbsPath), 0, -1)
+		} else {
+			fp, ioErr = os.Open(src)
+		}
+		if ioErr != nil {
+			return ioErr
 		}
 		defer fp.Close()
+		if strings.HasSuffix(src, ".gz") {
+			var err error
+			r, err = gzip.NewReader(fp)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+		} else {
+			r = fp
+		}
 	}
-	m := meta.NewClient(ctx.Args().Get(0), &meta.Config{Retries: 10, Strict: true})
+	m := meta.NewClient(metaUri, &meta.Config{Retries: 10, Strict: true})
 	if format, err := m.Load(false); err == nil {
-		return fmt.Errorf("Database %s is used by volume %s", utils.RemovePassword(ctx.Args().Get(0)), format.Name)
+		return fmt.Errorf("Database %s is used by volume %s", utils.RemovePassword(metaUri), format.Name)
 	}
-	if err := m.LoadMeta(fp); err != nil {
+	if err := m.LoadMeta(r); err != nil {
 		return err
 	}
 	if format, err := m.Load(true); err == nil {
@@ -74,6 +135,6 @@ func load(ctx *cli.Context) error {
 	} else {
 		return err
 	}
-	logger.Infof("Load metadata from %s succeed", ctx.Args().Get(1))
+	logger.Infof("Load metadata from %s succeed", src)
 	return nil
 }
diff --git a/cmd/main.go b/cmd/main.go
index 662f78d1..422ef6c0 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -285,12 +285,7 @@ func setup(c *cli.Context, n int) {
 }
 
 func removePassword(uri string) {
-	var uri2 string
-	if strings.Contains(uri, "://") {
-		uri2 = utils.RemovePassword(uri)
-	} else {
-		uri2 = utils.RemovePassword("redis://" + uri)
-	}
+	uri2 := utils.RemovePassword(uri)
 	if uri2 != uri {
 		for i, a := range os.Args {
 			if a == uri {
diff --git a/cmd/mount.go b/cmd/mount.go
index 0707c362..5d824bbc 100644
--- a/cmd/mount.go
+++ b/cmd/mount.go
@@ -284,6 +284,7 @@ func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config {
 	cfg := &meta.Config{
 		Retries:    c.Int("io-retries"),
 		Strict:     true,
+		MaxDeletes: c.Int("max-deletes"),
 		ReadOnly:   readOnly,
 		NoBGJob:    c.Bool("no-bgjob"),
 		OpenCache:  time.Duration(c.Float64("open-cache") * 1e9),
@@ -291,6 +292,9 @@ func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config {
 		MountPoint: mp,
 		Subdir:     c.String("subdir"),
 	}
+	if cfg.MaxDeletes == 0 {
+		logger.Warnf("Deleting object will be disabled since max-deletes is 0")
+	}
 	if cfg.Heartbeat < time.Second {
 		logger.Warnf("heartbeat should not be less than 1 second")
 		cfg.Heartbeat = time.Second
@@ -316,7 +320,6 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
 		GetTimeout:    time.Second * time.Duration(c.Int("get-timeout")),
 		PutTimeout:    time.Second * time.Duration(c.Int("put-timeout")),
 		MaxUpload:     c.Int("max-uploads"),
-		MaxDeletes:    c.Int("max-deletes"),
 		MaxRetries:    c.Int("io-retries"),
 		Writeback:     c.Bool("writeback"),
 		Prefetch:      c.Int("prefetch"),
diff --git a/cmd/objbench.go b/cmd/objbench.go
index bc139a6d..411c6d07 100644
--- a/cmd/objbench.go
+++ b/cmd/objbench.go
@@ -592,6 +592,9 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 			r = nspt
 		} else if err != nil {
 			r = err.Error()
+			if len(r) > 45 {
+				r = r[:45] + "..."
+			}
 			if colorful {
 				r = fmt.Sprintf("%s%dm%s%s", COLOR_SEQ, RED, r, RESET_SEQ)
 			}
@@ -677,15 +680,18 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 			return fmt.Errorf("put object failed: %s", err)
 		}
 		defer blob.Delete(key) //nolint:errcheck
-		if s, err := get(blob, key, 0, -1); err != nil && s != string(br) {
-			return fmt.Errorf("get object failed: %s", err)
+		if d, e := get(blob, key, 0, -1); e != nil || d != string(br) {
+			return fmt.Errorf(`failed to get an object: expect "hello", but got %v, error: %s`, d, e)
+		}
+		if d, e := get(blob, key, 0, 5); e != nil || d != string(br) {
+			return fmt.Errorf(`failed to get an object: expect "hello", but got %v, error: %s`, d, e)
 		}
 		return nil
 	})
 
 	runCase("get non-exist", func(blob object.ObjectStorage) error {
 		if _, err := blob.Get("not_exists_file", 0, -1); err == nil {
-			return fmt.Errorf("get not exists object should failed: %s", err)
+			return fmt.Errorf("get not existed object should failed: %s", err)
 		}
 		return nil
 	})
@@ -696,23 +702,30 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 			return fmt.Errorf("put object failed: %s", err)
 		}
 		defer blob.Delete(key) //nolint:errcheck
-		if d, e := get(blob, key, 0, -1); d != "hello" {
-			return fmt.Errorf("expect hello, but got %v, error: %s", d, e)
+
+		// get first
+		if d, e := get(blob, key, 0, 1); e != nil || d != "h" {
+			return fmt.Errorf(`failed to get the first byte:, expect "h", but got %q, error: %s`, d, e)
 		}
-		if d, e := get(blob, key, 2, -1); d != "llo" {
-			logger.Errorf("expect llo, but got %v, error: %s", d, e)
+		// get last
+		if d, e := get(blob, key, 4, 1); e != nil || d != "o" {
+			return fmt.Errorf(`failed to get the last byte: expect "o", but got %q, error: %s`, d, e)
 		}
-		if d, e := get(blob, key, 2, 3); d != "llo" {
-			return fmt.Errorf("expect llo, but got %v, error: %s", d, e)
+		// get last 3
+		if d, e := get(blob, key, 2, 3); e != nil || d != "llo" {
+			return fmt.Errorf(`failed to get the last three bytes: expect "llo", but got %q, error: %s`, d, e)
 		}
-		if d, e := get(blob, key, 2, 2); d != "ll" {
-			return fmt.Errorf("expect ll, but got %v, error: %s", d, e)
+		// get middle
+		if d, e := get(blob, key, 2, 2); e != nil || d != "ll" {
+			return fmt.Errorf(`failed to get two bytes: expect "ll", but got %q, error: %s`, d, e)
 		}
-		if d, e := get(blob, key, 4, 2); d != "o" {
-			return fmt.Errorf("out-of-range get: 'o', but got %v, error: %s", len(d), e)
+		// get the end out of range
+		if d, e := get(blob, key, 4, 2); e != nil || d != "o" {
+			return fmt.Errorf(`failed to get object with the end out of range, expect "o", but got %q, error: %s`, d, e)
 		}
-		if d, e := get(blob, key, 6, 2); d != "" {
-			return fmt.Errorf("out-of-range get: '', but got %v, error: %s", len(d), e)
+		// get the off out of range
+		if d, e := get(blob, key, 6, 2); e != nil || d != "" {
+			return fmt.Errorf(`failed to get object with the offset out of range, expect "", but got %q, error: %s`, d, e)
 		}
 		return nil
 	})
@@ -727,7 +740,7 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 			return fmt.Errorf("failed to head object %s", err)
 		} else {
 			if h.Key() != key {
-				return fmt.Errorf("expected get key is test but get %s", h.Key())
+				return fmt.Errorf("expected key 'test' but got %s", h.Key())
 			}
 		}
 		return nil
@@ -746,7 +759,7 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 		}
 
 		if err := blob.Delete(key); err != nil {
-			return fmt.Errorf("delete not exists %v", err)
+			return fmt.Errorf("delete not existed: %v", err)
 		}
 		return nil
 	})
@@ -851,6 +864,21 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 		return nil
 	})
 
+	runCase("special key", func(blob object.ObjectStorage) error {
+		key := "测试编码文件" + `{"name":"juicefs"}` + string('\u001F')
+		defer blob.Delete(key) //nolint:errcheck
+		if err := blob.Put(key, bytes.NewReader(nil)); err != nil {
+			return fmt.Errorf("put encode file failed: %s", err)
+		} else {
+			if resp, err := blob.List("", "测试编码文件", 1); err != nil && err != utils.ENOTSUP {
+				return fmt.Errorf("list encode file failed %s", err)
+			} else if len(resp) == 1 && resp[0].Key() != key {
+				return fmt.Errorf("list encode file failed: expect key %s, but got %s", key, resp[0].Key())
+			}
+		}
+		return nil
+	})
+
 	runCase("put a big object", func(blob object.ObjectStorage) error {
 		fsize := 256 << 20
 		buffL := 4 << 20
diff --git a/cmd/sync.go b/cmd/sync.go
index 69b294a2..dc384a21 100644
--- a/cmd/sync.go
+++ b/cmd/sync.go
@@ -315,6 +315,8 @@ func doSync(c *cli.Context) error {
 	// Windows support `\` and `/` as its separator, Unix only use `/`
 	srcURL := c.Args().Get(0)
 	dstURL := c.Args().Get(1)
+	removePassword(srcURL)
+	removePassword(dstURL)
 	if runtime.GOOS == "windows" {
 		if !strings.Contains(srcURL, "://") {
 			srcURL = strings.Replace(srcURL, "\\", "/", -1)
diff --git a/go.mod b/go.mod
index eba82fea..1781ff53 100644
--- a/go.mod
+++ b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/billziss-gh/cgofuse v1.5.0
 	github.com/ceph/go-ceph v0.4.0
 	github.com/colinmarc/hdfs/v2 v2.3.0
-	github.com/dgraph-io/badger/v3 v3.2103.2
+	github.com/dgraph-io/badger/v3 v3.2103.5
 	github.com/erikdubbelboer/gspt v0.0.0-20210805194459-ce36a5128377
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/go-sql-driver/mysql v1.6.0
@@ -61,16 +61,16 @@ require (
 	github.com/urfave/cli/v2 v2.4.0
 	github.com/vbauerster/mpb/v7 v7.0.3
 	github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8
-	github.com/volcengine/ve-tos-golang-sdk/v2 v2.4.2
+	github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.1
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
 	go.etcd.io/etcd v3.3.27+incompatible
 	go.etcd.io/etcd/client/v3 v3.5.2
-	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
+	golang.org/x/net v0.7.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	golang.org/x/text v0.3.7
+	golang.org/x/sys v0.5.0
+	golang.org/x/term v0.5.0
+	golang.org/x/text v0.7.0
 	google.golang.org/api v0.70.0
 	gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216
 	xorm.io/xorm v1.0.7
@@ -106,7 +106,7 @@ require (
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/dchest/siphash v1.2.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
-	github.com/dgraph-io/ristretto v0.1.0 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -245,3 +245,5 @@ replace github.com/vbauerster/mpb/v7 v7.0.3 => github.com/juicedata/mpb/v7 v7.0.
 replace google.golang.org/grpc v1.43.0 => google.golang.org/grpc v1.29.0
 
 replace xorm.io/xorm v1.0.7 => gitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a
+
+replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible
diff --git a/go.sum b/go.sum
index 68b7327a..e5662859 100644
--- a/go.sum
+++ b/go.sum
@@ -246,10 +246,10 @@ github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsP
 github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/decker502/dnspod-go v0.2.0/go.mod h1:qsurYu1FgxcDwfSwXJdLt4kRsBLZeosEb9uq4Sy+08g=
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
-github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8=
-github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M=
-github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
-github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
+github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg=
+github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -545,8 +545,6 @@ github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvh
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible h1:tANYIteuFrosKbRYUk1Yo/OGJjbt4x3OVg211Qc60M0=
-github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
 github.com/hungys/go-lz4 v0.0.0-20170805124057-19ff7f07f099 h1:heHZCso/ytvpYr+hp2cDxlZfA/jTw46aHSvT9kZnJ7o=
 github.com/hungys/go-lz4 v0.0.0-20170805124057-19ff7f07f099/go.mod h1:h44tqw4M3GN0Woo9KBStxJxm8huNi+9+tOHoeqSvhaY=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -593,6 +591,8 @@ github.com/juicedata/go-fuse/v2 v2.1.1-0.20221212033224-ed3172586347 h1:wvWzYUrx
 github.com/juicedata/go-fuse/v2 v2.1.1-0.20221212033224-ed3172586347/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA=
 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw=
+github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=
+github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=
 github.com/juicedata/minio v0.0.0-20220913070254-47325cb1d5fe h1:BcKE03BrzklTWcU9RtLsjahnXPhEKBF/xac/Q1flnVU=
 github.com/juicedata/minio v0.0.0-20220913070254-47325cb1d5fe/go.mod h1:8oMBmyEWA8aYwMwO7eUNOjvVNOhDeqDlio2RSv6T/4Q=
 github.com/juicedata/mpb/v7 v7.0.4-0.20220719014258-68df1356cfba h1:YSCPvyONPDp/ivKgRanFpNEHh3N5/0UsKmwbbKQIuGE=
@@ -971,8 +971,8 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6Jc
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ=
 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE=
-github.com/volcengine/ve-tos-golang-sdk/v2 v2.4.2 h1:QL1274Hd6K9VqPtqFAYprXfQh7VN/9IZXW0TyU40GJw=
-github.com/volcengine/ve-tos-golang-sdk/v2 v2.4.2/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=
+github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.1 h1:rxplp5ZoaLvRmtaTMPSxDnBzKMLpx0lEtm8Xhc7ut2M=
+github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.1/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=
 github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
 github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
@@ -1058,8 +1058,9 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI=
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
+golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1149,8 +1150,9 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1274,11 +1276,14 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1288,8 +1293,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/pkg/chunk/cached_store.go b/pkg/chunk/cached_store.go
index 8bf70d35..d38452c2 100644
--- a/pkg/chunk/cached_store.go
+++ b/pkg/chunk/cached_store.go
@@ -44,8 +44,10 @@ var (
 )
 
 type pendingItem struct {
-	key   string
-	fpath string
+	key       string
+	fpath     string    // full path of local file corresponding to the key
+	ts        time.Time // timestamp when this item is added
+	uploading bool
 }
 
 // slice for read and remove
@@ -201,6 +203,11 @@ func (s *rSlice) delete(indx int) error {
 	st := time.Now()
 	err := s.store.storage.Delete(key)
 	used := time.Since(st)
+	if err != nil && (strings.Contains(err.Error(), "NoSuchKey") ||
+		strings.Contains(err.Error(), "not found") ||
+		strings.Contains(err.Error(), "No such file")) {
+		err = nil
+	}
 	logger.Debugf("DELETE %v (%v, %.3fs)", key, err, used.Seconds())
 	if used > SlowRequest {
 		logger.Infof("slow request: DELETE %v (%v, %.3fs)", key, err, used.Seconds())
@@ -227,17 +234,12 @@ func (s *rSlice) Remove() error {
 		s.store.bcache.remove(key)
 	}
 
-	if s.store.conf.MaxDeletes == 0 {
-		return errors.New("skip deleting objects because MaxDeletes is 0")
-	}
 	var err error
-	s.store.currentDelete <- struct{}{}
 	for i := 0; i <= lastIndx; i++ {
 		if e := s.delete(i); e != nil {
 			err = e
 		}
 	}
-	<-s.store.currentDelete
 	return err
 }
 
@@ -443,7 +445,7 @@ func (s *wSlice) upload(indx int) {
 								}
 							}
 						} else { // add to delay list and wait for later scanning
-							s.store.addDelayedStaging(key, stagingPath, time.Now().Add(time.Second*30), false)
+							s.store.addDelayedStaging(key, stagingPath, time.Now(), false)
 						}
 						return
 					default:
@@ -525,7 +527,6 @@ type Config struct {
 	AutoCreate     bool
 	Compress       string
 	MaxUpload      int
-	MaxDeletes     int
 	MaxRetries     int
 	UploadLimit    int64 // bytes per second
 	DownloadLimit  int64 // bytes per second
@@ -548,9 +549,8 @@ type cachedStore struct {
 	conf          Config
 	group         *Controller
 	currentUpload chan bool
-	currentDelete chan struct{}
-	pendingCh     chan pendingItem
-	pendingKeys   map[string]time.Time
+	pendingCh     chan *pendingItem
+	pendingKeys   map[string]*pendingItem
 	pendingMutex  sync.Mutex
 	compressor    compress.Compressor
 	seekable      bool
@@ -657,11 +657,10 @@ func NewCachedStore(storage object.ObjectStorage, config Config, reg prometheus.
 		storage:       storage,
 		conf:          config,
 		currentUpload: make(chan bool, config.MaxUpload),
-		currentDelete: make(chan struct{}, config.MaxDeletes),
 		compressor:    compressor,
 		seekable:      compressor.CompressBound(0) == 0,
-		pendingCh:     make(chan pendingItem, 100*config.MaxUpload),
-		pendingKeys:   make(map[string]time.Time),
+		pendingCh:     make(chan *pendingItem, 100*config.MaxUpload),
+		pendingKeys:   make(map[string]*pendingItem),
 		group:         &Controller{},
 	}
 	if config.UploadLimit > 0 {
@@ -673,11 +672,10 @@ func NewCachedStore(storage object.ObjectStorage, config Config, reg prometheus.
 	}
 	store.initMetrics()
 	store.bcache = newCacheManager(&config, reg, func(key, fpath string, force bool) bool {
-		if force {
-			return store.addDelayedStaging(key, fpath, time.Time{}, true)
-		} else if fi, err := os.Stat(fpath); err == nil {
-			return store.addDelayedStaging(key, fpath, fi.ModTime(), false)
+		if fi, err := os.Stat(fpath); err == nil {
+			return store.addDelayedStaging(key, fpath, fi.ModTime(), force)
 		} else {
+			logger.Warnf("Stat staging block %s: %s", fpath, err)
 			return false
 		}
 	})
@@ -802,7 +800,7 @@ func (store *cachedStore) uploadStagingFile(key string, stagingPath string) {
 	}()
 
 	store.pendingMutex.Lock()
-	_, ok := store.pendingKeys[key]
+	item, ok := store.pendingKeys[key]
 	store.pendingMutex.Unlock()
 	if !ok {
 		logger.Debugf("Key %s is not needed, drop it", key)
@@ -839,16 +837,27 @@ func (store *cachedStore) uploadStagingFile(key string, stagingPath string) {
 				m.stageBlockBytes.Sub(float64(blen))
 			}
 		}
+	} else {
+		item.uploading = false
 	}
 }
 
 func (store *cachedStore) addDelayedStaging(key, stagingPath string, added time.Time, force bool) bool {
 	store.pendingMutex.Lock()
-	store.pendingKeys[key] = added
+	item := store.pendingKeys[key]
+	if item == nil {
+		item = &pendingItem{key, stagingPath, added, false}
+		store.pendingKeys[key] = item
+	}
 	store.pendingMutex.Unlock()
+	if item.uploading {
+		logger.Debugf("Key %s is ignored since it's already being uploaded", key)
+		return true
+	}
 	if force || time.Since(added) > store.conf.UploadDelay {
 		select {
-		case store.pendingCh <- pendingItem{key, stagingPath}:
+		case store.pendingCh <- item:
+			item.uploading = true
 			return true
 		default:
 		}
@@ -866,10 +875,11 @@ func (store *cachedStore) scanDelayedStaging() {
 	cutoff := time.Now().Add(-store.conf.UploadDelay)
 	store.pendingMutex.Lock()
 	defer store.pendingMutex.Unlock()
-	for key, added := range store.pendingKeys {
+	for _, item := range store.pendingKeys {
 		store.pendingMutex.Unlock()
-		if added.Before(cutoff) {
-			store.pendingCh <- pendingItem{key, store.bcache.stagePath(key)}
+		if !item.uploading && item.ts.Before(cutoff) {
+			item.uploading = true
+			store.pendingCh <- item
 		}
 		store.pendingMutex.Lock()
 	}
diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go
index 07ae2176..7006fce1 100644
--- a/pkg/fs/fs.go
+++ b/pkg/fs/fs.go
@@ -406,6 +406,13 @@ func (fs *FileSystem) Stat(ctx meta.Context, path string) (fi *FileStat, err sys
 	defer trace.StartRegion(context.TODO(), "fs.Stat").End()
 	l := vfs.NewLogContext(ctx)
 	defer func() { fs.log(l, "Stat (%s): %s", path, errstr(err)) }()
+	return fs.resolve(ctx, path, true)
+}
+
+func (fs *FileSystem) Lstat(ctx meta.Context, path string) (fi *FileStat, err syscall.Errno) {
+	defer trace.StartRegion(context.TODO(), "fs.Lstat").End()
+	l := vfs.NewLogContext(ctx)
+	defer func() { fs.log(l, "Lstat (%s): %s", path, errstr(err)) }()
 	return fs.resolve(ctx, path, false)
 }
 
@@ -716,6 +723,10 @@ func (fs *FileSystem) resolve(ctx meta.Context, p string, followLastSymlink bool
 			p = strings.TrimRight(p, "/")
 			ss := strings.Split(p, "/")
 			fi.name = ss[len(ss)-1]
+			if fi.IsSymlink() && followLastSymlink {
+				// fast resolve can't follow symlink
+				err = syscall.ENOTSUP
+			}
 		}
 		if err != syscall.ENOTSUP {
 			return
@@ -756,7 +767,6 @@ func (fs *FileSystem) resolve(ctx meta.Context, p string, followLastSymlink bool
 			return
 		}
 		fi = AttrToFileInfo(inode, attr)
-		fi.name = name
 		if (!resolved || followLastSymlink) && fi.IsSymlink() {
 			var buf []byte
 			err = fs.m.ReadLink(ctx, inode, &buf)
@@ -775,6 +785,7 @@ func (fs *FileSystem) resolve(ctx meta.Context, p string, followLastSymlink bool
 			inode = fi.Inode()
 			attr = fi.attr
 		}
+		fi.name = name
 		parent = inode
 	}
 	if parent == meta.RootInode {
diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go
index b3adea82..0e1b9eea 100644
--- a/pkg/gateway/gateway.go
+++ b/pkg/gateway/gateway.go
@@ -558,17 +558,20 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 		etag, _ = n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)
 	}
 	size := fi.Size()
+	var contentType string
 	if fi.IsDir() {
 		size = 0
+		contentType = "application/octet-stream"
 	}
 	return minio.ObjectInfo{
-		Bucket:  bucket,
-		Name:    object,
-		ModTime: fi.ModTime(),
-		Size:    size,
-		IsDir:   fi.IsDir(),
-		AccTime: fi.ModTime(),
-		ETag:    string(etag),
+		Bucket:      bucket,
+		Name:        object,
+		ModTime:     fi.ModTime(),
+		Size:        size,
+		IsDir:       fi.IsDir(),
+		AccTime:     fi.ModTime(),
+		ETag:        string(etag),
+		ContentType: contentType,
 	}, nil
 }
 
diff --git a/pkg/meta/base.go b/pkg/meta/base.go
index fc36ab12..17a85583 100644
--- a/pkg/meta/base.go
+++ b/pkg/meta/base.go
@@ -19,6 +19,7 @@ package meta
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"runtime"
 	"sort"
 	"strings"
@@ -50,7 +51,7 @@ type engine interface {
 	doLoad() ([]byte, error)
 
 	doNewSession(sinfo []byte) error
-	doRefreshSession()
+	doRefreshSession() error
 	doFindStaleSessions(limit int) ([]uint64, error) // limit < 0 means all
 	doCleanStaleSession(sid uint64) error
 
@@ -100,6 +101,7 @@ type baseMeta struct {
 	removedFiles map[Ino]bool
 	compacting   map[uint64]bool
 	maxDeleting  chan struct{}
+	dslices      chan Slice // slices to delete
 	symlinks     *sync.Map
 	msgCallbacks *msgCallbacks
 	umounting    bool
@@ -251,6 +253,24 @@ func (m *baseMeta) Load(checkVersion bool) (*Format, error) {
 	return &m.fmt, nil
 }
 
+func (m *baseMeta) newSessionInfo() []byte {
+	host, err := os.Hostname()
+	if err != nil {
+		logger.Warnf("Failed to get hostname: %s", err)
+		host = ""
+	}
+	buf, err := json.Marshal(&SessionInfo{
+		Version:    version.Version(),
+		HostName:   host,
+		MountPoint: m.conf.MountPoint,
+		ProcessID:  os.Getpid(),
+	})
+	if err != nil {
+		panic(err) // marshal SessionInfo should never fail
+	}
+	return buf
+}
+
 func (m *baseMeta) NewSession() error {
 	go m.refreshUsage()
 	if m.conf.ReadOnly {
@@ -263,18 +283,22 @@ func (m *baseMeta) NewSession() error {
 		return fmt.Errorf("get session ID: %s", err)
 	}
 	m.sid = uint64(v)
-	info := newSessionInfo()
-	info.MountPoint = m.conf.MountPoint
-	data, err := json.Marshal(info)
-	if err != nil {
-		return fmt.Errorf("json: %s", err)
-	}
-	if err = m.en.doNewSession(data); err != nil {
+	if err = m.en.doNewSession(m.newSessionInfo()); err != nil {
 		return fmt.Errorf("create session: %s", err)
 	}
 	logger.Infof("Create session %d OK with version: %s", m.sid, version.Version())
 
 	go m.refreshSession()
+	if m.conf.MaxDeletes > 0 {
+		m.dslices = make(chan Slice, m.conf.MaxDeletes*10240)
+		for i := 0; i < m.conf.MaxDeletes; i++ {
+			go func() {
+				for s := range m.dslices {
+					m.deleteSlice_(s.Id, s.Size)
+				}
+			}()
+		}
+	}
 	if !m.conf.NoBGJob {
 		go m.cleanupDeletedFiles()
 		go m.cleanupSlices()
@@ -295,7 +319,9 @@ func (m *baseMeta) refreshSession() {
 			m.sesMu.Unlock()
 			return
 		}
-		m.en.doRefreshSession()
+		if err := m.en.doRefreshSession(); err != nil {
+			logger.Errorf("Refresh session %d: %s", m.sid, err)
+		}
 		m.sesMu.Unlock()
 
 		old := m.fmt.UUID
@@ -977,16 +1003,24 @@ func (m *baseMeta) tryDeleteFileData(inode Ino, length uint64, force bool) {
 	}()
 }
 
+func (m *baseMeta) deleteSlice_(id uint64, size uint32) {
+	if err := m.newMsg(DeleteSlice, id, size); err != nil {
+		logger.Warnf("Delete data blocks of slice %d (%d bytes): %s", id, size, err)
+		return
+	}
+	if err := m.en.doDeleteSlice(id, size); err != nil {
+		logger.Errorf("Delete meta entry of slice %d (%d bytes): %s", id, size, err)
+	}
+}
+
 func (m *baseMeta) deleteSlice(id uint64, size uint32) {
-	if id == 0 {
+	if id == 0 || m.conf.MaxDeletes == 0 {
 		return
 	}
-	if err := m.newMsg(DeleteSlice, id, size); err == nil || strings.Contains(err.Error(), "NoSuchKey") || strings.Contains(err.Error(), "not found") {
-		if err = m.en.doDeleteSlice(id, size); err != nil {
-			logger.Errorf("delete slice %d: %s", id, err)
-		}
-	} else if !strings.Contains(err.Error(), "skip deleting") {
-		logger.Warnf("delete slice %d (%d bytes): %s", id, size, err)
+	if m.dslices != nil {
+		m.dslices <- Slice{Id: id, Size: size}
+	} else {
+		m.deleteSlice_(id, size)
 	}
 }
 
diff --git a/pkg/meta/config.go b/pkg/meta/config.go
index fe4a760b..36923ff5 100644
--- a/pkg/meta/config.go
+++ b/pkg/meta/config.go
@@ -34,6 +34,7 @@ import (
 type Config struct {
 	Strict      bool // update ctime
 	Retries     int
+	MaxDeletes  int
 	CaseInsensi bool
 	ReadOnly    bool
 	NoBGJob     bool // disable background jobs like clean-up, backup, etc.
diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go
index f048fef0..898abb6a 100644
--- a/pkg/meta/interface.go
+++ b/pkg/meta/interface.go
@@ -27,7 +27,6 @@ import (
 	"time"
 
 	"github.com/juicedata/juicefs/pkg/utils"
-	"github.com/juicedata/juicefs/pkg/version"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -411,15 +410,6 @@ func NewClient(uri string, conf *Config) Meta {
 	return m
 }
 
-func newSessionInfo() *SessionInfo {
-	host, err := os.Hostname()
-	if err != nil {
-		logger.Warnf("Failed to get hostname: %s", err)
-		host = ""
-	}
-	return &SessionInfo{Version: version.Version(), HostName: host, ProcessID: os.Getpid()}
-}
-
 // Get all paths of an inode
 func GetPaths(m Meta, ctx Context, inode Ino) []string {
 	if inode == RootInode {
diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go
index bc59309a..481cd6be 100644
--- a/pkg/meta/redis.go
+++ b/pkg/meta/redis.go
@@ -101,6 +101,7 @@ func newRedisMeta(driver, addr string, conf *Config) (Meta, error) {
 	readTimeout := query.duration("read-timeout", "read_timeout", time.Second*30)
 	writeTimeout := query.duration("write-timeout", "write_timeout", time.Second*5)
 	routeRead := query.pop("route-read")
+	skipVerify := query.pop("insecure-skip-verify")
 	u.RawQuery = values.Encode()
 
 	hosts := u.Host
@@ -108,6 +109,10 @@ func newRedisMeta(driver, addr string, conf *Config) (Meta, error) {
 	if err != nil {
 		return nil, fmt.Errorf("redis parse %s: %s", uri, err)
 	}
+	if opt.TLSConfig != nil {
+		opt.TLSConfig.ServerName = "" // use the host of each connection as ServerName
+		opt.TLSConfig.InsecureSkipVerify = skipVerify != ""
+	}
 	if opt.Password == "" {
 		opt.Password = os.Getenv("REDIS_PASSWORD")
 	}
@@ -1489,8 +1494,9 @@ func (m *redisMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentD
 			if rs[3] == nil {
 				logger.Warnf("no attribute for inode %d (%d, %s)", dino, parentDst, nameDst)
 				trash = 0
+			} else {
+				m.parseAttr([]byte(rs[3].(string)), &tattr)
 			}
-			m.parseAttr([]byte(rs[3].(string)), &tattr)
 			tattr.Ctime = now.Unix()
 			tattr.Ctimensec = uint32(now.Nanosecond())
 			if exchange {
@@ -1899,10 +1905,21 @@ func (m *redisMeta) doFindStaleSessions(limit int) ([]uint64, error) {
 	return sids, nil
 }
 
-func (m *redisMeta) doRefreshSession() {
-	m.rdb.ZAdd(Background, m.allSessions(), &redis.Z{
+func (m *redisMeta) doRefreshSession() error {
+	ctx := Background
+	ssid := strconv.FormatUint(m.sid, 10)
+	// we have to check sessionInfo here because the operations are not within a transaction
+	ok, err := m.rdb.HExists(ctx, m.sessionInfos(), ssid).Result()
+	if err == nil && !ok {
+		logger.Warnf("Session %d was stale and cleaned up, but now it comes back again", m.sid)
+		err = m.rdb.HSet(ctx, m.sessionInfos(), m.sid, m.newSessionInfo()).Err()
+	}
+	if err != nil {
+		return err
+	}
+	return m.rdb.ZAdd(ctx, m.allSessions(), &redis.Z{
 		Score:  float64(m.expireTime()),
-		Member: strconv.FormatUint(m.sid, 10)})
+		Member: ssid}).Err()
 }
 
 func (m *redisMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {
@@ -1944,7 +1961,7 @@ func (m *redisMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) s
 		return 0
 	}
 	defer m.timeit(time.Now())
-	vals, err := m.rdb.LRange(ctx, m.chunkKey(inode, indx), 0, 1000000).Result()
+	vals, err := m.rdb.LRange(ctx, m.chunkKey(inode, indx), 0, -1).Result()
 	if err != nil {
 		return errno(err)
 	}
@@ -2071,19 +2088,18 @@ func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino,
 		attr.Ctime = now.Unix()
 		attr.Ctimensec = uint32(now.Nanosecond())
 
-		p := tx.Pipeline()
+		var vals [][]string
 		for i := offIn / ChunkSize; i <= (offIn+size)/ChunkSize; i++ {
-			p.LRange(ctx, m.chunkKey(fin, uint32(i)), 0, 1000000)
-		}
-		vals, err := p.Exec(ctx)
-		if err != nil {
-			return err
+			val, err := tx.LRange(ctx, m.chunkKey(fin, uint32(i)), 0, -1).Result()
+			if err != nil {
+				return err
+			}
+			vals = append(vals, val)
 		}
 
 		_, err = tx.Pipelined(ctx, func(pipe redis.Pipeliner) error {
 			coff := offIn / ChunkSize * ChunkSize
-			for _, v := range vals {
-				sv := v.(*redis.StringSliceCmd).Val()
+			for _, sv := range vals {
 				// Add a zero chunk for hole
 				ss := readSlices(sv)
 				if ss == nil {
@@ -2330,9 +2346,9 @@ func (m *redisMeta) deleteChunk(inode Ino, indx uint32) error {
 	err := m.txn(ctx, func(tx *redis.Tx) error {
 		todel = todel[:0]
 		rs = rs[:0]
-		vals, err := tx.LRange(ctx, key, 0, 1000000).Result()
-		if err == redis.Nil {
-			return nil
+		vals, err := tx.LRange(ctx, key, 0, -1).Result()
+		if err != nil || len(vals) == 0 {
+			return err
 		}
 		slices := readSlices(vals)
 		if slices == nil {
@@ -2745,7 +2761,7 @@ func (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool,
 	p := m.rdb.Pipeline()
 	err := m.scan(ctx, "c*_*", func(keys []string) error {
 		for _, key := range keys {
-			_ = p.LRange(ctx, key, 0, 100000000)
+			_ = p.LRange(ctx, key, 0, -1)
 		}
 		cmds, err := p.Exec(ctx)
 		if err != nil {
@@ -2915,7 +2931,7 @@ func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {
 			xr[i] = p.HGetAll(ctx, m.xattrKey(inode))
 			switch e.Attr.Type {
 			case "regular":
-				cr[i] = p.LRange(ctx, m.chunkKey(inode, 0), 0, 1000000)
+				cr[i] = p.LRange(ctx, m.chunkKey(inode, 0), 0, -1)
 			case "directory":
 				dr[i] = p.HGetAll(ctx, m.entryKey(inode))
 			case "symlink":
@@ -3022,7 +3038,7 @@ func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {
 			}
 			for i := range cr {
 				c := lcs[i]
-				cr[i] = p.LRange(ctx, m.chunkKey(c.inode, c.indx), 0, 1000000)
+				cr[i] = p.LRange(ctx, m.chunkKey(c.inode, c.indx), 0, -1)
 			}
 			if _, err := p.Exec(ctx); err != nil {
 				return err
diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go
index 167af530..2e64b94c 100644
--- a/pkg/meta/sql.go
+++ b/pkg/meta/sql.go
@@ -1894,14 +1894,12 @@ func (m *dbMeta) doFindStaleSessions(limit int) ([]uint64, error) {
 	return sids, nil
 }
 
-func (m *dbMeta) doRefreshSession() {
-	_ = m.txn(func(ses *xorm.Session) error {
+func (m *dbMeta) doRefreshSession() error {
+	return m.txn(func(ses *xorm.Session) error {
 		n, err := ses.Cols("Expire").Update(&session2{Expire: m.expireTime()}, &session2{Sid: m.sid})
 		if err == nil && n == 0 {
-			err = fmt.Errorf("no session found matching sid: %d", m.sid)
-		}
-		if err != nil {
-			logger.Errorf("update session: %s", err)
+			logger.Warnf("Session %d was stale and cleaned up, but now it comes back again", m.sid)
+			err = mustInsert(ses, &session2{m.sid, m.expireTime(), m.newSessionInfo()})
 		}
 		return err
 	})
diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go
index 3714a279..58e5833e 100644
--- a/pkg/meta/tkv.go
+++ b/pkg/meta/tkv.go
@@ -59,6 +59,7 @@ type tkvClient interface {
 	reset(prefix []byte) error
 	close() error
 	shouldRetry(err error) bool
+	gc()
 }
 
 type kvMeta struct {
@@ -400,8 +401,16 @@ func (m *kvMeta) doNewSession(sinfo []byte) error {
 	return nil
 }
 
-func (m *kvMeta) doRefreshSession() {
-	_ = m.setValue(m.sessionKey(m.sid), m.packInt64(m.expireTime()))
+func (m *kvMeta) doRefreshSession() error {
+	return m.txn(func(tx kvTxn) error {
+		buf := tx.get(m.sessionKey(m.sid))
+		if buf == nil {
+			logger.Warnf("Session %d was stale and cleaned up, but now it comes back again", m.sid)
+			tx.set(m.sessionInfoKey(m.sid), m.newSessionInfo())
+		}
+		tx.set(m.sessionKey(m.sid), m.packInt64(m.expireTime()))
+		return nil
+	})
 }
 
 func (m *kvMeta) doCleanStaleSession(sid uint64) error {
@@ -1859,6 +1868,9 @@ func (m *kvMeta) doFindDeletedFiles(ts int64, limit int) (map[Ino]uint64, error)
 }
 
 func (m *kvMeta) doCleanupSlices() {
+	if m.Name() == "tikv" {
+		m.client.gc()
+	}
 	klen := 1 + 8 + 4
 	vals, _ := m.scanValues(m.fmtKey("K"), -1, func(k, v []byte) bool {
 		// filter out invalid ones
diff --git a/pkg/meta/tkv_badger.go b/pkg/meta/tkv_badger.go
index bab49d14..fe78f1a7 100644
--- a/pkg/meta/tkv_badger.go
+++ b/pkg/meta/tkv_badger.go
@@ -266,6 +266,8 @@ func (c *badgerClient) close() error {
 	return c.client.Close()
 }
 
+func (c *badgerClient) gc() {}
+
 func newBadgerClient(addr string) (tkvClient, error) {
 	opt := badger.DefaultOptions(addr)
 	opt.Logger = utils.GetLogger("badger")
diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go
index 0a24b655..bdbc7e7c 100644
--- a/pkg/meta/tkv_etcd.go
+++ b/pkg/meta/tkv_etcd.go
@@ -342,6 +342,8 @@ func (c *etcdClient) close() error {
 	return c.client.Close()
 }
 
+func (c *etcdClient) gc() {}
+
 func buildTlsConfig(u *url.URL) (*tls.Config, error) {
 	var tsinfo transport.TLSInfo
 	q := u.Query()
diff --git a/pkg/meta/tkv_mem.go b/pkg/meta/tkv_mem.go
index e1fe4a08..f2c8b848 100644
--- a/pkg/meta/tkv_mem.go
+++ b/pkg/meta/tkv_mem.go
@@ -332,3 +332,5 @@ func (c *memKV) reset(prefix []byte) error {
 func (c *memKV) close() error {
 	return nil
 }
+
+func (c *memKV) gc() {}
diff --git a/pkg/meta/tkv_tikv.go b/pkg/meta/tkv_tikv.go
index c2c48b1d..6c20aaf3 100644
--- a/pkg/meta/tkv_tikv.go
+++ b/pkg/meta/tkv_tikv.go
@@ -23,12 +23,14 @@ import (
 	"context"
 	"net/url"
 	"strings"
+	"time"
 
 	plog "github.com/pingcap/log"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/tikv/client-go/v2/config"
 	tikverr "github.com/tikv/client-go/v2/error"
+	"github.com/tikv/client-go/v2/oracle"
 	"github.com/tikv/client-go/v2/tikv"
 	"github.com/tikv/client-go/v2/txnkv"
 	"github.com/tikv/client-go/v2/txnkv/txnutil"
@@ -61,20 +63,30 @@ func newTikvClient(addr string) (tkvClient, error) {
 	if err != nil {
 		return nil, err
 	}
+	query := tUrl.Query()
 	config.UpdateGlobal(func(conf *config.Config) {
-		q := tUrl.Query()
 		conf.Security = config.NewSecurity(
-			q.Get("ca"),
-			q.Get("cert"),
-			q.Get("key"),
-			strings.Split(q.Get("verify-cn"), ","))
+			query.Get("ca"),
+			query.Get("cert"),
+			query.Get("key"),
+			strings.Split(query.Get("verify-cn"), ","))
 	})
+	interval := time.Hour * 3
+	if dur, err := time.ParseDuration(query.Get("gc-interval")); err == nil {
+		if dur != 0 && dur < time.Hour {
+			logger.Warnf("TiKV gc-interval (%s) is too short, and is reset to 1h", dur)
+			dur = time.Hour
+		}
+		interval = dur
+	}
+	logger.Infof("TiKV gc interval is set to %s", interval)
+
 	client, err := txnkv.NewClient(strings.Split(tUrl.Host, ","))
 	if err != nil {
 		return nil, err
 	}
 	prefix := strings.TrimLeft(tUrl.Path, "/")
-	return withPrefix(&tikvClient{client.KVStore}, append([]byte(prefix), 0xFD)), nil
+	return withPrefix(&tikvClient{client.KVStore, interval}, append([]byte(prefix), 0xFD)), nil
 }
 
 type tikvTxn struct {
@@ -224,7 +236,8 @@ func (tx *tikvTxn) dels(keys ...[]byte) {
 }
 
 type tikvClient struct {
-	client *tikv.KVStore
+	client     *tikv.KVStore
+	gcInterval time.Duration
 }
 
 func (c *tikvClient) name() string {
@@ -292,3 +305,17 @@ func (c *tikvClient) reset(prefix []byte) error {
 func (c *tikvClient) close() error {
 	return c.client.Close()
 }
+
+func (c *tikvClient) gc() {
+	if c.gcInterval == 0 {
+		return
+	}
+	go func() {
+		safePoint, err := c.client.GC(context.Background(), oracle.GoTimeToTS(time.Now().Add(-c.gcInterval)))
+		if err == nil {
+			logger.Debugf("TiKV GC returns new safe point: %d (%s)", safePoint, oracle.GetTimeFromTS(safePoint))
+		} else {
+			logger.Warnf("TiKV GC: %s", err)
+		}
+	}()
+}
diff --git a/pkg/object/b2.go b/pkg/object/b2.go
index 63bf95be..7933e24a 100644
--- a/pkg/object/b2.go
+++ b/pkg/object/b2.go
@@ -46,11 +46,17 @@ func (c *b2client) Create() error {
 }
 
 func (c *b2client) getFileInfo(key string) (*backblaze.File, error) {
-	f, r, err := c.bucket.DownloadFileRangeByName(key, &backblaze.FileRange{Start: 0, End: 1})
+	var f *backblaze.File
+	var r io.ReadCloser
+	var err error
+	f, r, err = c.bucket.DownloadFileRangeByName(key, &backblaze.FileRange{Start: 0, End: 1})
 	if err != nil {
-		if e, ok := err.(backblaze.B2Error); ok && e.Status == http.StatusNotFound {
-			err = os.ErrNotExist
+		//	get empty file info
+		if e, ok := err.(*backblaze.B2Error); ok && e.Status == http.StatusRequestedRangeNotSatisfiable {
+			f, r, err = c.bucket.DownloadFileRangeByName(key, nil)
 		}
+	}
+	if err != nil {
 		return nil, err
 	}
 	var buf [2]byte
diff --git a/pkg/object/ceph.go b/pkg/object/ceph.go
index bff390ff..24a48eab 100644
--- a/pkg/object/ceph.go
+++ b/pkg/object/ceph.go
@@ -162,9 +162,13 @@ func (c *ceph) Put(key string, in io.Reader) error {
 }
 
 func (c *ceph) Delete(key string) error {
-	return c.do(func(ctx *rados.IOContext) error {
+	err := c.do(func(ctx *rados.IOContext) error {
 		return ctx.Delete(key)
 	})
+	if err == rados.ErrNotFound {
+		err = nil
+	}
+	return err
 }
 
 func (c *ceph) Head(key string) (Object, error) {
diff --git a/pkg/object/cos.go b/pkg/object/cos.go
index 2557286b..b383767b 100644
--- a/pkg/object/cos.go
+++ b/pkg/object/cos.go
@@ -30,6 +30,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/tencentyun/cos-go-sdk-v5"
 )
 
@@ -76,20 +78,15 @@ func (c *COS) Head(key string) (Object, error) {
 }
 
 func (c *COS) Get(key string, off, limit int64) (io.ReadCloser, error) {
-	params := &cos.ObjectGetOptions{}
-	if off > 0 || limit > 0 {
-		var r string
-		if limit > 0 {
-			r = fmt.Sprintf("bytes=%d-%d", off, off+limit-1)
-		} else {
-			r = fmt.Sprintf("bytes=%d-", off)
-		}
-		params.Range = r
-	}
+	params := &cos.ObjectGetOptions{Range: getRange(off, limit)}
 	resp, err := c.c.Object.Get(ctx, key, params)
 	if err != nil {
 		return nil, err
 	}
+	if err = checkGetStatus(resp.StatusCode, params.Range != ""); err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
 	if off == 0 && limit == -1 {
 		resp.Body = verifyChecksum(resp.Body, resp.Header.Get(cosChecksumKey))
 	}
@@ -121,13 +118,16 @@ func (c *COS) Delete(key string) error {
 
 func (c *COS) List(prefix, marker string, limit int64) ([]Object, error) {
 	param := cos.BucketGetOptions{
-		Prefix:  prefix,
-		Marker:  marker,
-		MaxKeys: int(limit),
+		Prefix:       prefix,
+		Marker:       marker,
+		MaxKeys:      int(limit),
+		EncodingType: "url",
 	}
 	resp, _, err := c.c.Bucket.Get(ctx, &param)
 	for err == nil && len(resp.Contents) == 0 && resp.IsTruncated {
-		param.Marker = resp.NextMarker
+		if param.Marker, err = cos.DecodeURIComponent(resp.NextMarker); err != nil {
+			return nil, errors.WithMessagef(err, "failed to decode nextMarker %s", resp.NextMarker)
+		}
 		resp, _, err = c.c.Bucket.Get(ctx, &param)
 	}
 	if err != nil {
@@ -138,7 +138,11 @@ func (c *COS) List(prefix, marker string, limit int64) ([]Object, error) {
 	for i := 0; i < n; i++ {
 		o := resp.Contents[i]
 		t, _ := time.Parse(time.RFC3339, o.LastModified)
-		objs[i] = &obj{o.Key, int64(o.Size), t, strings.HasSuffix(o.Key, "/")}
+		key, err := cos.DecodeURIComponent(o.Key)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to decode key %s", o.Key)
+		}
+		objs[i] = &obj{key, int64(o.Size), t, strings.HasSuffix(key, "/")}
 	}
 	return objs, nil
 }
diff --git a/pkg/object/ibmcos.go b/pkg/object/ibmcos.go
index 25cf499a..761f3fb4 100644
--- a/pkg/object/ibmcos.go
+++ b/pkg/object/ibmcos.go
@@ -29,6 +29,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/IBM/ibm-cos-sdk-go/aws"
 	"github.com/IBM/ibm-cos-sdk-go/aws/awserr"
 	"github.com/IBM/ibm-cos-sdk-go/aws/credentials/ibmiam"
@@ -136,10 +138,11 @@ func (s *ibmcos) Delete(key string) error {
 
 func (s *ibmcos) List(prefix, marker string, limit int64) ([]Object, error) {
 	param := s3.ListObjectsInput{
-		Bucket:  &s.bucket,
-		Prefix:  &prefix,
-		Marker:  &marker,
-		MaxKeys: &limit,
+		Bucket:       &s.bucket,
+		Prefix:       &prefix,
+		Marker:       &marker,
+		MaxKeys:      &limit,
+		EncodingType: aws.String("url"),
 	}
 	resp, err := s.s3.ListObjects(&param)
 	if err != nil {
@@ -149,7 +152,11 @@ func (s *ibmcos) List(prefix, marker string, limit int64) ([]Object, error) {
 	objs := make([]Object, n)
 	for i := 0; i < n; i++ {
 		o := resp.Contents[i]
-		objs[i] = &obj{*o.Key, *o.Size, *o.LastModified, strings.HasSuffix(*o.Key, "/")}
+		oKey, err := url.QueryUnescape(*o.Key)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to decode key %s", *o.Key)
+		}
+		objs[i] = &obj{oKey, *o.Size, *o.LastModified, strings.HasSuffix(oKey, "/")}
 	}
 	return objs, nil
 }
diff --git a/pkg/object/ks3.go b/pkg/object/ks3.go
index ed5f5bd7..6db88f09 100644
--- a/pkg/object/ks3.go
+++ b/pkg/object/ks3.go
@@ -29,6 +29,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/juicedata/juicefs/pkg/utils"
 	"github.com/ks3sdklib/aws-sdk-go/aws"
@@ -141,10 +143,11 @@ func (s *ks3) Delete(key string) error {
 
 func (s *ks3) List(prefix, marker string, limit int64) ([]Object, error) {
 	param := s3.ListObjectsInput{
-		Bucket:  &s.bucket,
-		Prefix:  &prefix,
-		Marker:  &marker,
-		MaxKeys: &limit,
+		Bucket:       &s.bucket,
+		Prefix:       &prefix,
+		Marker:       &marker,
+		MaxKeys:      &limit,
+		EncodingType: aws.String("url"),
 	}
 	resp, err := s.s3.ListObjects(&param)
 	if err != nil {
@@ -154,7 +157,11 @@ func (s *ks3) List(prefix, marker string, limit int64) ([]Object, error) {
 	objs := make([]Object, n)
 	for i := 0; i < n; i++ {
 		o := resp.Contents[i]
-		objs[i] = &obj{*o.Key, *o.Size, *o.LastModified, strings.HasSuffix(*o.Key, "/")}
+		oKey, err := url.QueryUnescape(*o.Key)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to decode key %s", *o.Key)
+		}
+		objs[i] = &obj{oKey, *o.Size, *o.LastModified, strings.HasSuffix(oKey, "/")}
 	}
 	return objs, nil
 }
diff --git a/pkg/object/obs.go b/pkg/object/obs.go
index 95f36771..40c0e702 100644
--- a/pkg/object/obs.go
+++ b/pkg/object/obs.go
@@ -31,6 +31,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
 	"github.com/juicedata/juicefs/pkg/utils"
 	"golang.org/x/net/http/httpproxy"
@@ -85,14 +87,21 @@ func (s *obsClient) Get(key string, off, limit int64) (io.ReadCloser, error) {
 	params := &obs.GetObjectInput{}
 	params.Bucket = s.bucket
 	params.Key = key
-	params.RangeStart = off
-	if limit > 0 {
-		params.RangeEnd = off + limit - 1
+	var resp *obs.GetObjectOutput
+	var err error
+	rangeStr := getRange(off, limit)
+	if rangeStr != "" {
+		resp, err = s.c.GetObject(params, obs.WithHeader(obs.HEADER_RANGE, []string{rangeStr}))
+	} else {
+		resp, err = s.c.GetObject(params)
 	}
-	resp, err := s.c.GetObject(params)
 	if err != nil {
 		return nil, err
 	}
+	if err = checkGetStatus(resp.StatusCode, rangeStr != ""); err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
 	return resp.Body, nil
 }
 
@@ -165,6 +174,7 @@ func (s *obsClient) List(prefix, marker string, limit int64) ([]Object, error) {
 	}
 	input.Prefix = prefix
 	input.MaxKeys = int(limit)
+	input.EncodingType = "url"
 	resp, err := s.c.ListObjects(input)
 	if err != nil {
 		return nil, err
@@ -173,7 +183,11 @@ func (s *obsClient) List(prefix, marker string, limit int64) ([]Object, error) {
 	objs := make([]Object, n)
 	for i := 0; i < n; i++ {
 		o := resp.Contents[i]
-		objs[i] = &obj{o.Key, o.Size, o.LastModified, strings.HasSuffix(o.Key, "/")}
+		key, err := obs.UrlDecode(o.Key)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to decode key %s", o.Key)
+		}
+		objs[i] = &obj{key, o.Size, o.LastModified, strings.HasSuffix(key, "/")}
 	}
 	return objs, nil
 }
@@ -333,7 +347,7 @@ func newOBS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 	}
 	var checkEtag bool
 	if _, err = c.GetBucketEncryption(bucketName); err != nil {
-		if obsError, ok := err.(*obs.ObsError); ok && obsError.Code == "NoSuchEncryptionConfiguration" {
+		if obsError, ok := err.(obs.ObsError); ok && obsError.Code == "NoSuchEncryptionConfiguration" {
 			checkEtag = true
 		} else if !ok || obsError.Code != "NoSuchBucket" {
 			logger.Warnf("get bucket encryption: %q", err)
diff --git a/pkg/object/prefix.go b/pkg/object/prefix.go
index cbc467cd..82812908 100644
--- a/pkg/object/prefix.go
+++ b/pkg/object/prefix.go
@@ -70,6 +70,9 @@ func (p *withPrefix) Head(key string) (Object, error) {
 }
 
 func (p *withPrefix) Get(key string, off, limit int64) (io.ReadCloser, error) {
+	if off > 0 && limit < 0 {
+		return nil, fmt.Errorf("invalid range: %d-%d", off, limit)
+	}
 	return p.os.Get(p.prefix+key, off, limit)
 }
 
diff --git a/pkg/object/qingstor.go b/pkg/object/qingstor.go
index 56f2491a..4149b861 100644
--- a/pkg/object/qingstor.go
+++ b/pkg/object/qingstor.go
@@ -69,19 +69,18 @@ func (q *qingstor) Head(key string) (Object, error) {
 
 func (q *qingstor) Get(key string, off, limit int64) (io.ReadCloser, error) {
 	input := &qs.GetObjectInput{}
-	if off > 0 || limit > 0 {
-		var r string
-		if limit > 0 {
-			r = fmt.Sprintf("bytes=%d-%d", off, off+limit-1)
-		} else {
-			r = fmt.Sprintf("bytes=%d-", off)
-		}
-		input.Range = &r
+	rangeStr := getRange(off, limit)
+	if rangeStr != "" {
+		input.Range = &rangeStr
 	}
 	output, err := q.bucket.GetObject(key, input)
 	if err != nil {
 		return nil, err
 	}
+	if err = checkGetStatus(*output.StatusCode, rangeStr != ""); err != nil {
+		_ = output.Body.Close()
+		return nil, err
+	}
 	return output.Body, nil
 }
 
diff --git a/pkg/object/restful.go b/pkg/object/restful.go
index 98c9d680..1a6d0a55 100644
--- a/pkg/object/restful.go
+++ b/pkg/object/restful.go
@@ -181,15 +181,33 @@ func (s *RestfulStorage) Head(key string) (Object, error) {
 	}, nil
 }
 
-func (s *RestfulStorage) Get(key string, off, limit int64) (io.ReadCloser, error) {
-	headers := make(map[string]string)
+func getRange(off, limit int64) string {
 	if off > 0 || limit > 0 {
 		if limit > 0 {
-			headers["Range"] = fmt.Sprintf("bytes=%d-%d", off, off+limit-1)
+			return fmt.Sprintf("bytes=%d-%d", off, off+limit-1)
 		} else {
-			headers["Range"] = fmt.Sprintf("bytes=%d-", off)
+			return fmt.Sprintf("bytes=%d-", off)
 		}
 	}
+	return ""
+}
+
+func checkGetStatus(statusCode int, partial bool) error {
+	var expected = http.StatusOK
+	if partial {
+		expected = http.StatusPartialContent
+	}
+	if statusCode != expected {
+		return fmt.Errorf("expected status code %d, but got %d", expected, statusCode)
+	}
+	return nil
+}
+
+func (s *RestfulStorage) Get(key string, off, limit int64) (io.ReadCloser, error) {
+	headers := make(map[string]string)
+	if off > 0 || limit > 0 {
+		headers["Range"] = getRange(off, limit)
+	}
 	resp, err := s.request("GET", key, nil, headers)
 	if err != nil {
 		return nil, err
@@ -197,6 +215,10 @@ func (s *RestfulStorage) Get(key string, off, limit int64) (io.ReadCloser, error
 	if resp.StatusCode != 200 && resp.StatusCode != 206 {
 		return nil, parseError(resp)
 	}
+	if err = checkGetStatus(resp.StatusCode, len(headers) > 0); err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
 	return resp.Body, nil
 }
 
diff --git a/pkg/object/s3.go b/pkg/object/s3.go
index 0da9eeea..7ae2b38e 100644
--- a/pkg/object/s3.go
+++ b/pkg/object/s3.go
@@ -30,6 +30,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -162,7 +164,7 @@ func (s *s3client) Delete(key string) error {
 		Key:    &key,
 	}
 	_, err := s.s3.DeleteObject(&param)
-	if err != nil && strings.Contains(err.Error(), "NoSuckKey") {
+	if err != nil && strings.Contains(err.Error(), "NoSuchKey") {
 		err = nil
 	}
 	return err
@@ -170,10 +172,11 @@ func (s *s3client) Delete(key string) error {
 
 func (s *s3client) List(prefix, marker string, limit int64) ([]Object, error) {
 	param := s3.ListObjectsInput{
-		Bucket:  &s.bucket,
-		Prefix:  &prefix,
-		Marker:  &marker,
-		MaxKeys: &limit,
+		Bucket:       &s.bucket,
+		Prefix:       &prefix,
+		Marker:       &marker,
+		MaxKeys:      &limit,
+		EncodingType: aws.String("url"),
 	}
 	resp, err := s.s3.ListObjects(&param)
 	if err != nil {
@@ -183,14 +186,18 @@ func (s *s3client) List(prefix, marker string, limit int64) ([]Object, error) {
 	objs := make([]Object, n)
 	for i := 0; i < n; i++ {
 		o := resp.Contents[i]
-		if !strings.HasPrefix(*o.Key, prefix) || *o.Key < marker {
-			return nil, fmt.Errorf("found invalid key %s from List, prefix: %s, marker: %s", *o.Key, prefix, marker)
+		oKey, err := url.QueryUnescape(*o.Key)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to decode key %s", *o.Key)
+		}
+		if !strings.HasPrefix(oKey, prefix) || oKey < marker {
+			return nil, fmt.Errorf("found invalid key %s from List, prefix: %s, marker: %s", oKey, prefix, marker)
 		}
 		objs[i] = &obj{
-			*o.Key,
+			oKey,
 			*o.Size,
 			*o.LastModified,
-			strings.HasSuffix(*o.Key, "/"),
+			strings.HasSuffix(oKey, "/"),
 		}
 	}
 	return objs, nil
@@ -341,6 +348,9 @@ func parseRegion(endpoint string) string {
 	return region
 }
 
+var oracleCompileRegexp = `.*\.compat.objectstorage\.(.*)\.oraclecloud\.com`
+var OVHCompileRegexp = `^s3\.(\w*)(\.\w*)?\.cloud\.ovh\.net$`
+
 func newS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {
 	if !strings.Contains(endpoint, "://") {
 		if len(strings.Split(endpoint, ".")) > 1 && !strings.HasSuffix(endpoint, ".amazonaws.com") {
@@ -408,10 +418,13 @@ func newS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 				// compatible s3
 				bucketName = hostParts[0]
 				ep = hostParts[1]
-				oracleCompile := regexp.MustCompile(`.*\.compat.objectstorage\.(.*)\.oraclecloud\.com`)
-				if oracleCompile.MatchString(ep) {
-					if submatch := oracleCompile.FindStringSubmatch(ep); len(submatch) == 2 {
-						region = submatch[1]
+				for _, compileRegexp := range []string{oracleCompileRegexp, OVHCompileRegexp} {
+					compile := regexp.MustCompile(compileRegexp)
+					if compile.MatchString(ep) {
+						if submatch := compile.FindStringSubmatch(ep); len(submatch) >= 2 {
+							region = submatch[1]
+							break
+						}
 					}
 				}
 			}
@@ -433,6 +446,12 @@ func newS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 		DisableSSL: aws.Bool(!ssl),
 		HTTPClient: httpClient,
 	}
+
+	disable100Continue := strings.EqualFold(uri.Query().Get("disable-100-continue"), "true")
+	if disable100Continue {
+		awsConfig.S3Disable100Continue = aws.Bool(true)
+	}
+
 	if accessKey == "anonymous" {
 		awsConfig.Credentials = credentials.AnonymousCredentials
 	} else if accessKey != "" {
diff --git a/pkg/object/tos.go b/pkg/object/tos.go
index baf4bc1b..e8730edb 100644
--- a/pkg/object/tos.go
+++ b/pkg/object/tos.go
@@ -52,21 +52,19 @@ func (t tosClient) Create() error {
 }
 
 func (t tosClient) Get(key string, off, limit int64) (io.ReadCloser, error) {
-	var rangeEnd int64
-	if limit > 0 {
-		rangeEnd = off + limit - 1
-	} else {
-		rangeEnd = 0
-	}
+	rangeStr := getRange(off, limit)
 	resp, err := t.client.GetObjectV2(context.Background(), &tos.GetObjectV2Input{
-		Bucket:     t.bucket,
-		Key:        key,
-		RangeStart: off,
-		RangeEnd:   rangeEnd,
+		Bucket: t.bucket,
+		Key:    key,
+		Range:  rangeStr, // When Range and RangeStart & RangeEnd appear together, range is preferred
 	})
 	if err != nil {
 		return nil, err
 	}
+	if err = checkGetStatus(resp.StatusCode, rangeStr != ""); err != nil {
+		_ = resp.Content.Close()
+		return nil, err
+	}
 	return resp.Content, nil
 }
 
diff --git a/pkg/vfs/handle.go b/pkg/vfs/handle.go
index ffd57f4d..edfb52eb 100644
--- a/pkg/vfs/handle.go
+++ b/pkg/vfs/handle.go
@@ -54,6 +54,13 @@ type handle struct {
 	bctx    meta.Context
 }
 
+func (h *handle) Write(buf []byte) (int, error) {
+	h.Lock()
+	defer h.Unlock()
+	h.data = append(h.data, buf...)
+	return len(buf), nil
+}
+
 func (h *handle) addOp(ctx Context) {
 	h.Lock()
 	defer h.Unlock()
diff --git a/pkg/vfs/internal.go b/pkg/vfs/internal.go
index ae5e9353..0b8b3639 100644
--- a/pkg/vfs/internal.go
+++ b/pkg/vfs/internal.go
@@ -19,6 +19,7 @@ package vfs
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 	"strings"
@@ -183,7 +184,7 @@ func collectMetrics(registry *prometheus.Registry) []byte {
 	return w.Bytes()
 }
 
-func writeProgress(count, bytes *uint64, data *[]byte, done chan struct{}) {
+func writeProgress(count, bytes *uint64, out io.Writer, done chan struct{}) {
 	wb := utils.NewBuffer(17)
 	wb.Put8(meta.CPROGRESS)
 	if bytes == nil {
@@ -195,14 +196,14 @@ func writeProgress(count, bytes *uint64, data *[]byte, done chan struct{}) {
 		case <-ticker.C:
 			wb.Put64(atomic.LoadUint64(count))
 			wb.Put64(atomic.LoadUint64(bytes))
-			*data = append(*data, wb.Bytes()...)
+			_, _ = out.Write(wb.Bytes())
 			wb.Seek(1)
 		case <-done:
 			ticker.Stop()
 			if *count > 0 || *bytes > 0 {
 				wb.Put64(atomic.LoadUint64(count))
 				wb.Put64(atomic.LoadUint64(bytes))
-				*data = append(*data, wb.Bytes()...)
+				_, _ = out.Write(wb.Bytes())
 			}
 			return
 		}
@@ -247,7 +248,7 @@ func (v *VFS) caclObjects(id uint64, size, offset, length uint32) []*obj {
 	return objs
 }
 
-func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, data *[]byte) {
+func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, out io.Writer) {
 	switch cmd {
 	case meta.Rmr:
 		done := make(chan struct{})
@@ -262,13 +263,13 @@ func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, d
 			}
 			close(done)
 		}()
-		writeProgress(&count, nil, data, done)
+		writeProgress(&count, nil, out, done)
 		if st == 0 && v.InvalidateEntry != nil {
 			if st = v.InvalidateEntry(inode, name); st != 0 {
 				logger.Errorf("Invalidate entry %d/%s: %s", inode, name, st)
 			}
 		}
-		*data = append(*data, uint8(st))
+		_, _ = out.Write([]byte{uint8(st)})
 	case meta.Info:
 		var summary meta.Summary
 		inode := Ino(r.Get64())
@@ -286,7 +287,7 @@ func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, d
 		if r != 0 {
 			msg := r.Error()
 			wb.Put32(uint32(len(msg)))
-			*data = append(*data, append(wb.Bytes(), msg...)...)
+			_, _ = out.Write(append(wb.Bytes(), msg...))
 			return
 		}
 		var w = bytes.NewBuffer(nil)
@@ -328,7 +329,7 @@ func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, d
 			}
 		}
 		wb.Put32(uint32(w.Len()))
-		*data = append(*data, append(wb.Bytes(), w.Bytes()...)...)
+		_, _ = out.Write(append(wb.Bytes(), w.Bytes()...))
 	case meta.FillCache:
 		paths := strings.Split(string(r.Get(int(r.Get32()))), "\n")
 		concurrent := r.Get16()
@@ -340,13 +341,13 @@ func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, d
 				v.fillCache(ctx, paths, int(concurrent), &count, &bytes)
 				close(done)
 			}()
-			writeProgress(&count, &bytes, data, done)
+			writeProgress(&count, &bytes, out, done)
 		} else {
 			go v.fillCache(meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids()), paths, int(concurrent), nil, nil)
 		}
-		*data = append(*data, uint8(0))
+		_, _ = out.Write([]byte{0})
 	default:
 		logger.Warnf("unknown message type: %d", cmd)
-		*data = append(*data, uint8(syscall.EINVAL&0xff))
+		_, _ = out.Write([]byte{byte(syscall.EINVAL & 0xff)})
 	}
 }
diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go
index 945af841..d19984b0 100644
--- a/pkg/vfs/vfs.go
+++ b/pkg/vfs/vfs.go
@@ -494,6 +494,7 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i
 		if ino == logInode {
 			n = readAccessLog(fh, buf)
 		} else {
+			defer func() { logit(ctx, "read (%d,%d,%d,%d): %s (%d)", ino, size, off, fh, strerr(err), n) }()
 			if ino == controlInode && runtime.GOOS == "darwin" {
 				fh = v.getControlHandle(ctx.Pid())
 			}
@@ -502,25 +503,21 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i
 				err = syscall.EBADF
 				return
 			}
-			data := h.data
+			h.Lock()
+			defer h.Unlock()
 			if off < h.off {
-				data = nil
-			} else {
-				off -= h.off
+				logger.Errorf("read dropped data from %s: %d < %d", ino, off, h.off)
+				err = syscall.EIO
+				return
 			}
-			if int(off) < len(data) {
-				data = data[off:]
-				if int(size) < len(data) {
-					data = data[:size]
-				}
-				n = copy(buf, data)
+			if int(off-h.off) < len(h.data) {
+				n = copy(buf, h.data[off-h.off:])
 			}
-			if len(h.data) > 2<<20 {
+			if len(h.data) > 2<<20 && off-h.off > 1<<20 {
 				// drop first part to avoid OOM
 				h.off += 1 << 20
 				h.data = h.data[1<<20:]
 			}
-			logit(ctx, "read (%d,%d,%d,%d): %s (%d)", ino, size, off, fh, strerr(err), n)
 		}
 		return
 	}
@@ -577,6 +574,8 @@ func (v *VFS) Write(ctx Context, ino Ino, buf []byte, off, fh uint64) (err sysca
 	}
 
 	if ino == controlInode {
+		h.Lock()
+		defer h.Unlock()
 		h.pending = append(h.pending, buf...)
 		rb := utils.ReadBuffer(h.pending)
 		cmd := rb.Get32()
@@ -589,7 +588,7 @@ func (v *VFS) Write(ctx Context, ino Ino, buf []byte, off, fh uint64) (err sysca
 		h.pending = h.pending[:0]
 		if rb.Left() == size {
 			h.bctx = meta.NewContext(ctx.Pid(), ctx.Uid(), ctx.Gids())
-			go v.handleInternalMsg(h.bctx, cmd, rb, &h.data)
+			go v.handleInternalMsg(h.bctx, cmd, rb, h)
 		} else {
 			logger.Warnf("broken message: %d %d < %d", cmd, size, rb.Left())
 			h.data = append(h.data, uint8(syscall.EIO&0xff))
diff --git a/sdk/java/libjfs/main.go b/sdk/java/libjfs/main.go
index 08c6ea1f..b03a287f 100644
--- a/sdk/java/libjfs/main.go
+++ b/sdk/java/libjfs/main.go
@@ -380,12 +380,13 @@ func jfs_init(cname, jsonConf, user, group, superuser, supergroup *C.char) uintp
 		}
 
 		metaConf := &meta.Config{
-			Retries:   jConf.IORetries,
-			Strict:    true,
-			ReadOnly:  jConf.ReadOnly,
-			NoBGJob:   jConf.NoBGJob,
-			OpenCache: time.Duration(jConf.OpenCache * 1e9),
-			Heartbeat: time.Second * time.Duration(jConf.Heartbeat),
+			Retries:    jConf.IORetries,
+			Strict:     true,
+			MaxDeletes: jConf.MaxDeletes,
+			ReadOnly:   jConf.ReadOnly,
+			NoBGJob:    jConf.NoBGJob,
+			OpenCache:  time.Duration(jConf.OpenCache * 1e9),
+			Heartbeat:  time.Second * time.Duration(jConf.Heartbeat),
 		}
 		m := meta.NewClient(jConf.MetaURL, metaConf)
 		format, err := m.Load(true)
@@ -453,7 +454,6 @@ func jfs_init(cname, jsonConf, user, group, superuser, supergroup *C.char) uintp
 			AutoCreate:     jConf.AutoCreate,
 			CacheFullBlock: jConf.CacheFullBlock,
 			MaxUpload:      jConf.MaxUploads,
-			MaxDeletes:     jConf.MaxDeletes,
 			MaxRetries:     jConf.IORetries,
 			UploadLimit:    int64(jConf.UploadLimit) * 1e6 / 8,
 			DownloadLimit:  int64(jConf.DownloadLimit) * 1e6 / 8,
@@ -855,7 +855,7 @@ func jfs_lstat1(pid int, h uintptr, cpath *C.char, buf uintptr) int {
 	if w == nil {
 		return EINVAL
 	}
-	fi, err := w.Stat(w.withPid(pid), C.GoString(cpath))
+	fi, err := w.Lstat(w.withPid(pid), C.GoString(cpath))
 	if err != 0 {
 		return errno(err)
 	}


Test patch:
diff --git a/.github/workflows/sdktest.yml b/.github/workflows/sdktest.yml
index e5a1a738..54ed9022 100644
--- a/.github/workflows/sdktest.yml
+++ b/.github/workflows/sdktest.yml
@@ -54,6 +54,14 @@ jobs:
         run: |
           sudo ./juicefs format  localhost dev
 
+      - name: Juicefs Mount
+        run: |
+          sudo ./juicefs mount -d localhost /jfs
+          touch /jfs/inner_sym_target
+          echo "hello juicefs" > /jfs/inner_sym_target
+          cd /jfs
+          ln -s inner_sym_target inner_sym_link 
+
       - name: Sdk Test
         run: |
           make -C sdk/java/libjfs
diff --git a/pkg/chunk/cached_store_test.go b/pkg/chunk/cached_store_test.go
index dfdbc2d6..e04946b5 100644
--- a/pkg/chunk/cached_store_test.go
+++ b/pkg/chunk/cached_store_test.go
@@ -95,7 +95,6 @@ var defaultConf = Config{
 	CacheDir:   filepath.Join(os.TempDir(), "diskCache"),
 	CacheSize:  1,
 	MaxUpload:  1,
-	MaxDeletes: 1,
 	MaxRetries: 10,
 	PutTimeout: time.Second,
 	GetTimeout: time.Second * 2,
diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go
index 2d3725d8..4af2653d 100644
--- a/pkg/fs/fs_test.go
+++ b/pkg/fs/fs_test.go
@@ -169,9 +169,12 @@ func TestFileSystem(t *testing.T) {
 	if target, e := fs.Readlink(ctx, "/sym"); e != 0 || string(target) != "hello" {
 		t.Fatalf("readlink: %s", string(target))
 	}
-	if fi, err := fs.Stat(ctx, "/sym"); err != 0 || fi.name != "sym" {
+	if fi, err := fs.Stat(ctx, "/sym"); err != 0 || fi.name != "sym" || fi.IsSymlink() {
 		t.Fatalf("stat symlink: %s %+v", err, fi)
 	}
+	if fi, err := fs.Lstat(ctx, "/sym"); err != 0 || fi.name != "sym" || !fi.IsSymlink() {
+		t.Fatalf("lstat symlink: %s %+v", err, fi)
+	}
 	if err := fs.Delete(ctx, "/sym"); err != 0 {
 		t.Fatalf("delete /sym: %s", err)
 	}
@@ -276,11 +279,10 @@ func createTestFS(t *testing.T) *FileSystem {
 	}
 	_ = m.Init(format, true)
 	var conf = vfs.Config{
-		Meta: &meta.Config{},
+		Meta: &meta.Config{MaxDeletes: 2},
 		Chunk: &chunk.Config{
 			BlockSize:  format.BlockSize << 10,
 			MaxUpload:  1,
-			MaxDeletes: 1,
 			BufferSize: 100 << 20,
 		},
 		DirEntryTimeout: time.Millisecond * 100,
diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go
index 3f47f2ba..6eef8ec6 100644
--- a/pkg/meta/base_test.go
+++ b/pkg/meta/base_test.go
@@ -34,7 +34,7 @@ import (
 )
 
 func TestRedisClient(t *testing.T) {
-	var conf = Config{}
+	var conf = Config{MaxDeletes: 2}
 	m, err := newRedisMeta("redis", "127.0.0.1:6379/10", &conf)
 	if err != nil || m.Name() != "redis" {
 		t.Fatalf("create meta: %s", err)
@@ -43,7 +43,7 @@ func TestRedisClient(t *testing.T) {
 }
 
 func TestRedisCluster(t *testing.T) {
-	var conf = Config{}
+	var conf = Config{MaxDeletes: 2}
 	m, err := newRedisMeta("redis", "127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003/2", &conf)
 	if err != nil {
 		t.Fatalf("create meta: %s", err)
@@ -906,10 +906,10 @@ func testCompaction(t *testing.T, m Meta, trash bool) {
 		t.Fatalf("list all slices: %s", st)
 	}
 
-	l.Lock()
-	deletes := len(deleted)
-	l.Unlock()
 	if trash {
+		l.Lock()
+		deletes := len(deleted)
+		l.Unlock()
 		if deletes > 10 {
 			t.Fatalf("deleted slices %d is greater than 10", deletes)
 		}
@@ -917,10 +917,11 @@ func testCompaction(t *testing.T, m Meta, trash bool) {
 			t.Fatalf("list delayed slices %d is less than 200", len(sliceMap[1]))
 		}
 		m.(engine).doCleanupDelayedSlices(time.Now().Unix() + 1)
-		l.Lock()
-		deletes = len(deleted)
-		l.Unlock()
 	}
+	time.Sleep(time.Second * 3) // wait for all slices deleted
+	l.Lock()
+	deletes := len(deleted)
+	l.Unlock()
 	if deletes < 200 {
 		t.Fatalf("deleted slices %d is less than 200", deletes)
 	}
diff --git a/pkg/meta/benchmarks_test.go b/pkg/meta/benchmarks_test.go
index c88c757d..490efc18 100644
--- a/pkg/meta/benchmarks_test.go
+++ b/pkg/meta/benchmarks_test.go
@@ -631,16 +631,16 @@ func benchmarkAll(b *testing.B, m Meta) {
 }
 
 func BenchmarkRedis(b *testing.B) {
-	m := NewClient(redisAddr, &Config{})
+	m := NewClient(redisAddr, &Config{MaxDeletes: 2})
 	benchmarkAll(b, m)
 }
 
 func BenchmarkSQL(b *testing.B) {
-	m := NewClient(sqlAddr, &Config{})
+	m := NewClient(sqlAddr, &Config{MaxDeletes: 2})
 	benchmarkAll(b, m)
 }
 
 func BenchmarkTKV(b *testing.B) {
-	m := NewClient(tkvAddr, &Config{})
+	m := NewClient(tkvAddr, &Config{MaxDeletes: 2})
 	benchmarkAll(b, m)
 }
diff --git a/pkg/meta/sql_test.go b/pkg/meta/sql_test.go
index ff05e5d0..15413b04 100644
--- a/pkg/meta/sql_test.go
+++ b/pkg/meta/sql_test.go
@@ -23,7 +23,7 @@ import (
 )
 
 func TestSQLiteClient(t *testing.T) {
-	m, err := newSQLMeta("sqlite3", path.Join(t.TempDir(), "jfs-unit-test.db"), &Config{})
+	m, err := newSQLMeta("sqlite3", path.Join(t.TempDir(), "jfs-unit-test.db"), &Config{MaxDeletes: 2})
 	if err != nil || m.Name() != "sqlite3" {
 		t.Fatalf("create meta: %s", err)
 	}
@@ -31,7 +31,7 @@ func TestSQLiteClient(t *testing.T) {
 }
 
 func TestMySQLClient(t *testing.T) {
-	m, err := newSQLMeta("mysql", "root:@/dev", &Config{})
+	m, err := newSQLMeta("mysql", "root:@/dev", &Config{MaxDeletes: 2})
 	if err != nil || m.Name() != "mysql" {
 		t.Fatalf("create meta: %s", err)
 	}
@@ -39,7 +39,7 @@ func TestMySQLClient(t *testing.T) {
 }
 
 func TestPostgreSQLClient(t *testing.T) {
-	m, err := newSQLMeta("postgres", "localhost:5432/test?sslmode=disable", &Config{})
+	m, err := newSQLMeta("postgres", "localhost:5432/test?sslmode=disable", &Config{MaxDeletes: 2})
 	if err != nil || m.Name() != "postgres" {
 		t.Fatalf("create meta: %s", err)
 	}
diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go
index daa6e9b6..dd94f150 100644
--- a/pkg/meta/tkv_test.go
+++ b/pkg/meta/tkv_test.go
@@ -25,7 +25,7 @@ import (
 
 func TestMemKVClient(t *testing.T) {
 	_ = os.Remove(settingPath)
-	m, err := newKVMeta("memkv", "jfs-unit-test", &Config{})
+	m, err := newKVMeta("memkv", "jfs-unit-test", &Config{MaxDeletes: 2})
 	if err != nil || m.Name() != "memkv" {
 		t.Fatalf("create meta: %s", err)
 	}
@@ -33,7 +33,7 @@ func TestMemKVClient(t *testing.T) {
 }
 
 func TestTiKVClient(t *testing.T) {
-	m, err := newKVMeta("tikv", "127.0.0.1:2379/jfs-unit-test", &Config{})
+	m, err := newKVMeta("tikv", "127.0.0.1:2379/jfs-unit-test", &Config{MaxDeletes: 2})
 	if err != nil || m.Name() != "tikv" {
 		t.Fatalf("create meta: %s", err)
 	}
@@ -41,7 +41,7 @@ func TestTiKVClient(t *testing.T) {
 }
 
 func TestBadgerClient(t *testing.T) {
-	m, err := newKVMeta("badger", "badger", &Config{})
+	m, err := newKVMeta("badger", "badger", &Config{MaxDeletes: 2})
 	if err != nil || m.Name() != "badger" {
 		t.Fatalf("create meta: %s", err)
 	}
@@ -49,7 +49,7 @@ func TestBadgerClient(t *testing.T) {
 }
 
 func TestEtcdClient(t *testing.T) {
-	m, err := newKVMeta("etcd", "localhost:2379", &Config{})
+	m, err := newKVMeta("etcd", "localhost:2379", &Config{MaxDeletes: 2})
 	if err != nil {
 		t.Fatalf("create meta: %s", err)
 	}
diff --git a/pkg/object/object_storage_test.go b/pkg/object/object_storage_test.go
index 74832246..65ca7b6d 100644
--- a/pkg/object/object_storage_test.go
+++ b/pkg/object/object_storage_test.go
@@ -28,6 +28,7 @@ import (
 	"math"
 	"os"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
 	"testing"
@@ -80,6 +81,19 @@ func testStorage(t *testing.T, s ObjectStorage) {
 			t.Fatalf("delete failed: %s", err)
 		}
 	}()
+
+	key := "测试编码文件" + `{"name":"zhijian"}` + string('\u001F')
+	if err := s.Put(key, bytes.NewReader(nil)); err != nil {
+		t.Logf("PUT testEncodeFile failed: %s", err.Error())
+	} else {
+		if resp, err := s.List("", "测试编码文件", 1); err != nil && err != notSupported {
+			t.Logf("List testEncodeFile Failed: %s", err)
+		} else if len(resp) == 1 && resp[0].Key() != key {
+			t.Logf("List testEncodeFile Failed: expect key %s, but got %s", key, resp[0].Key())
+		}
+	}
+	_ = s.Delete(key)
+
 	k := "large"
 	defer s.Delete(k)
 
@@ -93,21 +107,34 @@ func testStorage(t *testing.T, s ObjectStorage) {
 		t.Fatalf("PUT failed: %s", err.Error())
 	}
 
+	// get all
 	if d, e := get(s, "test", 0, -1); d != "hello" {
 		t.Fatalf("expect hello, but got %v, error: %s", d, e)
 	}
-	if d, e := get(s, "test", 2, -1); d != "llo" {
-		t.Logf("expect llo, but got %v, error: %s", d, e)
+	if d, e := get(s, "test", 0, 5); d != "hello" {
+		t.Fatalf("expect hello, but got %v, error: %s", d, e)
+	}
+	// get first
+	if d, e := get(s, "test", 0, 1); d != "h" {
+		t.Fatalf("expect h, but got %v, error: %s", d, e)
+	}
+	// get last
+	if d, e := get(s, "test", 4, 1); d != "o" {
+		t.Fatalf("expect o, but got %v, error: %s", d, e)
 	}
+	// get last 3
 	if d, e := get(s, "test", 2, 3); d != "llo" {
 		t.Fatalf("expect llo, but got %v, error: %s", d, e)
 	}
+	// get middle
 	if d, e := get(s, "test", 2, 2); d != "ll" {
 		t.Fatalf("expect ll, but got %v, error: %s", d, e)
 	}
+	// get the end out of range
 	if d, e := get(s, "test", 4, 2); d != "o" {
 		t.Logf("out-of-range get: 'o', but got %v, error: %s", len(d), e)
 	}
+	// get the off out of range
 	if d, e := get(s, "test", 6, 2); d != "" {
 		t.Logf("out-of-range get: '', but got %v, error: %s", len(d), e)
 	}
@@ -329,6 +356,39 @@ func TestS3(t *testing.T) {
 	testStorage(t, s)
 }
 
+func TestOracleCompileRegexp(t *testing.T) {
+	ep := "axntujn0ebj1.compat.objectstorage.ap-singapore-1.oraclecloud.com"
+	oracleCompile := regexp.MustCompile(oracleCompileRegexp)
+	if oracleCompile.MatchString(ep) {
+		if submatch := oracleCompile.FindStringSubmatch(ep); len(submatch) >= 2 {
+			if submatch[1] != "ap-singapore-1" {
+				t.Fatalf("oracle endpoint parse failed")
+			}
+		} else {
+			t.Fatalf("oracle endpoint parse failed")
+		}
+	} else {
+		t.Fatalf("oracle endpoint parse failed")
+	}
+}
+
+func TestOVHCompileRegexp(t *testing.T) {
+	for _, ep := range []string{"s3.gra.cloud.ovh.net", "s3.gra.perf.cloud.ovh.net", "s3.gra.io.cloud.ovh.net"} {
+		ovhCompile := regexp.MustCompile(OVHCompileRegexp)
+		if ovhCompile.MatchString(ep) {
+			if submatch := ovhCompile.FindStringSubmatch(ep); len(submatch) >= 2 {
+				if submatch[1] != "gra" {
+					t.Fatalf("ovh endpoint parse failed")
+				}
+			} else {
+				t.Fatalf("ovh endpoint parse failed")
+			}
+		} else {
+			t.Fatalf("ovh endpoint parse failed")
+		}
+	}
+}
+
 func TestOSS(t *testing.T) {
 	if os.Getenv("ALICLOUD_ACCESS_KEY_ID") == "" {
 		t.SkipNow()
@@ -698,19 +758,20 @@ func TestTOS(t *testing.T) {
 }
 
 func TestMain(m *testing.M) {
-	// schema: S3 AWS_ENDPOINT=xxxxx  AWS_ACCESS_KEY_ID=xxxx  AWS_SECRET_ACCESS_KEY=xxxx
-	envFile := "/tmp/aksk.txt"
-	if _, err := os.Stat(envFile); err == nil {
-		file, _ := os.ReadFile(envFile)
-		for _, line := range strings.Split(strings.TrimSpace(string(file)), "\n") {
-			env := strings.Fields(line)
-			if len(env) <= 1 {
-				continue
-			}
-			for _, e := range env[1:] {
-				envkv := strings.SplitN(e, "=", 2)
-				if err := os.Setenv(envkv[0], envkv[1]); err != nil {
-					logger.Errorf("set env %s=%s error", envkv[0], envkv[1])
+	if envFile := os.Getenv("JUICEFS_ENV_FILE_FOR_TEST"); envFile != "" {
+		// schema: S3 AWS_ENDPOINT=xxxxx  AWS_ACCESS_KEY_ID=xxxx  AWS_SECRET_ACCESS_KEY=xxxx
+		if _, err := os.Stat(envFile); err == nil {
+			file, _ := os.ReadFile(envFile)
+			for _, line := range strings.Split(strings.TrimSpace(string(file)), "\n") {
+				env := strings.Fields(line)
+				if len(env) <= 1 {
+					continue
+				}
+				for _, e := range env[1:] {
+					envkv := strings.SplitN(e, "=", 2)
+					if err := os.Setenv(envkv[0], envkv[1]); err != nil {
+						logger.Errorf("set env %s=%s error", envkv[0], envkv[1])
+					}
 				}
 			}
 		}
diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go
index 06dbd92c..dfa80818 100644
--- a/pkg/utils/utils_test.go
+++ b/pkg/utils/utils_test.go
@@ -82,3 +82,32 @@ func TestTimeout(t *testing.T) {
 		t.Fatalf("slow function should be timeout: %s", err)
 	}
 }
+
+func TestRemovePassword(t *testing.T) {
+	testCase := []struct {
+		uri      string
+		expected string
+	}{
+		{"redis://:password@localhost:6379/0",
+			"redis://:****@localhost:6379/0",
+		},
+		{":password@localhost:6379/0",
+			":****@localhost:6379/0",
+		},
+		{"oss://ak:sk@zhijian-test2.oss-cn-hangzhou.aliyuncs.com",
+			"oss://ak:****@zhijian-test2.oss-cn-hangzhou.aliyuncs.com",
+		},
+		{"/tmp/file",
+			"/tmp/file",
+		},
+		{"file:///tmp/file",
+			"file:///tmp/file",
+		},
+		{"sftp:///tmp/file",
+			"sftp:///tmp/file",
+		},
+	}
+	for _, tc := range testCase {
+		assertEqual(t, RemovePassword(tc.uri), tc.expected)
+	}
+}
diff --git a/pkg/vfs/compact_test.go b/pkg/vfs/compact_test.go
index ca5e96f6..93a63c5d 100644
--- a/pkg/vfs/compact_test.go
+++ b/pkg/vfs/compact_test.go
@@ -30,7 +30,6 @@ func TestCompact(t *testing.T) {
 		BlockSize:  256 * 1024,
 		Compress:   "lz4",
 		MaxUpload:  2,
-		MaxDeletes: 1,
 		BufferSize: 30 << 20,
 		CacheSize:  10,
 		CacheDir:   "memory",
diff --git a/pkg/vfs/vfs_test.go b/pkg/vfs/vfs_test.go
index 1825458b..affbd526 100644
--- a/pkg/vfs/vfs_test.go
+++ b/pkg/vfs/vfs_test.go
@@ -41,6 +41,7 @@ func createTestVFS() (*VFS, object.ObjectStorage) {
 	metaConf := &meta.Config{
 		Retries:    10,
 		Strict:     true,
+		MaxDeletes: 2,
 		MountPoint: mp,
 	}
 	m := meta.NewClient("memkv://", metaConf)
@@ -63,7 +64,6 @@ func createTestVFS() (*VFS, object.ObjectStorage) {
 			BlockSize:  format.BlockSize * 1024,
 			Compress:   format.Compression,
 			MaxUpload:  2,
-			MaxDeletes: 1,
 			BufferSize: 30 << 20,
 			CacheSize:  10,
 			CacheDir:   "memory",
diff --git a/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java b/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
index 404ceb56..dde8e0d6 100644
--- a/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
+++ b/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
@@ -711,4 +711,11 @@ public class JuiceFileSystemTest extends TestCase {
       counter.getAndIncrement();
     }
   }
+
+  public void testInnerSymlink() throws Exception {
+    //echo "hello juicefs" > inner_sym_link
+    FileStatus status = fs.getFileStatus(new Path("/inner_sym_link"));
+    assertEquals("inner_sym_link", status.getPath().getName());
+    assertEquals(14, status.getLen());
+  }
 }