Repository: juicedata/juicefs
Pull request #3056 (2022-12-06 05:47:11)

CI: add chaos test.


Patch:
diff --git a/.github/actions/fio/action.yml b/.github/actions/fio/action.yml
deleted file mode 100644
index fead90b3..00000000
--- a/.github/actions/fio/action.yml
+++ /dev/null
@@ -1,165 +0,0 @@
-name: 'Fio Test Action'
-description: 'Fio Test Action'
-inputs:
-  meta_url:  
-    description: 'meta url'
-    required: true
-    default: 'redis://localhost/1'
-  storage:  
-    description: 'storage'
-    required: true
-    default: 'minio'
-  bucket:  
-    description: 'bucket'
-    required: true
-    default: 'http://localhost:9000/fio-test'
-  access_key:  
-    description: 'access_key'
-    required: true
-    default: 'minioadmin'
-  secret_key:  
-    description: 'secret_key'
-    required: true
-    default: 'minioadmin'
-  volume_name: 
-    description: 'volume name'
-    required: true
-    default: 'fio-test'
-  mount_point: 
-    description: 'mount_point'
-    required: true
-    default: '/tmp/fio-test'
-  fio_job: 
-    description: 'fio job, include name, options and mount args'
-    required: true
-    default: 
-  version_count:
-    description: 'how many version to test besides dev'
-    required: true
-    default: 1
-  mysql_password:
-    description: 'mysql password'
-    required: true
-    default: ''
-  PYROSCOPE_AUTH_TOKEN:
-    description: 'PYROSCOPE_AUTH_TOKEN'
-    required: true
-    default: ''
-
-runs:
-  using: "composite"
-  steps:
-    - name: Install tools
-      run: |
-        set -x 
-        sudo apt install fio
-        wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc
-        chmod +x mc 
-        [[ ${{ inputs.meta_url }} == redis://* ]] && docker run --name my-redis -p 6379:6379 -d redis
-        [[ ${{ inputs.storage }} == minio ]] && docker run -d -p 9000:9000 --name minio \
-                     -e "MINIO_ACCESS_KEY=minioadmin" \
-                     -e "MINIO_SECRET_KEY=minioadmin" \
-                     -v /tmp/data:/data \
-                     -v /tmp/config:/root/.minio \
-                     minio/minio server /data
-      shell: bash
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.18.x'
-
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cache/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-          
-    - name: Build linux target
-      run: |
-        make juicefs
-      shell: bash
-
-    - name: Clean Before
-      run: |
-        rm /var/jfsCache/ -rf || true
-        if [ -d ${{ inputs.mount_point }} ]; then
-          ./juicefs umount ${{ inputs.mount_point }} || true
-        fi
-        uuid=$(./juicefs status ${{ inputs.meta_url }} | grep UUID | cut -d '"' -f 4) || true
-        if [ -n "$uuid" ];then
-          sudo ./juicefs destroy --force ${{ inputs.meta_url }} $uuid
-        fi
-      shell: bash
-    - name: Fio Benchmark 
-      run: |
-        set -o pipefail
-        PYROSCOPE_URL=http://172.27.0.1:4040
-        export PYROSCOPE_AUTH_TOKEN=${{inputs.PYROSCOPE_AUTH_TOKEN}}
-        urls=("juicefs")
-        urls+=($(curl -s https://api.github.com/repos/juicedata/juicefs/releases | grep browser_download_url | grep linux-amd64.tar.gz | awk -F\" '{print $4}' | head -${{inputs.version_count}}))         
-        for url in "${urls[@]}"; do
-          if [[ $url == http* ]]; then  
-            wget -q $url
-            bin_name=$(echo $(basename $url) | sed 's/.tar.gz//g')
-            mkdir jfs || true
-            tar -zxf $(basename $url) -C jfs
-            mv jfs/juicefs $bin_name
-            rm $(basename $url)
-            rm jfs -rf
-          fi
-        done
-        job="${{inputs.fio_job}}"
-        set -x
-        for url in "${urls[@]}"; do
-          echo version: $(./$bin_name -V), job: $job,  
-          bin_name=$(echo $(basename $url) | sed 's/.tar.gz//g') 
-          juicefs_version=$(./$bin_name -V|cut -b 17- | sed 's/:/-/g')
-          name=$(echo $job | awk -F: '{print $1}' | xargs)
-          fio_arg=$(echo $job | awk -F: '{print $2}' | xargs)
-          mount_arg=$(echo $job | awk -F: '{print $3}' | xargs)
-          ./$bin_name format --help | grep "trash-days" && trash_day="--trash-days 0" || trash_day=""
-          ./$bin_name --help | grep "pyroscope" && pyroscope="--pyroscope $PYROSCOPE_URL" || pyroscope=""
-          ./$bin_name format $trash_day --storage ${{inputs.storage}} --bucket  ${{inputs.bucket}}  --access-key ${{inputs.access_key}} --secret-key ${{inputs.secret_key}}  ${{inputs.meta_url}} ${{inputs.volume_name}}
-          ./$bin_name mount -d ${{ inputs.meta_url }} ${{ inputs.mount_point }} --no-usage-report $pyroscope  $mount_arg
-          if [[ "$name" =~ ^big-file-rand-read.* ]]; then
-            block_size=$(echo $name | awk -F- '{print $NF}' | xargs)
-            echo block_size is $block_size
-            fio --name=big-file-rand-read-preload --directory=${{ inputs.mount_point }} --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=$block_size --pre_read=1
-            sudo sync 
-            sudo bash -c  "echo 3 > /proc/sys/vm/drop_caches"
-          fi
-          echo "start fio"
-          fio --name=$name --directory=${{ inputs.mount_point }} $fio_arg > "$name-$juicefs_version.log"
-          echo "finish fio"
-          bw_str=$(tail -1 $name-$juicefs_version.log | awk '{print $2}' | awk -F '=' '{print $2}' )
-          bw=$(echo $bw_str | sed 's/.iB.*//g') 
-          if [[ $bw_str == *KiB* ]]; then
-            bw=$(echo "scale=2; $bw/1024.0" | bc -l)
-          elif [[ $bw_str == *GiB* ]]; then
-            bw=$(echo "scale=2; $bw*1024.0" | bc -l)
-          fi
-          meta=`echo ${{inputs.meta_url}} | awk -F: '{print $1}'`
-          hostname | grep bench  && runner="self_runner" || runner='github_runner'
-          mysql -h 8.210.231.144 -u juicedata -p${{inputs.mysql_password}} -e "use  test_result;  INSERT INTO fio_test (name, bandwidth,  juicefs_version, size, nrfiles, ref_name, created_date, github_revision, workflow_url, meta_engine, storage, runner) values ('$name', $bw, '$juicefs_version', '1G', 10000, '${{ github.ref_name }}', '$(date +"%Y-%m-%d %H:%M:%S")', '${{github.sha}}', 'https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}',  '$meta', '${{inputs.storage}}', '$runner'); "
-          ./$bin_name umount -f ${{ inputs.mount_point }}
-          uuid=$(./juicefs status ${{ inputs.meta_url }} | grep UUID | cut -d '"' -f 4)
-          if [ -n "$uuid" ];then
-            sudo ./juicefs destroy --force ${{ inputs.meta_url }} $uuid
-          fi
-          # mc rb  myminio/$name --force --dangerous || printf "Warining; remove bucket failed: %s, exit code: %s" "myminio/$name" "$?"
-        done
-        # mysql -h 8.210.231.144 -u juicedata -p${{inputs.mysql_password}} -e "use  test_result; LOAD DATA INFILE '/var/lib/mysql-files/result.csv' INTO TABLE fio_test FIELDS TERMINATED BY ',' (name, bandwidth,  juicefs_version, size, nrfiles, ref_name, created_date, github_revision, workflow_url, meta_engine, storage, runner); " 
-      shell: bash
-
-    - name: log
-      if: ${{ always() }}
-      shell: bash
-      run: | 
-        if [ -f ~/.juicefs/juicefs.log ]; then
-          tail -300 ~/.juicefs/juicefs.log
-          grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
-        fi
\ No newline at end of file
diff --git a/.github/scripts/chaos/dynamic.yaml b/.github/scripts/chaos/dynamic.yaml
new file mode 100644
index 00000000..300bb976
--- /dev/null
+++ b/.github/scripts/chaos/dynamic.yaml
@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dynamic-ce
+  labels:
+    juicefs-app-type: dynamic-ce
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      juicefs-app-type: dynamic-ce
+  template:
+    metadata:
+      labels:
+        juicefs-app-type: dynamic-ce
+    spec:
+      containers:
+      - name: vdbench
+        image: zwwhdlsdocker/vdbench:latest
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+          - mountPath: /data
+            name: data
+          - mountPath: /vdbench/config
+            name: vdbench-cfg
+          - mountPath: /vdbench/output
+            name: output
+        command: ["sh", "-c", "./vdbench -f /vdbench/config/vdbench.vdb -v"]
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: dynamic-ce
+      - name: output
+        hostPath:
+          path: /root/vdbench/output
+      - name: vdbench-cfg
+        configMap:
+          name: dynamic-ce
+          items:
+          - key: "vdbench.vdb"
+            path: "vdbench.vdb"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dynamic-ce
+data:
+  vdbench.vdb: |
+    messagescan=no
+    fsd=fsd1,anchor=/data,depth=1,width=1,files=20000,size=4k,openflags=o_direct
+    fwd=fwd1,fsd=fsd1,operation=write,xfersize=4k,fileio=random,fileselect=random,threads=1
+    rd=rd1,fwd=fwd1,fwdrate=max,format=yes,elapsed=60,interval=2
diff --git a/.github/scripts/chaos/juicefs-csi-driver.Dockerfile b/.github/scripts/chaos/juicefs-csi-driver.Dockerfile
new file mode 100644
index 00000000..cafb4e7e
--- /dev/null
+++ b/.github/scripts/chaos/juicefs-csi-driver.Dockerfile
@@ -0,0 +1,31 @@
+FROM golang:1.18-buster as builder
+
+ARG GOPROXY
+# refs/remotes/pull/3056/merge
+ARG GITHUB_REF
+# 4ac69613b5919142d87f21a64ca744ae537192d6
+ARG GITHUB_SHA
+ARG JUICEFS_REPO_URL=https://github.com/juicedata/juicefs
+
+WORKDIR /workspace
+ENV GOPROXY=${GOPROXY:-https://proxy.golang.org}
+ENV STATIC=1
+
+RUN apt-get update && apt-get install -y musl-tools upx-ucl && \
+    cd /workspace && git clone --depth=1 $JUICEFS_REPO_URL && \
+    cd juicefs && git fetch --no-tags --prune origin +$GITHUB_SHA:$GITHUB_REF && \
+    git checkout $GITHUB_REF && \
+    make juicefs
+
+FROM juicedata/juicefs-csi-driver:nightly
+
+WORKDIR /app
+COPY --from=builder /workspace/juicefs/juicefs /usr/local/bin/
+
+RUN ls -l /usr/local/bin/juicefs
+
+RUN /usr/local/bin/juicefs --version
+RUN echo GITHUB_REF is $GITHUB_REF
+RUN echo GITHUB_SHA is $GITHUB_SHA
+
+# ENTRYPOINT ["/tini", "--", "/bin/juicefs-csi-driver"]
diff --git a/.github/scripts/chaos/juicefs.Dockerfile b/.github/scripts/chaos/juicefs.Dockerfile
new file mode 100644
index 00000000..419f7f0a
--- /dev/null
+++ b/.github/scripts/chaos/juicefs.Dockerfile
@@ -0,0 +1,5 @@
+FROM juicedata/mount:nightly
+COPY ./juicefs /usr/local/bin/juicefs
+# RUN apt-get update && apt-get install -y musl-tools upx-ucl && STATIC=1 make
+# RUN cp -f juicefs /usr/local/bin/juicefs
+RUN /usr/local/bin/juicefs version
\ No newline at end of file
diff --git a/.github/scripts/chaos/minio.yaml b/.github/scripts/chaos/minio.yaml
new file mode 100644
index 00000000..410bd363
--- /dev/null
+++ b/.github/scripts/chaos/minio.yaml
@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: minio-server
+  namespace: kube-system
+  labels:
+    app: minio-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio-server
+  serviceName: minio
+  template:
+    metadata:
+      labels:
+        app: minio-server
+    spec:
+      containers:
+      - name: minio
+        image: minio/minio
+        env:
+        - name: MINIO_ROOT_USER
+          value: minioadmin
+        - name: MINIO_ROOT_PASSWORD
+          value: minioadmin
+        args:
+        - server
+        - /data
+        volumeMounts:
+        - mountPath: /data
+          name: minio-data
+        ports:
+        - containerPort: 9000
+          name: sever
+      volumes:
+      - name: minio-data
+        hostPath:
+          path: /data/minio-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector:
+    app: minio-server
+  ports:
+  - protocol: TCP
+    port: 9000
+    targetPort: 9000
+    nodePort: 31275
+    name: server
\ No newline at end of file
diff --git a/.github/scripts/chaos/pvc.yaml b/.github/scripts/chaos/pvc.yaml
new file mode 100644
index 00000000..07e496d5
--- /dev/null
+++ b/.github/scripts/chaos/pvc.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: dynamic-ce
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Pi
+  storageClassName: dynamic-ce
\ No newline at end of file
diff --git a/.github/scripts/chaos/redis.yaml b/.github/scripts/chaos/redis.yaml
new file mode 100644
index 00000000..47feff6c
--- /dev/null
+++ b/.github/scripts/chaos/redis.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis-server
+  namespace: kube-system
+  labels:
+    app: redis-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis-server
+  serviceName: redis
+  template:
+    metadata:
+      labels:
+        app: redis-server
+    spec:
+      containers:
+      - name: redis
+        image: redis
+        volumeMounts:
+        - mountPath: /data
+          name: redis-data
+        resources:
+          requests:
+            memory: 100m
+        ports:
+        - containerPort: 6379
+      volumes:
+      - name: redis-data
+        hostPath:
+          path: /data/redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector:
+    app: redis-server
+  ports:
+  - protocol: TCP
+    port: 6379
+    targetPort: 6379
+    nodePort: 31274
diff --git a/.github/scripts/chaos/sc.yaml b/.github/scripts/chaos/sc.yaml
new file mode 100644
index 00000000..221c04da
--- /dev/null
+++ b/.github/scripts/chaos/sc.yaml
@@ -0,0 +1,34 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: dynamic-ce
+parameters:
+  csi.storage.k8s.io/node-publish-secret-name: dynamic-ce
+  csi.storage.k8s.io/node-publish-secret-namespace: kube-system
+  csi.storage.k8s.io/provisioner-secret-name: dynamic-ce
+  csi.storage.k8s.io/provisioner-secret-namespace: kube-system
+  juicefs/mount-cpu-limit: 5000m
+  juicefs/mount-memory-limit: 1Gi
+  juicefs/mount-cpu-request: 100m
+  juicefs/mount-memory-request: 500Mi
+  juicefs/mount-image: juicedata/mount:ci
+#mountOptions:
+#  - cache-dir=/var/foo:/var/foo1:/var/foo2
+provisioner: csi.juicefs.com
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+---
+apiVersion: v1
+stringData:
+  access-key: minioadmin
+  bucket: http://minio.kube-system:9000/minio/dynamic-ce
+  name: dynamic-ce
+  metaurl: redis://redis.kube-system:6379/0
+  secret-key: minioadmin
+  storage: minio
+  format-options: trash-days=0,block-size=4096
+kind: Secret
+metadata:
+  name: dynamic-ce
+  namespace: kube-system
+type: Opaque
diff --git a/.github/scripts/chaos/workflow.yaml b/.github/scripts/chaos/workflow.yaml
new file mode 100644
index 00000000..41e13a20
--- /dev/null
+++ b/.github/scripts/chaos/workflow.yaml
@@ -0,0 +1,224 @@
+apiVersion: chaos-mesh.org/v1alpha1
+kind: Workflow
+metadata:
+  name: juicefs-workflow
+spec:
+  entry: the-entry
+  templates:
+    - name: the-entry
+      templateType: Parallel
+      children:
+        # - minio-delay
+        # - minio-io
+        # - minio-memory
+        # - minio-cpu
+        # - minio-bandwidth
+        # - redis-bandwidth
+        # - redis-io
+        # - redis-delay
+        # - redis-memory
+        # - redis-cpu
+        # - juicefs-bandwidth
+        # - juicefs-memory
+        # - juicefs-cpu
+        # - juicefs-delay
+    # minio 带宽
+    - name: minio-bandwidth
+      templateType: NetworkChaos
+      deadline: 20s
+      networkChaos:
+        action: bandwidth
+        mode: all
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: minio-server
+        bandwidth:
+          rate: '500bps'
+          limit: 100
+          buffer: 10000
+    # minio 网络延迟
+    - name: minio-delay
+      templateType: NetworkChaos
+      networkChaos:
+        action: delay
+        mode: all
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: minio-server
+        delay:
+          latency: '500ms'
+          correlation: '50'
+          jitter: '500ms'
+    # minio 磁盘读写延迟
+    - name: minio-io
+      templateType: IOChaos
+      ioChaos:
+        action: latency
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: minio-server
+        volumePath: /data
+        delay: '1s'
+    # minio 内存压力
+    - name: minio-memory
+      templateType: StressChaos
+      stressChaos:
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: minio-server
+        stressors:
+          memory:
+            workers: 4
+            size: '128MB'
+    # minio cpu 压力
+    - name: minio-cpu
+      templateType: StressChaos
+      stressChaos:
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: minio-server
+        stressors:
+          cpu:
+            workers: 4
+            load: 100
+    # redis 带宽
+    - name: redis-bandwidth
+      templateType: NetworkChaos
+      networkChaos:
+        action: bandwidth
+        mode: all
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: redis-server
+        bandwidth:
+          rate: '200mbps'
+          limit: 100
+          buffer: 10000
+    - name: redis-delay
+      templateType: NetworkChaos
+      networkChaos:
+        action: delay
+        mode: all
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: redis-server
+        delay:
+          latency: '100ms'
+          correlation: '50'
+          jitter: '500ms'
+    # redis 磁盘读写延迟
+    - name: redis-io
+      templateType: IOChaos
+      ioChaos:
+        action: latency
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: redis-server
+        volumePath: /redis
+        delay: '1s'
+    # redis 内存压力
+    - name: redis-memory
+      templateType: StressChaos
+      stressChaos:
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: redis-server
+        stressors:
+          memory:
+            workers: 4
+            size: '2GB'
+    # redis cpu 压力
+    - name: redis-cpu
+      templateType: StressChaos
+      stressChaos:
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app: redis-server
+        stressors:
+          cpu:
+            workers: 4
+            load: 100
+    # 客户端带宽
+    - name: juicefs-bandwidth
+      templateType: NetworkChaos
+      deadline: 20s
+      networkChaos:
+        action: bandwidth
+        mode: all
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app.kubernetes.io/name: juicefs-mount
+        bandwidth:
+          rate: '100bps'
+          limit: 100
+          buffer: 10000
+    - name: juicefs-delay
+      templateType: NetworkChaos
+      networkChaos:
+        action: delay
+        mode: all
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app.kubernetes.io/name: juicefs-mount
+        delay:
+          latency: '100ms'
+          correlation: '50'
+          jitter: '500ms'
+    # 客户端内存压力
+    - name: juicefs-memory
+      templateType: StressChaos
+      stressChaos:
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app.kubernetes.io/name: juicefs-mount
+        stressors:
+          memory:
+            workers: 4
+            size: '1GB'
+    # 客户端cpu压力
+    - name: juicefs-cpu
+      templateType: StressChaos
+      stressChaos:
+        mode: one
+        selector:
+          namespaces:
+            - kube-system
+          labelSelectors:
+            app.kubernetes.io/name: juicefs-mount
+        stressors:
+          cpu:
+            workers: 4
+            load: 100
diff --git a/.github/workflows/chaos.yml b/.github/workflows/chaos.yml
new file mode 100644
index 00000000..f05036a2
--- /dev/null
+++ b/.github/workflows/chaos.yml
@@ -0,0 +1,219 @@
+name: "chaos-test"
+
+on:
+  push:
+    branches:
+      - release-**
+      # - main
+    paths-ignore:
+      - docs/**
+  pull_request:
+    branches:
+      - release-**
+      - main
+    paths-ignore:
+      - docs/**
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
+  schedule:
+    - cron:  '0 */6 * * *'        
+
+jobs:
+  chaos-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # chaos: ["minio-io", "minio-memory", "minio-cpu", "minio-bandwidth", "redis-bandwidth", "redis-io", "redis-delay", "redis-memory", "redis-cpu", "juicefs-bandwidth", "juicefs-memory", "juicefs-cpu", "juicefs-delay"]
+        chaos: ["minio-io", "minio-memory", "minio-cpu", "minio-bandwidth",  "redis-io", "redis-delay", "redis-memory", "redis-cpu", "juicefs-bandwidth", "juicefs-memory", "juicefs-cpu", "juicefs-delay"]
+        # chaos: ["minio-io"]
+    steps:        
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18.x'
+          cache: true
+
+      - name: Build 
+        run: | 
+          sudo apt-get update && sudo apt-get install -y musl-tools upx-ucl
+          export STATIC=1
+          make juicefs
+
+      - name: Creating kind cluster
+        uses: helm/kind-action@v1.5.0
+
+      - name: Print cluster information
+        run: |
+          kubectl config view
+          kubectl cluster-info
+          kubectl get nodes
+          kubectl get pods -n kube-system
+          helm version
+          kubectl version
+
+      # - name: Build And Load CSI Docker Image
+      #   run: |
+      #     echo GITHUB_REF is $GITHUB_REF
+      #     echo GITHUB_SHA is $GITHUB_SHA
+      #     helm repo add juicefs https://juicedata.github.io/charts/
+      #     helm repo update
+      #     APP_VERSION=$(helm search repo juicefs/juicefs-csi-driver --versions | grep juicefs | head -1 | awk -F" " '{print $3}')
+      #     echo APP_VERSION is $APP_VERSION
+      #     docker build --build-arg GITHUB_REF=$GITHUB_REF --build-arg GITHUB_SHA=$GITHUB_SHA -f .github/scripts/chaos/juicefs-csi-driver.Dockerfile -t juicedata/juicefs-csi-driver:v$APP_VERSION .
+      #     kind load docker-image juicedata/juicefs-csi-driver:v$APP_VERSION --name chart-testing
+
+      - name: Build And Load CSI Docker Image
+        run: |
+          docker build -f .github/scripts/chaos/juicefs.Dockerfile -t juicedata/mount:ci .
+          helm repo add juicefs https://juicedata.github.io/charts/
+          helm repo update
+          kind load docker-image juicedata/mount:ci --name chart-testing
+
+      - name: Install JuiceFS CSI Driver
+        run: |
+          CHART_VERSION=$(helm search repo juicefs/juicefs-csi-driver --versions | grep juicefs | head -1 | awk -F" " '{print $2}')
+          echo CHART_VERSION is $CHART_VERSION
+          helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system --version $CHART_VERSION
+          kubectl -n kube-system get pods -l app.kubernetes.io/name=juicefs-csi-driver
+
+      - name: Deploy redis
+        run: |
+          kubectl apply -f .github/scripts/chaos/redis.yaml
+
+      - name: Deploy minio
+        run: |
+          kubectl apply -f .github/scripts/chaos/minio.yaml
+
+      - name: Mount Juicefs 
+        run: | 
+          kubectl apply -f .github/scripts/chaos/sc.yaml
+          kubectl apply -f .github/scripts/chaos/pvc.yaml
+
+      - name: Start vdbenh
+        run: |
+          kubectl apply -f .github/scripts/chaos/dynamic.yaml
+
+      - name: Get Config Base64
+        run: | 
+          chaos=${{matrix.chaos}}
+          sed -i "s/# - $chaos/- $chaos/g" .github/scripts/chaos/workflow.yaml 
+          cat .github/scripts/chaos/workflow.yaml 
+          base64=$(base64  .github/scripts/chaos/workflow.yaml  | xargs | sed -e 's: ::g' )
+          echo "base64 is $base64"
+          echo "workflow_base64=$base64" >> $GITHUB_ENV
+
+      - name: Run chaos mesh action
+        uses: chaos-mesh/chaos-mesh-action@master
+        env:
+          CHAOS_MESH_VERSION: v2.5.0
+          CFG_BASE64: ${{env.workflow_base64}}
+
+      - name: Verify 
+        run: |
+          sleep 120
+          kubectl get pods --all-namespaces
+          apps=("dynamic-ce" "juicefs-csi-node" "juicefs-csi-controller" "juicefs-chart-testing-control-plane-pvc"  "redis" "minio")
+          for app in ${apps[@]}; do
+            echo app is $app
+            kubectl get pods --all-namespaces | grep $app | grep Running
+            if [ $? != 0 ]; then
+              echo status of $app is not expected.
+              exit 1
+            fi
+          done
+
+      - name: Check csi controller log
+        if: ${{ always() }}
+        run: |
+          kubectl describe pvc dynamic-ce
+          kubectl -n kube-system get po -l app=juicefs-csi-controller
+          kubectl -n kube-system logs juicefs-csi-controller-0 juicefs-plugin
+
+      - name: Check csi node log
+        if: ${{ always() }}
+        run: |
+          POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep juicefs-csi-node)
+          echo POD_NAME is $POD_NAME
+          kubectl -n kube-system describe po $POD_NAME
+          kubectl -n kube-system logs $POD_NAME -c juicefs-plugin > csi_node.log
+          cat csi_node.log
+          # grep -i "error" csi_node.log && exit 1 || true
+
+      - name: Check mount point pod
+        if: ${{ always() }}
+        run: | 
+          POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep juicefs-chart-testing-control-plane-pvc)
+          echo POD_NAME is $POD_NAME
+          kubectl -n kube-system describe po $POD_NAME
+          kubectl logs -n kube-system $POD_NAME > juicefs.log
+          cat juicefs.log
+          grep "<FATAL>:" juicefs.log && exit 1 || true
+
+      - name: Check vdbench log
+        if: ${{ always() }}
+        run: | 
+          POD_NAME=$(kubectl get pods -n default -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep dynamic-ce )
+          echo POD_NAME is $POD_NAME
+          kubectl -n default describe po $POD_NAME
+          kubectl logs -n default $POD_NAME > vdbench.log
+          cat vdbench.log
+          # grep -i "error" vdbench.log && exit 1 || true
+
+      - name: Check Redis log
+        if: ${{ always() }}
+        run: | 
+          POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep redis )
+          echo POD_NAME is $POD_NAME
+          kubectl -n kube-system describe po $POD_NAME
+          kubectl logs -n kube-system $POD_NAME > redis.log
+          cat redis.log
+          # grep -i "error" redis.log && exit 1 || true
+
+      - name: Check Minio log
+        if: ${{ always() }}
+        run: | 
+          POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep minio )
+          echo POD_NAME is $POD_NAME
+          kubectl -n kube-system describe po $POD_NAME
+          kubectl logs -n kube-system $POD_NAME > minio.log
+          cat minio.log
+          # grep -i "error" minio.log && exit 1 || true
+
+      - name: Setup upterm session
+        # if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure()
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
+
+  success-all-test:
+    runs-on: ubuntu-latest
+    needs: [chaos-test]
+    if: always()
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v3
+      - uses: actions/checkout@v3
+
+      - name: Check Failure
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: exit 1
+
+      - name: Send Slack Notification
+        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        uses: juicedata/slack-notify-action@main
+        with:
+          channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
+          slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
+
+      - name: Success
+        if: ${{ success() }}
+        run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 26bd79bd..86ff286d 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -43,10 +43,10 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Set up Java
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v3
       with:
         distribution: 'adopt'
         java-version: '8'
diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml
index b1be6ffa..cf457fd9 100644
--- a/.github/workflows/config.yml
+++ b/.github/workflows/config.yml
@@ -34,7 +34,7 @@ jobs:
           - 6379:6379
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/encoding_mysql.yml b/.github/workflows/encoding_mysql.yml
index 9b94aa8c..98c4d759 100644
--- a/.github/workflows/encoding_mysql.yml
+++ b/.github/workflows/encoding_mysql.yml
@@ -21,19 +21,19 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
       - name: Set Variable
         id: vars
         run: |
-          echo ::set-output name=META_URL1::"mysql://root:root@\\(127.0.0.1\\)/mysql_encoding1"
-          echo ::set-output name=META_URL2::"mysql://root:root@\\(127.0.0.1\\)/mysql_encoding2"
-          echo ::set-output name=MOUNT_POINT1::/tmp/juicefs-mysql-encoding-test1
-          echo ::set-output name=MOUNT_POINT2::/tmp/juicefs-mysql-encoding-test2
-          echo ::set-output name=BUCKET::/var/jfs/
-          echo ::set-output name=NAME::mysql-encoding-test
+          echo "META_URL1=mysql://root:root@\\(127.0.0.1\\)/mysql_encoding1" >> $GITHUB_OUTPUT
+          echo "META_URL2=mysql://root:root@\\(127.0.0.1\\)/mysql_encoding2" >> $GITHUB_OUTPUT
+          echo "MOUNT_POINT1=/tmp/juicefs-mysql-encoding-test1" >> $GITHUB_OUTPUT
+          echo "MOUNT_POINT2=/tmp/juicefs-mysql-encoding-test2" >> $GITHUB_OUTPUT
+          echo "BUCKET=/var/jfs/" >> $GITHUB_OUTPUT
+          echo "NAME=mysql-encoding-test" >> $GITHUB_OUTPUT
 
       - name: Download old linux target
         run: |
diff --git a/.github/workflows/fio_benchmark.yml b/.github/workflows/fio_benchmark.yml
index c7055823..345b21da 100644
--- a/.github/workflows/fio_benchmark.yml
+++ b/.github/workflows/fio_benchmark.yml
@@ -4,338 +4,214 @@ on:
   schedule:
     - cron:  '0 0 * * *'
   workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
 
 jobs:
-  big-file-sequential-read:
+  fio_benchmark:
     if: github.repository == 'juicedata/juicefs'
     runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-sequential-read:  --rw=read --refill_buffers --bs=256k --size=1G"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-sequential-write:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-sequential-write:  --rw=write --refill_buffers --bs=256k  --size=1G"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-multi-read-1:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-multi-read-1:  --rw=read --refill_buffers --bs=256k --size=1G --numjobs=1"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
 
-  big-file-multi-read-4:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-multi-read-4:  --rw=read --refill_buffers --bs=256k --size=1G --numjobs=4"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-multi-read-16:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-multi-read-16:  --rw=read --refill_buffers --bs=256k --size=1G --numjobs=16"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-multi-write-1:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-multi-write-1:       --rw=write --refill_buffers --bs=256k --size=1G --numjobs=1"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-multi-write-4:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-multi-write-4:       --rw=write --refill_buffers --bs=256k --size=1G --numjobs=4"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-multi-write-16:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-multi-write-16:       --rw=write --refill_buffers --bs=256k --size=1G --numjobs=16"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  big-file-rand-read-4k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-rand-read-4k:       --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=4k" 
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-        
-  big-file-rand-read-256k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-rand-read-256k:       --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=256k" 
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-
-  big-file-random-write-16k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-random-write-16k:    --rw=randwrite --refill_buffers --size=1G --bs=16k"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
+    strategy:
+      fail-fast: false
+      matrix:
+        meta: ['redis']
+        fio_job : ['big-file-sequential-read', 'big-file-sequential-write', 'big-file-multi-read-1', 'big-file-multi-read-4', 'big-file-multi-read-16', 
+          'big-file-multi-write-1', 'big-file-multi-write-4', 'big-file-multi-write-16', 'big-file-rand-read-4k', 'big-file-rand-read-256k', 
+          'big-file-random-write-16k', 'big-file-random-write-256k', 'small-file-seq-read-4k', 'small-file-seq-read-256k', 
+          'small-file-seq-write-4k', 'small-file-seq-write-256k', 'small-file-multi-read-1', 'small-file-multi-read-4', 
+          'small-file-multi-read-16', 'small-file-multi-write-1', 'small-file-multi-write-4', 'small-file-multi-write-16'] 
 
-  big-file-random-write-256k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "big-file-random-write-256k:    --rw=randwrite --refill_buffers --size=1G --bs=256k"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
 
-  small-file-seq-read-4k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v3
         with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-seq-read-4k:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 :--cache-size=0"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
+          go-version: '1.18.x'
+          cache: true
+            
+      - name: Build linux target
+        run: |
+          make juicefs
 
-  small-file-seq-read-256k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-seq-read-256k:      --rw=read --file_service_type=sequential --bs=256k --filesize=256k --nrfiles=10000 :--cache-size=0"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
+      - name: Install tools
+        run: |
+          sudo apt install fio
+          wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc
+          chmod +x mc 
 
+      - name: Start meta and storage
+        run: |
+          chmod +x .github/scripts/start_meta_engine.sh
+          source .github/scripts/start_meta_engine.sh
+          start_meta_engine ${{matrix.meta}} minio
         
-  small-file-seq-write-4k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-seq-write-4k:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 :--writeback"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-seq-write-256k:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-seq-write-256k:     --rw=write --file_service_type=sequential --bs=256k --filesize=256k --nrfiles=10000 :--writeback"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-multi-read-1:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-multi-read-1:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=1"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-multi-read-4:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-multi-read-4:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=4"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-multi-read-16:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-multi-read-16:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=16"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-multi-write-1:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-multi-write-1:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=1"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-multi-write-4:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-multi-write-4:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=4"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
-
-  small-file-multi-write-16:
-    if: github.repository == 'juicedata/juicefs'
-    runs-on: [ubuntu-20.04]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Test 
-        uses: ./.github/actions/fio
-        with:
-          fio_job: "small-file-multi-write-16:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=16"
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
-          PYROSCOPE_AUTH_TOKEN: ${{secrets.PYROSCOPE_AUTH_TOKEN}}
+      - name: Download juicefs bin
+        id: download
+        run: |
+          latest_url=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep browser_download_url | grep linux-amd64.tar.gz | awk -F\" '{print $4}' )
+          echo latest_url is $latest_url
+          echo "latest_url=$latest_url" >> $GITHUB_OUTPUT
+          urls=("juicefs"  $latest_url)
+          for url in "${urls[@]}"; do
+            if [[ $url == http* ]]; then  
+              wget -q $url
+              bin_name=$(echo $(basename $url) | sed 's/.tar.gz//g')
+              mkdir jfs || true
+              tar -zxf $(basename $url) -C jfs
+              mv jfs/juicefs $bin_name
+              rm $(basename $url)
+              rm jfs -rf
+            fi
+          done
+
+      - name: Generate fio job
+        id: gen_fio_job
+        run: |
+          echo matrix.fio_job is ${{matrix.fio_job}}
+          case "${{matrix.fio_job}}" in
+            "big-file-sequential-read") fio_job="big-file-sequential-read:  --rw=read --refill_buffers --bs=256k --size=1G"
+            ;;
+            "big-file-sequential-write") fio_job="big-file-sequential-write:  --rw=write --refill_buffers --bs=256k  --size=1G"
+            ;;
+            "big-file-multi-read-1") fio_job="big-file-multi-read-1:  --rw=read --refill_buffers --bs=256k --size=1G --numjobs=1"
+            ;;
+            "big-file-multi-read-4") fio_job="big-file-multi-read-4:  --rw=read --refill_buffers --bs=256k --size=1G --numjobs=4"
+            ;;
+            "big-file-multi-read-16") fio_job="big-file-multi-read-16:  --rw=read --refill_buffers --bs=256k --size=1G --numjobs=16"
+            ;;
+            "big-file-multi-write-1") fio_job="big-file-multi-write-1:       --rw=write --refill_buffers --bs=256k --size=1G --numjobs=1"
+            ;;
+            "big-file-multi-write-4") fio_job="big-file-multi-write-4:       --rw=write --refill_buffers --bs=256k --size=1G --numjobs=4"
+            ;;
+            "big-file-multi-write-16") fio_job="big-file-multi-write-16:       --rw=write --refill_buffers --bs=256k --size=1G --numjobs=16"
+            ;;
+            "big-file-rand-read-4k") fio_job="big-file-rand-read-4k:       --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=4k"
+            ;;
+            "big-file-rand-read-256k") fio_job="big-file-rand-read-256k:       --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=256k"
+            ;;
+            "big-file-random-write-16k") fio_job="big-file-random-write-16k:    --rw=randwrite --refill_buffers --size=1G --bs=16k"
+            ;;
+            "big-file-random-write-256k") fio_job="big-file-random-write-256k:    --rw=randwrite --refill_buffers --size=1G --bs=256k"
+            ;;
+            "small-file-seq-read-4k") fio_job="small-file-seq-read-4k:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 :--cache-size=0"
+            ;;
+            "small-file-seq-read-256k") fio_job="small-file-seq-read-256k:      --rw=read --file_service_type=sequential --bs=256k --filesize=256k --nrfiles=10000 :--cache-size=0"
+            ;;
+            "small-file-seq-write-4k") fio_job="small-file-seq-write-4k:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 :--writeback"
+            ;;
+            "small-file-seq-write-256k") fio_job="small-file-seq-write-256k:     --rw=write --file_service_type=sequential --bs=256k --filesize=256k --nrfiles=10000 :--writeback"
+            ;;
+            "small-file-multi-read-1") fio_job="small-file-multi-read-1:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=1"
+            ;;
+            "small-file-multi-read-4") fio_job="small-file-multi-read-4:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=4"
+            ;;
+            "small-file-multi-read-16") fio_job="small-file-multi-read-16:      --rw=read --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=16"
+            ;;
+            "small-file-multi-write-1") fio_job="small-file-multi-write-1:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=1"
+            ;;
+            "small-file-multi-write-4") fio_job="small-file-multi-write-4:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=4"
+            ;;
+            "small-file-multi-write-16") fio_job="small-file-multi-write-16:     --rw=write --file_service_type=sequential --bs=4k --filesize=4k --nrfiles=10000 --numjobs=16"
+            ;;
+          esac
+          echo fio_job is $fio_job
+          echo "fio_job=$fio_job" >> $GITHUB_OUTPUT
+
+      - name: Fio Benchmark 
+        run: |
+          # set -o pipefail
+          echo "Fio Benchmark"
+          PYROSCOPE_URL=http://172.27.0.1:4040
+          export PYROSCOPE_AUTH_TOKEN=${{secrets.PYROSCOPE_AUTH_TOKEN}}
+
+          source .github/scripts/start_meta_engine.sh
+          meta_url=$(get_meta_url ${{matrix.meta}})
+          mount_point=/tmp/fio-benchmark
+
+          latest_url=${{ steps.download.outputs.latest_url }}
+          echo latest_url is $latest_url
+          urls=("juicefs" $latest_url)
+
+          fio_job="${{ steps.gen_fio_job.outputs.fio_job }}"
+          echo fio_job is $fio_job
+
+          for url in "${urls[@]}"; do
+            bin_name=$(echo $(basename $url) | sed 's/.tar.gz//g') 
+            echo version: $(./$bin_name -V) 
+            juicefs_version=$(./$bin_name -V|cut -b 17- | sed 's/:/-/g')
+            name=$(echo $fio_job | awk -F: '{print $1}' | xargs)
+            fio_arg=$(echo $fio_job | awk -F: '{print $2}' | xargs)
+            mount_arg=$(echo $fio_job | awk -F: '{print $3}' | xargs)
+            ./$bin_name format --help | grep "trash-days" && trash_day="--trash-days 0" || trash_day=""
+            ./$bin_name --help | grep "pyroscope" && pyroscope="--pyroscope $PYROSCOPE_URL" || pyroscope=""
+            ./$bin_name format $trash_day --storage minio --bucket  http://localhost:9000/fio-test  --access-key minioadmin --secret-key minioadmin  $meta_url  fio-benchmark
+            ./$bin_name mount -d $meta_url $mount_point --no-usage-report $pyroscope  $mount_arg
+            if [[ "$name" =~ ^big-file-rand-read.* ]]; then
+              block_size=$(echo $name | awk -F- '{print $NF}' | xargs)
+              echo block_size is $block_size
+              fio --name=big-file-rand-read-preload --directory=$mount_point --rw=randread --refill_buffers --size=1G --filename=randread.bin --bs=$block_size --pre_read=1
+              sudo sync 
+              sudo bash -c  "echo 3 > /proc/sys/vm/drop_caches"
+            fi
+            echo "start fio"
+            fio --name=$name --directory=$mount_point $fio_arg > "$name-$juicefs_version.log"
+            echo "finish fio"
+            bw_str=$(tail -1 $name-$juicefs_version.log | awk '{print $2}' | awk -F '=' '{print $2}' )
+            bw=$(echo $bw_str | sed 's/.iB.*//g') 
+            if [[ $bw_str == *KiB* ]]; then
+              bw=$(echo "scale=2; $bw/1024.0" | bc -l)
+            elif [[ $bw_str == *GiB* ]]; then
+              bw=$(echo "scale=2; $bw*1024.0" | bc -l)
+            fi
+            meta=`echo $meta_url | awk -F: '{print $1}'`
+            hostname | grep bench  && runner="self_runner" || runner='github_runner'
+            mysql -h 8.210.231.144 -u juicedata -p${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} -e "use  test_result;  INSERT INTO fio_test (name, bandwidth,  juicefs_version, size, nrfiles, ref_name, created_date, github_revision, workflow_url, meta_engine, storage, runner) values ('$name', $bw, '$juicefs_version', '1G', 10000, '${{ github.ref_name }}', '$(date +"%Y-%m-%d %H:%M:%S")', '${{github.sha}}', 'https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}',  '$meta', '${{inputs.storage}}', '$runner'); "
+            ./$bin_name umount -f $mount_point
+            uuid=$(./juicefs status $meta_url | grep UUID | cut -d '"' -f 4)
+            if [ -n "$uuid" ];then
+              sudo ./juicefs destroy --force $meta_url $uuid
+            fi
+            # mc rb  myminio/$name --force --dangerous || printf "Warining; remove bucket failed: %s, exit code: %s" "myminio/$name" "$?"
+          done
+        shell: bash
+
+      - name: log
+        if: ${{ always() }}
+        shell: bash
+        run: | 
+          if [ -f ~/.juicefs/juicefs.log ]; then
+            tail -300 ~/.juicefs/juicefs.log
+            grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
+          fi
+
+  success-all-test:
+    runs-on: ubuntu-latest
+    needs: [fio_benchmark]
+    if: always()
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v2
+      - uses: actions/checkout@v3
+
+      - name: Check Failure
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: exit 1
+
+      - name: Send Slack Notification
+        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        uses: juicedata/slack-notify-action@main
+        with:
+          channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
+          slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
+
+      - name: Success
+        if: ${{ success() }}
+        run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
index c15be8c1..638adb48 100644
--- a/.github/workflows/format.yml
+++ b/.github/workflows/format.yml
@@ -34,7 +34,7 @@ jobs:
           - 6379:6379
     steps:        
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/fsrand.yml b/.github/workflows/fsrand.yml
index ad7960e8..c9ea6515 100644
--- a/.github/workflows/fsrand.yml
+++ b/.github/workflows/fsrand.yml
@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -29,9 +29,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml
index 6d54a808..e643b503 100644
--- a/.github/workflows/gateway.yml
+++ b/.github/workflows/gateway.yml
@@ -65,7 +65,7 @@ jobs:
 
     steps: 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
       
@@ -73,9 +73,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/gc.yml b/.github/workflows/gc.yml
index 8a24afa6..e3335784 100644
--- a/.github/workflows/gc.yml
+++ b/.github/workflows/gc.yml
@@ -37,7 +37,7 @@ jobs:
           - 6379:6379
     steps:        
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
index 14f4279e..3c132873 100644
--- a/.github/workflows/load.yml
+++ b/.github/workflows/load.yml
@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -29,9 +29,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/ltpfs.yml b/.github/workflows/ltpfs.yml
index 91f50a3d..fb54e7eb 100644
--- a/.github/workflows/ltpfs.yml
+++ b/.github/workflows/ltpfs.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/ltpsyscallshead.yml b/.github/workflows/ltpsyscallshead.yml
index 86df023e..8fcbca9a 100644
--- a/.github/workflows/ltpsyscallshead.yml
+++ b/.github/workflows/ltpsyscallshead.yml
@@ -28,7 +28,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -36,9 +36,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/mongodb_mmap.yml b/.github/workflows/mongodb_mmap.yml
index 47799325..6db82c01 100644
--- a/.github/workflows/mongodb_mmap.yml
+++ b/.github/workflows/mongodb_mmap.yml
@@ -25,7 +25,7 @@ jobs:
           architecture: 'x64'
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/redis_compile.yml b/.github/workflows/redis_compile.yml
index 8649e89f..c9c7a87d 100644
--- a/.github/workflows/redis_compile.yml
+++ b/.github/workflows/redis_compile.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 80fc2dcf..c6f9eb71 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -10,12 +10,12 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: '1.18.x'
 
       - name: Set up Java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '8'
@@ -26,7 +26,7 @@ jobs:
           gpg-passphrase: MAVEN_GPG_PASSPHRASE
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -39,7 +39,7 @@ jobs:
 
       - name: Cache local Maven repository
         id: cache-maven
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
diff --git a/.github/workflows/rmfiles.yml b/.github/workflows/rmfiles.yml
index 9e71ebca..570b83fb 100644
--- a/.github/workflows/rmfiles.yml
+++ b/.github/workflows/rmfiles.yml
@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
       
@@ -24,9 +24,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index 994f1dcc..ba69c78f 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -32,7 +32,7 @@ jobs:
           - 6379:6379
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -40,7 +40,7 @@ jobs:
         uses: ./.github/actions/build
 
       - name: Checkout source code for test
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: juicedata/juicefs
           path: jfs_source
@@ -98,7 +98,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
           
@@ -106,7 +106,7 @@ jobs:
         uses: ./.github/actions/build
 
       - name: Checkout source code for test
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: juicedata/juicefs
           path: jfs_source
@@ -143,7 +143,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -153,7 +153,7 @@ jobs:
           target: ${{steps.vars.outputs.target}}
 
       - name: Checkout source code for test
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: juicedata/juicefs
           path: jfs_source
diff --git a/.github/workflows/sysbench.yml b/.github/workflows/sysbench.yml
index 225a8d3a..e3d64d8a 100644
--- a/.github/workflows/sysbench.yml
+++ b/.github/workflows/sysbench.yml
@@ -31,9 +31,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml
index b3f8d97b..b926bfe3 100644
--- a/.github/workflows/tpcds.yml
+++ b/.github/workflows/tpcds.yml
@@ -30,13 +30,13 @@ jobs:
       - run: sudo swapoff -a
 
       - name: Set up Java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '8'
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/vdbench.yml b/.github/workflows/vdbench.yml
index b63c6106..33c05a19 100644
--- a/.github/workflows/vdbench.yml
+++ b/.github/workflows/vdbench.yml
@@ -23,9 +23,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
index 4a632e99..926865e5 100644
--- a/.github/workflows/verify.yml
+++ b/.github/workflows/verify.yml
@@ -23,10 +23,10 @@ jobs:
   lint:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: '1.18.x'
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -34,29 +34,21 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: '1.18.x'
+        cache: true
       id: go
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
+      
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install g++-multilib gcc-mingw-w64
 
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cache/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-
     - name: Build linux target
       run: make
 
diff --git a/.github/workflows/version_compatible_hypo.yml b/.github/workflows/version_compatible_hypo.yml
index aabec041..7497f7da 100644
--- a/.github/workflows/version_compatible_hypo.yml
+++ b/.github/workflows/version_compatible_hypo.yml
@@ -51,9 +51,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build new version
@@ -106,9 +106,11 @@ jobs:
           export META=${{matrix.meta}}
           export STORAGE=${{matrix.storage}}
           new_version=`./juicefs --version | awk -F" " '{print $3}' | awk -F+ '{print $1}'`
+          echo new_version is $new_version
           mv juicefs juicefs-$new_version
           export NEW_JFS_BIN="juicefs-$new_version"
           old_version=`${{matrix.old_juicefs_version}}/juicefs --version | awk -F" " '{print $3}' | awk -F+ '{print $1}'`
+          echo old_version is $old_version
           mv ${{matrix.old_juicefs_version}}/juicefs juicefs-$old_version
           export OLD_JFS_BIN="juicefs-$old_version"
           python3 .github/scripts/testVersionCompatible.py
@@ -139,7 +141,7 @@ jobs:
 
       - name: Setup upterm session
         if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
-        timeout-minutes: 1
+        timeout-minutes: 60
         uses: lhotari/action-upterm@v1
 
 
@@ -148,7 +150,7 @@ jobs:
     needs: [vc-hypo]
     if: always()
     steps:
-      - uses: technote-space/workflow-conclusion-action@v2
+      - uses: technote-space/workflow-conclusion-action@v3
       - uses: actions/checkout@v3
 
       - name: Check Failure
diff --git a/cmd/debug.go b/cmd/debug.go
index 4a0757b7..5dbd2b58 100644
--- a/cmd/debug.go
+++ b/cmd/debug.go
@@ -121,7 +121,7 @@ func copyFile(srcPath, destPath string, requireRootPrivileges bool) error {
 }
 
 func getCmdMount(mp string) (uid, pid, cmd string, err error) {
-	psArgs := []string{"/bin/sh", "-c", fmt.Sprintf("ps -ef | grep -v grep | grep -E \"%s mount | /sbin/mount.juicefs\" | grep %s", os.Args[0], mp)}
+	psArgs := []string{"/bin/sh", "-c", fmt.Sprintf("ps -ef | grep -v grep | grep mount | grep %s", mp)}
 	ret, err := exec.Command(psArgs[0], psArgs[1:]...).CombinedOutput()
 	if err != nil {
 		return "", "", "", fmt.Errorf("failed to execute command `%s`: %v", strings.Join(psArgs, " "), err)
@@ -218,7 +218,7 @@ func getPprofPort(pid, amp string, requireRootPrivileges bool) (int, error) {
 	if err := json.Unmarshal(content, &cfg); err != nil {
 		logger.Warnf("failed to unmarshal config file: %v", err)
 	}
-	if cfg.Port.DebugAgent != "" {
+	if cfg.Port != nil {
 		if len(strings.Split(cfg.Port.DebugAgent, ":")) >= 2 {
 			if port, err := strconv.Atoi(strings.Split(cfg.Port.DebugAgent, ":")[1]); err != nil {
 				logger.Warnf("failed to parse debug agent port: %v", err)
@@ -455,6 +455,14 @@ func collectLog(ctx *cli.Context, cmd string, requireRootPrivileges bool, currDi
 		logger.Warnf("Collecting log currently only support Linux/macOS")
 		return nil
 	}
+	matched, err := regexp.MatchString(`.*\s-d\s.*`, cmd)
+	if err != nil {
+		return fmt.Errorf("failed to match cmd: %v", err)
+	}
+	if !matched {
+		logger.Warnf("The juicefs mount by foreground, the log will not be collected")
+		return nil
+	}
 	logPath, err := getLogPath(cmd, requireRootPrivileges)
 	if err != nil {
 		return fmt.Errorf("failed to get log path: %v", err)
@@ -474,9 +482,7 @@ func collectSysInfo(ctx *cli.Context, currDir string) error {
 
 	result := fmt.Sprintf(`Platform: 
 %s %s
-%s
-JuiceFS Version:
-%s`, runtime.GOOS, runtime.GOARCH, sysInfo, ctx.App.Version)
+%s`, runtime.GOOS, runtime.GOARCH, sysInfo)
 
 	sysPath := filepath.Join(currDir, "system-info.log")
 	sysFile, err := os.Create(sysPath)
@@ -543,7 +549,7 @@ func debug(ctx *cli.Context) error {
 	}
 
 	if err := collectSysInfo(ctx, currDir); err != nil {
-		return err
+		logger.Errorf("Failed to collect system info: %v", err)
 	}
 
 	uid, pid, cmd, err := getCmdMount(amp)
@@ -560,15 +566,15 @@ func debug(ctx *cli.Context) error {
 
 	var wg sync.WaitGroup
 	if err := collectSpecialFile(ctx, amp, currDir, requireRootPrivileges, &wg); err != nil {
-		return err
+		logger.Errorf("Failed to collect special file: %v", err)
 	}
 
 	if err := collectLog(ctx, cmd, requireRootPrivileges, currDir); err != nil {
-		return err
+		logger.Errorf("Failed to collect log: %v", err)
 	}
 
 	if err := collectPprof(ctx, cmd, pid, amp, requireRootPrivileges, currDir, &wg); err != nil {
-		return err
+		logger.Errorf("Failed to collect pprof: %v", err)
 	}
 
 	wg.Wait()
diff --git a/cmd/flags.go b/cmd/flags.go
index 2d1258fc..06d346d9 100644
--- a/cmd/flags.go
+++ b/cmd/flags.go
@@ -106,7 +106,7 @@ func clientFlags() []cli.Flag {
 		},
 		&cli.IntFlag{
 			Name:  "max-deletes",
-			Value: 2,
+			Value: 10,
 			Usage: "number of threads to delete objects",
 		},
 		&cli.IntFlag{
diff --git a/cmd/format.go b/cmd/format.go
index a8e022a3..065ca1fc 100644
--- a/cmd/format.go
+++ b/cmd/format.go
@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -197,7 +196,6 @@ func createStorage(format meta.Format) (object.ObjectStorage, error) {
 	object.UserAgent = "JuiceFS-" + version.Version()
 	var blob object.ObjectStorage
 	var err error
-	object.GetHttpClient().Transport.(*http.Transport).TLSClientConfig = &tls.Config{}
 	if u, err := url.Parse(format.Bucket); err == nil {
 		values := u.Query()
 		if values.Get("tls-insecure-skip-verify") != "" {
diff --git a/cmd/gateway.go b/cmd/gateway.go
index e34989b5..543cfdec 100644
--- a/cmd/gateway.go
+++ b/cmd/gateway.go
@@ -22,7 +22,9 @@ package cmd
 import (
 	_ "net/http/pprof"
 	"os"
+	"os/signal"
 	"strconv"
+	"syscall"
 	"time"
 
 	"github.com/juicedata/juicefs/pkg/chunk"
@@ -208,7 +210,18 @@ func initForSvc(c *cli.Context, mp string, metaUrl string) (*vfs.Config, *fs.Fil
 	if err != nil {
 		logger.Fatalf("new session: %s", err)
 	}
-
+	// Go will catch all the signals
+	signal.Ignore(syscall.SIGPIPE)
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
+	go func() {
+		sig := <-signalChan
+		logger.Infof("Received signal %s, exiting...", sig.String())
+		if err := metaCli.CloseSession(); err != nil {
+			logger.Fatalf("close session failed: %s", err)
+		}
+		os.Exit(0)
+	}()
 	vfsConf := getVfsConf(c, metaConf, format, chunkConf)
 	vfsConf.AccessLog = c.String("access-log")
 	vfsConf.AttrTimeout = time.Millisecond * time.Duration(c.Float64("attr-cache")*1000)
diff --git a/cmd/mount.go b/cmd/mount.go
index a4b6d410..b3bf24d6 100644
--- a/cmd/mount.go
+++ b/cmd/mount.go
@@ -615,6 +615,9 @@ func mount(c *cli.Context) error {
 	if c.Bool("background") && os.Getenv("JFS_FOREGROUND") == "" {
 		daemonRun(c, addr, vfsConf, metaCli)
 	} else {
+		if c.IsSet("log") {
+			logger.Warnf("--log flag is ignored in foreground mode, the log output will be Stderr")
+		}
 		go checkMountpoint(vfsConf.Format.Name, mp, c.String("log"), false)
 	}
 
@@ -628,5 +631,7 @@ func mount(c *cli.Context) error {
 	v := vfs.NewVFS(vfsConf, metaCli, store, registerer, registry)
 	initBackgroundTasks(c, vfsConf, metaConf, metaCli, blob, registerer, registry)
 	mount_main(v, c)
-	return metaCli.CloseSession()
+	err = metaCli.CloseSession()
+	logger.Infof("The juicefs mount process exit successfully, mountpoint: %s", metaConf.MountPoint)
+	return err
 }
diff --git a/cmd/objbench.go b/cmd/objbench.go
index ec6bcaa0..634fc3a6 100644
--- a/cmd/objbench.go
+++ b/cmd/objbench.go
@@ -177,7 +177,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("smallput", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("smallput", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
@@ -190,7 +190,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("smallget", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("smallget", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
@@ -204,7 +204,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("put", float64(bSize>>20*bCount)/cost, cost*1000/float64(bCount), 2, colorful)
+					line[1], line[2] = colorize("put", float64(bSize>>20*bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful)
 					line[1] += " MiB/s"
 					line[2] += " ms/object"
 				}
@@ -223,7 +223,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("get", float64(bSize>>20*bCount)/cost, cost*1000/float64(bCount), 2, colorful)
+					line[1], line[2] = colorize("get", float64(bSize>>20*bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful)
 					line[1] += " MiB/s"
 					line[2] += " ms/object"
 				}
@@ -236,7 +236,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("list", float64(sCount)*100/cost, cost*10, 2, colorful)
+					line[1], line[2] = colorize("list", float64(sCount)*100/cost, float64(threads)*cost*10, 2, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/op"
 				}
@@ -249,7 +249,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("head", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("head", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
@@ -262,7 +262,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("chtimes", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("chtimes", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
@@ -275,7 +275,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("chmod", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("chmod", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
@@ -288,7 +288,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("chown", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("chown", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
@@ -301,7 +301,7 @@ func objbench(ctx *cli.Context) error {
 			getResult: func(cost float64) []string {
 				line := []string{"", nspt, nspt}
 				if cost > 0 {
-					line[1], line[2] = colorize("delete", float64(sCount)/cost, cost*1000/float64(sCount), 1, colorful)
+					line[1], line[2] = colorize("delete", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful)
 					line[1] += " objects/s"
 					line[2] += " ms/object"
 				}
diff --git a/cmd/status.go b/cmd/status.go
index 515f293f..b82ed00d 100644
--- a/cmd/status.go
+++ b/cmd/status.go
@@ -64,14 +64,14 @@ type sections struct {
 }
 
 type statistic struct {
-	UsedSpace         uint64
-	AvailableSpace    uint64
-	UsedInodes        uint64
-	AvailableInodes   uint64
-	DeletedSliceCount int64 `json:",omitempty"`
-	DeletedSliceSize  int64 `json:",omitempty"`
-	DeletedFileCount  int64 `json:",omitempty"`
-	DeletedFileSize   int64 `json:",omitempty"`
+	UsedSpace               uint64
+	AvailableSpace          uint64
+	UsedInodes              uint64
+	AvailableInodes         uint64
+	TrashSliceCount         int64 `json:",omitempty"`
+	TrashSliceSize          int64 `json:",omitempty"`
+	PendingDeletedFileCount int64 `json:",omitempty"`
+	PendingDeletedFileSize  int64 `json:",omitempty"`
 }
 
 func printJson(v interface{}) {
@@ -115,9 +115,9 @@ func status(ctx *cli.Context) error {
 
 	if ctx.Bool("more") {
 		progress := utils.NewProgress(false, false)
-		slicesSpinner := progress.AddDoubleSpinner("Delayed Slices")
+		slicesSpinner := progress.AddDoubleSpinner("Trash Slices")
 		defer slicesSpinner.Done()
-		fileSpinner := progress.AddDoubleSpinner("Delayed Files")
+		fileSpinner := progress.AddDoubleSpinner("Pending Deleted Files")
 		defer fileSpinner.Done()
 
 		err = m.ScanDeletedObject(
@@ -137,8 +137,8 @@ func status(ctx *cli.Context) error {
 			logger.Fatalf("statistic: %s", err)
 			return err
 		}
-		stat.DeletedSliceCount, stat.DeletedSliceSize = slicesSpinner.Current()
-		stat.DeletedFileCount, stat.DeletedFileSize = fileSpinner.Current()
+		stat.TrashSliceCount, stat.TrashSliceSize = slicesSpinner.Current()
+		stat.PendingDeletedFileCount, stat.PendingDeletedFileSize = fileSpinner.Current()
 	}
 
 	printJson(&sections{format, sessions, stat})
diff --git a/docs/en/administration/metadata/redis_best_practices.md b/docs/en/administration/metadata/redis_best_practices.md
index c7ac13c0..ffcb5c68 100644
--- a/docs/en/administration/metadata/redis_best_practices.md
+++ b/docs/en/administration/metadata/redis_best_practices.md
@@ -188,7 +188,9 @@ If you want to use a Redis compatible product as the metadata engine, you need t
 + [DECRBY](https://redis.io/commands/decrby/)
 + [DEL](https://redis.io/commands/del/)
 + [GET](https://redis.io/commands/get/)
++ [INCR](https://redis.io/commands/incr/)
 + [INCRBY](https://redis.io/commands/incrby/)
++ [DECR](https://redis.io/commands/decr/)
 + [MGET](https://redis.io/commands/mget/)
 + [MSET](https://redis.io/commands/mset/)
 + [SETNX](https://redis.io/commands/setnx/)
diff --git a/docs/en/deployment/hadoop_java_sdk.md b/docs/en/deployment/hadoop_java_sdk.md
index 458e45f4..6849d873 100644
--- a/docs/en/deployment/hadoop_java_sdk.md
+++ b/docs/en/deployment/hadoop_java_sdk.md
@@ -163,7 +163,7 @@ Please refer to the following table to set the relevant parameters of the JuiceF
 | Configuration            | Default Value | Description                                     |
 |--------------------------|---------------|-------------------------------------------------|
 | `juicefs.max-uploads`    | 20            | The max number of connections to upload         |
-| `juicefs.max-deletes`    | 2             | The max number of connections to delete         |
+| `juicefs.max-deletes`    | 10            | The max number of connections to delete         |
 | `juicefs.get-timeout`    | 5             | The max number of seconds to download an object |
 | `juicefs.put-timeout`    | 60            | The max number of seconds to upload an object   |
 | `juicefs.memory-size`    | 300           | Total read/write buffering in MiB               |
diff --git a/docs/en/reference/command_reference.md b/docs/en/reference/command_reference.md
index 8f3259c5..2c117732 100644
--- a/docs/en/reference/command_reference.md
+++ b/docs/en/reference/command_reference.md
@@ -275,7 +275,7 @@ number of retries after network failure (default: 10)
 number of connections to upload (default: 20)
 
 `--max-deletes value`<br />
-number of threads to delete objects (default: 2)
+number of threads to delete objects (default: 10)
 
 `--buffer-size value`<br />
 total read/write buffering in MiB (default: 300)
@@ -405,7 +405,7 @@ number of retries after network failure (default: 10)
 number of connections to upload (default: 20)
 
 `--max-deletes value`<br />
-number of threads to delete objects (default: 2)
+number of threads to delete objects (default: 10)
 
 `--buffer-size value`<br />
 total read/write buffering in MiB (default: 300)
@@ -532,7 +532,7 @@ number of retries after network failure (default: 10)
 number of connections to upload (default: 20)
 
 `--max-deletes value`<br />
-number of threads to delete objects (default: 2)
+number of threads to delete objects (default: 10)
 
 `--buffer-size value`<br />
 total read/write buffering in MiB (default: 300)
diff --git a/docs/zh_cn/administration/metadata/redis_best_practices.md b/docs/zh_cn/administration/metadata/redis_best_practices.md
index 27df1926..3b8f6c94 100644
--- a/docs/zh_cn/administration/metadata/redis_best_practices.md
+++ b/docs/zh_cn/administration/metadata/redis_best_practices.md
@@ -189,7 +189,9 @@ Redis 对数据备份非常友好，因为您可以在数据库运行时复制 R
 + [DECRBY](https://redis.io/commands/decrby/)
 + [DEL](https://redis.io/commands/del/)
 + [GET](https://redis.io/commands/get/)
++ [INCR](https://redis.io/commands/incr/)
 + [INCRBY](https://redis.io/commands/incrby/)
++ [DECR](https://redis.io/commands/decr/)
 + [MGET](https://redis.io/commands/mget/)
 + [MSET](https://redis.io/commands/mset/)
 + [SETNX](https://redis.io/commands/setnx/)
diff --git a/docs/zh_cn/community/adopters.md b/docs/zh_cn/community/adopters.md
index 786bfa5c..8bf5396d 100644
--- a/docs/zh_cn/community/adopters.md
+++ b/docs/zh_cn/community/adopters.md
@@ -7,9 +7,9 @@ slug: /adopters
 | 公司／团队                                       | 使用环境   | 使用场景               | 案例                                                                                                                                                                                                  |
 |--------------------------------------------------|------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [小米](https://www.mi.com)                       | Production | AI                     |                                                                                                                                                                                                       |
-| [理想汽车](https://www.lixiang.com)              | Production | 大数据，AI             | [JuiceFS 在理想汽车的使用和展望](https://juicefs.com/zh-cn/blog/li-auto-with-juicefs)<br />[理想汽车：从 Hadoop 到云原生的演进与思考](https://juicefs.com/zh-cn/blog/liauto-case-hadoop-cloudnatrive) |
+| [理想汽车](https://www.lixiang.com)              | Production | 大数据，AI             | [JuiceFS 在理想汽车的使用和展望](https://juicefs.com/zh-cn/blog/li-auto-with-juicefs)，<br />[理想汽车：从 Hadoop 到云原生的演进与思考](https://juicefs.com/zh-cn/blog/liauto-case-hadoop-cloudnatrive) |
 | [Shopee](https://shopee.com)                     | Production | 大数据                 | [Shopee x JuiceFS：ClickHouse 冷热数据分离存储架构与实践](https://juicefs.com/zh-cn/blog/shopee-clickhouse-with-juicefs)                                                                              |
-| [知乎](https://www.zhihu.com)                    | Production | 大数据                 | [知乎 x JuiceFS：利用 JuiceFS 给 Flink 容器启动加速](https://juicefs.com/zh-cn/blog/zhihu-flink-with-juicefs)                                                                                         |
+| [知乎](https://www.zhihu.com)                    | Production | 大数据                 | [知乎 x JuiceFS：利用 JuiceFS 给 Flink 容器启动加速](https://juicefs.com/zh-cn/blog/zhihu-flink-with-juicefs)，<br />[利用 JuiceFS 动态注入 Protobuf JAR 包](https://zhuanlan.zhihu.com/p/586120009)                                                                                         |
 | [尧信](https://www.yaoxinhd.com)                 | Production | 大数据，共享文件存储   |                                                                                                                                                                                                       |
 | [旷视](https://megvii.com)                       | Production | AI                     |                                                                                                                                                                                                       |
 | [航天宏图](https://www.piesat.cn)                | Production | 共享文件存储           |                                                                                                                                                                                                       |
@@ -38,6 +38,25 @@ slug: /adopters
 | [云从科技](https://www.cloudwalk.com)            | Production | AI                     |                                                                                                                                                                                                       |
 | 名之梦                                           | Production | AI                     |                                                                                                                                                                                                       |
 | [建信金融科技](https://www.ccbft.com)            | Production | AI，共享文件存储       |                                                                                                                                                                                                       |
+| [思谋 Smartmore](https://cn.smartmore.com/) | Production | AI |    |
+| [汇丰银行 HSBC](https://www.hsbc.com.cn/) | Production | 共享文件存储 |   |
+| [中国电信](http://www.chinatelecom.com.cn/) | Production | 大数据 |     |
+| [酷数智能](http://www.kurudata.com/) | Production | 共享文件存储 |    |
+| [朗新集团](https://www.longshine.com/) | Production | 大数据，共享文件存储 |    |
+| [网易邮箱](https://mail.163.com/) | Production | 大数据 |    |
+| [梧桐车联](https://www.auto-pai.com/) | Production | AI，大数据 |    |
+| [南昌维网科技](https://www.vwell.cn/) | Production | 共享数据存储 |    |
+| [声网](https://www.agora.io/cn/) | Production | 大数据 |    |
+| [南京鹏云网络](https://www.pengyunnetwork.cn/) | Production | 共享文件存储 |    |
+| [聚云位智 LinkoopDB](http://www.datapps.cn/) | Production | 大数据 |    |
+| [之江实验室](https://www.zhejianglab.com/) | Production | AI，共享文件存储 |    |
+| [国家天文科学数据中心](https://nadc.china-vo.org/) | Production | 共享文件存储 |    |
+| [艾莎医学](https://www.ashermed.com/) | Production | 共享文件存储 |    |
+| [蝉妈妈数据](https://www.chanmama.com/) | Production | AI |    |
+| [NodeReal](https://nodereal.io/) | Production | 共享文件存储 |    |
+| [不鸣科技](https://www.boomingtech.com/) | Production | 共享文件存储 |    |
+| [博依特科技](https://www.poi-t.com/) | Production | 大数据 |    |
+| [九曳供应链](https://www.jiuyescm.com/) | Production | 大数据 |    |
 | [顺丰速运](https://www.sf-express.com)           | Testing    | 大数据，共享文件存储   |                                                                                                                                                                                                       |
 | [BIGO](https://bigo.tv)                          | Testing    | AI                     |                                                                                                                                                                                                       |
 | [蒙商消费金融](https://www.mengshangxiaofei.com) | Testing    | 共享文件存储           |                                                                                                                                                                                                       |
diff --git a/docs/zh_cn/community/articles.md b/docs/zh_cn/community/articles.md
index 08dcb5ed..79a09054 100644
--- a/docs/zh_cn/community/articles.md
+++ b/docs/zh_cn/community/articles.md
@@ -20,7 +20,10 @@ JuiceFS 广泛适用于各种数据存储和共享场景，本页汇总来自世
 
 ## 大数据
 
+
+- [Protobuf 在知乎大数据场景的应用，利用 JuiceFS 动态注入 JAR 包](https://zhuanlan.zhihu.com/p/586120009)，2022-11-23，胡梦宇@知乎
 - [金山云：基于 JuiceFS 的 Elasticsearch 温冷热数据管理实践](https://juicefs.com/zh-cn/blog/user-stories/juicefs-elasticsearch-cold-heat-data-management)，2022-11-17，侯学峰@金山云
+- [JuiceFS替代HDFS，苦HDFS小文件久矣](https://zhuanlan.zhihu.com/p/569586606)，2022-10-08，久耶供应链 大数据总监
 - [JuiceFS 在 Elasticsearch/ClickHouse 温冷数据存储中的实践](https://juicefs.com/zh-cn/blog/solutions/juicefs-elasticsearch-clickhouse-hot-cold-data-storage)，2022-09-30，高昌健
 - [从 Hadoop 到云原生，大数据平台如何做存算分离](https://juicefs.com/zh-cn/blog/solutions/hadoop-to-cloud-native-separation-of-compute-and-storage-for-big-data-platform)，2022-09-14，苏锐
 - [理想汽车：从 Hadoop 到云原生的演进与思考](https://juicefs.com/zh-cn/blog/user-stories/li-auto-case-hadoop-cloud-native)，2022-08-30，聂磊@理想汽车
diff --git a/docs/zh_cn/deployment/hadoop_java_sdk.md b/docs/zh_cn/deployment/hadoop_java_sdk.md
index 1a061355..8367198a 100644
--- a/docs/zh_cn/deployment/hadoop_java_sdk.md
+++ b/docs/zh_cn/deployment/hadoop_java_sdk.md
@@ -164,7 +164,7 @@ make win
 | 配置项                      | 默认值     | 描述                     |
 |--------------------------|---------|------------------------|
 | `juicefs.max-uploads`    | 20      | 上传数据的最大连接数             |
-| `juicefs.max-deletes`    | 2       | 删除数据的最大连接数             |
+| `juicefs.max-deletes`    | 10      | 删除数据的最大连接数             |
 | `juicefs.get-timeout`    | 5       | 下载一个对象的超时时间，单位为秒。      |
 | `juicefs.put-timeout`    | 60      | 上传一个对象的超时时间，单位为秒。      |
 | `juicefs.memory-size`    | 300     | 读写数据的缓冲区最大空间，单位为 MiB。  |
diff --git a/docs/zh_cn/reference/command_reference.md b/docs/zh_cn/reference/command_reference.md
index 72fcb519..c5d6f57c 100644
--- a/docs/zh_cn/reference/command_reference.md
+++ b/docs/zh_cn/reference/command_reference.md
@@ -277,7 +277,7 @@ consul 注册中心地址 (默认："127.0.0.1:8500")
 上传对象的连接数 (默认：20)
 
 `--max-deletes value`<br />
-删除对象的连接数 (默认：2)
+删除对象的连接数 (默认：10)
 
 `--buffer-size value`<br />
 读写缓存的总大小；单位为 MiB (默认：300)
@@ -407,7 +407,7 @@ juicefs gateway [command options] META-URL ADDRESS
 上传对象的连接数 (默认：20)
 
 `--max-deletes value`<br />
-删除对象的连接数 (默认：2)
+删除对象的连接数 (默认：10)
 
 `--buffer-size value`<br />
 读写缓存的总大小；单位为 MiB (默认：300)
@@ -533,7 +533,7 @@ juicefs webdav [command options] META-URL ADDRESS
 上传对象的连接数 (默认：20)
 
 `--max-deletes value`<br />
-删除对象的连接数 (默认：2)
+删除对象的连接数 (默认：10)
 
 `--buffer-size value`<br />
 读写缓存的总大小；单位为 MiB (默认：300)
diff --git a/pkg/chunk/disk_cache.go b/pkg/chunk/disk_cache.go
index a7364dc4..98091ca4 100644
--- a/pkg/chunk/disk_cache.go
+++ b/pkg/chunk/disk_cache.go
@@ -371,9 +371,6 @@ func (cache *cacheStore) uploaded(key string, size int) {
 
 // locked
 func (cache *cacheStore) cleanup() {
-	if !cache.scanned {
-		return
-	}
 	goal := cache.capacity * 95 / 100
 	num := len(cache.keys) * 99 / 100
 	// make sure we have enough free space after cleanup
diff --git a/pkg/fuse/fuse.go b/pkg/fuse/fuse.go
index 86423adb..c3f32cb9 100644
--- a/pkg/fuse/fuse.go
+++ b/pkg/fuse/fuse.go
@@ -19,6 +19,7 @@ package fuse
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"runtime"
 	"strings"
 	"syscall"
@@ -470,6 +471,13 @@ func Serve(v *vfs.VFS, options string, xattrs, ioctl bool) error {
 	}
 	fssrv, err := fuse.NewServer(imp, conf.Meta.MountPoint, &opt)
 	if err != nil {
+		if execErr, ok := err.(*exec.Error); ok {
+			if pathErr, ok := execErr.Unwrap().(*os.PathError); ok &&
+				strings.Contains(pathErr.Path, "fusermount") &&
+				pathErr.Unwrap() == syscall.ENOENT {
+				return fmt.Errorf("fuse is not installed. Please install it first")
+			}
+		}
 		return fmt.Errorf("fuse: %s", err)
 	}
 	if runtime.GOOS == "linux" {
diff --git a/pkg/meta/base.go b/pkg/meta/base.go
index 81797ecb..6328354f 100644
--- a/pkg/meta/base.go
+++ b/pkg/meta/base.go
@@ -64,7 +64,7 @@ type engine interface {
 	doFindDeletedFiles(ts int64, limit int) (map[Ino]uint64, error) // limit < 0 means all
 	doDeleteFileData(inode Ino, length uint64)
 	doCleanupSlices()
-	doCleanupDelayedSlices(edge int64, limit int) (int, error)
+	doCleanupDelayedSlices(edge int64) (int, error)
 	doDeleteSlice(id uint64, size uint32) error
 
 	doGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno
@@ -358,7 +358,7 @@ func (m *baseMeta) refresh() {
 		if m.conf.ReadOnly || m.conf.NoBGJob {
 			continue
 		}
-		if ok, err := m.en.setIfSmall("lastCleanupSessions", time.Now().Unix(), int64(m.conf.Heartbeat/time.Second)); err != nil {
+		if ok, err := m.en.setIfSmall("lastCleanupSessions", time.Now().Unix(), int64((m.conf.Heartbeat * 9 / 10).Seconds())); err != nil {
 			logger.Warnf("checking counter lastCleanupSessions: %s", err)
 		} else if ok {
 			go m.CleanStaleSessions()
@@ -428,7 +428,7 @@ func (m *baseMeta) flushStats() {
 func (m *baseMeta) cleanupDeletedFiles() {
 	for {
 		utils.SleepWithJitter(time.Minute)
-		if ok, err := m.en.setIfSmall("lastCleanupFiles", time.Now().Unix(), 60); err != nil {
+		if ok, err := m.en.setIfSmall("lastCleanupFiles", time.Now().Unix(), int64(time.Minute.Seconds())*9/10); err != nil {
 			logger.Warnf("checking counter lastCleanupFiles: %s", err)
 		} else if ok {
 			files, err := m.en.doFindDeletedFiles(time.Now().Add(-time.Hour).Unix(), 10000)
@@ -447,7 +447,7 @@ func (m *baseMeta) cleanupDeletedFiles() {
 func (m *baseMeta) cleanupSlices() {
 	for {
 		utils.SleepWithJitter(time.Hour)
-		if ok, err := m.en.setIfSmall("nextCleanupSlices", time.Now().Unix(), 3600); err != nil {
+		if ok, err := m.en.setIfSmall("nextCleanupSlices", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil {
 			logger.Warnf("checking counter nextCleanupSlices: %s", err)
 		} else if ok {
 			m.en.doCleanupSlices()
@@ -1267,26 +1267,30 @@ func (m *baseMeta) CompactAll(ctx Context, threads int, bar *utils.Bar) syscall.
 	return 0
 }
 
-func (m *baseMeta) fileDeleted(opened bool, inode Ino, length uint64) {
+func (m *baseMeta) fileDeleted(opened, force bool, inode Ino, length uint64) {
 	if opened {
 		m.Lock()
 		m.removedFiles[inode] = true
 		m.Unlock()
 	} else {
-		m.tryDeleteFileData(inode, length)
+		m.tryDeleteFileData(inode, length, force)
 	}
 }
 
-func (m *baseMeta) tryDeleteFileData(inode Ino, length uint64) {
-	select {
-	case m.maxDeleting <- struct{}{}:
-		go func() {
-			m.en.doDeleteFileData(inode, length)
-			<-m.maxDeleting
-		}()
-	default:
-		// will be cleanup later
+func (m *baseMeta) tryDeleteFileData(inode Ino, length uint64, force bool) {
+	if force {
+		m.maxDeleting <- struct{}{}
+	} else {
+		select {
+		case m.maxDeleting <- struct{}{}:
+		default:
+			return // will be cleanup later
+		}
 	}
+	go func() {
+		m.en.doDeleteFileData(inode, length)
+		<-m.maxDeleting
+	}()
 }
 
 func (m *baseMeta) deleteSlice(id uint64, size uint32) {
@@ -1356,7 +1360,7 @@ func (m *baseMeta) cleanupTrash() {
 			}
 			continue
 		}
-		if ok, err := m.en.setIfSmall("lastCleanupTrash", time.Now().Unix(), 3600); err != nil {
+		if ok, err := m.en.setIfSmall("lastCleanupTrash", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil {
 			logger.Warnf("checking counter lastCleanupTrash: %s", err)
 		} else if ok {
 			go m.doCleanupTrash(false)
@@ -1446,15 +1450,10 @@ func (m *baseMeta) cleanupDelayedSlices() {
 	now := time.Now()
 	edge := now.Unix() - int64(m.fmt.TrashDays)*24*3600
 	logger.Debugf("Cleanup delayed slices: started with edge %d", edge)
-	if count, err := m.en.doCleanupDelayedSlices(edge, 3e5); err == nil {
-		msg := fmt.Sprintf("Cleanup delayed slices: deleted %d slices in %v", count, time.Since(now))
-		if count >= 3e5 {
-			logger.Warnf("%s (reached max limit, stop)", msg)
-		} else {
-			logger.Debugf(msg)
-		}
-	} else {
+	if count, err := m.en.doCleanupDelayedSlices(edge); err != nil {
 		logger.Warnf("Cleanup delayed slices: deleted %d slices in %v, but got error: %s", count, time.Since(now), err)
+	} else if count > 0 {
+		logger.Infof("Cleanup delayed slices: deleted %d slices in %v", count, time.Since(now))
 	}
 }
 
diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go
index 765421d8..70ad5b54 100644
--- a/pkg/meta/redis.go
+++ b/pkg/meta/redis.go
@@ -1326,7 +1326,7 @@ func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno
 	}, m.entryKey(parent), m.inodeKey(parent))
 	if err == nil && trash == 0 {
 		if _type == TypeFile && attr.Nlink == 0 {
-			m.fileDeleted(opened, inode, attr.Length)
+			m.fileDeleted(opened, isTrash(parent), inode, attr.Length)
 		}
 		m.updateStats(newSpace, newInode)
 	}
@@ -1672,7 +1672,7 @@ func (m *redisMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentD
 	}, m.entryKey(parentSrc), m.inodeKey(parentSrc), m.entryKey(parentDst), m.inodeKey(parentDst))
 	if err == nil && !exchange && trash == 0 {
 		if dino > 0 && dtyp == TypeFile && tattr.Nlink == 0 {
-			m.fileDeleted(opened, dino, tattr.Length)
+			m.fileDeleted(opened, false, dino, tattr.Length)
 		}
 		m.updateStats(newSpace, newInode)
 	}
@@ -1961,7 +1961,7 @@ func (m *redisMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {
 	})
 	if err == nil {
 		m.updateStats(newSpace, -1)
-		m.tryDeleteFileData(inode, attr.Length)
+		m.tryDeleteFileData(inode, attr.Length, false)
 	}
 	return err
 }
@@ -2437,8 +2437,9 @@ func (m *redisMeta) doDeleteFileData_(inode Ino, length uint64, tracking string)
 	_ = m.rdb.ZRem(ctx, m.delfiles(), tracking)
 }
 
-func (r *redisMeta) doCleanupDelayedSlices(edge int64, limit int) (int, error) {
+func (r *redisMeta) doCleanupDelayedSlices(edge int64) (int, error) {
 	ctx := Background
+	start := time.Now()
 	stop := fmt.Errorf("reach limit")
 	var count int
 	var ss []Slice
@@ -2489,9 +2490,9 @@ func (r *redisMeta) doCleanupDelayedSlices(edge int64, limit int) (int, error) {
 					count++
 				}
 			}
-			if count >= limit {
-				return stop
-			}
+		}
+		if time.Since(start) > 50*time.Minute {
+			return stop
 		}
 		return nil
 	})
diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go
index cccc075f..7640b25f 100644
--- a/pkg/meta/sql.go
+++ b/pkg/meta/sql.go
@@ -1367,7 +1367,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno {
 	}, parent)
 	if err == nil && trash == 0 {
 		if n.Type == TypeFile && n.Nlink == 0 {
-			m.fileDeleted(opened, n.Inode, n.Length)
+			m.fileDeleted(opened, isTrash(parent), n.Inode, n.Length)
 		}
 		m.updateStats(newSpace, newInode)
 	}
@@ -1741,7 +1741,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
 	}, parentSrc)
 	if err == nil && !exchange && trash == 0 {
 		if dino > 0 && dn.Type == TypeFile && dn.Nlink == 0 {
-			m.fileDeleted(opened, dino, dn.Length)
+			m.fileDeleted(opened, false, dino, dn.Length)
 		}
 		m.updateStats(newSpace, newInode)
 	}
@@ -1983,7 +1983,7 @@ func (m *dbMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {
 	})
 	if err == nil {
 		m.updateStats(newSpace, -1)
-		m.tryDeleteFileData(inode, n.Length)
+		m.tryDeleteFileData(inode, n.Length, false)
 	}
 	return err
 }
@@ -2328,54 +2328,58 @@ func (m *dbMeta) doDeleteFileData(inode Ino, length uint64) {
 	})
 }
 
-func (m *dbMeta) doCleanupDelayedSlices(edge int64, limit int) (int, error) {
-	var result []delslices
-	_ = m.roTxn(func(s *xorm.Session) error {
-		result = nil
-		return s.Where("deleted < ?", edge).Limit(limit, 0).Find(&result)
-	})
-
+func (m *dbMeta) doCleanupDelayedSlices(edge int64) (int, error) {
+	start := time.Now()
 	var count int
 	var ss []Slice
-	for _, ds := range result {
-		if err := m.txn(func(ses *xorm.Session) error {
-			ss = ss[:0]
-			ds := delslices{Id: ds.Id}
-			if ok, e := ses.ForUpdate().Get(&ds); e != nil {
-				return e
-			} else if !ok {
-				return nil
-			}
-			m.decodeDelayedSlices(ds.Slices, &ss)
-			if len(ss) == 0 {
-				return fmt.Errorf("invalid value for delayed slices %d: %v", ds.Id, ds.Slices)
-			}
-			for _, s := range ss {
-				if _, e := ses.Exec("update jfs_chunk_ref set refs=refs-1 where chunkid=? and size=?", s.Id, s.Size); e != nil {
+	var result []delslices
+	var batch int = 1e6
+	for {
+		_ = m.roTxn(func(s *xorm.Session) error {
+			result = result[:0]
+			return s.Where("deleted < ?", edge).Limit(batch, 0).Find(&result)
+		})
+
+		for _, ds := range result {
+			if err := m.txn(func(ses *xorm.Session) error {
+				ss = ss[:0]
+				ds := delslices{Id: ds.Id}
+				if ok, e := ses.ForUpdate().Get(&ds); e != nil {
 					return e
+				} else if !ok {
+					return nil
+				}
+				m.decodeDelayedSlices(ds.Slices, &ss)
+				if len(ss) == 0 {
+					return fmt.Errorf("invalid value for delayed slices %d: %v", ds.Id, ds.Slices)
+				}
+				for _, s := range ss {
+					if _, e := ses.Exec("update jfs_chunk_ref set refs=refs-1 where chunkid=? and size=?", s.Id, s.Size); e != nil {
+						return e
+					}
 				}
+				_, e := ses.Delete(&delslices{Id: ds.Id})
+				return e
+			}); err != nil {
+				logger.Warnf("Cleanup delayed slices %d: %s", ds.Id, err)
+				continue
 			}
-			_, e := ses.Delete(&delslices{Id: ds.Id})
-			return e
-		}); err != nil {
-			logger.Warnf("Cleanup delayed slices %d: %s", ds.Id, err)
-			continue
-		}
-		for _, s := range ss {
-			var ref = sliceRef{Id: s.Id}
-			err := m.roTxn(func(s *xorm.Session) error {
-				ok, err := s.Get(&ref)
-				if err == nil && !ok {
-					err = errors.New("not found")
+			for _, s := range ss {
+				var ref = sliceRef{Id: s.Id}
+				err := m.roTxn(func(s *xorm.Session) error {
+					ok, err := s.Get(&ref)
+					if err == nil && !ok {
+						err = errors.New("not found")
+					}
+					return err
+				})
+				if err == nil && ref.Refs <= 0 {
+					m.deleteSlice(s.Id, s.Size)
+					count++
 				}
-				return err
-			})
-			if err == nil && ref.Refs <= 0 {
-				m.deleteSlice(s.Id, s.Size)
-				count++
 			}
 		}
-		if count >= limit {
+		if len(result) < batch || time.Since(start) > 50*time.Minute {
 			break
 		}
 	}
diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go
index fbfd0416..081827b0 100644
--- a/pkg/meta/tkv.go
+++ b/pkg/meta/tkv.go
@@ -43,6 +43,7 @@ type kvTxn interface {
 	gets(keys ...[]byte) [][]byte
 	scanRange(begin, end []byte) map[string][]byte
 	scanKeys(prefix []byte) [][]byte
+	scanKeysRange(begin, end []byte, limit int, filter func(k []byte) bool) [][]byte
 	scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte
 	exist(prefix []byte) bool
 	set(key, value []byte)
@@ -1207,7 +1208,7 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno {
 	}, parent)
 	if err == nil && trash == 0 {
 		if _type == TypeFile && attr.Nlink == 0 {
-			m.fileDeleted(opened, inode, attr.Length)
+			m.fileDeleted(opened, isTrash(parent), inode, attr.Length)
 		}
 		m.updateStats(newSpace, newInode)
 	}
@@ -1508,7 +1509,7 @@ func (m *kvMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
 	}, parentSrc)
 	if err == nil && !exchange && trash == 0 {
 		if dino > 0 && dtyp == TypeFile && tattr.Nlink == 0 {
-			m.fileDeleted(opened, dino, tattr.Length)
+			m.fileDeleted(opened, false, dino, tattr.Length)
 		}
 		m.updateStats(newSpace, newInode)
 	}
@@ -1666,7 +1667,7 @@ func (m *kvMeta) doDeleteSustainedInode(sid uint64, inode Ino) error {
 	})
 	if err == nil {
 		m.updateStats(newSpace, -1)
-		m.tryDeleteFileData(inode, attr.Length)
+		m.tryDeleteFileData(inode, attr.Length, false)
 	}
 	return err
 }
@@ -1969,52 +1970,52 @@ func (m *kvMeta) doDeleteFileData(inode Ino, length uint64) {
 	_ = m.deleteKeys(m.delfileKey(inode, length))
 }
 
-func (m *kvMeta) doCleanupDelayedSlices(edge int64, limit int) (int, error) {
-	var keys [][]byte
-	if err := m.client.txn(func(tx kvTxn) error {
-		vals := tx.scanRange(m.delSliceKey(0, 0), m.delSliceKey(edge, 0))
-		for k := range vals {
-			if len(k) != 1+8+8 { // delayed slices: Lttttttttcccccccc
-				continue
-			}
-			keys = append(keys, []byte(k))
-		}
-		return nil
-	}); err != nil {
-		logger.Warnf("Scan delayed slices: %s", err)
-		return 0, err
-	}
-
+func (m *kvMeta) doCleanupDelayedSlices(edge int64) (int, error) {
+	start := time.Now()
 	var count int
 	var ss []Slice
 	var rs []int64
-	for _, key := range keys {
-		if err := m.txn(func(tx kvTxn) error {
-			ss, rs = ss[:0], rs[:0]
-			buf := tx.get(key)
-			if len(buf) == 0 {
-				return nil
-			}
-			m.decodeDelayedSlices(buf, &ss)
-			if len(ss) == 0 {
-				return fmt.Errorf("invalid value for delayed slices %s: %v", key, buf)
-			}
-			for _, s := range ss {
-				rs = append(rs, tx.incrBy(m.sliceKey(s.Id, s.Size), -1))
-			}
-			tx.dels(key)
+	var keys [][]byte
+	var batch int = 1e5
+	for {
+		if err := m.client.txn(func(tx kvTxn) error {
+			keys = tx.scanKeysRange(m.delSliceKey(0, 0), m.delSliceKey(edge, 0), batch, func(k []byte) bool {
+				return len(k) == 1+8+8 // delayed slices: Lttttttttcccccccc
+			})
 			return nil
 		}); err != nil {
-			logger.Warnf("Cleanup delayed slices %s: %s", key, err)
-			continue
+			logger.Warnf("Scan delayed slices: %s", err)
+			return count, err
 		}
-		for i, s := range ss {
-			if rs[i] < 0 {
-				m.deleteSlice(s.Id, s.Size)
-				count++
+
+		for _, key := range keys {
+			if err := m.txn(func(tx kvTxn) error {
+				ss, rs = ss[:0], rs[:0]
+				buf := tx.get(key)
+				if len(buf) == 0 {
+					return nil
+				}
+				m.decodeDelayedSlices(buf, &ss)
+				if len(ss) == 0 {
+					return fmt.Errorf("invalid value for delayed slices %s: %v", key, buf)
+				}
+				for _, s := range ss {
+					rs = append(rs, tx.incrBy(m.sliceKey(s.Id, s.Size), -1))
+				}
+				tx.dels(key)
+				return nil
+			}); err != nil {
+				logger.Warnf("Cleanup delayed slices %s: %s", key, err)
+				continue
+			}
+			for i, s := range ss {
+				if rs[i] < 0 {
+					m.deleteSlice(s.Id, s.Size)
+					count++
+				}
 			}
 		}
-		if count >= limit {
+		if len(keys) < batch || time.Since(start) > 50*time.Minute {
 			break
 		}
 	}
diff --git a/pkg/meta/tkv_badger.go b/pkg/meta/tkv_badger.go
index 9a020576..79222f50 100644
--- a/pkg/meta/tkv_badger.go
+++ b/pkg/meta/tkv_badger.go
@@ -92,6 +92,30 @@ func (tx *badgerTxn) scanKeys(prefix []byte) [][]byte {
 	return ret
 }
 
+func (tx *badgerTxn) scanKeysRange(begin, end []byte, limit int, filter func(k []byte) bool) [][]byte {
+	if limit == 0 {
+		return nil
+	}
+	it := tx.t.NewIterator(badger.IteratorOptions{
+		PrefetchValues: true,
+		PrefetchSize:   1024,
+	})
+	defer it.Close()
+	var ret [][]byte
+	for it.Rewind(); it.Valid(); it.Next() {
+		key := it.Item().KeyCopy(nil)
+		if filter == nil || filter(key) {
+			ret = append(ret, key)
+			if limit > 0 {
+				if limit--; limit == 0 {
+					break
+				}
+			}
+		}
+	}
+	return ret
+}
+
 func (tx *badgerTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
 	if limit == 0 {
 		return nil
diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go
index 383961d8..0a24b655 100644
--- a/pkg/meta/tkv_etcd.go
+++ b/pkg/meta/tkv_etcd.go
@@ -141,6 +141,28 @@ func (tx *etcdTxn) scanKeys(prefix []byte) [][]byte {
 	return keys
 }
 
+func (tx *etcdTxn) scanKeysRange(begin_, end_ []byte, limit int, filter func(k []byte) bool) [][]byte {
+	if limit == 0 {
+		return nil
+	}
+	resp, err := tx.kv.Get(tx.ctx, string(begin_), etcd.WithRange(string(end_)), etcd.WithKeysOnly())
+	if err != nil {
+		panic(fmt.Errorf("get range [%v-%v): %s", string(begin_), string(end_), err))
+	}
+	var keys [][]byte
+	for _, kv := range resp.Kvs {
+		if filter == nil || filter(kv.Key) {
+			k := string(kv.Key)
+			tx.observed[k] = kv.ModRevision
+			keys = append(keys, kv.Key)
+			if limit > 0 && len(keys) >= limit {
+				break
+			}
+		}
+	}
+	return keys
+}
+
 func (tx *etcdTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
 	if limit == 0 {
 		return nil
diff --git a/pkg/meta/tkv_fdb.go b/pkg/meta/tkv_fdb.go
index e81eb580..4ad05609 100644
--- a/pkg/meta/tkv_fdb.go
+++ b/pkg/meta/tkv_fdb.go
@@ -160,6 +160,23 @@ func (tx *fdbTxn) scanKeys(prefix []byte) [][]byte {
 	return ret
 }
 
+func (tx *fdbTxn) scanKeysRange(begin, end []byte, limit int, filter func(k []byte) bool) [][]byte {
+	ret := make([][]byte, 0)
+	iter := tx.range0(begin, end)
+	for iter.Advance() {
+		r := iter.MustGet()
+		if filter == nil || filter(r.Key) {
+			ret = append(ret, r.Key)
+			if limit > 0 {
+				if limit--; limit == 0 {
+					break
+				}
+			}
+		}
+	}
+	return ret
+}
+
 func (tx *fdbTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
 	ret := make(map[string][]byte)
 	iter := tx.range0(prefix, nextKey(prefix))
diff --git a/pkg/meta/tkv_mem.go b/pkg/meta/tkv_mem.go
index 6afcf611..3d08238d 100644
--- a/pkg/meta/tkv_mem.go
+++ b/pkg/meta/tkv_mem.go
@@ -132,6 +132,34 @@ func (tx *memTxn) scanKeys(prefix_ []byte) [][]byte {
 	return keys
 }
 
+func (tx *memTxn) scanKeysRange(begin_, end_ []byte, limit int, filter func(k []byte) bool) [][]byte {
+	if limit == 0 {
+		return nil
+	}
+	tx.store.Lock()
+	defer tx.store.Unlock()
+	begin := string(begin_)
+	end := string(end_)
+	var keys [][]byte
+	tx.store.items.AscendGreaterOrEqual(&kvItem{key: begin}, func(i btree.Item) bool {
+		it := i.(*kvItem)
+		if end == "" || it.key < end {
+			if filter == nil || filter([]byte(it.key)) {
+				tx.observed[it.key] = it.ver
+				keys = append(keys, []byte(it.key))
+				if limit > 0 {
+					if limit--; limit == 0 {
+						return false
+					}
+				}
+			}
+			return true
+		}
+		return false
+	})
+	return keys
+}
+
 func (tx *memTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
 	if limit == 0 {
 		return nil
diff --git a/pkg/meta/tkv_prefix.go b/pkg/meta/tkv_prefix.go
index 9ee06398..ee0560b2 100644
--- a/pkg/meta/tkv_prefix.go
+++ b/pkg/meta/tkv_prefix.go
@@ -62,6 +62,19 @@ func (tx *prefixTxn) scanKeys(prefix []byte) [][]byte {
 	return keys
 }
 
+func (tx *prefixTxn) scanKeysRange(begin, end []byte, limit int, filter func(k []byte) bool) [][]byte {
+	keys := tx.kvTxn.scanKeysRange(tx.realKey(begin), tx.realKey(end), limit, func(k []byte) bool {
+		if filter == nil {
+			return true
+		}
+		return filter(tx.origKey(k))
+	})
+	for i, k := range keys {
+		keys[i] = tx.origKey(k)
+	}
+	return keys
+}
+
 func (tx *prefixTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
 	r := tx.kvTxn.scanValues(tx.realKey(prefix), limit, func(k, v []byte) bool {
 		if filter == nil {
diff --git a/pkg/meta/tkv_tikv.go b/pkg/meta/tkv_tikv.go
index ea9cd56c..a453e487 100644
--- a/pkg/meta/tkv_tikv.go
+++ b/pkg/meta/tkv_tikv.go
@@ -22,6 +22,7 @@ package meta
 import (
 	"context"
 	"net/url"
+	"os"
 	"strings"
 
 	plog "github.com/pingcap/log"
@@ -32,6 +33,7 @@ import (
 	"github.com/tikv/client-go/v2/tikv"
 	"github.com/tikv/client-go/v2/txnkv"
 	"github.com/tikv/client-go/v2/txnkv/txnutil"
+	"go.uber.org/zap"
 )
 
 func init() {
@@ -54,7 +56,7 @@ func newTikvClient(addr string) (tkvClient, error) {
 	default:
 		plvl = "dpanic"
 	}
-	l, prop, _ := plog.InitLogger(&plog.Config{Level: plvl})
+	l, prop, _ := plog.InitLogger(&plog.Config{Level: plvl}, zap.Fields(zap.String("component", "tikv"), zap.Int("pid", os.Getpid())))
 	plog.ReplaceGlobals(l, prop)
 
 	tUrl, err := url.Parse("tikv://" + addr)
@@ -153,6 +155,33 @@ func (tx *tikvTxn) scanKeys(prefix []byte) [][]byte {
 	return ret
 }
 
+func (tx *tikvTxn) scanKeysRange(begin, end []byte, limit int, filter func(k []byte) bool) [][]byte {
+	if limit == 0 {
+		return nil
+	}
+	it, err := tx.Iter(begin, end)
+	if err != nil {
+		panic(err)
+	}
+	defer it.Close()
+	var ret [][]byte
+	for it.Valid() {
+		key := it.Key()
+		if filter == nil || filter(key) {
+			ret = append(ret, it.Key())
+			if limit > 0 {
+				if limit--; limit == 0 {
+					break
+				}
+			}
+		}
+		if err = it.Next(); err != nil {
+			panic(err)
+		}
+	}
+	return ret
+}
+
 func (tx *tikvTxn) scanValues(prefix []byte, limit int, filter func(k, v []byte) bool) map[string][]byte {
 	return tx.scanRange0(prefix, nextKey(prefix), limit, filter)
 }
diff --git a/pkg/object/ks3.go b/pkg/object/ks3.go
index c77af244..10203e8a 100644
--- a/pkg/object/ks3.go
+++ b/pkg/object/ks3.go
@@ -270,9 +270,11 @@ func newKS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 	bucket := hostParts[0]
 	region := hostParts[1][3:]
 	region = strings.TrimLeft(region, "-")
+	var pathStyle bool = true
 	if strings.HasSuffix(uri.Host, "ksyun.com") || strings.HasSuffix(uri.Host, "ksyuncs.com") {
 		region = strings.TrimSuffix(region, "-internal")
 		region = ks3Regions[region]
+		pathStyle = false
 	} else if envRegion := os.Getenv("AWS_REGION"); envRegion != "" {
 		region = envRegion
 	}
@@ -294,7 +296,7 @@ func newKS3(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 		Endpoint:         strings.SplitN(uri.Host, ".", 2)[1],
 		DisableSSL:       !ssl,
 		HTTPClient:       httpClient,
-		S3ForcePathStyle: true,
+		S3ForcePathStyle: pathStyle,
 		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, token),
 	}
 
diff --git a/pkg/object/obs.go b/pkg/object/obs.go
index fd93f0f1..60d291da 100644
--- a/pkg/object/obs.go
+++ b/pkg/object/obs.go
@@ -41,9 +41,10 @@ import (
 const obsDefaultRegion = "cn-north-1"
 
 type obsClient struct {
-	bucket string
-	region string
-	c      *obs.ObsClient
+	bucket    string
+	region    string
+	checkEtag bool
+	c         *obs.ObsClient
 }
 
 func (s *obsClient) String() string {
@@ -135,7 +136,7 @@ func (s *obsClient) Put(key string, in io.Reader) error {
 	params.ContentMD5 = base64.StdEncoding.EncodeToString(sum[:])
 	params.ContentType = mimeType
 	resp, err := s.c.PutObject(params)
-	if err == nil && strings.Trim(resp.ETag, "\"") != obs.Hex(sum) {
+	if err == nil && s.checkEtag && strings.Trim(resp.ETag, "\"") != obs.Hex(sum) {
 		err = fmt.Errorf("unexpected ETag: %s != %s", strings.Trim(resp.ETag, "\""), obs.Hex(sum))
 	}
 	return err
@@ -212,7 +213,7 @@ func (s *obsClient) UploadPart(key string, uploadID string, num int, body []byte
 	sum := md5.Sum(body)
 	params.ContentMD5 = base64.StdEncoding.EncodeToString(sum[:])
 	resp, err := s.c.UploadPart(params)
-	if err == nil && strings.Trim(resp.ETag, "\"") != obs.Hex(sum[:]) {
+	if err == nil && s.checkEtag && strings.Trim(resp.ETag, "\"") != obs.Hex(sum[:]) {
 		err = fmt.Errorf("unexpected ETag: %s != %s", strings.Trim(resp.ETag, "\""), obs.Hex(sum[:]))
 	}
 	if err != nil {
@@ -339,7 +340,15 @@ func newOBS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 	if err != nil {
 		return nil, fmt.Errorf("fail to initialize OBS: %q", err)
 	}
-	return &obsClient{bucketName, region, c}, nil
+	var checkEtag bool
+	if _, err = c.GetBucketEncryption(bucketName); err != nil {
+		if obsError, ok := err.(*obs.ObsError); ok && obsError.Code == "NoSuchEncryptionConfiguration" {
+			checkEtag = true
+		} else if !ok || obsError.Code != "NoSuchBucket" {
+			logger.Warnf("get bucket encryption: %q", err)
+		}
+	}
+	return &obsClient{bucket: bucketName, region: region, checkEtag: checkEtag, c: c}, nil
 }
 
 func init() {
diff --git a/pkg/object/restful.go b/pkg/object/restful.go
index d4657bf2..3c8bb029 100644
--- a/pkg/object/restful.go
+++ b/pkg/object/restful.go
@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/hmac"
 	"crypto/sha1"
+	"crypto/tls"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -77,6 +78,7 @@ func init() {
 				return nil, err
 			},
 			DisableCompression: true,
+			TLSClientConfig:    &tls.Config{},
 		},
 		Timeout: time.Hour,
 	}
diff --git a/sdk/java/src/main/java/io/juicefs/JuiceFileSystem.java b/sdk/java/src/main/java/io/juicefs/JuiceFileSystem.java
index d6299deb..6cfc033e 100644
--- a/sdk/java/src/main/java/io/juicefs/JuiceFileSystem.java
+++ b/sdk/java/src/main/java/io/juicefs/JuiceFileSystem.java
@@ -45,6 +45,7 @@ public class JuiceFileSystem extends FilterFileSystem {
   private static boolean distcpPatched = false;
 
   private ScheduledExecutorService emptier;
+  private FileSystem emptierFs;
 
   static {
     PatchUtil.patchBefore("org.apache.flink.runtime.fs.hdfs.HadoopRecoverableFsDataOutputStream",
@@ -81,7 +82,7 @@ public class JuiceFileSystem extends FilterFileSystem {
 
     try {
       UserGroupInformation superUser = UserGroupInformation.createRemoteUser(getConf(conf, "superuser", "hdfs"));
-      FileSystem emptierFs = superUser.doAs((PrivilegedExceptionAction<FileSystem>) () -> {
+      emptierFs = superUser.doAs((PrivilegedExceptionAction<FileSystem>) () -> {
         JuiceFileSystemImpl fs = new JuiceFileSystemImpl();
         fs.initialize(uri, conf);
         return fs;
@@ -155,6 +156,7 @@ public class JuiceFileSystem extends FilterFileSystem {
   public void close() throws IOException {
     if (this.emptier != null) {
       emptier.shutdownNow();
+      emptierFs.close();
     }
     super.close();
   }
diff --git a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
index 01664464..9970ac87 100644
--- a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
+++ b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
@@ -324,7 +324,7 @@ public class JuiceFileSystemImpl extends FileSystem {
     obj.put("metacache", Boolean.valueOf(getConf(conf, "metacache", "true")));
     obj.put("autoCreate", Boolean.valueOf(getConf(conf, "auto-create-cache-dir", "true")));
     obj.put("maxUploads", Integer.valueOf(getConf(conf, "max-uploads", "20")));
-    obj.put("maxDeletes", Integer.valueOf(getConf(conf, "max-deletes", "2")));
+    obj.put("maxDeletes", Integer.valueOf(getConf(conf, "max-deletes", "10")));
     obj.put("uploadLimit", Integer.valueOf(getConf(conf, "upload-limit", "0")));
     obj.put("downloadLimit", Integer.valueOf(getConf(conf, "download-limit", "0")));
     obj.put("ioRetries", Integer.valueOf(getConf(conf, "io-retries", "10")));


Test patch:
diff --git a/.github/scripts/testVersionCompatible.py b/.github/scripts/testVersionCompatible.py
index 2f03388a..81737c88 100644
--- a/.github/scripts/testVersionCompatible.py
+++ b/.github/scripts/testVersionCompatible.py
@@ -12,13 +12,16 @@ import unittest
 from xmlrpc.client import boolean
 from hypothesis.stateful import rule, precondition, RuleBasedStateMachine
 from hypothesis import assume, strategies as st
+from hypothesis import seed
 from packaging import version
 from minio import Minio
 import uuid
 from utils import *
 from fsrand import *
 from cmptree import *
+import random
 
+@seed(random.randint(10000, 1000000))
 class JuicefsMachine(RuleBasedStateMachine):
     MIN_CLIENT_VERSIONS = ['0.0.1', '0.0.17','1.0.0-beta1', '1.0.0-rc1']
     MAX_CLIENT_VERSIONS = ['1.1.0', '1.2.0', '2.0.0']
@@ -34,6 +37,7 @@ class JuicefsMachine(RuleBasedStateMachine):
 
     def __init__(self):
         super(JuicefsMachine, self).__init__()
+        print(f"seed is: {self._hypothesis_internal_use_seed}")
         self.run_id = uuid.uuid4().hex
         print(f'\ninit with run_id: {self.run_id}')
         with open(os.path.expanduser('~/command.log'), 'a') as f:
@@ -741,9 +745,14 @@ class JuicefsMachine(RuleBasedStateMachine):
         print('webdav succeed')
 
     def greater_than_version_formatted(self, ver):
+        print(f'ver is {ver}, formatted_by is {self.formatted_by}')
+        if not self.formatted_by:
+            return True
         return version.parse('-'.join(ver.split('-')[1:])) >=  version.parse('-'.join(self.formatted_by.split('-')[1:]))
 
     def greater_than_version_dumped(self, ver):
+        if not self.dumped_by:
+            return True
         return version.parse('-'.join(ver.split('-')[1:])) >=  version.parse('-'.join(self.dumped_by.split('-')[1:]))
 
     def greater_than_version_mounted(self, ver):
diff --git a/.github/workflows/aitests.yml b/.github/workflows/aitests.yml
index f56e1726..4259f0c3 100644
--- a/.github/workflows/aitests.yml
+++ b/.github/workflows/aitests.yml
@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/aitests_limitcache.yml b/.github/workflows/aitests_limitcache.yml
index a77fa90e..7a218ea4 100644
--- a/.github/workflows/aitests_limitcache.yml
+++ b/.github/workflows/aitests_limitcache.yml
@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
           
diff --git a/.github/workflows/elastictest.yml b/.github/workflows/elastictest.yml
index da800e2c..2ea2f6a6 100644
--- a/.github/workflows/elastictest.yml
+++ b/.github/workflows/elastictest.yml
@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml
index 8f6520d9..3ecff042 100644
--- a/.github/workflows/integrationtests.yml
+++ b/.github/workflows/integrationtests.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/mdtest.yml b/.github/workflows/mdtest.yml
index 3b5d89e2..f96c2b1e 100644
--- a/.github/workflows/mdtest.yml
+++ b/.github/workflows/mdtest.yml
@@ -28,18 +28,18 @@ jobs:
     runs-on: [self-hosted, daily-build, bench-02]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
       - name: Set Variable
         id: vars
         run: |
-          echo ::set-output name=META_URL::redis://mymaster,172.27.0.1,172.27.0.2,172.27.0.3:26379/7
-          echo ::set-output name=MOUNT_POINT::/tmp/mdtest
-          echo ::set-output name=BACKWARD_VERSIONS::1
-          echo ::set-output name=PYROSCOPE_URL::http://172.27.0.1:4040
-
+          echo "META_URL=redis://mymaster,172.27.0.1,172.27.0.2,172.27.0.3:26379/7" >> $GITHUB_OUTPUT
+          echo "MOUNT_POINT=/tmp/mdtest" >> $GITHUB_OUTPUT
+          echo "BACKWARD_VERSIONS=1" >> $GITHUB_OUTPUT
+          echo "PYROSCOPE_URL=http://172.27.0.1:4040" >> $GITHUB_OUTPUT
+          
       - name: Build
         uses: ./.github/actions/build
 
diff --git a/.github/workflows/mutate-test.yml b/.github/workflows/mutate-test.yml
index 03084af6..a7c11531 100644
--- a/.github/workflows/mutate-test.yml
+++ b/.github/workflows/mutate-test.yml
@@ -34,7 +34,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: '1.18.x'
 
diff --git a/.github/workflows/mysqltest_bigtable.yml b/.github/workflows/mysqltest_bigtable.yml
index e2763bf6..581e8581 100644
--- a/.github/workflows/mysqltest_bigtable.yml
+++ b/.github/workflows/mysqltest_bigtable.yml
@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -86,11 +86,12 @@ jobs:
         run: |
           if [ "${{matrix.size}}" == "big"]; then
             echo "threads=4" >>/tmp/config.cfg
-            sudo sysbench --config-file=/tmp/config.cfg oltp_point_select --tables=4 --table-size=5000000 prepare
-            sudo sysbench --config-file=/tmp/config.cfg oltp_read_only --tables=4 --time=300 --table-size=5000000 run
-            sudo sysbench --config-file=/tmp/config.cfg oltp_write_only --tables=4 --time=300 --table-size=5000000 run
-            sudo sysbench --config-file=/tmp/config.cfg oltp_read_write --tables=4 --time=600 --table-size=5000000 run
-            sudo sysbench --config-file=/tmp/config.cfg oltp_point_select --tables=4 --table-size=5000000 cleanup
+            TABLE_SIZE=1000000
+            sudo sysbench --config-file=/tmp/config.cfg oltp_point_select --tables=4 --table-size=$TABLE_SIZE prepare
+            sudo sysbench --config-file=/tmp/config.cfg oltp_read_only --tables=4 --time=300 --table-size=$TABLE_SIZE run
+            sudo sysbench --config-file=/tmp/config.cfg oltp_write_only --tables=4 --time=300 --table-size=$TABLE_SIZE run
+            sudo sysbench --config-file=/tmp/config.cfg oltp_read_write --tables=4 --time=600 --table-size=$TABLE_SIZE run
+            sudo sysbench --config-file=/tmp/config.cfg oltp_point_select --tables=4 --table-size=$TABLE_SIZE cleanup
           else
             echo "threads=64" >>/tmp/config.cfg
             sudo mysql -uroot -proot  -e "use mysql;set global max_prepared_stmt_count = 100000;"
diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml
index 41534030..4cdd467c 100644
--- a/.github/workflows/pjdfstest.yml
+++ b/.github/workflows/pjdfstest.yml
@@ -18,7 +18,7 @@ on:
       - '**.md'
       - '.github/**'
   schedule:
-    - cron:  '0 17 * * *'
+    - cron:  '0 19 * * *'
   workflow_dispatch:
     inputs:
       debug:
@@ -36,13 +36,13 @@ jobs:
           echo "github.event_name is ${{github.event_name}}"
           echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
           if [ "${{github.event_name}}" == "schedule"  ]; then
-            echo '::set-output name=meta_matrix::["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]'
+            echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "pull_request"  ]; then
-            echo '::set-output name=meta_matrix::["redis"]'
+            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
-            echo '::set-output name=meta_matrix::["tikv", "tidb"]'
+            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
           else
-            echo '::set-output name=meta_matrix::["redis"]'
+            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
           fi
     outputs:
       meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
@@ -62,7 +62,7 @@ jobs:
           perl-version: '5.34'
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -70,9 +70,9 @@ jobs:
         id: vars
         run: |
           if [ "${{matrix.meta}}" == "fdb" ]; then
-            echo ::set-output name=target::"juicefs.fdb"
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=target::"juicefs"
+            echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
 
       - name: Build
diff --git a/.github/workflows/sdktest.yml b/.github/workflows/sdktest.yml
index 670d76d2..cd352cef 100644
--- a/.github/workflows/sdktest.yml
+++ b/.github/workflows/sdktest.yml
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
index e87f2b99..afecd50e 100644
--- a/.github/workflows/unittests.yml
+++ b/.github/workflows/unittests.yml
@@ -116,7 +116,7 @@ jobs:
 
 
       - name: Code Coverage
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           files: ./coverage.txt
 
diff --git a/.github/workflows/wintest.yml b/.github/workflows/wintest.yml
index 67b38102..b7006868 100644
--- a/.github/workflows/wintest.yml
+++ b/.github/workflows/wintest.yml
@@ -23,12 +23,12 @@ jobs:
       Actions_Allow_Unsecure_Commands: true
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: '1.18.x'
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
@@ -43,7 +43,7 @@ jobs:
 
       - name: Cache WinFsp
         id: Cache-WinFsp
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: 'C:\wfsp\'
           key: ${{ runner.os }}-winfsp-key
diff --git a/cmd/fsck_test.go b/cmd/fsck_test.go
index 3f97481d..7d9f07c2 100644
--- a/cmd/fsck_test.go
+++ b/cmd/fsck_test.go
@@ -23,6 +23,7 @@ import (
 )
 
 func TestFsck(t *testing.T) {
+	t.Skipf("this test is not stable, skip it")
 	mountTemp(t, nil, nil, nil)
 	defer umountTemp(t)
 
diff --git a/cmd/gc_test.go b/cmd/gc_test.go
index cbb88620..cd1cf0e6 100644
--- a/cmd/gc_test.go
+++ b/cmd/gc_test.go
@@ -64,6 +64,7 @@ func getFileCount(dir string) int {
 }
 
 func TestGc(t *testing.T) {
+	t.Skipf("this test is not stable, skip it")
 	var bucket string
 	mountTemp(t, &bucket, []string{"--trash-days=0"}, nil)
 	defer umountTemp(t)
diff --git a/cmd/info_test.go b/cmd/info_test.go
index cf9afc05..74eea2e8 100644
--- a/cmd/info_test.go
+++ b/cmd/info_test.go
@@ -27,6 +27,7 @@ import (
 )
 
 func TestInfo(t *testing.T) {
+	t.Skipf("this test is not stable, skip it")
 	Convey("TestInfo", t, func() {
 		Convey("TestInfo", func() {
 			tmpFile, err := os.CreateTemp("/tmp", "")
diff --git a/cmd/warmup_test.go b/cmd/warmup_test.go
index 3076c0ff..6f36b222 100644
--- a/cmd/warmup_test.go
+++ b/cmd/warmup_test.go
@@ -27,6 +27,7 @@ import (
 )
 
 func TestWarmup(t *testing.T) {
+	t.Skipf("this test is not stable, skip it")
 	mountTemp(t, nil, nil, nil)
 	defer umountTemp(t)
 
diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go
index 8e2ab054..b98d0f04 100644
--- a/pkg/meta/base_test.go
+++ b/pkg/meta/base_test.go
@@ -999,7 +999,7 @@ func testCompaction(t *testing.T, m Meta, trash bool) {
 		if len(sliceMap[1]) < 200 {
 			t.Fatalf("list delayed slices %d is less than 200", len(sliceMap[1]))
 		}
-		m.(engine).doCleanupDelayedSlices(time.Now().Unix()+1, 1000)
+		m.(engine).doCleanupDelayedSlices(time.Now().Unix() + 1)
 		l.Lock()
 		deletes = len(deleted)
 		l.Unlock()
diff --git a/pkg/meta/tkv_test.go b/pkg/meta/tkv_test.go
index 6d18a844..125f6036 100644
--- a/pkg/meta/tkv_test.go
+++ b/pkg/meta/tkv_test.go
@@ -104,6 +104,12 @@ func testTKV(t *testing.T, c tkvClient) {
 	if len(keys) != 2 || string(keys[0]) != "k" || string(keys[1]) != "k2" {
 		t.Fatalf("keys: %+v", keys)
 	}
+	txn(func(kt kvTxn) {
+		keys = kt.scanKeysRange([]byte("a"), []byte("z"), -1, func(k []byte) bool { return len(k) == 1 })
+	})
+	if len(keys) != 2 || string(keys[0]) != "k" || string(keys[1]) != "v" {
+		t.Fatalf("keys: %+v", keys)
+	}
 	txn(func(kt kvTxn) { keys = kt.scanKeys(nil) })
 	if len(keys) != 3 || string(keys[0]) != "k" || string(keys[1]) != "k2" || string(keys[2]) != "v" {
 		t.Fatalf("keys: %+v", keys)
diff --git a/pkg/object/object_storage_test.go b/pkg/object/object_storage_test.go
index 8e48589f..80e07a6c 100644
--- a/pkg/object/object_storage_test.go
+++ b/pkg/object/object_storage_test.go
@@ -21,13 +21,11 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"math"
-	"net/http"
 	"os"
 	"reflect"
 	"regexp"
@@ -785,6 +783,5 @@ func TestMain(m *testing.M) {
 			}
 		}
 	}
-	httpClient.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: false}
 	m.Run()
 }