Repository: minio/minio (language: go)

Issue:
stale multipart folders in .minio.sys is not cleaned up
## Expected Behavior
.minio.sys should be cleaned up periodically

## Current Behavior
there is a about 20% of volume space on each minio drive wasted in .minio.sys directory on most of the nodes
trash and multipart data are growing and not being cleaned up
**multipart** directory contains 15M files (59GB in total) and listing this directory takes 8minutes and disk usage calculation takes 50 minutes (on fast ssd storage)
**tmp-old** directory contains 1.3M files/dirs (1.1TB in total) 

## Possible Solution
could we delete the files directly on filesystem?

## Steps to Reproduce (for bugs)
directories were filing up over the time from time of creation

## Context
some put operations are already failing for not enough space, but it should have many TBs still available.
buckets are un-versioned

## Regression
no

## Your Environment

[root@minio01n01:/mnt/ebs/minio2/.minio.sys] uname -a
Linux minio01n01 5.15.16-1.el8.x86_64 #1 SMP Sun Jan 23 20:26:05 CET 2022 x86_64 x86_64 x86_64 GNU/Linux

[root@minio01n01:/mnt/ebs/minio2/.minio.sys/multipart] time ls -l | wc -l
15027108

real    6m22.408s
user    1m39.316s
sys     3m0.987s


[root@minio01n01:/mnt/ebs/minio2/.minio.sys/multipart] time ls -ltr | head
total 60108448
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 dff52d2ad6a8a8ee3d8458b13b1f1fd0cf5d96ee428d51142cd79295ad1ace63
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 ae8f4914dbfc4742d7ab9779b20ffcb932fcd28ea18b00be407cc7742d49c067
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 a0c5bd131fdb78a88bc72f758a394308f1c6d5912b63a588840f8579ed0f7c9a
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 d40776b5446902c9619623f82ef7476bef7c196affe17adca4a2fa9b58ac6da4
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 d782ffdf043966361fcaf54f59362bd6a58f9538abb86af14d2c17a63c885687
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 9af516cece9138ebb253048066b3a249b3ad23c48bfd2eb26a281fbe50bdd331
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 e1e01ce76c53be3ef4b8d50d94aea69298ebcc813d1aca6d982d9e41fd4b8780
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 455050e50a52200474b312006701dd7b7931a18920c27c92f811f1e49b906a61
drwxr-xr-x. 2 minio minio 4096 Jan 31  2022 4c49a09683d8917de15271cfe6d6871190fbcd5c4d84c76eb8a719432d81674d

real    4m17.273s
user    0m19.135s
sys     2m42.146s


[root@minio01n01:/mnt/ebs/minio2/.minio.sys/multipart] time du -skh . 
59G     .

real    43m20.772s
user    0m41.829s
sys     6m14.462s


[root@minio01n01:/mnt/ebs/minio2/.minio.sys/tmp-old/7f98ff0f-22b9-47b1-904b-58f867e92d03/.trash] ll -rt | head
total 15317120
-rw-r--r--. 1 minio minio  90362 Feb  1  2022 8e317970-aec8-4c5f-be51-7b3c3d3f1e2b
drwxr-xr-x. 2 minio minio   4096 Feb  1  2022 ff027dbb-ce97-42ed-9940-c7bd74297a30
-rw-r--r--. 1 minio minio    368 Feb  1  2022 59a39a10-80e5-4a56-aa6d-0b595682d7e8
drwxr-xr-x. 2 minio minio   4096 Feb  1  2022 31a33396-e6f6-4da8-8f5d-881e7887b917
-rw-r--r--. 1 minio minio    431 Feb  1  2022 9085af01-6d9f-4780-b3a4-7ab3f3c05016
-rw-r--r--. 1 minio minio    368 Apr 10  2022 bded8f55-f9e7-4a4b-a3e1-0cd47efb4ff7
drwxr-xr-x. 2 minio minio   4096 Jun  2 08:01 7ce21bd4-e831-46ad-bb0d-f2d2874d19dd
-rw-r--r--. 1 minio minio    420 Jun  2 08:01 bb2e12bb-c66c-4284-ac45-915c472adcb8
drwxr-xr-x. 2 minio minio   4096 Jun  4 08:01 ef4ac6a3-0d6a-4776-b72b-114844126a1a


[root@minio01n01:/mnt/ebs/minio2/.minio.sys/tmp-old/7f98ff0f-22b9-47b1-904b-58f867e92d03/.trash] time du -skh .
1.1T	.

real	2m12.488s
user	0m3.141s
sys	0m24.579s


[root@minio01n01:/mnt/ebs/minio2/.minio.sys/tmp-old/7f98ff0f-22b9-47b1-904b-58f867e92d03/.trash] time ls -la | wc -l
1304198

real	0m47.756s
user	0m7.661s
sys	0m21.106s


[root@minio01n01:/mnt/ebs/minio2/.minio.sys] df -h .
Filesystem      Size  Used Avail Use% Mounted on
/dev/vdc        5.0T  4.7T   83G  99% /mnt/ebs/minio2


[root@minio01n01:/mnt/ebs/minio2/.minio.sys] mc admin heal -v minio-na3
Servers status:

Pool 1st:
  minio01n01.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n02.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n03.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB

  minio01n04.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n05.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB

  minio01n06.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.2 TiB/4.7 TiB

  minio01n07.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB

  minio01n08.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB

  minio01n09.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.2 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.2 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB

  minio01n10.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n11.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB

  minio01n12.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.7 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB

  minio01n13.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB

  minio01n14.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB

  minio01n15.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 3.6 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 3.7 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.6 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.7 TiB/4.7 TiB

  minio01n16.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB

Pool 2nd:
  minio01n17.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n18.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB

  minio01n19.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n20.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB

  minio01n21.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.6 TiB/4.7 TiB

  minio01n22.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB

  minio01n23.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB

  minio01n24.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB

  minio01n25.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB

  minio01n26.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.1 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB

  minio01n27.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.5 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB

  minio01n28.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB

  minio01n29.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.7 TiB/4.7 TiB

  minio01n30.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.8 TiB/4.7 TiB

  minio01n31.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.0 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 3.9 TiB/4.7 TiB

  minio01n32.int.na3.********.com:443:
  +  /mnt/ebs/minio1 : OK
  |__  Capacity: 4.4 TiB/4.7 TiB
  +  /mnt/ebs/minio2 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio3 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB
  +  /mnt/ebs/minio4 : OK
  |__  Capacity: 4.3 TiB/4.7 TiB


Summary:

No active healing is detected among disks.


[root@minio01n01:/mnt/ebs/minio2/.minio.sys] mc admin info minio-na3
●  minio01n01.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n02.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n03.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n04.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n05.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n06.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n07.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n08.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n09.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n10.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n11.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n12.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n13.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n14.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n15.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n16.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 1

●  minio01n17.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n18.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n19.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n20.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n21.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n22.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n23.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n24.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n25.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n26.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n27.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n28.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n29.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n30.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n31.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

●  minio01n32.int.na3.********.com:443
   Uptime: 3 days 
   Version: 2022-10-21T22:37:48Z
   Network: 32/32 OK 
   Drives: 4/4 OK 
   Pool: 2

Pools:
   1st, Erasure sets: 4, Drives per erasure set: 16
   2nd, Erasure sets: 4, Drives per erasure set: 16

326 TiB Used, 3 Buckets, 20,945,259 Objects
128 drives online, 0 drives offline

[root@minio01n01:/mnt/ebs/minio2/.minio.sys] mc admin config export minio-na3 | grep -v '^#'
site name= region= 
api requests_max=0 requests_deadline=10s cluster_deadline=10s cors_allow_origin=* remote_transport_deadline=2h list_quorum=strict transition_workers=100 stale_uploads_cleanup_interval=6h stale_uploads_expiry=24h delete_cleanup_interval=5m replication_priority=auto disable_odirect=off gzip_objects=off gzip_objects=off gzip_objects=off 
scanner delay=10 max_wait=15s cycle=1m 
subnet license= api_key= proxy= 
storage_class standard= rrs=EC:2 
heal bitrotscan=off max_sleep=1s max_io=100 



Discussion / hints:
Should be addressed via #15979 
@harshavardhana , https://github.com/minio/minio/pull/15979 seems to only take care of tmp-old directory ? What about multipart as well?
Right now, using npd script to keep cleaning the empty directories under .minio.sys/multipart . Have been thinking to file a bug, did not have time to test a consistent repro.

@mshanmu `stale_uploads_cleanup_interval=6h stale_uploads_expiry=24h` in your config above controls your upload purging.
@klauspost multipart cleanup obviously not working as expected as there are files/dirs from January
> @harshavardhana , #15979 seems to only take care of tmp-old directory ? What about multipart as well? Right now, using npd script to keep cleaning the empty directories under .minio.sys/multipart . Have been thinking to file a bug, did not have time to test a consistent repro.

Multipart must work all the time.
> @klauspost multipart cleanup obviously not working as expected as there are files/dirs from January

We will take a look I don't see anything suspicious there, you must be using an older release that created such a state.
in case it is not deleted automatically, is it safe to run something like ?
`find  /mnt/ebs/minio[1234]/.minio.sys/multipart -mtime +7 -exec rm -rf {} \;`
@jan-cipa There is nothing that is 100% safe. You are welcome to try. Maybe start by moving it to another folder and observe for problems before you delete.
 @harshavardhana / @klauspost  ,  am using a older minio image (6 months old), so, take it for what is worth, but, I am consistently seeing tons of older empty directories under multipart directory, even though these are supposedly to be cleaned up every day.

@jan-cipa , Since you have the latest minio image, can you please verify how old are the file & directories under .minio.sys/multipart directory?

@jan-cipa , we have been using ```find <multipart directory> -type d -empty -mtime +7 | xargs rmdir```  command and it has been working well for the past 6 months.
@mshanmu 
number of files+dirs per day since cluster creation

ll -rt /mnt/ebs/minio2/.minio.sys/multipart | awk '{print $6 " " $7}' | uniq -c
    496 Jan 31
  52407 Feb 1
 153906 Feb 2
  45822 Feb 3
  50889 Feb 4
  47635 Feb 5
  45921 Feb 6
  47429 Feb 7
  43740 Feb 8
  43305 Feb 9
  42865 Feb 10
  40246 Feb 11
  77844 Feb 12
   6411 Feb 13
  93294 Feb 14
 128864 Feb 15
  64211 Feb 16
  87672 Feb 17
  36480 Feb 18
   1743 Feb 19
   5206 Feb 20
  49105 Feb 21
 144967 Feb 22
  99131 Feb 23
 103067 Feb 24
   3401 Feb 25
  52834 Feb 26
  36172 Feb 27
  38080 Feb 28
  32100 Mar 1
  40583 Mar 2
  50742 Mar 3
 101946 Mar 4
  87412 Mar 5
   6714 Mar 6
   4634 Mar 7
   2736 Mar 8
 379589 Mar 9
   3047 Mar 10
 109556 Mar 11
  37861 Mar 12
  41586 Mar 13
  40914 Mar 14
  37539 Mar 15
  31866 Mar 16
  48409 Mar 17
  37379 Mar 18
  31421 Mar 19
  46664 Mar 20
  35833 Mar 21
  39413 Mar 22
  41714 Mar 23
  45540 Mar 24
  75714 Mar 25
 105049 Mar 26
  41637 Mar 27
  44198 Mar 28
  50887 Mar 29
  54084 Mar 30
  55486 Mar 31
  50114 Apr 1
  38960 Apr 2
  59339 Apr 3
  46881 Apr 4
  45206 Apr 5
  51425 Apr 6
  50995 Apr 7
  52460 Apr 8
  38959 Apr 9
  54892 Apr 10
  59156 Apr 11
  78866 Apr 12
  57896 Apr 13
  53972 Apr 14
  85009 Apr 15
  34095 Apr 16
  78328 Apr 17
  55361 Apr 18
  57996 Apr 19
  47705 Apr 20
  65466 Apr 21
  77872 Apr 22
  52179 Apr 23
  41346 Apr 24
  42708 Apr 25
  52405 Apr 26
  89242 Apr 27
  77073 Apr 28
  43049 Apr 29
  38250 Apr 30
  45422 May 1
  41252 May 2
  51171 May 3
  51070 May 4
  48665 May 5
  58092 May 6
  41210 May 7
  38624 May 8
  39141 May 9
  43860 May 10
  48341 May 11
  44133 May 12
  46863 May 13
  43765 May 14
  39701 May 15
  10647 May 16
  69508 May 17
  47314 May 18
  46932 May 19
  80519 May 20
  32402 May 21
  50913 May 22
  37150 May 23
  42457 May 24
  42976 May 25
  67112 May 26
  52454 May 27
  80319 May 28
  70156 May 29
  44835 May 30
  47028 May 31
  65519 Jun 1
  46601 Jun 2
  43767 Jun 3
  30942 Jun 4
  70155 Jun 5
  46171 Jun 6
  44745 Jun 7
  48998 Jun 8
  59251 Jun 9
  63422 Jun 10
  47957 Jun 11
  63913 Jun 12
  42156 Jun 13
  51758 Jun 14
  62617 Jun 15
  56920 Jun 16
  36378 Jun 17
  50422 Jun 18
  54343 Jun 19
  50882 Jun 20
  51748 Jun 21
  87617 Jun 22
  96304 Jun 23
  61829 Jun 24
  37512 Jun 25
  76468 Jun 26
  40462 Jun 27
  49119 Jun 28
  40172 Jun 29
  49315 Jun 30
  45204 Jul 1
  34117 Jul 2
  49260 Jul 3
  43284 Jul 4
  46512 Jul 5
 129544 Jul 6
  61016 Jul 7
  54960 Jul 8
  44224 Jul 9
  49018 Jul 10
  50859 Jul 11
  61035 Jul 12
  53852 Jul 13
  55557 Jul 14
  61606 Jul 15
  80096 Jul 16
  50951 Jul 17
  52647 Jul 18
  47030 Jul 19
  47547 Jul 20
  52557 Jul 21
  48775 Jul 22
  43274 Jul 23
  53510 Jul 24
  42376 Jul 25
  52688 Jul 26
  48361 Jul 27
  45633 Jul 28
  57521 Jul 29
  30630 Jul 30
  62416 Jul 31
  46435 Aug 1
  51905 Aug 2
  57406 Aug 3
  62361 Aug 4
  47832 Aug 5
  27826 Aug 6
  59776 Aug 7
  44069 Aug 8
  58587 Aug 9
  77212 Aug 10
  77482 Aug 11
  46327 Aug 12
  34080 Aug 13
  60696 Aug 14
  47498 Aug 15
  56130 Aug 16
  52052 Aug 17
  50917 Aug 18
  67371 Aug 19
  40184 Aug 20
 131311 Aug 21
  52948 Aug 22
  37554 Aug 23
  68247 Aug 24
  49121 Aug 25
  81440 Aug 26
  32618 Aug 27
  66717 Aug 28
  51956 Aug 29
  56239 Aug 30
  51192 Aug 31
  57194 Sep 1
  59516 Sep 2
  15340 Sep 3
  68531 Sep 4
  49814 Sep 5
  49843 Sep 6
  43503 Sep 7
  43601 Sep 8
  60242 Sep 9
  27974 Sep 10
  66557 Sep 11
  41212 Sep 12
  51621 Sep 13
  50604 Sep 14
  36614 Sep 15
  58784 Sep 16
  31632 Sep 17
 109552 Sep 18
  63273 Sep 19
  46825 Sep 20
  44521 Sep 21
   4542 Sep 22
   5433 Sep 23
   2657 Sep 24
   3001 Sep 25
  68776 Sep 26
  37991 Sep 27
   9479 Sep 28
 246880 Sep 29
 164182 Sep 30
  44854 Oct 1
  92844 Oct 2
  52431 Oct 3
  39568 Oct 4
  55050 Oct 5
  47681 Oct 6
  77983 Oct 7
  31158 Oct 8
  93562 Oct 9
  46228 Oct 10
  46559 Oct 11
  52788 Oct 12
  64464 Oct 13
  48874 Oct 14
  46228 Oct 15
  62069 Oct 16
  47392 Oct 17
  47640 Oct 18
  45922 Oct 19
  66878 Oct 20
 125424 Oct 21
  51860 Oct 22
  14080 Oct 23
  49816 Oct 24
  92818 Oct 25
 286609 Oct 26
  80250 Oct 27
  17500 Oct 28
   2834 Oct 29
   2925 Oct 30
   1489 Oct 31
@harshavardhana / @klauspost , I think what @jan-cipa posted seems to prove that multipart directory is not getting properly cleaned every day.

> ll -rt /mnt/ebs/minio2/.minio.sys/multipart | awk '{print $6 " " $7}' | uniq -c

Are there any files inside this or is it just empty folders? 
> Are there any files inside this or is it just empty folders?

@mshanmu this question is for you as well.. 

Fix patch:
diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go
index 8e1e58c55883a..5f0d6895398ac 100644
--- a/cmd/erasure-healing.go
+++ b/cmd/erasure-healing.go
@@ -622,7 +622,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	}
 
-	defer er.renameAll(context.Background(), minioMetaTmpBucket, tmpID)
+	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tmpID)
 
 	// Rename from tmp location to the actual location.
 	for i, disk := range outDatedDisks {
diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go
index fd94b02f7a388..b5071988215f9 100644
--- a/cmd/erasure-multipart.go
+++ b/cmd/erasure-multipart.go
@@ -175,21 +175,6 @@ func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Du
 	wg.Wait()
 }
 
-func (er erasureObjects) renameAll(ctx context.Context, bucket, prefix string) {
-	var wg sync.WaitGroup
-	for _, disk := range er.getDisks() {
-		if disk == nil {
-			continue
-		}
-		wg.Add(1)
-		go func(disk StorageAPI) {
-			defer wg.Done()
-			disk.RenameFile(ctx, bucket, prefix, minioMetaTmpDeletedBucket, mustGetUUID())
-		}(disk)
-	}
-	wg.Wait()
-}
-
 func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 	var wg sync.WaitGroup
 	for _, disk := range er.getDisks() {
@@ -218,11 +203,12 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
 			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", false)
 			if err != nil {
+				er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 				return nil
 			}
 			wait := er.deletedCleanupSleeper.Timer(ctx)
 			if now.Sub(fi.ModTime) > expiry {
-				er.renameAll(ctx, minioMetaMultipartBucket, uploadIDPath)
+				er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 			}
 			wait()
 			return nil
@@ -641,7 +627,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.renameAll(context.Background(), minioMetaTmpBucket, tmpPart)
+			er.deleteAll(context.Background(), minioMetaTmpBucket, tmpPart)
 		}
 	}()
 
@@ -1274,7 +1260,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 	}
 
 	// Cleanup all uploaded parts.
-	er.renameAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
+	er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
 
 	// Successfully purged.
 	return nil
diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go
index f62ac08e55100..5d859e2dac437 100644
--- a/cmd/erasure-object.go
+++ b/cmd/erasure-object.go
@@ -1072,7 +1072,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.renameAll(context.Background(), minioMetaTmpBucket, tempObj)
+			er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 		}
 	}()
 
diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go
index c4b33dc355a49..22db3a992326f 100644
--- a/cmd/erasure-server-pool.go
+++ b/cmd/erasure-server-pool.go
@@ -1699,7 +1699,7 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 func (z *erasureServerPools) renameAll(ctx context.Context, bucket, prefix string) {
 	for _, servers := range z.serverPools {
 		for _, set := range servers.sets {
-			set.renameAll(ctx, bucket, prefix)
+			set.deleteAll(ctx, bucket, prefix)
 		}
 	}
 }


Test patch: