Repository: minio/minio (language: go)

Issue:
Scanner deletes empty buckets
## NOTE
If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.

<!--- Provide a general summary of the issue in the Title above -->
It appears that the scanner will sometimes delete empty buckets. This happens reliably on simple buckets that previously contained files and are now empty, but can also be manually triggered by running heal with the `-remove` flag (if that is not its intended behavior).

I believe this is because the bucket's data folder is being deleted. The configuration folder in `.minio.sys/buckets` remains intact and recreating the data folder restores configuration.

While trying to reproduce at first I was able to even have new empty buckets deleted by the scanner reliably, then only ones that contained a file previously, then only buckets that contain a file, have the scanner run, delete the file, next scanner run deletes the bucket.

## Expected Behavior
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->
Empty buckets should not be deleted and remain intact.

## Current Behavior
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
The bucket is deleted.

## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
<!--- or ideas how to implement the addition or change -->

## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->

Scanner:
1. `mc mb cluster/testbucket`
2. `touch testfile && mc cp testfile cluster/testbucket/`
3. wait until scanner completes run cycle (`mc admin scanner trace cluster | grep testbucket`)
4. `mc rm cluster/testbucket/testfile`
5. wait until scanner completes run cycle (`mc admin scanner trace cluster | grep testbucket`)
6. `mc ls cluster/testbucket # Bucket does not exist`

Manually:
1. `mc mb play/testbucketpleaseignore`
2. `mc admin heal play/testbucketpleaseignore -remove`
3. `mc ls play/testbucketpleaseignore # Bucket does not exist`

## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->

## Regression
<!-- Is this issue a regression? (Yes / No) -->
<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
Yes, since RELEASE.2024-01-11T07-46-16Z

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used (`minio --version`): 2024-01-16T16:07:38Z
* Server setup and configuration: 4 nodes with 1 disk each



Discussion / hints:

Fix patch:
diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go
index 87eb07f0b4ebb..13af5741edea9 100644
--- a/cmd/erasure-server-pool.go
+++ b/cmd/erasure-server-pool.go
@@ -1931,76 +1931,12 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 }
 
 func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
-	r := madmin.HealResultItem{
-		Type:   madmin.HealItemBucket,
-		Bucket: bucket,
-	}
-
 	// Attempt heal on the bucket metadata, ignore any failures
 	hopts := opts
 	hopts.Recreate = false
 	defer z.HealObject(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), "", hopts)
 
-	type DiskStat struct {
-		VolInfos []VolInfo
-		Errs     []error
-	}
-
-	for _, pool := range z.serverPools {
-		// map of node wise disk stats
-		diskStats := make(map[string]DiskStat)
-		for _, set := range pool.sets {
-			for _, disk := range set.getDisks() {
-				if disk == OfflineDisk {
-					continue
-				}
-				vi, err := disk.StatVol(ctx, bucket)
-				hostName := disk.Hostname()
-				if disk.IsLocal() {
-					hostName = "local"
-				}
-				ds, ok := diskStats[hostName]
-				if !ok {
-					newds := DiskStat{
-						VolInfos: []VolInfo{vi},
-						Errs:     []error{err},
-					}
-					diskStats[hostName] = newds
-				} else {
-					ds.VolInfos = append(ds.VolInfos, vi)
-					ds.Errs = append(ds.Errs, err)
-					diskStats[hostName] = ds
-				}
-			}
-		}
-		nodeCount := len(diskStats)
-		bktNotFoundCount := 0
-		for _, ds := range diskStats {
-			if isAllBucketsNotFound(ds.Errs) {
-				bktNotFoundCount++
-			}
-		}
-		// if the bucket if not found on more than hslf the no of nodes, its dangling
-		if bktNotFoundCount > nodeCount/2 {
-			opts.Remove = true
-		} else {
-			opts.Recreate = true
-		}
-
-		result, err := z.s3Peer.HealBucket(ctx, bucket, opts)
-		if err != nil {
-			if _, ok := err.(BucketNotFound); ok {
-				continue
-			}
-			return result, err
-		}
-		r.DiskCount += result.DiskCount
-		r.SetCount += result.SetCount
-		r.Before.Drives = append(r.Before.Drives, result.Before.Drives...)
-		r.After.Drives = append(r.After.Drives, result.After.Drives...)
-	}
-
-	return r, nil
+	return z.s3Peer.HealBucket(ctx, bucket, opts)
 }
 
 // Walk a bucket, optionally prefix recursively, until we have returned
diff --git a/cmd/peer-s3-client.go b/cmd/peer-s3-client.go
index 3c851caf7a3a7..37b05adf79953 100644
--- a/cmd/peer-s3-client.go
+++ b/cmd/peer-s3-client.go
@@ -135,6 +135,35 @@ func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys {
 func (sys *S3PeerSys) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	g := errgroup.WithNErrs(len(sys.peerClients))
 
+	for idx, client := range sys.peerClients {
+		idx := idx
+		client := client
+		g.Go(func() error {
+			if client == nil {
+				return errPeerOffline
+			}
+			_, err := client.GetBucketInfo(ctx, bucket, BucketOptions{})
+			return err
+		}, idx)
+	}
+
+	errs := g.Wait()
+
+	var poolErrs []error
+	for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ {
+		perPoolErrs := make([]error, 0, len(sys.peerClients))
+		for i, client := range sys.peerClients {
+			if slices.Contains(client.GetPools(), poolIdx) {
+				perPoolErrs = append(perPoolErrs, errs[i])
+			}
+		}
+		quorum := len(perPoolErrs) / 2
+		poolErrs = append(poolErrs, reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum))
+	}
+
+	opts.Remove = isAllBucketsNotFound(poolErrs)
+	opts.Recreate = !opts.Remove
+
 	healBucketResults := make([]madmin.HealResultItem, len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		idx := idx
@@ -152,7 +181,7 @@ func (sys *S3PeerSys) HealBucket(ctx context.Context, bucket string, opts madmin
 		}, idx)
 	}
 
-	errs := g.Wait()
+	errs = g.Wait()
 
 	for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ {
 		perPoolErrs := make([]error, 0, len(sys.peerClients))
diff --git a/docs/distributed/decom-compressed-sse-s3.sh b/docs/distributed/decom-compressed-sse-s3.sh
index 2e3fdcf644d22..d7d2f10f03960 100755
--- a/docs/distributed/decom-compressed-sse-s3.sh
+++ b/docs/distributed/decom-compressed-sse-s3.sh
@@ -21,7 +21,7 @@ export MINIO_KMS_AUTO_ENCRYPTION=on
 export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
 export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 
-(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
 pid=$!
 
 sleep 30
@@ -52,8 +52,11 @@ policy_count=$(./mc admin policy list myminio/ | wc -l)
 
 kill $pid
 
-(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
-pid=$!
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) &
+pid_1=$!
+
+(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) &
+pid_2=$!
 
 sleep 30
 
@@ -82,20 +85,25 @@ fi
 ./mc ls -r myminio/versioned/ >expanded_ns.txt
 ./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt
 
-./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1}
 
 until $(./mc admin decom status myminio/ | grep -q Complete); do
 	echo "waiting for decom to finish..."
 	sleep 1
 done
 
-kill $pid
+kill $pid_1
+kill $pid_2
 
-(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) &
+sleep 5
+
+(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) &
 pid=$!
 
 sleep 30
 
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/"
+
 decom_user_count=$(./mc admin user list myminio/ | wc -l)
 decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
 
diff --git a/docs/distributed/decom-encrypted-sse-s3.sh b/docs/distributed/decom-encrypted-sse-s3.sh
index 1c4b6eb0c1a66..58a10f855982d 100755
--- a/docs/distributed/decom-encrypted-sse-s3.sh
+++ b/docs/distributed/decom-encrypted-sse-s3.sh
@@ -16,7 +16,7 @@ export CI=true
 export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
 export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 
-(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
 pid=$!
 
 sleep 30
@@ -49,8 +49,11 @@ policy_count=$(./mc admin policy list myminio/ | wc -l)
 
 kill $pid
 
-(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
-pid=$!
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) &
+pid_1=$!
+
+(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) &
+pid_2=$!
 
 sleep 30
 
@@ -86,20 +89,25 @@ fi
 ./mc ls -r myminio/versioned/ >expanded_ns.txt
 ./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt
 
-./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1}
 
 until $(./mc admin decom status myminio/ | grep -q Complete); do
 	echo "waiting for decom to finish..."
 	sleep 1
 done
 
-kill $pid
+kill $pid_1
+kill $pid_2
 
-(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) &
+sleep 5
+
+(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) &
 pid=$!
 
 sleep 30
 
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/"
+
 decom_user_count=$(./mc admin user list myminio/ | wc -l)
 decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
 
diff --git a/docs/distributed/decom-encrypted.sh b/docs/distributed/decom-encrypted.sh
index 375e39b579fc3..ea95c415b4aff 100755
--- a/docs/distributed/decom-encrypted.sh
+++ b/docs/distributed/decom-encrypted.sh
@@ -16,7 +16,7 @@ export CI=true
 export MINIO_KMS_AUTO_ENCRYPTION=on
 export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
 
-(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
 pid=$!
 
 sleep 30
@@ -48,8 +48,12 @@ user_count=$(./mc admin user list myminio/ | wc -l)
 policy_count=$(./mc admin policy list myminio/ | wc -l)
 
 kill $pid
-(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
-pid=$!
+
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) &
+pid_1=$!
+
+(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) &
+pid_2=$!
 
 sleep 30
 
@@ -78,20 +82,25 @@ fi
 ./mc ls -r myminio/versioned/ >expanded_ns.txt
 ./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt
 
-./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1}
 
 until $(./mc admin decom status myminio/ | grep -q Complete); do
 	echo "waiting for decom to finish..."
 	sleep 1
 done
 
-kill $pid
+kill $pid_1
+kill $pid_2
+
+sleep 5
 
-(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null) &
+(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) &
 pid=$!
 
 sleep 30
 
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/"
+
 decom_user_count=$(./mc admin user list myminio/ | wc -l)
 decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
 
diff --git a/docs/distributed/decom.sh b/docs/distributed/decom.sh
index b3635ecd28ef2..39107f6060d69 100755
--- a/docs/distributed/decom.sh
+++ b/docs/distributed/decom.sh
@@ -16,7 +16,7 @@ fi
 export CI=true
 export MINIO_SCANNER_SPEED=fastest
 
-(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/tmp/decom.log) &
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/tmp/decom.log) &
 pid=$!
 
 sleep 30
@@ -48,16 +48,16 @@ user_count=$(./mc admin user list myminio/ | wc -l)
 policy_count=$(./mc admin policy list myminio/ | wc -l)
 
 ## create a warm tier instance
-(minio server /tmp/xltier/{1...4}/disk{0...1} --address :9001 2>&1 >/dev/null) &
+(minio server /tmp/xltier/{1...4}/disk{0...1} --address :9002 2>&1 >/dev/null) &
 sleep 30
 
-export MC_HOST_mytier="http://minioadmin:minioadmin@localhost:9001/"
+export MC_HOST_mytier="http://minioadmin:minioadmin@localhost:9002/"
 
 ./mc mb -l myminio/bucket2
 ./mc mb -l mytier/tiered
 
 ## create a tier and set up ilm policy to tier immediately
-./mc admin tier add minio myminio TIER1 --endpoint http://localhost:9001 --access-key minioadmin --secret-key minioadmin --bucket tiered --prefix prefix5/
+./mc admin tier add minio myminio TIER1 --endpoint http://localhost:9002 --access-key minioadmin --secret-key minioadmin --bucket tiered --prefix prefix5/
 ./mc ilm add myminio/bucket2 --transition-days 0 --transition-tier TIER1 --transition-days 0
 
 ## mirror some content to bucket2 and capture versions tiered
@@ -70,8 +70,12 @@ sleep 30
 ./mc ls -r --versions mytier/tiered/ >tiered_ns_versions.txt
 
 kill $pid
-(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
-pid=$!
+
+(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) &
+pid_1=$!
+
+(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) &
+pid_2=$!
 
 sleep 30
 
@@ -100,25 +104,30 @@ fi
 ./mc ls -r myminio/versioned/ >expanded_ns.txt
 ./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt
 
-./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1}
 
 count=0
 until $(./mc admin decom status myminio/ | grep -q Complete); do
 	echo "waiting for decom to finish..."
 	count=$((count + 1))
 	if [ ${count} -eq 120 ]; then
-		./mc cat /tmp/expanded.log
+		./mc cat /tmp/expanded_*.log
 	fi
 	sleep 1
 done
 
-kill $pid
+kill $pid_1
+kill $pid_2
+
+sleep 5
 
-(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null) &
+(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) &
 pid=$!
 
 sleep 30
 
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/"
+
 decom_user_count=$(./mc admin user list myminio/ | wc -l)
 decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
 


Test patch: