Repository: minio/minio (language: go)

Issue:
"file version not found" during healing
## NOTE
If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.
"file version not found" during healing
Probably related/or root cause of [problem](https://github.com/minio/minio/issues/18992) reported some time ago. 
<!--- Provide a general summary of the issue in the Title above -->

## Expected Behavior
Drive healing is really successful and all objects are redundant. 
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->

## Current Behavior
Drive healing not successful. 
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->

## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
<!--- or ideas how to implement the addition or change -->

## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->
1. Set up lab, vm based deployment, 4 nodes, 8 drives each
```
$ mcli admin info minio-debug-http
●  s54000:9000
   Uptime: 24 seconds
   Version: 2024-02-24T17:11:14Z
   Network: 4/4 OK
   Drives: 8/8 OK
   Pool: 1

●  s54001:9000
   Uptime: 24 seconds
   Version: 2024-02-24T17:11:14Z
   Network: 4/4 OK
   Drives: 8/8 OK
   Pool: 1

●  s54002:9000
   Uptime: 24 seconds
   Version: 2024-02-24T17:11:14Z
   Network: 4/4 OK
   Drives: 8/8 OK
   Pool: 1

●  s54003:9000
   Uptime: 24 seconds
   Version: 2024-02-24T17:11:14Z
   Network: 4/4 OK
   Drives: 8/8 OK
   Pool: 1
```
2. Run mongodb pbm backup (all objects saved succesfully, can be read) 
 ```
pbm config
pitr:
  enabled: false
  oplogSpanMin: 0
  compression: s2
storage:
  type: s3
  s3:
    region: pl-kra
    endpointUrl: http://minio-debug-http:9000
    bucket: mongobackup-debug
    prefix: tmp-roman-cord-ivs-t
    credentials:
      access-key-id: '***'
      secret-access-key: '***'
    uploadPartSize: 10000000
    maxUploadParts: 10000
    insecureSkipTLSVerify: false
    retryer:
      numMaxRetries: 10
      minRetryDelay: 1s
      maxRetryDelay: 10m0s
```
4. Capture fs usage and see what is saved
```
root@s54000:/jbod-ssd/storage0# df -h | grep jbod
/dev/xvdf1       50G  3.7G   47G   8% /jbod-ssd/storage3
/dev/xvdg1       50G  3.6G   47G   8% /jbod-ssd/storage4
/dev/xvdi1       50G  3.6G   47G   8% /jbod-ssd/storage6
/dev/xvdb1       50G  3.7G   47G   8% /jbod-ssd/storage0
/dev/xvde1       50G  3.7G   47G   8% /jbod-ssd/storage2
/dev/xvdh1       50G  3.6G   47G   8% /jbod-ssd/storage5
/dev/xvdc1       50G  3.7G   47G   8% /jbod-ssd/storage1
/dev/xvdj1       50G  3.6G   47G   8% /jbod-ssd/storage7

root@s54001:~# df -h | grep jbod
/dev/xvdc1       50G  3.7G   47G   8% /jbod-ssd/storage1
/dev/xvdf1       50G  3.7G   47G   8% /jbod-ssd/storage3
/dev/xvdh1       50G  3.6G   47G   8% /jbod-ssd/storage5
/dev/xvdj1       50G  3.6G   47G   8% /jbod-ssd/storage7
/dev/xvdi1       50G  3.6G   47G   8% /jbod-ssd/storage6
/dev/xvdg1       50G  3.6G   47G   8% /jbod-ssd/storage4
/dev/xvdb1       50G  3.7G   47G   8% /jbod-ssd/storage0
/dev/xvde1       50G  3.7G   47G   8% /jbod-ssd/storage2


root@s54002:~# df -h | grep jbod
/dev/xvdh1       50G  3.6G   47G   8% /jbod-ssd/storage5
/dev/xvdf1       50G  3.7G   47G   8% /jbod-ssd/storage3
/dev/xvdc1       50G  3.7G   47G   8% /jbod-ssd/storage1
/dev/xvdj1       50G  3.6G   47G   8% /jbod-ssd/storage7
/dev/xvde1       50G  3.7G   47G   8% /jbod-ssd/storage2
/dev/xvdi1       50G  3.6G   47G   8% /jbod-ssd/storage6
/dev/xvdg1       50G  3.6G   47G   8% /jbod-ssd/storage4
/dev/xvdb1       50G  3.7G   47G   8% /jbod-ssd/storage0


root@s54003:~# df -h | grep jbod
/dev/xvdb1       50G  3.7G   47G   8% /jbod-ssd/storage0
/dev/xvdc1       50G  3.7G   47G   8% /jbod-ssd/storage1
/dev/xvdf1       50G  3.7G   47G   8% /jbod-ssd/storage3
/dev/xvdg1       50G  3.6G   47G   8% /jbod-ssd/storage4
/dev/xvdj1       50G  3.6G   47G   8% /jbod-ssd/storage7
/dev/xvdh1       50G  3.6G   47G   8% /jbod-ssd/storage5
/dev/xvdi1       50G  3.6G   47G   8% /jbod-ssd/storage6
/dev/xvde1       50G  3.7G   47G   8% /jbod-ssd/storage2

$ mcli ls  minio-debug-http/mongobackup-debug/tmp-roman-cord-ivs-t
[2024-02-26 22:33:47 CET]     5B STANDARD .pbm.init
[2024-02-26 22:46:27 CET]  19KiB STANDARD 2024-02-26T21:41:16Z.pbm.json
[2024-02-26 23:15:02 CET]     0B 2024-02-26T21:41:16Z/

$ mcli ls -r  minio-debug-http/mongobackup-debug/tmp-roman-cord-ivs-t | wc -l
102

$ mcli stat minio-debug-http/mongobackup-debug
Name      : mongobackup-debug
Date      : 2024-02-26 23:25:27 CET
Size      : N/A
Type      : folder

Properties:
  Versioning: Un-versioned
  Location: us-east-1
  Anonymous: Disabled
  ILM: Disabled

Usage:
      Total size: 76 GiB
   Objects count: 102
  Versions count: 0

```
5. Simulate drive failure
```
root@s54002:~# service minio stop
root@s54002:~# rm -rf /jbod-ssd/storage5/*
root@s54002:~# rm -rf /jbod-ssd/storage5/.minio.sys/
root@s54002:~# service minio start
```
6. Logs : 
```
Feb 26 22:49:46 s54002 minio[356062]: Healing drive '/jbod-ssd/storage5' - use 4 parallel workers.
Feb 26 22:49:46 s54002 minio[356062]: Healing of drive '/jbod-ssd/storage5' complete (healed: 4, failed: 0).
```
7. Check if filesystem returned to pre-failure state
```root@s54002:~# df -h
Filesystem      Size  Used Avail Use% Mounted on
tmpfs           792M  2.3M  790M   1% /run
/dev/xvda2      9.8G  6.6G  2.8G  71% /
tmpfs           3.9G   40K  3.9G   1% /dev/shm
tmpfs           5.0M     0  5.0M   0% /run/lock
/dev/xvdh1       50G  390M   50G   1% /jbod-ssd/storage5. -----> was 3.6G before
/dev/xvdf1       50G  3.7G   47G   8% /jbod-ssd/storage3
/dev/xvdc1       50G  3.7G   47G   8% /jbod-ssd/storage1
/dev/xvdj1       50G  3.6G   47G   8% /jbod-ssd/storage7
/dev/xvde1       50G  3.7G   47G   8% /jbod-ssd/storage2
/dev/xvdi1       50G  3.6G   47G   8% /jbod-ssd/storage6
/dev/xvdg1       50G  3.6G   47G   8% /jbod-ssd/storage4
/dev/xvdb1       50G  3.7G   47G   8% /jbod-ssd/storage0

```
8. Saved object can be downloaded, as we have read quorum but after i shutdown another node quorum is lost (what prove that healing wasn't successful) 
9. _mcli admin trace --call healing  minio-debug-http_ logs attached (complains about file version not found)
[trace-healing.txt](https://github.com/minio/minio/files/14411975/trace-healing.txt)

## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
<!--- or ideas how to implement the addition or change -->
Something wrong with object names? (2024-02-26T21:41:16Z .pbm.init)? As when testing less fancy names (not from mongo backup) healing works as expected. 

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used (`minio version RELEASE.2024-02-24T17-11-14Z (commit-id=c2b54d92f66ea02642be4ede26f2b7ecac1f2056)
Runtime: go1.21.7 linux/amd64
License: GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>
Copyright: 2015-2024 MinIO, Inc.`):
* Server setup and configuration: lab setup based on vm's 4 nodes, 8 drives each
* Operating System and version (`5.15.0-91-generic #101-Ubuntu SMP Tue Nov 14 13:30:08 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux`):



Discussion / hints:
This looks weird @lukasz-borek will check what might be going on
i can't seem to reproduce this @lukasz-borek - can you consistently do this? 
Also tried with the same object names, not able to reproduce either. 
@lukasz-borek please share the server log from the node 

```
root@s54002:~# service minio stop
root@s54002:~# rm -rf /jbod-ssd/storage5/*
root@s54002:~# rm -rf /jbod-ssd/storage5/.minio.sys/
root@s54002:~# service minio start
```

can you do that?

Fix patch:
diff --git a/cmd/endpoint.go b/cmd/endpoint.go
index c0fe711a060bc..7c69ba37dcd75 100644
--- a/cmd/endpoint.go
+++ b/cmd/endpoint.go
@@ -75,6 +75,7 @@ type Endpoint struct {
 	PoolIdx, SetIdx, DiskIdx int
 }
 
+// Equal returns true if endpoint == ep
 func (endpoint Endpoint) Equal(ep Endpoint) bool {
 	if endpoint.IsLocal == ep.IsLocal && endpoint.PoolIdx == ep.PoolIdx && endpoint.SetIdx == ep.SetIdx && endpoint.DiskIdx == ep.DiskIdx {
 		if endpoint.Path == ep.Path && endpoint.Host == ep.Host {
diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go
index 165b8a61b3a65..6e7412db988cc 100644
--- a/cmd/erasure-multipart.go
+++ b/cmd/erasure-multipart.go
@@ -33,6 +33,7 @@ import (
 
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -461,6 +462,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	// if storageClass is standard no need to save it as part of metadata.
+	if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(userDefined, xhttp.AmzStorageClass)
+	}
+
 	if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() {
 		userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String()
 	}
diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go
index 91413a3699ff9..49bdd035b68ff 100644
--- a/cmd/erasure-object.go
+++ b/cmd/erasure-object.go
@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
@@ -1509,6 +1510,11 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	// if storageClass is standard no need to save it as part of metadata.
+	if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(userDefined, xhttp.AmzStorageClass)
+	}
+
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go
index 4d064eb1b5892..9f5412caecc8b 100644
--- a/cmd/storage-rest-server.go
+++ b/cmd/storage-rest-server.go
@@ -81,7 +81,7 @@ func getStorageViaEndpoint(endpoint Endpoint) StorageAPI {
 	if len(globalLocalSetDrives) == 0 {
 		for _, drive := range globalLocalDrives {
 			if drive != nil && drive.Endpoint().Equal(endpoint) {
-                               return drive
+				return drive
 			}
 		}
 	}
diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go
index 4b234b31446eb..a51441b7919c5 100644
--- a/cmd/xl-storage-format-v2.go
+++ b/cmd/xl-storage-format-v2.go
@@ -35,6 +35,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/storageclass"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
@@ -639,6 +640,9 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string, allParts bool) (FileInfo
 		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 			continue
 		}
+		if equals(k, "x-amz-storage-class") && v == storageclass.STANDARD {
+			continue
+		}
 
 		fi.Metadata[k] = v
 	}
@@ -655,6 +659,9 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string, allParts bool) (FileInfo
 				continue
 			}
 		}
+		if equals(k, "x-amz-storage-class") && string(v) == storageclass.STANDARD {
+			continue
+		}
 		switch {
 		case strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower), equals(k, VersionPurgeStatusKey):
 			fi.Metadata[k] = string(v)


Test patch: