Repository: minio/minio (language: go)

Issue:
panic due to  Invalid Tier Object delete marker versionId  
## NOTE
If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.

<!--- Provide a general summary of the issue in the Title above -->
With master commit id 31188e932749bf53434ca7f5c515ac8ab955b9ea
```
panic: Invalid Tier Object delete marker versionId  invalid UUID length: 0

goroutine 220585 [running]:
github.com/minio/minio/cmd.xlMetaV2Object.InitFreeVersion({{0x9, 0xe0, 0xc0, 0x6f, 0x95, 0x3e, 0x49, 0x8, 0x9c, 0x4, ...}, ...}, ...)
       /Users/kp/code/src/github.com/minio/minio/cmd/xl-storage-free-version.go:36 +0x3d4
github.com/minio/minio/cmd.(*xlMetaV2).DeleteVersion(_, {{0x0, 0x0}, {0x0, 0x0}, {0x1401c71d0e0, 0x24}, 0x0, 0x0, {0x0, ...}, ...})
       /Users/kp/code/src/github.com/minio/minio/cmd/xl-storage-format-v2.go:1401 +0x15c0
github.com/minio/minio/cmd.(*xlStorage).DeleteVersion(_, {_, _}, {_, _}, {_, _}, {{0x0, 0x0}, {0x0, ...}, ...}, ...)
       /Users/kp/code/src/github.com/minio/minio/cmd/xl-storage.go:1091 +0x468
github.com/minio/minio/cmd.(*xlStorageDiskIDCheck).DeleteVersion(_, {_, _}, {_, _}, {_, _}, {{0x0, 0x0}, {0x0, ...}, ...}, ...)
       /Users/kp/code/src/github.com/minio/minio/cmd/xl-storage-disk-id-check.go:456 +0x1b4
github.com/minio/minio/cmd.erasureObjects.deleteIfDangling.func1()
       /Users/kp/code/src/github.com/minio/minio/cmd/erasure-object.go:495 +0x94
github.com/minio/minio/internal/sync/errgroup.(*Group).Go.func1()
       /Users/kp/code/src/github.com/minio/minio/internal/sync/errgroup/errgroup.go:123 +0x1bc
created by github.com/minio/minio/internal/sync/errgroup.(*Group).Go
       /Users/kp/code/src/github.com/minio/minio/internal/sync/errgroup/errgroup.go:105 +0xb4
```

Also seeing another panic in the second pool due to send on closed channel
```
Restarting on service signal
panic: "PUT /bucket/confess/02_14_23_16%3A41/pfx2/GxbYcZEq7sAhgJcJzSxXXR": send on closed channel
goroutine 3117 [running]:
runtime/debug.Stack()
	/opt/homebrew/opt/go/libexec/src/runtime/debug/stack.go:24 +0x68
github.com/minio/minio/cmd.setCriticalErrorHandler.func1.1()
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:574 +0x68
panic({0x106bb7fe0, 0x106f8b448})
	/opt/homebrew/opt/go/libexec/src/runtime/panic.go:838 +0x204
github.com/minio/minio/cmd.(*transitionState).queueTransitionTask(_, {{0x1400e257365, 0x6}, {0x140286c4840, 0x32}, {0x18c87798, 0xedb7e2159, 0x0}, 0x1000, 0x0, ...}, ...)
	/Users/kp/code/src/github.com/minio/minio/cmd/bucket-lifecycle.go:171 +0xcc
github.com/minio/minio/cmd.enqueueTransitionImmediate({{0x1400e257365, 0x6}, {0x140286c4840, 0x32}, {0x18c87798, 0xedb7e2159, 0x0}, 0x1000, 0x0, {0x1400c63f680, ...}, ...})
	/Users/kp/code/src/github.com/minio/minio/cmd/bucket-lifecycle.go:316 +0x1f4
github.com/minio/minio/cmd.objectAPIHandlers.PutObjectHandler({0x106f82020?, 0x106f82008?}, {0x106fa3880, 0x14023f59bd8}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/object-handlers.go:1930 +0x3d30
net/http.HandlerFunc.ServeHTTP(...)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084
github.com/minio/minio/cmd.httpTrace.func1({0x106fa3880, 0x14023f59bd8}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/http-tracer.go:183 +0x1f4
net/http.HandlerFunc.ServeHTTP(0x106a95da0?, {0x106fa3880?, 0x14023f59bd8?}, 0x4?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/klauspost/compress/gzhttp.NewWrapper.func1.1({0x106fa3e80, 0x1402b1abc30}, 0x106bebae0?)
	/Users/kp/go/pkg/mod/github.com/klauspost/compress@v1.15.15/gzhttp/compress.go:412 +0x3bc
net/http.HandlerFunc.ServeHTTP(...)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084
github.com/minio/minio/cmd.maxClients.func1({0x106fa3e80, 0x1402b1abc30}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/handler-api.go:275 +0x32c
net/http.HandlerFunc.ServeHTTP(...)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084
github.com/minio/minio/cmd.collectAPIStats.func1({0x106fa3e80, 0x1402b1abb80}, 0x0?)
	/Users/kp/code/src/github.com/minio/minio/cmd/handler-utils.go:355 +0xe8
net/http.HandlerFunc.ServeHTTP(0x10?, {0x106fa3e80?, 0x1402b1abb80?}, 0x140073acb48?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setBucketForwardingHandler.func1({0x106fa3e80, 0x1402b1abb80}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:479 +0x208
net/http.HandlerFunc.ServeHTTP(0x14027268628?, {0x106fa3e80?, 0x1402b1abb80?}, 0x102840270?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setUploadForwardingHandler.func1({0x106fa3e80, 0x1402b1abb80}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:592 +0x174
net/http.HandlerFunc.ServeHTTP(0x106fa3ee0?, {0x106fa3e80?, 0x1402b1abb80?}, 0xa?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.addCustomHeaders.func1({0x106fa3ee0, 0x1400dc50d38}, 0x14028103420?)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:557 +0x474
net/http.HandlerFunc.ServeHTTP(0x106e94960?, {0x106fa3ee0?, 0x1400dc50d38?}, 0x39?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setRequestValidityHandler.func1({0x106fa3ee0, 0x1400dc50d38}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:467 +0xd58
net/http.HandlerFunc.ServeHTTP(0x1401f56e280?, {0x106fa3ee0?, 0x1400dc50d38?}, 0x104451844?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setHTTPStatsHandler.func1({0x106fa66d0?, 0x14024ffa8c0}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:305 +0x118
net/http.HandlerFunc.ServeHTTP(0x14027268e58?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x14024ca3f80?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setRequestLimitHandler.func1({0x106fa66d0?, 0x14024ffa8c0}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:130 +0x420
net/http.HandlerFunc.ServeHTTP(0x10444d7e2?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x104250214?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setCrossDomainPolicy.func1({0x106fa66d0?, 0x14024ffa8c0?}, 0x0?)
	/Users/kp/code/src/github.com/minio/minio/cmd/crossdomain-xml-handler.go:43 +0x108
net/http.HandlerFunc.ServeHTTP(0x14029ec1b00?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x1?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setBrowserRedirectHandler.func1({0x106fa66d0, 0x14024ffa8c0}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:161 +0x140
net/http.HandlerFunc.ServeHTTP(0x0?, {0x106fa66d0?, 0x14024ffa8c0?}, 0xc0aee78?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setAuthHandler.func1({0x106fa66d0, 0x14024ffa8c0}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/auth-handler.go:607 +0x47c
net/http.HandlerFunc.ServeHTTP(0x3a?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x102facce8?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.httpTracer.func1({0x106fa66d0, 0x14024ffa8c0}, 0x14029ec1b00)
	/Users/kp/code/src/github.com/minio/minio/cmd/http-tracer.go:71 +0x5a4
net/http.HandlerFunc.ServeHTTP(0x14029ec1a00?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x106a95da0?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/mux.(*Router).ServeHTTP(0x140006a6f00, {0x106fa66d0, 0x14024ffa8c0}, 0x14029ec1a00)
	/Users/kp/go/pkg/mod/github.com/minio/mux@v1.9.0/mux.go:214 +0x1b8
github.com/rs/cors.(*Cors).Handler.func1({0x106fa66d0, 0x14024ffa8c0}, 0x14029ec1a00)
	/Users/kp/go/pkg/mod/github.com/rs/cors@v1.8.3/cors.go:236 +0x1e8
net/http.HandlerFunc.ServeHTTP(0x140272699a8?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x102ae0b68?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/cmd.setCriticalErrorHandler.func1({0x106fa66d0?, 0x14024ffa8c0?}, 0x800?)
	/Users/kp/code/src/github.com/minio/minio/cmd/generic-handlers.go:581 +0x7c
net/http.HandlerFunc.ServeHTTP(0x1400507a000?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x102ae87d8?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
github.com/minio/minio/internal/http.(*Server).Start.func1({0x106fa66d0?, 0x14024ffa8c0?}, 0x14027269a88?)
	/Users/kp/code/src/github.com/minio/minio/internal/http/server.go:111 +0xbc
net/http.HandlerFunc.ServeHTTP(0x0?, {0x106fa66d0?, 0x14024ffa8c0?}, 0x1028406e0?)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2084 +0x3c
net/http.serverHandler.ServeHTTP({0x106f9f6e8?}, {0x106fa66d0, 0x14024ffa8c0}, 0x14029ec1a00)
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:2916 +0x3fc
net/http.(*conn).serve(0x14004b77860, {0x106fa80b8, 0x14000562270})
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:1966 +0x56c
created by net/http.(*Server).Serve
	/opt/homebrew/opt/go/libexec/src/net/http/server.go:3071 +0x450
```

## Expected Behavior
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->

## Current Behavior
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
server crashes
## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
<!--- or ideas how to implement the addition or change -->
not sure - seems like a race

## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->

1.  tiering set up on a versioned bucket with immediate transition
2. testing confess tool which performs a series of put, get/head, versioned deletes in parallel. Can probably be repro'd with just a for loop of these operations.
3. perform `mc admin service restart <alias> 
4.

## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->

## Regression
<!-- Is this issue a regression? (Yes / No) --> Don't know
<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in --> 
* Version used (`minio --version`): 31188e932749bf53434ca7f5c515ac8ab955b9ea master branch
* Server setup and configuration: distributed multi pool setup for both clusters
* Operating System and version (`uname -a`): 



Discussion / hints:

Fix patch:
diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go
index 32e9c60607291..17140f5a98de1 100644
--- a/cmd/bucket-lifecycle.go
+++ b/cmd/bucket-lifecycle.go
@@ -152,7 +152,6 @@ type transitionTask struct {
 }
 
 type transitionState struct {
-	once         sync.Once
 	transitionCh chan transitionTask
 
 	ctx        context.Context
@@ -169,10 +168,7 @@ type transitionState struct {
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 	select {
-	case <-GlobalContext.Done():
-		t.once.Do(func() {
-			close(t.transitionCh)
-		})
+	case <-t.ctx.Done():
 	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
 	default:
 	}
@@ -214,20 +210,20 @@ func (t *transitionState) ActiveTasks() int {
 }
 
 // worker waits for transition tasks
-func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
+func (t *transitionState) worker(objectAPI ObjectLayer) {
 	for {
 		select {
 		case <-t.killCh:
 			return
-		case <-ctx.Done():
+		case <-t.ctx.Done():
 			return
 		case task, ok := <-t.transitionCh:
 			if !ok {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
+				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
 					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
@@ -278,7 +274,7 @@ func (t *transitionState) UpdateWorkers(n int) {
 
 func (t *transitionState) updateWorkers(n int) {
 	for t.numWorkers < n {
-		go t.worker(t.ctx, t.objAPI)
+		go t.worker(t.objAPI)
 		t.numWorkers++
 	}
 
diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go
index a4614a53bf51b..d71f6e2fd12de 100644
--- a/cmd/erasure-object.go
+++ b/cmd/erasure-object.go
@@ -483,7 +483,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			fi.VersionID = opts.VersionID
 		}
-
+		fi.SetTierFreeVersionID(mustGetUUID())
 		disks := er.getDisks()
 		g := errgroup.WithNErrs(len(disks))
 		for index := range disks {


Test patch: