Repository: minio/minio (language: go)

Issue:
"assignment to entry in nil map" on PUT
My Loki deployment tries to write chunks to minio but fails. Minio logs the following panic. Loki is configured to use a service account on the root user with the following policy:

```
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket",
                "s3:PutObject",
                "s3:GetObject",
                "s3:DeleteObject"
            ],
            "Resource": [
                "arn:aws:s3:::loki-ruler",
                "arn:aws:s3:::loki-ruler/*",
                "arn:aws:s3:::loki-chunks",
                "arn:aws:s3:::loki-chunks/*"
            ]
        }
    ]
}
```

```
panic: "PUT /loki-chunks/fake/59db3987253e39ac/18e065b8efb%3A18e06c96bfc%3A1c350418": assignment to entry in nil map
goroutine 3269134 [running]:
runtime/debug.Stack()
	runtime/debug/stack.go:24 +0x5e
github.com/minio/minio/cmd.serverMain.func8.setCriticalErrorHandler.func2.1()
	github.com/minio/minio/cmd/generic-handlers.go:555 +0x97
panic({0x287e200?, 0x4cd9460?})
	runtime/panic.go:914 +0x21f
github.com/minio/minio/cmd.(*IAMObjectStore).loadUser(0xc000f59940, {0x4d003b8, 0xc000efd310}, {0x2d0191c, 0x11}, 0x0?, 0x0?)
	github.com/minio/minio/cmd/iam-object-store.go:269 +0x289
github.com/minio/minio/cmd.(*siteReplicatorCred).Get(0x647cc20, {0x4d003b8, 0xc000efd310})
	github.com/minio/minio/cmd/site-replication.go:6213 +0x25b
github.com/minio/minio/cmd.getTokenSigningKey()
	github.com/minio/minio/cmd/sts-handlers.go:197 +0x91
github.com/minio/minio/cmd.checkClaimsFromToken(0xc14f58ed00, {{0xc14f0d20b0, 0x8}, {0xc1f59dda10, 0x28}, {0xc14f0ce2c0, 0x2ba}, {0x0, 0x0, 0x0}, ...})
	github.com/minio/minio/cmd/auth-handler.go:311 +0x511
github.com/minio/minio/cmd.checkKeyValid(0xc14f58ed00, {0xc14ef0131c, 0x8})
	github.com/minio/minio/cmd/signature-v4-utils.go:171 +0x24c
github.com/minio/minio/cmd.getReqAccessKeyV4(0xc14f58ed00, {0x0, 0x0}, {0x2ce135a, 0x2})
	github.com/minio/minio/cmd/signature-v4-parser.go:66 +0x350
github.com/minio/minio/cmd.isPutActionAllowed({0x4d00380, 0xc14f23fce0}, 0x9?, {0xc14f1d9745, 0xb}, {0xc14f586580, 0x36}, 0xc14f58ed00, {0x2cf4785, 0xc})
	github.com/minio/minio/cmd/auth-handler.go:753 +0x255
github.com/minio/minio/cmd.objectAPIHandlers.PutObjectHandler({0xc14f1c7610?}, {0x4cfab00?, 0xc1123f2820}, 0xc14f58ed00)
	github.com/minio/minio/cmd/object-handlers.go:2050 +0x8af
net/http.HandlerFunc.ServeHTTP(0x53e964a?, {0x4cfab00?, 0xc1123f2820?}, 0x69d92c?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.s3APIMiddleware.func1.httpTraceHdrs.httpTrace.func3({0x4cfab00, 0xc1123f2820}, 0xc14d225e00?)
	github.com/minio/minio/cmd/http-tracer.go:189 +0xef
net/http.HandlerFunc.ServeHTTP(0x6407720?, {0x4cfab00?, 0xc1123f2820?}, 0x4?)
	net/http/server.go:2136 +0x29
github.com/klauspost/compress/gzhttp.NewWrapper.func1.1({0x4cf9650, 0xc14d225e00}, 0xc14f582ae0?)
	github.com/klauspost/compress@v1.17.6/gzhttp/compress.go:497 +0x37d
net/http.HandlerFunc.ServeHTTP(0xc14b595548?, {0x4cf9650?, 0xc14d225e00?}, 0x1?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.s3APIMiddleware.func1.maxClients.func1({0x4cf9650, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/handler-api.go:334 +0x47e
net/http.HandlerFunc.ServeHTTP(0x647cef0?, {0x4cf9650?, 0xc14d225e00?}, 0x53e9677?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.s3APIMiddleware.func1.collectAPIStats.func2({0x4cf9650?, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/handler-utils.go:306 +0x44e
github.com/minio/minio/cmd.s3APIMiddleware.func1({0x4cf9650, 0xc14d225e00}, 0x1b?)
	github.com/minio/minio/cmd/api-router.go:246 +0x174
net/http.HandlerFunc.ServeHTTP(0xc14f23f920?, {0x4cf9650?, 0xc14d225e00?}, 0x9485db7c144f5ac?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setBucketForwardingMiddleware.func1({0x4cf9650, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/generic-handlers.go:460 +0x2fc
net/http.HandlerFunc.ServeHTTP(0xc14f58ed00?, {0x4cf9650?, 0xc14d225e00?}, 0x8?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setUploadForwardingMiddleware.func1({0x4cf9650?, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/generic-handlers.go:610 +0x6b4
net/http.HandlerFunc.ServeHTTP(0xc14f1dedc1?, {0x4cf9650?, 0xc14d225e00?}, 0xc14b5964c0?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setRequestValidityMiddleware.func1({0x4cf9650?, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/generic-handlers.go:443 +0x168d
net/http.HandlerFunc.ServeHTTP(0xc14f23fa40?, {0x4cf9650?, 0xc14d225e00?}, 0xe2?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setRequestLimitMiddleware.func1({0x4cf9650?, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/generic-handlers.go:134 +0x662
net/http.HandlerFunc.ServeHTTP(0x2cefbc4?, {0x4cf9650?, 0xc14d225e00?}, 0x22606eb?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setCrossDomainPolicyMiddleware.func1({0x4cf9650?, 0xc14d225e00?}, 0x6?)
	github.com/minio/minio/cmd/crossdomain-xml-handler.go:42 +0xf1
net/http.HandlerFunc.ServeHTTP(0xc14f58ed00?, {0x4cf9650?, 0xc14d225e00?}, 0xc14f1c7600?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setBrowserRedirectMiddleware.func1({0x4cf9650, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/generic-handlers.go:164 +0x19c
net/http.HandlerFunc.ServeHTTP(0x0?, {0x4cf9650?, 0xc14d225e00?}, 0x3883d791?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.setAuthMiddleware.func1({0x4cf9650?, 0xc14d225e00}, 0xc14f58ed00)
	github.com/minio/minio/cmd/auth-handler.go:649 +0x6a2
net/http.HandlerFunc.ServeHTTP(0x4d00380?, {0x4cf9650?, 0xc14d225e00?}, 0x4cd7410?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.httpTracerMiddleware.func1({0x4cfa6b0, 0xc14f194b60}, 0xc14f58ec00)
	github.com/minio/minio/cmd/http-tracer.go:89 +0x3b4
net/http.HandlerFunc.ServeHTTP(0x2a7d040?, {0x4cfa6b0?, 0xc14f194b60?}, 0xa?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.addCustomHeadersMiddleware.func1({0x4cfa6b0, 0xc14f194b60}, 0xc14f23f9e0?)
	github.com/minio/minio/cmd/generic-handlers.go:538 +0x438
net/http.HandlerFunc.ServeHTTP(0xc14f58eb00?, {0x4cfa6b0?, 0xc14f194b60?}, 0x69d93f?)
	net/http/server.go:2136 +0x29
github.com/minio/mux.(*Router).ServeHTTP(0xc000000300, {0x4cfa6b0, 0xc14f194b60}, 0xc14f58eb00)
	github.com/minio/mux@v1.9.0/mux.go:214 +0x1f7
github.com/minio/minio/cmd.corsHandler.(*Cors).Handler.func2({0x4cfa6b0, 0xc14f194b60}, 0xc14f58eb00)
	github.com/rs/cors@v1.10.1/cors.go:281 +0x184
net/http.HandlerFunc.ServeHTTP(0x0?, {0x4cfa6b0?, 0xc14f194b60?}, 0x0?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/cmd.serverMain.func8.setCriticalErrorHandler.func2({0x4cfa6b0?, 0xc14f194b60?}, 0xc14f58eb00?)
	github.com/minio/minio/cmd/generic-handlers.go:562 +0x78
net/http.HandlerFunc.ServeHTTP(0xc14f58c5f0?, {0x4cfa6b0?, 0xc14f194b60?}, 0x727b25?)
	net/http/server.go:2136 +0x29
github.com/minio/minio/internal/http.(*Server).Init.func1({0x4cfa6b0?, 0xc14f194b60?}, 0xc14f58c600?)
	github.com/minio/minio/internal/http/server.go:127 +0x258
net/http.HandlerFunc.ServeHTTP(0x410c25?, {0x4cfa6b0?, 0xc14f194b60?}, 0xc14f194b01?)
	net/http/server.go:2136 +0x29
net/http.serverHandler.ServeHTTP({0x4cf28b8?}, {0x4cfa6b0?, 0xc14f194b60?}, 0x6?)
	net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc14b3f8120, {0x4d00380, 0xc0009ae810})
	net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 215
	net/http/server.go:3086 +0x5cb
```

This is not something that continuous to happen, maybe it was during a restart or so, not entirely clear.

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used (`minio --version`): RELEASE.2024-03-05T04-48-44Z
* Server setup and configuration: deployed by operator (5.0.12), replicating to a single-node setup on the same version deployed with docker-compose.
* Operating System and version (`uname -a`): `Linux  6.1.0-18-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.76-1 (2024-02-01) x86_64 GNU/Linux`



Discussion / hints:

Fix patch:
diff --git a/cmd/site-replication.go b/cmd/site-replication.go
index 392c08c28884f..6b30c59eeddfc 100644
--- a/cmd/site-replication.go
+++ b/cmd/site-replication.go
@@ -6207,12 +6207,11 @@ func (s *siteReplicatorCred) Get(ctx context.Context) (auth.Credentials, error)
 		return s.Creds, nil
 	}
 	s.RUnlock()
-	s.Lock()
-	defer s.Unlock()
-	var m map[string]UserIdentity
+	m := make(map[string]UserIdentity)
 	if err := globalIAMSys.store.loadUser(ctx, siteReplicatorSvcAcc, svcUser, m); err != nil {
 		return auth.Credentials{}, err
 	}
+	s.Set(m[siteReplicatorSvcAcc].Credentials)
 	return m[siteReplicatorSvcAcc].Credentials, nil
 }
 


Test patch: