Repository: minio/minio (language: go)

Issue:
"A timeout occurred while trying to lock a resource, please reduce your request rate" when trying to list objects
## NOTE
code:
```
public void test() throws Exception {
        log.info("start test object list");
        long start = System.currentTimeMillis();
        int count = this.listAll("");
        log.info("test object list complete, spend: {}s, count: {}", (System.currentTimeMillis() - start) / 1000, count);
    }

    private int listAll(String prefix) {
        int count = 0;
        Iterable<Result<Item>> parts = minioClient.listObjects(ListObjectsArgs.builder()
                .prefix(prefix)
                .bucket("test")
                .recursive(true)
                .build());
        Iterator<Result<Item>> iterator = parts.iterator();
        while (iterator.hasNext()) {
            try {
                iterator.next().get();
                count++;
            } catch (Exception e) {
                log.info("list object error: {}", e.getMessage(), e);
            }
        }
        return count;
    }
```
console log:
```
2022-08-09 21:26:03.047  INFO 68233 --- [   scheduling-1] com.service.TestService            : list object error: A timeout occurred while trying to lock a resource, please reduce your request rate

io.minio.errors.ErrorResponseException: A timeout occurred while trying to lock a resource, please reduce your request rate
	at io.minio.S3Base$1.onResponse(S3Base.java:690) ~[minio-8.4.2.jar:8.4.2]
	at okhttp3.internal.connection.RealCall$AsyncCall.run(RealCall.kt:519) ~[okhttp-4.10.0.jar:na]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312]
	at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312]

2022-08-09 21:26:03.047  INFO 68233 --- [   scheduling-1] com.service.TestService            : test object list complete, spend: 10s, count: 2000
```

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used (`minio --version`): RELEASE.2022-08-05T23-27-09Z
* Server setup and configuration: 
* Operating System and version (`uname -a`): ubuntu18.04 4.15.0-180-generic



Discussion / hints:
MinIO Server Log:
```
API: ListObjectsV2(bucket=test)
Time: 02:09:23 UTC 08/10/2022
DeploymentID: 5bc4bac3-0087-447d-ab43-27ced0f0350a
RequestID: 1709D9C070110FAF
RemoteHost: 100.72.162.151
Host: minio:9000
UserAgent: MinIO (Linux; amd64) minio-java/8.4.2
Error: context deadline exceeded (context.deadlineExceededError)
       5: internal/logger/logger.go:259:logger.LogIf()
       4: cmd/erasure-single-drive.go:2907:cmd.(*erasureSingle).ListObjects()
       3: cmd/erasure-single-drive.go:2942:cmd.(*erasureSingle).ListObjectsV2()
       2: cmd/bucket-listobjects-handlers.go:238:cmd.objectAPIHandlers.ListObjectsV2Handler()
       1: net/http/server.go:2084:http.HandlerFunc.ServeHTTP()
```
@itzhoujun can you provide `mc support profile` output when you run your java code against MinIO? 

Also please explain your setup how many disks do you have?


@harshavardhana one server and one disk.
[profile.zip](https://github.com/minio/minio/files/9295957/profile.zip)
 
> API: ListObjectsV2(bucket=test)
> Time: 02:09:23 UTC 08/10/2022
> DeploymentID: 5bc4bac3-0087-447d-ab43-27ced0f0350a
> RequestID: 1709D9C070110FAF
> RemoteHost: 100.72.162.151
> Host: minio:9000
> UserAgent: MinIO (Linux; amd64) minio-java/8.4.2
> Error: context deadline exceeded (context.deadlineExceededError)
>        5: internal/logger/logger.go:259:logger.LogIf()
>        4: cmd/erasure-single-drive.go:2907:cmd.(*erasureSingle).ListObjects()
>        3: cmd/erasure-single-drive.go:2942:cmd.(*erasureSingle).ListObjectsV2()
>        2: cmd/bucket-listobjects-handlers.go:238:cmd.objectAPIHandlers.ListObjectsV2Handler()
>        1: net/http/server.go:2084:http.HandlerFunc.ServeHTTP()

This is helpful @itzhoujun thanks

Fix patch:
diff --git a/cmd/metacache-server-pool.go b/cmd/metacache-server-pool.go
index 26c19b748222f..91f572b76951c 100644
--- a/cmd/metacache-server-pool.go
+++ b/cmd/metacache-server-pool.go
@@ -112,14 +112,14 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	if o.ID != "" && !o.Transient {
 		// Create or ping with handout...
 		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
-		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-		defer cancel()
 		var c *metacache
 		if rpc == nil {
 			resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
 			c = &resp
 		} else {
-			c, err = rpc.GetMetacacheListing(ctx, *o)
+			rctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			c, err = rpc.GetMetacacheListing(rctx, *o)
+			cancel()
 		}
 		if err != nil {
 			if errors.Is(err, context.Canceled) {
@@ -318,58 +318,32 @@ func (es *erasureSingle) listPath(ctx context.Context, o *listPathOptions) (entr
 	//
 	// If we don't have a list id we must ask the server if it has a cache or create a new.
 	if o.ID != "" && !o.Transient {
-		// Create or ping with handout...
-		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
-		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-		defer cancel()
-		var c *metacache
-		if rpc == nil {
-			resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
-			c = &resp
-		} else {
-			c, err = rpc.GetMetacacheListing(ctx, *o)
+		resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
+		c := &resp
+		if c.fileNotFound {
+			// No cache found, no entries found.
+			return entries, io.EOF
 		}
-		if err != nil {
-			if errors.Is(err, context.Canceled) {
-				// Context is canceled, return at once.
-				// request canceled, no entries to return
-				return entries, io.EOF
-			}
-			if !errors.Is(err, context.DeadlineExceeded) {
-				o.debugln("listPath: got error", err)
-			}
-			o.Transient = true
+		if c.status == scanStateError || c.status == scanStateNone {
+			o.ID = ""
 			o.Create = false
-			o.ID = mustGetUUID()
+			o.debugln("scan status", c.status, " - waiting a roundtrip to create")
 		} else {
-			if c.fileNotFound {
-				// No cache found, no entries found.
-				return entries, io.EOF
-			}
-			if c.status == scanStateError || c.status == scanStateNone {
-				o.ID = ""
-				o.Create = false
-				o.debugln("scan status", c.status, " - waiting a roundtrip to create")
-			} else {
-				// Continue listing
-				o.ID = c.id
-				go func(meta metacache) {
-					// Continuously update while we wait.
-					t := time.NewTicker(metacacheMaxClientWait / 10)
-					defer t.Stop()
-					select {
-					case <-ctx.Done():
-						// Request is done, stop updating.
-						return
-					case <-t.C:
-						meta.lastHandout = time.Now()
-						if rpc == nil {
-							meta, _ = localMetacacheMgr.updateCacheEntry(meta)
-						}
-						meta, _ = rpc.UpdateMetacacheListing(ctx, meta)
-					}
-				}(*c)
-			}
+			// Continue listing
+			o.ID = c.id
+			go func(meta metacache) {
+				// Continuously update while we wait.
+				t := time.NewTicker(metacacheMaxClientWait / 10)
+				defer t.Stop()
+				select {
+				case <-ctx.Done():
+					// Request is done, stop updating.
+					return
+				case <-t.C:
+					meta.lastHandout = time.Now()
+					meta, _ = localMetacacheMgr.updateCacheEntry(meta)
+				}
+			}(*c)
 		}
 
 		// We have an existing list ID, continue streaming.
@@ -399,21 +373,7 @@ func (es *erasureSingle) listPath(ctx context.Context, o *listPathOptions) (entr
 			return entries, err
 		}
 		entries.truncate(0)
-		go func() {
-			rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
-			if rpc != nil {
-				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-				defer cancel()
-				c, err := rpc.GetMetacacheListing(ctx, *o)
-				if err == nil {
-					c.error = "no longer used"
-					c.status = scanStateError
-					rpc.UpdateMetacacheListing(ctx, *c)
-				}
-			}
-		}()
 		o.ID = ""
-
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Resuming listing from drives failed %w, proceeding to do raw listing", err))
 		}


Test patch: