Repository: go
Issue: runtime: fatal error: checkdead: runnable g
Labels: NeedsInvestigation,compiler/runtime

<!--
Please answer these questions before submitting your issue. Thanks!
-->

### What version of Go are you using (`go version`)?

<pre>
$ go version
go version go1.20.1 linux/amd64
</pre>

### Does this issue reproduce with the latest release?
only once, run 142d in production

![image](https://github.com/golang/go/assets/34152252/cf22e147-88c8-495b-b622-fa2163c49fa4)




### What operating system and processor architecture are you using (`go env`)?
run application at k8s, docker image from centos7 x86_64 GNU/Linux

<details><summary><code>go env</code> Output</summary><br><pre>
$ go env

</pre></details>

### What did you do?

<!--
If possible, provide a recipe for reproducing the error.
A complete runnable program is good.
A link on go.dev/play is best.
-->

I didn't do anything，it`s an grpc service，normal  running 142d, but get a panic today

### What did you see instead?

Below is some panic log, if needed，i can provide complete runtime panic output log

```
runtime: checkdead: find g 18322774 in status 1
fatal error: checkdead: runnable g

runtime stack:
runtime.throw({0x117acd1?, 0x0?})
	/usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0xc000191d58 sp=0xc000191d28 pc=0x436fdd
runtime.checkdead.func1(0xc0065d3520)
	/usr/local/go/src/runtime/proc.go:5216 +0xe5 fp=0xc000191d90 sp=0xc000191d58 pc=0x444c85
runtime.forEachG(0xc000191df0)
	/usr/local/go/src/runtime/proc.go:591 +0x5c fp=0xc000191dc0 sp=0xc000191d90 pc=0x43a77c
runtime.checkdead()
	/usr/local/go/src/runtime/proc.go:5203 +0xf7 fp=0xc000191e10 sp=0xc000191dc0 pc=0x444977
runtime.mput(0xc000191e40?)
	/usr/local/go/src/runtime/proc.go:5696 +0x33 fp=0xc000191e20 sp=0xc000191e10 pc=0x445ef3
runtime.stopm()
	/usr/local/go/src/runtime/proc.go:2332 +0x67 fp=0xc000191e50 sp=0xc000191e20 pc=0x43dba7
runtime.findRunnable()
	/usr/local/go/src/runtime/proc.go:3007 +0xabc fp=0xc000191f58 sp=0xc000191e50 pc=0x43f41c
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:3360 +0xb1 fp=0xc000191f90 sp=0xc000191f58 pc=0x440251
runtime.park_m(0xc0003264e0?)
	/usr/local/go/src/runtime/proc.go:3511 +0x12d fp=0xc000191fc0 sp=0xc000191f90 pc=0x44076d
runtime.mcall()
	/usr/local/go/src/runtime/asm_amd64.s:452 +0x43 fp=0xc000191fd0 sp=0xc000191fc0 pc=0x469ce3

goroutine 1 [IO wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000193a08 sp=0xc0001939e8 pc=0x439cf6
runtime.netpollblock(0x7f7ff55f7e38?, 0x40482f?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc000193a40 sp=0xc000193a08 pc=0x432657
internal/poll.runtime_pollWait(0x7f7ff5f5a0c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc000193a60 sp=0xc000193a40 pc=0x4665c9
internal/poll.(*pollDesc).wait(0xc0010c7a00?, 0x4?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc000193a88 sp=0xc000193a60 pc=0x4d9e72
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0010c7a00)
	/usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd fp=0xc000193b30 sp=0xc000193a88 pc=0x4df77d
net.(*netFD).accept(0xc0010c7a00)
	/usr/local/go/src/net/fd_unix.go:172 +0x35 fp=0xc000193be8 sp=0xc000193b30 pc=0x616635
net.(*TCPListener).accept(0xc0006a60c0)
	/usr/local/go/src/net/tcpsock_posix.go:148 +0x25 fp=0xc000193c10 sp=0xc000193be8 pc=0x6300a5
net.(*TCPListener).Accept(0xc0006a60c0)
	/usr/local/go/src/net/tcpsock.go:297 +0x3d fp=0xc000193c40 sp=0xc000193c10 pc=0x62f19d
google.golang.org/grpc.(*Server).Serve(0xc000342000, {0x12d3000, 0xc0006a60c0})
	/gobuild/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362 fp=0xc000193d58 sp=0xc000193c40 pc=0x8e77c2
internal/pkg/middleware/grpc.(*GrpcServer).Serve(0xc0008bae60)
	/middleware/grpc/grpc_server.go:97 +0xc5 fp=0xc000193db0 sp=0xc000193d58 pc=0x94bee5
main.main()
	/app/main.go:94 +0x4a9 fp=0xc000193f80 sp=0xc000193db0 pc=0xf091a9
runtime.main()
	/usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc000193fe0 sp=0xc000193f80 pc=0x4398c7
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000193fe8 sp=0xc000193fe0 pc=0x46bde1

goroutine 2 [force gc (idle), 239 minutes]:
runtime.gopark(0x14d316204e4f521?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e8fb0 sp=0xc0000e8f90 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc0000e8fe0 sp=0xc0000e8fb0 pc=0x439b30
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e8fe8 sp=0xc0000e8fe0 pc=0x46bde1
created by runtime.init.6
	/usr/local/go/src/runtime/proc.go:293 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e9780 sp=0xc0000e9760 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.bgsweep(0x0?)
	/usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc0000e97c8 sp=0xc0000e9780 pc=0x42437e
runtime.gcenable.func1()
	/usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc0000e97e0 sp=0xc0000e97c8 pc=0x419606
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e97e8 sp=0xc0000e97e0 pc=0x46bde1
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:178 +0x6b

goroutine 4 [runnable]:
runtime.gopark(0x14d3e54d28b3b6e?, 0x105aabe?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e9f70 sp=0xc0000e9f50 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x1b42920)
	/usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc0000e9fa0 sp=0xc0000e9f70 pc=0x422273
runtime.bgscavenge(0x0?)
	/usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc0000e9fc8 sp=0xc0000e9fa0 pc=0x422865
runtime.gcenable.func2()
	/usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc0000e9fe0 sp=0xc0000e9fc8 pc=0x4195a6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e9fe8 sp=0xc0000e9fe0 pc=0x46bde1
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:179 +0xaa

goroutine 17 [finalizer wait]:
runtime.gopark(0x0?, 0x11ce3d8?, 0x20?, 0x80?, 0x2000000020?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e8628 sp=0xc0000e8608 pc=0x439cf6
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc0000e87e0 sp=0xc0000e8628 pc=0x418647
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e87e8 sp=0xc0000e87e0 pc=0x46bde1
created by runtime.createfing
	/usr/local/go/src/runtime/mfinal.go:163 +0x45

goroutine 22 [chan receive, 205514 minutes]:
runtime.gopark(0x4064f1?, 0x1017420?, 0x90?, 0x47?, 0xc00030e6b0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e46a8 sp=0xc0000e4688 pc=0x439cf6
runtime.chanrecv(0xc0001b27e0, 0xc0000e4788, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc0000e4738 sp=0xc0000e46a8 pc=0x406f9d
runtime.chanrecv2(0x405d3d?, 0xc0000e47a8?)
	/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc0000e4760 sp=0xc0000e4738 pc=0x406ad8
github.com/alecthomas/log4go.(*ConsoleLogWriter).run(0xc0001aeea0, {0x12ce360, 0xc0001b0008})
	/gobuild/go/pkg/mod/github.com/alecthomas/log4go@v0.0.0-20180109082532-d146e6b86faa/termlog.go:33 +0xa5 fp=0xc0000e47b8 sp=0xc0000e4760 pc=0xe3bb65
github.com/alecthomas/log4go.NewConsoleLogWriter.func1()
	/gobuild/go/pkg/mod/github.com/alecthomas/log4go@v0.0.0-20180109082532-d146e6b86faa/termlog.go:26 +0x2e fp=0xc0000e47e0 sp=0xc0000e47b8 pc=0xe3ba2e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e47e8 sp=0xc0000e47e0 pc=0x46bde1
created by github.com/alecthomas/log4go.NewConsoleLogWriter
	/gobuild/go/pkg/mod/github.com/alecthomas/log4go@v0.0.0-20180109082532-d146e6b86faa/termlog.go:26 +0xf5

goroutine 8 [select, 205514 minutes]:
runtime.gopark(0xc0000ea788?, 0x2?, 0x88?, 0xda?, 0xc0000ea784?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000ea610 sp=0xc0000ea5f0 pc=0x439cf6
runtime.selectgo(0xc0000ea788, 0xc0000ea780, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0000ea750 sp=0xc0000ea610 pc=0x449e9e
database/sql.(*DB).connectionOpener(0xc0002ed2b0, {0x12d39a0, 0xc000116190})
	/usr/local/go/src/database/sql/sql.go:1218 +0x8d fp=0xc0000ea7b8 sp=0xc0000ea750 pc=0x78300d
database/sql.OpenDB.func1()
	/usr/local/go/src/database/sql/sql.go:791 +0x2e fp=0xc0000ea7e0 sp=0xc0000ea7b8 pc=0x78138e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000ea7e8 sp=0xc0000ea7e0 pc=0x46bde1
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:791 +0x18d

goroutine 14 [select, 205514 minutes]:
runtime.gopark(0xc0000ebf80?, 0x2?, 0x0?, 0x0?, 0xc0000ebf40?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000ebdc0 sp=0xc0000ebda0 pc=0x439cf6
runtime.selectgo(0xc0000ebf80, 0xc0000ebf3c, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0000ebf00 sp=0xc0000ebdc0 pc=0x449e9e
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/gobuild/go/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/connection.go:614 +0xaa fp=0xc0000ebfe0 sp=0xc0000ebf00 pc=0x95d06a
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000ebfe8 sp=0xc0000ebfe0 pc=0x46bde1
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/gobuild/go/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/connection.go:611 +0x10a
```