Repository: go
Issue: cmd/go: trim build cache incrementally
Labels: NeedsInvestigation

Can't reproduce reliably, but...

After running `go test`, I am sometimes seeing the test complete successfully (prints "ok ... pkg ... time"), and then it appears to hang for an indefinite period of time. Sending SIGINT does not terminate the process. Sending SIGTERM does. This happens with std lib tests (strings, crypto/x509). Each time I passed the `-v` flag, although I'm not sure that that is necessary. Using commit 5c13cab36b4667cc1a42667b16b8f049016586e0. This behavior is new, although this is the first time I've run tests in a few days or a week. And it doesn't reproduce 100%, so it's a bit muddled.

I attached lldb to two hung binaries and got similar backtraces from each.

<details>
 <summary>First backtrace</summary>
<pre>
$ lldb -p 55409
(lldb) process attach --pid 55409
Process 55409 stopped
* thread #1, queue = 'com.apple.main-thread', stop reason = signal SIGSTOP
    frame #0: 0x00007fff71de4ce6 libsystem_kernel.dylib`__psynch_cvwait + 10
libsystem_kernel.dylib`__psynch_cvwait:
->  0x7fff71de4ce6 <+10>: jae    0x7fff71de4cf0            ; <+20>
    0x7fff71de4ce8 <+12>: movq   %rax, %rdi
    0x7fff71de4ceb <+15>: jmp    0x7fff71de2a89            ; cerror_nocancel
    0x7fff71de4cf0 <+20>: retq   
Target 0: (go) stopped.

Executable module set to "/Users/josh/go/tip/bin/go".
Architecture set to: x86_64h-apple-macosx-.
(lldb) bt
* thread #1, queue = 'com.apple.main-thread', stop reason = signal SIGSTOP
  * frame #0: 0x00007fff71de4ce6 libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007fff71ea6185 libsystem_pthread.dylib`_pthread_cond_wait + 701
    frame #2: 0x000000000106ceb0 go`runtime.pthread_cond_wait_trampoline + 16
    frame #3: 0x000000000106ad8d go`runtime.asmcgocall + 173
    frame #4: 0x0000000001054c0c go`runtime.libcCall + 108
    frame #5: 0x00000000010557d9 go`runtime.pthread_cond_wait + 57
    frame #6: 0x0000000001032dcd go`runtime.semasleep + 141
    frame #7: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #8: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #9: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #10: 0x000000000103edd7 go`runtime.schedule + 727
    frame #11: 0x000000000103f37d go`runtime.park_m + 157
    frame #12: 0x000000000106933b go`runtime.mcall + 91
    frame #13: 0x00000000010557d9 go`runtime.pthread_cond_wait + 57
    frame #14: 0x0000000001032dcd go`runtime.semasleep + 141
    frame #15: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #16: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #17: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #18: 0x000000000103edd7 go`runtime.schedule + 727
    frame #19: 0x000000000103f37d go`runtime.park_m + 157
    frame #20: 0x000000000106933b go`runtime.mcall + 91
    frame #21: 0x0000000001032dcd go`runtime.semasleep + 141
    frame #22: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #23: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #24: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #25: 0x000000000103edd7 go`runtime.schedule + 727
    frame #26: 0x000000000103f37d go`runtime.park_m + 157
    frame #27: 0x000000000106933b go`runtime.mcall + 91
    frame #28: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #29: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #30: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #31: 0x000000000103edd7 go`runtime.schedule + 727
    frame #32: 0x000000000103f37d go`runtime.park_m + 157
    frame #33: 0x000000000106933b go`runtime.mcall + 91
    frame #34: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #35: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #36: 0x000000000103edd7 go`runtime.schedule + 727
    frame #37: 0x000000000103f37d go`runtime.park_m + 157
    frame #38: 0x000000000106933b go`runtime.mcall + 91
    frame #39: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #40: 0x000000000103edd7 go`runtime.schedule + 727
    frame #41: 0x000000000103f37d go`runtime.park_m + 157
    frame #42: 0x000000000106933b go`runtime.mcall + 91
    frame #43: 0x000000000103edd7 go`runtime.schedule + 727
    frame #44: 0x000000000103f37d go`runtime.park_m + 157
    frame #45: 0x000000000106933b go`runtime.mcall + 91
    frame #46: 0x000000000103f37d go`runtime.park_m + 157
    frame #47: 0x000000000106933b go`runtime.mcall + 91
    frame #48: 0x000000000106933b go`runtime.mcall + 91
    frame #49: 0x000000000101edff go`runtime.gcBgMarkWorker + 255
    frame #50: 0x000000000106b121 go`runtime.goexit + 1
    frame #51: 0x000000000106b121 go`runtime.goexit + 1
(lldb) ^D
</pre>
</details>

<details>
 <summary>Second backtrace</summary>
<pre>
$ lldb -p 55473
(lldb) process attach --pid 55473
Process 55473 stopped
* thread #1, queue = 'com.apple.main-thread', stop reason = signal SIGSTOP
    frame #0: 0x00007fff71de4ce6 libsystem_kernel.dylib`__psynch_cvwait + 10
libsystem_kernel.dylib`__psynch_cvwait:
->  0x7fff71de4ce6 <+10>: jae    0x7fff71de4cf0            ; <+20>
    0x7fff71de4ce8 <+12>: movq   %rax, %rdi
    0x7fff71de4ceb <+15>: jmp    0x7fff71de2a89            ; cerror_nocancel
    0x7fff71de4cf0 <+20>: retq   
Target 0: (go) stopped.

Executable module set to "/Users/josh/go/tip/bin/go".
Architecture set to: x86_64h-apple-macosx-.
(lldb) bt
* thread #1, queue = 'com.apple.main-thread', stop reason = signal SIGSTOP
  * frame #0: 0x00007fff71de4ce6 libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007fff71ea6185 libsystem_pthread.dylib`_pthread_cond_wait + 701
    frame #2: 0x000000000106ceb0 go`runtime.pthread_cond_wait_trampoline + 16
    frame #3: 0x000000000106ad8d go`runtime.asmcgocall + 173
    frame #4: 0x0000000001054c0c go`runtime.libcCall + 108
    frame #5: 0x00000000010557d9 go`runtime.pthread_cond_wait + 57
    frame #6: 0x0000000001032dcd go`runtime.semasleep + 141
    frame #7: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #8: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #9: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #10: 0x000000000103edd7 go`runtime.schedule + 727
    frame #11: 0x000000000103f37d go`runtime.park_m + 157
    frame #12: 0x000000000106933b go`runtime.mcall + 91
    frame #13: 0x00000000010557d9 go`runtime.pthread_cond_wait + 57
    frame #14: 0x0000000001032dcd go`runtime.semasleep + 141
    frame #15: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #16: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #17: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #18: 0x000000000103edd7 go`runtime.schedule + 727
    frame #19: 0x000000000103f37d go`runtime.park_m + 157
    frame #20: 0x000000000106933b go`runtime.mcall + 91
    frame #21: 0x0000000001032dcd go`runtime.semasleep + 141
    frame #22: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #23: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #24: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #25: 0x000000000103edd7 go`runtime.schedule + 727
    frame #26: 0x000000000103f37d go`runtime.park_m + 157
    frame #27: 0x000000000106933b go`runtime.mcall + 91
    frame #28: 0x000000000100dca7 go`runtime.notesleep + 231
    frame #29: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #30: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #31: 0x000000000103edd7 go`runtime.schedule + 727
    frame #32: 0x000000000103f37d go`runtime.park_m + 157
    frame #33: 0x000000000106933b go`runtime.mcall + 91
    frame #34: 0x000000000103c9a5 go`runtime.stopm + 197
    frame #35: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #36: 0x000000000103edd7 go`runtime.schedule + 727
    frame #37: 0x000000000103f37d go`runtime.park_m + 157
    frame #38: 0x000000000106933b go`runtime.mcall + 91
    frame #39: 0x000000000103e15f go`runtime.findrunnable + 2687
    frame #40: 0x000000000103edd7 go`runtime.schedule + 727
    frame #41: 0x000000000103f37d go`runtime.park_m + 157
    frame #42: 0x000000000106933b go`runtime.mcall + 91
    frame #43: 0x000000000103edd7 go`runtime.schedule + 727
    frame #44: 0x000000000103f37d go`runtime.park_m + 157
    frame #45: 0x000000000106933b go`runtime.mcall + 91
    frame #46: 0x000000000103f37d go`runtime.park_m + 157
    frame #47: 0x000000000106933b go`runtime.mcall + 91
    frame #48: 0x000000000106933b go`runtime.mcall + 91
    frame #49: 0x000000000101edff go`runtime.gcBgMarkWorker + 255
    frame #50: 0x000000000106b121 go`runtime.goexit + 1
    frame #51: 0x000000000106b121 go`runtime.goexit + 1
(lldb) ^D
</pre>
</details>