coreos/fleet — pull request #553 by jonboulle (2014-06-20 01:16:36+00:00)
Title: Panic during fleetctl submit (in  (r *EtcdRegistry) storeOrGetUnit)

When I run `fleetctl submit path/to/my.service`, I get the following panic stacktrace, from [registry/unit.go line 34](https://github.com/coreos/fleet/blob/8c37793a57f30003a1beda477215e2165e7c77fc/registry/unit.go#L34) (which is line 37 in the stack trace after I added a call to log.Errorf):

```
panic: interface conversion: error is *errors.errorString, not etcd.Error

goroutine 16 [running]:
runtime.panic(0x77c840, 0xc2080dc200)
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/panic.c:279 +0xf5
github.com/coreos/fleet/registry.(*EtcdRegistry).storeOrGetUnit(0xc20807aaa0, 0xc2080c63f0, 0xc208033680, 0x471, 0x7f851aa081b0, 0xc20817a8c0)
/home/sqs/src/github.com/coreos/fleet/registry/unit.go:37 +0x456
github.com/coreos/fleet/registry.(*EtcdRegistry).CreateJob(0xc20807aaa0, 0xc2081660a0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/registry/job.go:196 +0x71
main.createJob(0x7fff3ba22c48, 0xc, 0xc20807aea0, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/fleetctl/fleetctl.go:360 +0xa0
main.lazyCreateJobs(0xc20800e020, 0x1, 0x1, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/fleetctl/fleetctl.go:454 +0x5f4
main.runSubmitUnits(0xc20800e020, 0x1, 0x1, 0x0)
/home/sqs/src/github.com/coreos/fleet/fleetctl/submit.go:31 +0x65
main.main()
/home/sqs/src/github.com/coreos/fleet/fleetctl/fleetctl.go:212 +0x8c4

goroutine 19 [finalizer wait]:
runtime.park(0x425ab0, 0xa07830, 0xa05ac9)
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/proc.c:1369 +0x89
runtime.parkunlock(0xa07830, 0xa05ac9)
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/proc.c:1385 +0x3b
runfinq()
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/mgc0.c:2644 +0xcf
runtime.goexit()
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/proc.c:1445

goroutine 20 [chan receive]:
github.com/coreos/fleet/third_party/github.com/golang/glog.(*loggingT).flushDaemon(0xa09600)
/home/sqs/src/github.com/coreos/fleet/third_party/github.com/golang/glog/glog.go:839 +0x75
created by github.com/coreos/fleet/third_party/github.com/golang/glog.init·1
/home/sqs/src/github.com/coreos/fleet/third_party/github.com/golang/glog/glog.go:406 +0x2b2

goroutine 33 [syscall]:
runtime.goexit()
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/proc.c:1445

goroutine 22 [IO wait]:
net.runtime_pollWait(0x7f851aa0dd68, 0x72, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc2080e0140, 0x72, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc2080e0140, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).Read(0xc2080e00e0, 0xc2080f8000, 0x1000, 0x1000, 0x0, 0x7f851aa0c600, 0xb)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/fd_unix.go:232 +0x34c
net.(*conn).Read(0xc2080d6030, 0xc2080f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/net.go:122 +0xe7
bufio.(*Reader).fill(0xc208038300)
/home/sqs/.gvm/gos/go1.3/src/pkg/bufio/bufio.go:97 +0x1b3
bufio.(*Reader).Read(0xc208038300, 0xc208084320, 0x5, 0x5, 0x5, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/bufio/bufio.go:175 +0x230
io.ReadAtLeast(0x7f851aa0e348, 0xc208038300, 0xc208084320, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/io/io.go:289 +0xf7
io.ReadFull(0x7f851aa0e348, 0xc208038300, 0xc208084320, 0x5, 0x5, 0x8d4, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/io/io.go:307 +0x71
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*streamPacketCipher).readPacket(0xc208084300, 0xc20000002b, 0x7f851aa0e348, 0xc208038300, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/cipher.go:143 +0xd8
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*connectionState).readPacket(0xc2080038c0, 0xc208038300, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/transport.go:110 +0xe7
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*transport).readPacket(0xc2080038c0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/transport.go:106 +0x67
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*handshakeTransport).readOnePacket(0xc2080fc000, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/handshake.go:158 +0x105
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*handshakeTransport).readLoop(0xc2080fc000)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/handshake.go:138 +0x27
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.newClientTransport
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/handshake.go:101 +0xf2

goroutine 34 [chan receive]:
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*handshakeTransport).readPacket(0xc2080fc000, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/handshake.go:129 +0x86
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*mux).onePacket(0xc2080ec310, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/mux.go:220 +0x66
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*mux).loop(0xc2080ec310)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/mux.go:195 +0x49
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.newMux
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/mux.go:124 +0x113

goroutine 35 [chan receive]:
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*Client).handleGlobalRequests(0xc208074d80, 0xc2080e60e0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:130 +0x4d
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.NewClient
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:53 +0xb6

goroutine 36 [chan receive]:
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*Client).handleChannelOpens(0xc208074d80, 0xc208104420)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:139 +0x64
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.NewClient
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:54 +0xdb

goroutine 37 [semacquire]:
sync.runtime_Syncsemacquire(0xc208074c90)
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/sema.goc:257 +0xc0
sync.(*Cond).Wait(0xc208074c80)
/home/sqs/.gvm/gos/go1.3/src/pkg/sync/cond.go:62 +0x9d
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*mux).Wait(0xc2080ec310, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/mux.go:109 +0xb5
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.func·003()
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:56 +0x47
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.NewClient
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:58 +0x108

goroutine 38 [chan receive]:
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*forwardList).handleChannels(0xc208074d90, 0xc208104580)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/tcpip.go:157 +0x6a
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.NewClient
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/client.go:59 +0x15b

goroutine 50 [chan receive]:
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/agent.func·002()
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/agent/forward.go:38 +0x61
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/agent.ForwardToAgent
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/agent/forward.go:49 +0x17a

goroutine 23 [chan receive]:
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.DiscardRequests(0xc2080b40e0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/connection.go:81 +0x4d
created by github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*Client).dial
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/tcpip.go:328 +0x1d6

goroutine 24 [semacquire]:
sync.runtime_Syncsemacquire(0xc2080dc410)
/home/sqs/.gvm/gos/go1.3/src/pkg/runtime/sema.goc:257 +0xc0
sync.(*Cond).Wait(0xc2080dc400)
/home/sqs/.gvm/gos/go1.3/src/pkg/sync/cond.go:62 +0x9d
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*buffer).Read(0xc2080cc7e0, 0xc208153000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/buffer.go:96 +0x2f4
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*channel).ReadExtended(0xc2080d2370, 0xc208153000, 0x1000, 0x1000, 0x0, 0x4, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/channel.go:308 +0x82
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*channel).Read(0xc2080d2370, 0xc208153000, 0x1000, 0x1000, 0xc2080b79b0, 0x0, 0x0)
/home/sqs/src/github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh/channel.go:465 +0x96
github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh.(*tcpChanConn).Read(0xc20800ff50, 0xc208153000, 0x1000, 0x1000, 0xc208160580, 0x0, 0x0)
:252 +0x7b
net/http.noteEOFReader.Read(0x7f851aa12c38, 0xc20800ff50, 0xc2080d2738, 0xc208153000, 0x1000, 0x1000, 0xa1a000, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:1203 +0x72
net/http.(*noteEOFReader).Read(0xc20807b260, 0xc208153000, 0x1000, 0x1000, 0xc20807e170, 0x0, 0x0)
:124 +0xca
bufio.(*Reader).fill(0xc2080388a0)
/home/sqs/.gvm/gos/go1.3/src/pkg/bufio/bufio.go:97 +0x1b3
bufio.(*Reader).Peek(0xc2080388a0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/bufio/bufio.go:132 +0x101
net/http.(*persistConn).readLoop(0xc2080d26e0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:782 +0x95
created by net/http.(*Transport).dialConn
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:600 +0x93f

goroutine 25 [select]:
net/http.(*persistConn).writeLoop(0xc2080d26e0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:885 +0x38f
created by net/http.(*Transport).dialConn
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:601 +0x957

goroutine 40 [select]:
net/http.(*persistConn).roundTrip(0xc2080d26e0, 0xc2080630d0, 0x0, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:1015 +0x6db
net/http.(*Transport).RoundTrip(0xc208084600, 0xc208000d00, 0x7f85180f0f28, 0x0, 0x0)
/home/sqs/.gvm/gos/go1.3/src/pkg/net/http/transport.go:208 +0x49a
github.com/coreos/fleet/etcd.func·001()
/home/sqs/src/github.com/coreos/fleet/etcd/client.go:104 +0x53
created by github.com/coreos/fleet/etcd.(*client).requestHTTP
/home/sqs/src/github.com/coreos/fleet/etcd/client.go:118 +0x1b6

```

This is with a client fleetctl at commit 8c37793a57f30003a1beda477215e2165e7c77fc hitting a server on CoreOS 349.0.0 (alpha).

If I add a log.Errorf call, the err that is causing the panic is:

```
E0619 17:05:22.634808 29535 unit.go:35] ERROR! cancelled
```

My etcd cluster backing fleet appears to be in a bad state (1 out of 3 nodes has dropped inexplicably), which may be causing this issue. But this line of code should maybe be patched to not assume only a certain error type.

Other info: `go version go1.3 linux/amd64` `Linux qq 3.11.0-23-generic #40-Ubuntu SMP Wed Jun 4 21:05:23 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux`