influxdb/influxdb — pull request #1100 by toddboom (2014-11-07 16:04:39+00:00)
Title: Removing shard spaces in a cluster causes a continuous panic

Upgrading a cluster from v0.7.1 and while trying to create shard spaces, I managed to get the cluster config into a weird state. This seemed to happen on whichever node was elected as the leader, so they would just take turns dying.

```
[11/06/14 17:19:32] [INFO] Loading configuration file /opt/influxdb/shared/config.toml

+---------------------------------------------+
| _____ __ _ _____ ____ |
| |_ _| / _| | | __ \| _ \ |
| | | _ __ | |_| |_ ___ _| | | | |_) | |
| | | | '_ \| _| | | | \ \/ / | | | _ < |
| _| |_| | | | | | | |_| |> <| |__| | |_) | |
| |_____|_| |_|_| |_|\__,_/_/\_\_____/|____/ |
+---------------------------------------------+

panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x0 pc=0x6ff804]

goroutine 36 [running]:
runtime.panic(0xc21400, 0x14b4b93)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/panic.c:279 +0xf5
github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace(0xc208029380, 0xc208208110, 0xe, 0xc208208120, 0xa, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1345 +0x5a4
github.com/influxdb/influxdb/coordinator.(*DropShardSpaceCommand).Apply(0xc208211680, 0x7fe388c39ee8, 0xc2080bcc60, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/command.go:472 +0xbf
github.com/influxdb/influxdb/_vendor/raft.func·006(0xc2081b8740, 0x7fe388c36ae8, 0xc208211680, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:206 +0x422
github.com/influxdb/influxdb/_vendor/raft.(*Log).setCommitIndex(0xc20811b420, 0xb19c, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/log.go:375 +0x3e5
github.com/influxdb/influxdb/_vendor/raft.(*server).processAppendEntriesResponse(0xc2080bcc60, 0xc2082a7980)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:1028 +0x52f
github.com/influxdb/influxdb/_vendor/raft.(*server).leaderLoop(0xc2080bcc60)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:851 +0x539
github.com/influxdb/influxdb/_vendor/raft.(*server).loop(0xc2080bcc60)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:609 +0x3d1
github.com/influxdb/influxdb/_vendor/raft.func·007()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:470 +0x5d
created by github.com/influxdb/influxdb/_vendor/raft.(*server).Start
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:471 +0x3b8

goroutine 16 [sleep]:
time.Sleep(0x12a05f200)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/server.(*Server).ListenAndServe(0xc208024120, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/server/server.go:100 +0x14d
main.main()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/daemon/influxd.go:191 +0xfcf

goroutine 19 [finalizer wait]:
runtime.park(0x4ffb50, 0x14cd350, 0x14b7909)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/proc.c:1369 +0x89
runtime.parkunlock(0x14cd350, 0x14b7909)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/proc.c:1385 +0x3b
runfinq()
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/mgc0.c:2644 +0xcf
runtime.goexit()
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/proc.c:1445

goroutine 20 [syscall]:
os/signal.loop()
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/os/signal/signal_unix.go:21 +0x1e
created by os/signal.init·1
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/os/signal/signal_unix.go:27 +0x32

goroutine 21 [chan receive]:
code.google.com/p/log4go.ConsoleLogWriter.run(0xc208072000, 0x7fe388c36418, 0xc20803c008)
/home/jvshahid/codez/gocodez/src/code.google.com/p/log4go/termlog.go:27 +0x79
created by code.google.com/p/log4go.NewConsoleLogWriter
/home/jvshahid/codez/gocodez/src/code.google.com/p/log4go/termlog.go:19 +0x68

goroutine 17 [syscall]:
runtime.goexit()
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/proc.c:1445

goroutine 22 [runnable]:
code.google.com/p/log4go.func·002()
/home/jvshahid/codez/gocodez/src/code.google.com/p/log4go/filelog.go:84 +0x8ac
created by code.google.com/p/log4go.NewFileLogWriter
/home/jvshahid/codez/gocodez/src/code.google.com/p/log4go/filelog.go:116 +0x2c4

goroutine 23 [chan receive]:
github.com/influxdb/influxdb/wal.(*WAL).processEntries(0xc208026e00)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/wal/wal.go:252 +0x64
created by github.com/influxdb/influxdb/wal.NewWAL
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/wal/wal.go:103 +0xa53

goroutine 24 [chan receive]:
main.waitForSignals(0x7fe388c38ca8, 0xc208024120)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/daemon/null_profiler.go:23 +0x14c
created by main.startProfiler
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/daemon/null_profiler.go:15 +0x4b

goroutine 25 [IO wait]:
net.runtime_pollWait(0x7fe388c39cd0, 0x72, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc20811a1b0, 0x72, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc20811a1b0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).accept(0xc20811a150, 0xdc0910, 0x0, 0x7fe388c362b8, 0xb)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_unix.go:419 +0x343
net.(*TCPListener).AcceptTCP(0xc20803c038, 0x8, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/tcpsock_posix.go:234 +0x5d
net.(*TCPListener).Accept(0xc20803c038, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/tcpsock_posix.go:244 +0x4b
net/http.(*Server).Serve(0xc208004600, 0x7fe388c38d28, 0xc20803c038, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/server.go:1698 +0x91
github.com/influxdb/influxdb/coordinator.func·006()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:579 +0x3a
created by github.com/influxdb/influxdb/coordinator.(*RaftServer).Serve
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:583 +0x44c

goroutine 27 [select]:
github.com/influxdb/influxdb/cluster.(*WriteBuffer).handleWrites(0xc2081001c0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:75 +0xd3
created by github.com/influxdb/influxdb/cluster.NewWriteBuffer
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:43 +0x286

goroutine 28 [sleep]:
time.Sleep(0xbebc200)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).readResponses(0xc208048aa0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:194 +0x34a
github.com/influxdb/influxdb/coordinator.func·004()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:69 +0x3f
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:70 +0x96

goroutine 29 [sleep]:
time.Sleep(0xdf8475800)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).peridicallySweepTimedOutRequests(0xc208048aa0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:283 +0x37
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:71 +0xb1

goroutine 30 [sleep]:
time.Sleep(0xbebc200)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/cluster.(*ClusterServer).heartbeat(0xc208046a80)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:149 +0x201
created by github.com/influxdb/influxdb/cluster.(*ClusterServer).StartHeartbeat
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:65 +0x54

goroutine 31 [select]:
github.com/influxdb/influxdb/cluster.(*WriteBuffer).handleWrites(0xc208100230)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:75 +0xd3
created by github.com/influxdb/influxdb/cluster.NewWriteBuffer
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:43 +0x286

goroutine 33 [sleep]:
time.Sleep(0xbebc200)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).readResponses(0xc208048b40)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:186 +0x12e
github.com/influxdb/influxdb/coordinator.func·004()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:69 +0x3f
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:70 +0x96

goroutine 34 [sleep]:
time.Sleep(0xdf8475800)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).peridicallySweepTimedOutRequests(0xc208048b40)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:283 +0x37
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:71 +0xb1

goroutine 35 [sleep]:
time.Sleep(0x77359400)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/cluster.(*ClusterServer).handleHeartbeatError(0xc208046b80, 0x7fe388c321b0, 0xc2082a49b0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:185 +0x1b8
github.com/influxdb/influxdb/cluster.(*ClusterServer).heartbeat(0xc208046b80)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:139 +0x14f
created by github.com/influxdb/influxdb/cluster.(*ClusterServer).StartHeartbeat
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:65 +0x54

goroutine 37 [select]:
github.com/influxdb/influxdb/coordinator.(*RaftServer).CompactLog(0xc20804e840)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:348 +0x332
created by github.com/influxdb/influxdb/coordinator.(*RaftServer).startRaft
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:406 +0x512

goroutine 40 [IO wait]:
net.runtime_pollWait(0x7fe388c39b70, 0x72, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc20811a840, 0x72, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc20811a840, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).Read(0xc20811a7e0, 0xc2080b3000, 0x1000, 0x1000, 0x0, 0x7fe388c362b8, 0xb)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_unix.go:242 +0x34c
net.(*conn).Read(0xc20803c0d8, 0xc2080b3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/net.go:122 +0xe7
net/http.(*liveSwitchReader).Read(0xc20828c228, 0xc2080b3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/server.go:206 +0xaf
io.(*LimitedReader).Read(0xc2080a01e0, 0xc2080b3000, 0x1000, 0x1000, 0xc2080d87b0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/io/io.go:399 +0xd0
bufio.(*Reader).fill(0xc208004c60)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/bufio/bufio.go:97 +0x1b3
bufio.(*Reader).ReadSlice(0xc208004c60, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/bufio/bufio.go:298 +0x22c
bufio.(*Reader).ReadLine(0xc208004c60, 0x0, 0x0, 0x0, 0x50e300, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/bufio/bufio.go:326 +0x69
net/textproto.(*Reader).readLineSlice(0xc2080d8630, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/textproto/reader.go:55 +0x9d
net/textproto.(*Reader).ReadLine(0xc2080d8630, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/textproto/reader.go:36 +0x4e
net/http.ReadRequest(0xc208004c60, 0xc208001790, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/request.go:556 +0xc7
net/http.(*conn).readRequest(0xc20828c200, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/server.go:577 +0x276
net/http.(*conn).serve(0xc20828c200)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/server.go:1132 +0x61e
created by net/http.(*Server).Serve
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/server.go:1721 +0x313

goroutine 45 [IO wait]:
net.runtime_pollWait(0x7fe388c39a10, 0x72, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc20811b250, 0x72, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc20811b250, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).Read(0xc20811b1f0, 0xc20829f000, 0x1000, 0x1000, 0x0, 0x7fe388c362b8, 0xb)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_unix.go:242 +0x34c
net.(*conn).Read(0xc20803c188, 0xc20829f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/net.go:122 +0xe7
net/http.noteEOFReader.Read(0x7fe388c3e900, 0xc20803c188, 0xc20804eaa8, 0xc20829f000, 0x1000, 0x1000, 0x14f5aa0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/transport.go:1203 +0x72
net/http.(*noteEOFReader).Read(0xc2080a0700, 0xc20829f000, 0x1000, 0x1000, 0xc208024290, 0x0, 0x0)
:124 +0xca
bufio.(*Reader).fill(0xc208005920)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/bufio/bufio.go:97 +0x1b3
bufio.(*Reader).Peek(0xc208005920, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/bufio/bufio.go:132 +0x101
net/http.(*persistConn).readLoop(0xc20804ea50)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/transport.go:782 +0x95
created by net/http.(*Transport).dialConn
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/transport.go:600 +0x93f

goroutine 47 [select]:
github.com/influxdb/influxdb/coordinator.(*RaftServer).raftLeaderLoop(0xc20804e840, 0xc2082a8940, 0xc2082a8900)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:465 +0x34c
created by github.com/influxdb/influxdb/coordinator.(*RaftServer).raftEventHandler
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:453 +0x268

goroutine 46 [select]:
net/http.(*persistConn).writeLoop(0xc20804ea50)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/transport.go:885 +0x38f
created by net/http.(*Transport).dialConn
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/http/transport.go:601 +0x957

goroutine 59 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 49 [select]:
github.com/influxdb/influxdb/_vendor/raft.(*Peer).heartbeat(0xc2081005b0, 0xc208005bc0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/peer.go:148 +0x636
github.com/influxdb/influxdb/_vendor/raft.func·005()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/peer.go:97 +0x76
created by github.com/influxdb/influxdb/_vendor/raft.(*Peer).startHeartbeat
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/peer.go:98 +0x13d

goroutine 50 [select]:
github.com/influxdb/influxdb/_vendor/raft.(*server).send(0xc2080bcc60, 0xc2f900, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:630 +0x279
github.com/influxdb/influxdb/_vendor/raft.(*server).Do(0xc2080bcc60, 0x7fe388c3f158, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:897 +0x7c
github.com/influxdb/influxdb/_vendor/raft.func·010()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:828 +0xab
created by github.com/influxdb/influxdb/_vendor/raft.(*server).leaderLoop
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:829 +0x257

goroutine 54 [select]:
github.com/influxdb/influxdb/_vendor/raft.(*Peer).heartbeat(0xc2082ab490, 0xc20820e720)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/peer.go:148 +0x636
github.com/influxdb/influxdb/_vendor/raft.func·005()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/peer.go:97 +0x76
created by github.com/influxdb/influxdb/_vendor/raft.(*Peer).startHeartbeat
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/_vendor/raft/peer.go:98 +0x13d

goroutine 55 [IO wait]:
net.runtime_pollWait(0x7fe388c39c20, 0x72, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc2082ab950, 0x72, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc2082ab950, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).Read(0xc2082ab8f0, 0xc2082a5950, 0x4, 0x8, 0x0, 0x7fe388c362b8, 0xb)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/fd_unix.go:242 +0x34c
net.(*conn).Read(0xc20803c470, 0xc2082a5950, 0x4, 0x8, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/net/net.go:122 +0xe7
io.ReadAtLeast(0x7fe388c3e900, 0xc20803c470, 0xc2082a5950, 0x4, 0x8, 0x4, 0x0, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/io/io.go:289 +0xf7
io.ReadFull(0x7fe388c3e900, 0xc20803c470, 0xc2082a5950, 0x4, 0x8, 0x5f1844, 0x0, 0x0)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/io/io.go:307 +0x71
encoding/binary.Read(0x7fe388c3e900, 0xc20803c470, 0x7fe388c3e928, 0x0, 0xadd8c0, 0xc2082a5678, 0x0, 0x0)

▽
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/encoding/binary/binary.go:147 +0x12b

▽
#! /usr/bin/env bash
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).readResponses(0xc208049040)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:191 +0x23f
github.com/influxdb/influxdb/coordinator.func·004()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:69 +0x3f
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:70 +0x96

goroutine 56 [sleep]:
time.Sleep(0xdf8475800)
/home/jvshahid/.gvm/gos/go1.3.3/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).peridicallySweepTimedOutRequests(0xc208049040)
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:283 +0x37
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:71 +0xb1

goroutine 60 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 61 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 62 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 63 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 64 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 65 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 66 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570

goroutine 67 [runnable]:
github.com/influxdb/influxdb/cluster.func·008()
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1350
created by github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).RemoveShardSpace
/home/jvshahid/codez/gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:1354 +0x570
```