func (ns *nodeState) startReadyLoop(rfn ReadyFn) {
	ns.readyCh = make(chan interface{})
	go func() {
		done := false
		for !done {
			if ready, backoff := rfn(ns.node); ready {
				close(ns.readyCh)
				done = true
			} else if backoff == 0 {
				done = true
			} else {
				select {
				case <-ns.removedCh:
					done = true
				case <-time.After(backoff):
					break
				}
			}
		}
	}()
}
Starts a goroutine loop checking node readiness, waiting 'backoff' time between checks, and exiting if node is fully removed.
package server
import (
	"fmt"
	"time"
	"github.com/davecgh/go-spew/spew"
	log "github.com/sirupsen/logrus"
	cc "github.com/twitter/scoot/cloud/cluster"
	"github.com/twitter/scoot/common"
	"github.com/twitter/scoot/common/stats"
)
const noJob = ""
const noTask = ""
const defaultMaxLostDuration = time.Minute
const defaultMaxFlakyDuration = 15 * time.Minute
var nilTime = time.Time{}
type ReadyFn func(cc.Node) (ready bool, backoffDuration time.Duration)
type clusterState struct {
	nodesUpdatesCh   chan []cc.NodeUpdate
	nodes            map[cc.NodeId]*nodeState  
	suspendedNodes   map[cc.NodeId]*nodeState  
	offlinedNodes    map[cc.NodeId]*nodeState  
	nodeGroups       map[string]*nodeGroup     
	maxLostDuration  time.Duration             
	maxFlakyDuration time.Duration             
	readyFn          ReadyFn                   
	numRunning       int                       
	stats            stats.StatsReceiver       
	nopUpdateCnt     int
}
func (c *clusterState) isOfflined(ns *nodeState) bool {
	if _, ok := c.offlinedNodes[ns.node.Id()]; ok {
		return true
	}
	return false
}
type nodeGroup struct {
	idle map[cc.NodeId]*nodeState
	busy map[cc.NodeId]*nodeState
}
func newNodeGroup() *nodeGroup {
	return &nodeGroup{idle: map[cc.NodeId]*nodeState{}, busy: map[cc.NodeId]*nodeState{}}
}
type nodeState struct {
	node        cc.Node
	runningJob  string
	runningTask string
	snapshotId  string
	timeLost    time.Time         
	timeFlaky   time.Time         
	readyCh     chan interface{}  
	removedCh   chan interface{}  
}
func (n *nodeState) String() string {
	return fmt.Sprintf("{node:%s, jobId:%s, taskId:%s, snapshotId:%s, timeLost:%v, timeFlaky:%v, ready:%t}",
		spew.Sdump(n.node), n.runningJob, n.runningTask, n.snapshotId, n.timeLost, n.timeFlaky, (n.readyCh == nil))
}
func (ns *nodeState) suspended() bool {
	return ns.readyCh != nil || ns.timeLost != nilTime || ns.timeFlaky != nilTime
}
func (ns *nodeState) ready() bool {
	ready := (ns.readyCh == nil)
	if !ready {
		select {
		case <-ns.readyCh:
			ready = true
		default:
		}
	}
	return ready
}