func (i *InmemStore) StoreLogs(logs []*Log) error {
	i.l.Lock()
	defer i.l.Unlock()
	for _, l := range logs {
		i.logs[l.Index] = l
		if i.lowIndex == 0 {
			i.lowIndex = l.Index
		}
		if l.Index > i.highIndex {
			i.highIndex = l.Index
		}
	}
	return nil
}

func ( i * InmemStore ) StoreLogs ( logs [ ] * Log ) error { i . l . Lock ( ) defer i . l . Unlock ( ) for _ , l := range logs { i . logs [ l . Index ] = l if i . lowIndex == 0 { i . lowIndex = l . Index } if l . Index > i . highIndex { i . highIndex = l . Index } } return nil }

// StoreLogs implements the LogStore interface.

StoreLogs implements the LogStore interface .


func (i *InmemStore) DeleteRange(min, max uint64) error {
	i.l.Lock()
	defer i.l.Unlock()
	for j := min; j <= max; j++ {
		delete(i.logs, j)
	}
	if min <= i.lowIndex {
		i.lowIndex = max + 1
	}
	if max >= i.highIndex {
		i.highIndex = min - 1
	}
	if i.lowIndex > i.highIndex {
		i.lowIndex = 0
		i.highIndex = 0
	}
	return nil
}

func ( i * InmemStore ) DeleteRange ( min , max uint64 ) error { i . l . Lock ( ) defer i . l . Unlock ( ) for j := min ; j <= max ; j ++ { delete ( i . logs , j ) } if min <= i . lowIndex { i . lowIndex = max + 1 } if max >= i . highIndex { i . highIndex = min - 1 } if i . lowIndex > i . highIndex { i . lowIndex = 0 i . highIndex = 0 } return nil }

// DeleteRange implements the LogStore interface.

DeleteRange implements the LogStore interface .


func (i *InmemStore) Set(key []byte, val []byte) error {
	i.l.Lock()
	defer i.l.Unlock()
	i.kv[string(key)] = val
	return nil
}

func ( i * InmemStore ) Set ( key [ ] byte , val [ ] byte ) error { i . l . Lock ( ) defer i . l . Unlock ( ) i . kv [ string ( key ) ] = val return nil }

// Set implements the StableStore interface.

Set implements the StableStore interface .


func (i *InmemStore) Get(key []byte) ([]byte, error) {
	i.l.RLock()
	defer i.l.RUnlock()
	val := i.kv[string(key)]
	if val == nil {
		return nil, errors.New("not found")
	}
	return val, nil
}

func ( i * InmemStore ) Get ( key [ ] byte ) ( [ ] byte , error ) { i . l . RLock ( ) defer i . l . RUnlock ( ) val := i . kv [ string ( key ) ] if val == nil { return nil , errors . New ( " " ) } return val , nil }

// Get implements the StableStore interface.

Get implements the StableStore interface .


func (i *InmemStore) SetUint64(key []byte, val uint64) error {
	i.l.Lock()
	defer i.l.Unlock()
	i.kvInt[string(key)] = val
	return nil
}

func ( i * InmemStore ) SetUint64 ( key [ ] byte , val uint64 ) error { i . l . Lock ( ) defer i . l . Unlock ( ) i . kvInt [ string ( key ) ] = val return nil }

// SetUint64 implements the StableStore interface.

SetUint64 implements the StableStore interface .


func (i *InmemStore) GetUint64(key []byte) (uint64, error) {
	i.l.RLock()
	defer i.l.RUnlock()
	return i.kvInt[string(key)], nil
}

func ( i * InmemStore ) GetUint64 ( key [ ] byte ) ( uint64 , error ) { i . l . RLock ( ) defer i . l . RUnlock ( ) return i . kvInt [ string ( key ) ] , nil }

// GetUint64 implements the StableStore interface.

GetUint64 implements the StableStore interface .


func NewLogCache(capacity int, store LogStore) (*LogCache, error) {
	if capacity <= 0 {
		return nil, fmt.Errorf("capacity must be positive")
	}
	c := &LogCache{
		store: store,
		cache: make([]*Log, capacity),
	}
	return c, nil
}

func NewLogCache ( capacity int , store LogStore ) ( * LogCache , error ) { if capacity <= 0 { return nil , fmt . Errorf ( " " ) } c := & LogCache { store : store , cache : make ( [ ] * Log , capacity ) , } return c , nil }

// NewLogCache is used to create a new LogCache with the
// given capacity and backend store.

NewLogCache is used to create a new LogCache with the given capacity and backend store .


func (i *InmemTransport) Connect(peer ServerAddress, t Transport) {
	trans := t.(*InmemTransport)
	i.Lock()
	defer i.Unlock()
	i.peers[peer] = trans
}

func ( i * InmemTransport ) Connect ( peer ServerAddress , t Transport ) { trans := t . ( * InmemTransport ) i . Lock ( ) defer i . Unlock ( ) i . peers [ peer ] = trans }

// Connect is used to connect this transport to another transport for
// a given peer name. This allows for local routing.

Connect is used to connect this transport to another transport for a given peer name . This allows for local routing .


func (i *InmemTransport) Disconnect(peer ServerAddress) {
	i.Lock()
	defer i.Unlock()
	delete(i.peers, peer)

	// Disconnect any pipelines
	n := len(i.pipelines)
	for idx := 0; idx < n; idx++ {
		if i.pipelines[idx].peerAddr == peer {
			i.pipelines[idx].Close()
			i.pipelines[idx], i.pipelines[n-1] = i.pipelines[n-1], nil
			idx--
			n--
		}
	}
	i.pipelines = i.pipelines[:n]
}

func
(
i
*
InmemTransport
)
Disconnect
(
peer
ServerAddress
)
{
i
.
Lock
(
)
defer
i
.
Unlock
(
)
delete
(
i
.
peers
,
peer
)
// Disconnect any pipelines
n
:=
len
(
i
.
pipelines
)
for
idx
:=
0
;
idx
<
n
;
idx
++
{
if
i
.
pipelines
[
idx
]
.
peerAddr
==
peer
{
i
.
pipelines
[
idx
]
.
Close
(
)
i
.
pipelines
[
idx
]
,
i
.
pipelines
[
n
1
]
=
i
.
pipelines
[
n
1
]
,
idx
n
}
}
i
.
pipelines
=
i
.
pipelines
[
:
n
]
}

// Disconnect is used to remove the ability to route to a given peer.

Disconnect is used to remove the ability to route to a given peer .


func (i *InmemTransport) DisconnectAll() {
	i.Lock()
	defer i.Unlock()
	i.peers = make(map[ServerAddress]*InmemTransport)

	// Handle pipelines
	for _, pipeline := range i.pipelines {
		pipeline.Close()
	}
	i.pipelines = nil
}

func
(
i
*
InmemTransport
)
DisconnectAll
(
)
{
i
.
Lock
(
)
defer
i
.
Unlock
(
)
i
.
peers
=
make
(
map
[
ServerAddress
]
*
InmemTransport
)
// Handle pipelines
for
_
,
pipeline
:=
range
i
.
pipelines
{
pipeline
.
Close
(
)
}
i
.
pipelines
=
}

// DisconnectAll is used to remove all routes to peers.

DisconnectAll is used to remove all routes to peers .


func (r *RPC) Respond(resp interface{}, err error) {
	r.RespChan <- RPCResponse{resp, err}
}

func ( r * RPC ) Respond ( resp interface { } , err error ) { r . RespChan <- RPCResponse { resp , err } }

// Respond is used to respond with a response, error or both

Respond is used to respond with a response error or both


func (u *userSnapshotFuture) Open() (*SnapshotMeta, io.ReadCloser, error) {
	if u.opener == nil {
		return nil, nil, fmt.Errorf("no snapshot available")
	} else {
		// Invalidate the opener so it can't get called multiple times,
		// which isn't generally safe.
		defer func() {
			u.opener = nil
		}()
		return u.opener()
	}
}

// Open is a function you can call to access the underlying snapshot and its
// metadata.

Open is a function you can call to access the underlying snapshot and its metadata .


func (v *verifyFuture) vote(leader bool) {
	v.voteLock.Lock()
	defer v.voteLock.Unlock()

	// Guard against having notified already
	if v.notifyCh == nil {
		return
	}

	if leader {
		v.votes++
		if v.votes >= v.quorumSize {
			v.notifyCh <- v
			v.notifyCh = nil
		}
	} else {
		v.notifyCh <- v
		v.notifyCh = nil
	}
}

// vote is used to respond to a verifyFuture.
// This may block when responding on the notifyCh.

vote is used to respond to a verifyFuture . This may block when responding on the notifyCh .


func (s *followerReplication) notifyAll(leader bool) {
	// Clear the waiting notifies minimizing lock time
	s.notifyLock.Lock()
	n := s.notify
	s.notify = make(map[*verifyFuture]struct{})
	s.notifyLock.Unlock()

	// Submit our votes
	for v, _ := range n {
		v.vote(leader)
	}
}

// notifyAll is used to notify all the waiting verify futures
// if the follower believes we are still the leader.

notifyAll is used to notify all the waiting verify futures if the follower believes we are still the leader .


func (s *followerReplication) cleanNotify(v *verifyFuture) {
	s.notifyLock.Lock()
	delete(s.notify, v)
	s.notifyLock.Unlock()
}

func ( s * followerReplication ) cleanNotify ( v * verifyFuture ) { s . notifyLock . Lock ( ) delete ( s . notify , v ) s . notifyLock . Unlock ( ) }

// cleanNotify is used to delete notify, .


func (s *followerReplication) LastContact() time.Time {
	s.lastContactLock.RLock()
	last := s.lastContact
	s.lastContactLock.RUnlock()
	return last
}

func ( s * followerReplication ) LastContact ( ) time . Time { s . lastContactLock . RLock ( ) last := s . lastContact s . lastContactLock . RUnlock ( ) return last }

// LastContact returns the time of last contact.

LastContact returns the time of last contact .


func (s *followerReplication) setLastContact() {
	s.lastContactLock.Lock()
	s.lastContact = time.Now()
	s.lastContactLock.Unlock()
}

func ( s * followerReplication ) setLastContact ( ) { s . lastContactLock . Lock ( ) s . lastContact = time . Now ( ) s . lastContactLock . Unlock ( ) }

// setLastContact sets the last contact to the current time.

setLastContact sets the last contact to the current time .


func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	stopHeartbeat := make(chan struct{})
	defer close(stopHeartbeat)
	r.goFunc(func() { r.heartbeat(s, stopHeartbeat) })

RPC:
	shouldStop := false
	for !shouldStop {
		select {
		case maxIndex := <-s.stopCh:
			// Make a best effort to replicate up to this index
			if maxIndex > 0 {
				r.replicateTo(s, maxIndex)
			}
			return
		case <-s.triggerCh:
			lastLogIdx, _ := r.getLastLog()
			shouldStop = r.replicateTo(s, lastLogIdx)
		// This is _not_ our heartbeat mechanism but is to ensure
		// followers quickly learn the leader's commit index when
		// raft commits stop flowing naturally. The actual heartbeats
		// can't do this to keep them unblocked by disk IO on the
		// follower. See https://github.com/hashicorp/raft/issues/282.
		case <-randomTimeout(r.conf.CommitTimeout):
			lastLogIdx, _ := r.getLastLog()
			shouldStop = r.replicateTo(s, lastLogIdx)
		}

		// If things looks healthy, switch to pipeline mode
		if !shouldStop && s.allowPipeline {
			goto PIPELINE
		}
	}
	return

PIPELINE:
	// Disable until re-enabled
	s.allowPipeline = false

	// Replicates using a pipeline for high performance. This method
	// is not able to gracefully recover from errors, and so we fall back
	// to standard mode on failure.
	if err := r.pipelineReplicate(s); err != nil {
		if err != ErrPipelineReplicationNotSupported {
			r.logger.Error(fmt.Sprintf("Failed to start pipeline replication to %s: %s", s.peer, err))
		}
	}
	goto RPC
}

// replicate is a long running routine that replicates log entries to a single
// follower.

replicate is a long running routine that replicates log entries to a single follower .


func (r *Raft) pipelineReplicate(s *followerReplication) error {
	// Create a new pipeline
	pipeline, err := r.trans.AppendEntriesPipeline(s.peer.ID, s.peer.Address)
	if err != nil {
		return err
	}
	defer pipeline.Close()

	// Log start and stop of pipeline
	r.logger.Info(fmt.Sprintf("pipelining replication to peer %v", s.peer))
	defer r.logger.Info(fmt.Sprintf("aborting pipeline replication to peer %v", s.peer))

	// Create a shutdown and finish channel
	stopCh := make(chan struct{})
	finishCh := make(chan struct{})

	// Start a dedicated decoder
	r.goFunc(func() { r.pipelineDecode(s, pipeline, stopCh, finishCh) })

	// Start pipeline sends at the last good nextIndex
	nextIndex := s.nextIndex

	shouldStop := false
SEND:
	for !shouldStop {
		select {
		case <-finishCh:
			break SEND
		case maxIndex := <-s.stopCh:
			// Make a best effort to replicate up to this index
			if maxIndex > 0 {
				r.pipelineSend(s, pipeline, &nextIndex, maxIndex)
			}
			break SEND
		case <-s.triggerCh:
			lastLogIdx, _ := r.getLastLog()
			shouldStop = r.pipelineSend(s, pipeline, &nextIndex, lastLogIdx)
		case <-randomTimeout(r.conf.CommitTimeout):
			lastLogIdx, _ := r.getLastLog()
			shouldStop = r.pipelineSend(s, pipeline, &nextIndex, lastLogIdx)
		}
	}

	// Stop our decoder, and wait for it to finish
	close(stopCh)
	select {
	case <-finishCh:
	case <-r.shutdownCh:
	}
	return nil
}

// pipelineReplicate is used when we have synchronized our state with the follower,
// and want to switch to a higher performance pipeline mode of replication.
// We only pipeline AppendEntries commands, and if we ever hit an error, we fall
// back to the standard replication which can handle more complex situations.

pipelineReplicate is used when we have synchronized our state with the follower and want to switch to a higher performance pipeline mode of replication . We only pipeline AppendEntries commands and if we ever hit an error we fall back to the standard replication which can handle more complex situations .


func (r *Raft) pipelineSend(s *followerReplication, p AppendPipeline, nextIdx *uint64, lastIndex uint64) (shouldStop bool) {
	// Create a new append request
	req := new(AppendEntriesRequest)
	if err := r.setupAppendEntries(s, req, *nextIdx, lastIndex); err != nil {
		return true
	}

	// Pipeline the append entries
	if _, err := p.AppendEntries(req, new(AppendEntriesResponse)); err != nil {
		r.logger.Error(fmt.Sprintf("Failed to pipeline AppendEntries to %v: %v", s.peer, err))
		return true
	}

	// Increase the next send log to avoid re-sending old logs
	if n := len(req.Entries); n > 0 {
		last := req.Entries[n-1]
		*nextIdx = last.Index + 1
	}
	return false
}

// pipelineSend is used to send data over a pipeline. It is a helper to
// pipelineReplicate.

pipelineSend is used to send data over a pipeline . It is a helper to pipelineReplicate .


func (r *Raft) pipelineDecode(s *followerReplication, p AppendPipeline, stopCh, finishCh chan struct{}) {
	defer close(finishCh)
	respCh := p.Consumer()
	for {
		select {
		case ready := <-respCh:
			req, resp := ready.Request(), ready.Response()
			appendStats(string(s.peer.ID), ready.Start(), float32(len(req.Entries)))

			// Check for a newer term, stop running
			if resp.Term > req.Term {
				r.handleStaleTerm(s)
				return
			}

			// Update the last contact
			s.setLastContact()

			// Abort pipeline if not successful
			if !resp.Success {
				return
			}

			// Update our replication state
			updateLastAppended(s, req)
		case <-stopCh:
			return
		}
	}
}

func
(
r
*
Raft
)
pipelineDecode
(
s
*
followerReplication
,
p
AppendPipeline
,
stopCh
,
finishCh
chan
struct
{
}
)
{
defer
close
(
finishCh
)
respCh
:=
p
.
Consumer
(
)
for
{
select
{
case
ready
:=
<-
respCh
:
req
,
resp
:=
ready
.
Request
(
)
,
ready
.
Response
(
)
appendStats
(
string
(
s
.
peer
.
ID
)
,
ready
.
Start
(
)
,
float32
(
len
(
req
.
Entries
)
)
)
// Check for a newer term, stop running
if
resp
.
Term
>
req
.
Term
{
r
.
handleStaleTerm
(
s
)
return
}
// Update the last contact
s
.
setLastContact
(
)
// Abort pipeline if not successful
if
!
resp
.
Success
{
return
}
// Update our replication state
updateLastAppended
(
s
,
req
)
case
<-
stopCh
:
return
}
}
}

// pipelineDecode is used to decode the responses of pipelined requests.

pipelineDecode is used to decode the responses of pipelined requests .


func (r *Raft) setupAppendEntries(s *followerReplication, req *AppendEntriesRequest, nextIndex, lastIndex uint64) error {
	req.RPCHeader = r.getRPCHeader()
	req.Term = s.currentTerm
	req.Leader = r.trans.EncodePeer(r.localID, r.localAddr)
	req.LeaderCommitIndex = r.getCommitIndex()
	if err := r.setPreviousLog(req, nextIndex); err != nil {
		return err
	}
	if err := r.setNewLogs(req, nextIndex, lastIndex); err != nil {
		return err
	}
	return nil
}

func ( r * Raft ) setupAppendEntries ( s * followerReplication , req * AppendEntriesRequest , nextIndex , lastIndex uint64 ) error { req . RPCHeader = r . getRPCHeader ( ) req . Term = s . currentTerm req . Leader = r . trans . EncodePeer ( r . localID , r . localAddr ) req . LeaderCommitIndex = r . getCommitIndex ( ) if err := r . setPreviousLog ( req , nextIndex ) ; err != nil { return err } if err := r . setNewLogs ( req , nextIndex , lastIndex ) ; err != nil { return err } return nil }

// setupAppendEntries is used to setup an append entries request.

setupAppendEntries is used to setup an append entries request .


func (r *Raft) setPreviousLog(req *AppendEntriesRequest, nextIndex uint64) error {
	// Guard for the first index, since there is no 0 log entry
	// Guard against the previous index being a snapshot as well
	lastSnapIdx, lastSnapTerm := r.getLastSnapshot()
	if nextIndex == 1 {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0

	} else if (nextIndex - 1) == lastSnapIdx {
		req.PrevLogEntry = lastSnapIdx
		req.PrevLogTerm = lastSnapTerm

	} else {
		var l Log
		if err := r.logs.GetLog(nextIndex-1, &l); err != nil {
			r.logger.Error(fmt.Sprintf("Failed to get log at index %d: %v", nextIndex-1, err))
			return err
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	}
	return nil
}

// setPreviousLog is used to setup the PrevLogEntry and PrevLogTerm for an
// AppendEntriesRequest given the next index to replicate.

setPreviousLog is used to setup the PrevLogEntry and PrevLogTerm for an AppendEntriesRequest given the next index to replicate .


func (r *Raft) setNewLogs(req *AppendEntriesRequest, nextIndex, lastIndex uint64) error {
	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
	maxIndex := min(nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			r.logger.Error(fmt.Sprintf("Failed to get log at index %d: %v", i, err))
			return err
		}
		req.Entries = append(req.Entries, oldLog)
	}
	return nil
}

// setNewLogs is used to setup the logs which should be appended for a request.

setNewLogs is used to setup the logs which should be appended for a request .


func appendStats(peer string, start time.Time, logs float32) {
	metrics.MeasureSince([]string{"raft", "replication", "appendEntries", "rpc", peer}, start)
	metrics.IncrCounter([]string{"raft", "replication", "appendEntries", "logs", peer}, logs)
}

func appendStats ( peer string , start time . Time , logs float32 ) { metrics . MeasureSince ( [ ] string { " " , " " , " " , " " , peer } , start ) metrics . IncrCounter ( [ ] string { " " , " " , " " , " " , peer } , logs ) }

// appendStats is used to emit stats about an AppendEntries invocation.

appendStats is used to emit stats about an AppendEntries invocation .


func (r *Raft) handleStaleTerm(s *followerReplication) {
	r.logger.Error(fmt.Sprintf("peer %v has newer term, stopping replication", s.peer))
	s.notifyAll(false) // No longer leader
	asyncNotifyCh(s.stepDown)
}

func
(
r
*
Raft
)
handleStaleTerm
(
s
*
followerReplication
)
{
r
.
logger
.
Error
(
fmt
.
Sprintf
(
"
"
,
s
.
peer
)
)
s
.
notifyAll
(
false
)
// No longer leader
asyncNotifyCh
(
s
.
stepDown
)
}

// handleStaleTerm is used when a follower indicates that we have a stale term.

handleStaleTerm is used when a follower indicates that we have a stale term .


func (t *transport) AppendEntries(id raft.ServerID, target raft.ServerAddress, args *raft.AppendEntriesRequest, resp *raft.AppendEntriesResponse) error {
	ae := appendEntries{
		source:      t.node,
		target:      target,
		firstIndex:  firstIndex(args),
		lastIndex:   lastIndex(args),
		commitIndex: args.LeaderCommitIndex,
	}
	if len(t.ae) < cap(t.ae) {
		t.ae = append(t.ae, ae)
	}
	return t.sendRPC(string(target), args, resp)
}

func ( t * transport ) AppendEntries ( id raft . ServerID , target raft . ServerAddress , args * raft . AppendEntriesRequest , resp * raft . AppendEntriesResponse ) error { ae := appendEntries { source : t . node , target : target , firstIndex : firstIndex ( args ) , lastIndex : lastIndex ( args ) , commitIndex : args . LeaderCommitIndex , } if len ( t . ae ) < cap ( t . ae ) { t . ae = append ( t . ae , ae ) } return t . sendRPC ( string ( target ) , args , resp ) }

// AppendEntries sends the appropriate RPC to the target node.

AppendEntries sends the appropriate RPC to the target node .


func (t *transport) RequestVote(id raft.ServerID, target raft.ServerAddress, args *raft.RequestVoteRequest, resp *raft.RequestVoteResponse) error {
	return t.sendRPC(string(target), args, resp)
}

func ( t * transport ) RequestVote ( id raft . ServerID , target raft . ServerAddress , args * raft . RequestVoteRequest , resp * raft . RequestVoteResponse ) error { return t . sendRPC ( string ( target ) , args , resp ) }

// RequestVote sends the appropriate RPC to the target node.

RequestVote sends the appropriate RPC to the target node .


func (t *transport) InstallSnapshot(id raft.ServerID, target raft.ServerAddress, args *raft.InstallSnapshotRequest, resp *raft.InstallSnapshotResponse, data io.Reader) error {
	t.log.Printf("INSTALL SNAPSHOT *************************************")
	return errors.New("huh")
}

func ( t * transport ) InstallSnapshot ( id raft . ServerID , target raft . ServerAddress , args * raft . InstallSnapshotRequest , resp * raft . InstallSnapshotResponse , data io . Reader ) error { t . log . Printf ( " " ) return errors . New ( " " ) }

// InstallSnapshot is used to push a snapshot down to a follower. The data is read from
// the ReadCloser and streamed to the client.

InstallSnapshot is used to push a snapshot down to a follower . The data is read from the ReadCloser and streamed to the client .


func (t *transport) EncodePeer(id raft.ServerID, p raft.ServerAddress) []byte {
	return []byte(p)
}

func ( t * transport ) EncodePeer ( id raft . ServerID , p raft . ServerAddress ) [ ] byte { return [ ] byte ( p ) }

// EncodePeer is used to serialize a peer name.

EncodePeer is used to serialize a peer name .


func (t *transport) DecodePeer(p []byte) raft.ServerAddress {
	return raft.ServerAddress(p)
}

func ( t * transport ) DecodePeer ( p [ ] byte ) raft . ServerAddress { return raft . ServerAddress ( p ) }

// DecodePeer is used to deserialize a peer name.

DecodePeer is used to deserialize a peer name .


func (p *pipeline) AppendEntries(args *raft.AppendEntriesRequest, resp *raft.AppendEntriesResponse) (raft.AppendFuture, error) {
	e := &appendEntry{
		req:      args,
		res:      resp,
		start:    time.Now(),
		ready:    make(chan error),
		consumer: p.consumer,
	}
	p.work <- e
	return e, nil
}

func ( p * pipeline ) AppendEntries ( args * raft . AppendEntriesRequest , resp * raft . AppendEntriesResponse ) ( raft . AppendFuture , error ) { e := & appendEntry { req : args , res : resp , start : time . Now ( ) , ready : make ( chan error ) , consumer : p . consumer , } p . work <- e return e , nil }

// AppendEntries is used to add another request to the pipeline.
// The send may block which is an effective form of back-pressure.

AppendEntries is used to add another request to the pipeline . The send may block which is an effective form of back - pressure .


func ReadPeersJSON(path string) (Configuration, error) {
	// Read in the file.
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return Configuration{}, err
	}

	// Parse it as JSON.
	var peers []string
	dec := json.NewDecoder(bytes.NewReader(buf))
	if err := dec.Decode(&peers); err != nil {
		return Configuration{}, err
	}

	// Map it into the new-style configuration structure. We can only specify
	// voter roles here, and the ID has to be the same as the address.
	var configuration Configuration
	for _, peer := range peers {
		server := Server{
			Suffrage: Voter,
			ID:       ServerID(peer),
			Address:  ServerAddress(peer),
		}
		configuration.Servers = append(configuration.Servers, server)
	}

	// We should only ingest valid configurations.
	if err := checkConfiguration(configuration); err != nil {
		return Configuration{}, err
	}
	return configuration, nil
}

// ReadPeersJSON consumes a legacy peers.json file in the format of the old JSON
// peer store and creates a new-style configuration structure. This can be used
// to migrate this data or perform manual recovery when running protocol versions
// that can interoperate with older, unversioned Raft servers. This should not be
// used once server IDs are in use, because the old peers.json file didn't have
// support for these, nor non-voter suffrage types.

ReadPeersJSON consumes a legacy peers . json file in the format of the old JSON peer store and creates a new - style configuration structure . This can be used to migrate this data or perform manual recovery when running protocol versions that can interoperate with older unversioned Raft servers . This should not be used once server IDs are in use because the old peers . json file didn t have support for these nor non - voter suffrage types .


func ReadConfigJSON(path string) (Configuration, error) {
	// Read in the file.
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return Configuration{}, err
	}

	// Parse it as JSON.
	var peers []configEntry
	dec := json.NewDecoder(bytes.NewReader(buf))
	if err := dec.Decode(&peers); err != nil {
		return Configuration{}, err
	}

	// Map it into the new-style configuration structure.
	var configuration Configuration
	for _, peer := range peers {
		suffrage := Voter
		if peer.NonVoter {
			suffrage = Nonvoter
		}
		server := Server{
			Suffrage: suffrage,
			ID:       peer.ID,
			Address:  peer.Address,
		}
		configuration.Servers = append(configuration.Servers, server)
	}

	// We should only ingest valid configurations.
	if err := checkConfiguration(configuration); err != nil {
		return Configuration{}, err
	}
	return configuration, nil
}

// ReadConfigJSON reads a new-style peers.json and returns a configuration
// structure. This can be used to perform manual recovery when running protocol
// versions that use server IDs.

ReadConfigJSON reads a new - style peers . json and returns a configuration structure . This can be used to perform manual recovery when running protocol versions that use server IDs .


func NewTCPTransport(
	bindAddr string,
	advertise net.Addr,
	maxPool int,
	timeout time.Duration,
	logOutput io.Writer,
) (*NetworkTransport, error) {
	return newTCPTransport(bindAddr, advertise, func(stream StreamLayer) *NetworkTransport {
		return NewNetworkTransport(stream, maxPool, timeout, logOutput)
	})
}

func NewTCPTransport ( bindAddr string , advertise net . Addr , maxPool int , timeout time . Duration , logOutput io . Writer , ) ( * NetworkTransport , error ) { return newTCPTransport ( bindAddr , advertise , func ( stream StreamLayer ) * NetworkTransport { return NewNetworkTransport ( stream , maxPool , timeout , logOutput ) } ) }

// NewTCPTransport returns a NetworkTransport that is built on top of
// a TCP streaming transport layer.

NewTCPTransport returns a NetworkTransport that is built on top of a TCP streaming transport layer .


func NewTCPTransportWithLogger(
	bindAddr string,
	advertise net.Addr,
	maxPool int,
	timeout time.Duration,
	logger *log.Logger,
) (*NetworkTransport, error) {
	return newTCPTransport(bindAddr, advertise, func(stream StreamLayer) *NetworkTransport {
		return NewNetworkTransportWithLogger(stream, maxPool, timeout, logger)
	})
}

func NewTCPTransportWithLogger ( bindAddr string , advertise net . Addr , maxPool int , timeout time . Duration , logger * log . Logger , ) ( * NetworkTransport , error ) { return newTCPTransport ( bindAddr , advertise , func ( stream StreamLayer ) * NetworkTransport { return NewNetworkTransportWithLogger ( stream , maxPool , timeout , logger ) } ) }

// NewTCPTransportWithLogger returns a NetworkTransport that is built on top of
// a TCP streaming transport layer, with log output going to the supplied Logger

NewTCPTransportWithLogger returns a NetworkTransport that is built on top of a TCP streaming transport layer with log output going to the supplied Logger


func NewTCPTransportWithConfig(
	bindAddr string,
	advertise net.Addr,
	config *NetworkTransportConfig,
) (*NetworkTransport, error) {
	return newTCPTransport(bindAddr, advertise, func(stream StreamLayer) *NetworkTransport {
		config.Stream = stream
		return NewNetworkTransportWithConfig(config)
	})
}

func NewTCPTransportWithConfig ( bindAddr string , advertise net . Addr , config * NetworkTransportConfig , ) ( * NetworkTransport , error ) { return newTCPTransport ( bindAddr , advertise , func ( stream StreamLayer ) * NetworkTransport { config . Stream = stream return NewNetworkTransportWithConfig ( config ) } ) }

// NewTCPTransportWithConfig returns a NetworkTransport that is built on top of
// a TCP streaming transport layer, using the given config struct.

NewTCPTransportWithConfig returns a NetworkTransport that is built on top of a TCP streaming transport layer using the given config struct .


func (t *TCPStreamLayer) Dial(address ServerAddress, timeout time.Duration) (net.Conn, error) {
	return net.DialTimeout("tcp", string(address), timeout)
}

func ( t * TCPStreamLayer ) Dial ( address ServerAddress , timeout time . Duration ) ( net . Conn , error ) { return net . DialTimeout ( " " , string ( address ) , timeout ) }

// Dial implements the StreamLayer interface.

Dial implements the StreamLayer interface .


func (t *TCPStreamLayer) Accept() (c net.Conn, err error) {
	return t.listener.Accept()
}

func ( t * TCPStreamLayer ) Accept ( ) ( c net . Conn , err error ) { return t . listener . Accept ( ) }

// Accept implements the net.Listener interface.

Accept implements the net . Listener interface .


func (t *TCPStreamLayer) Addr() net.Addr {
	// Use an advertise addr if provided
	if t.advertise != nil {
		return t.advertise
	}
	return t.listener.Addr()
}

func
(
t
*
TCPStreamLayer
)
Addr
(
)
net
.
Addr
{
// Use an advertise addr if provided
if
t
.
advertise
!=
{
return
t
.
advertise
}
return
t
.
listener
.
Addr
(
)
}

// Addr implements the net.Listener interface.

Addr implements the net . Listener interface .


func (r *Raft) restoreSnapshot() error {
	snapshots, err := r.snapshots.List()
	if err != nil {
		r.logger.Error(fmt.Sprintf("Failed to list snapshots: %v", err))
		return err
	}

	// Try to load in order of newest to oldest
	for _, snapshot := range snapshots {
		_, source, err := r.snapshots.Open(snapshot.ID)
		if err != nil {
			r.logger.Error(fmt.Sprintf("Failed to open snapshot %v: %v", snapshot.ID, err))
			continue
		}
		defer source.Close()

		if err := r.fsm.Restore(source); err != nil {
			r.logger.Error(fmt.Sprintf("Failed to restore snapshot %v: %v", snapshot.ID, err))
			continue
		}

		// Log success
		r.logger.Info(fmt.Sprintf("Restored from snapshot %v", snapshot.ID))

		// Update the lastApplied so we don't replay old logs
		r.setLastApplied(snapshot.Index)

		// Update the last stable snapshot info
		r.setLastSnapshot(snapshot.Index, snapshot.Term)

		// Update the configuration
		if snapshot.Version > 0 {
			r.configurations.committed = snapshot.Configuration
			r.configurations.committedIndex = snapshot.ConfigurationIndex
			r.configurations.latest = snapshot.Configuration
			r.configurations.latestIndex = snapshot.ConfigurationIndex
		} else {
			configuration := decodePeers(snapshot.Peers, r.trans)
			r.configurations.committed = configuration
			r.configurations.committedIndex = snapshot.Index
			r.configurations.latest = configuration
			r.configurations.latestIndex = snapshot.Index
		}

		// Success!
		return nil
	}

	// If we had snapshots and failed to load them, its an error
	if len(snapshots) > 0 {
		return fmt.Errorf("failed to load any existing snapshots")
	}
	return nil
}

// restoreSnapshot attempts to restore the latest snapshots, and fails if none
// of them can be restored. This is called at initialization time, and is
// completely unsafe to call at any other time.

restoreSnapshot attempts to restore the latest snapshots and fails if none of them can be restored . This is called at initialization time and is completely unsafe to call at any other time .


func (r *Raft) BootstrapCluster(configuration Configuration) Future {
	bootstrapReq := &bootstrapFuture{}
	bootstrapReq.init()
	bootstrapReq.configuration = configuration
	select {
	case <-r.shutdownCh:
		return errorFuture{ErrRaftShutdown}
	case r.bootstrapCh <- bootstrapReq:
		return bootstrapReq
	}
}

func ( r * Raft ) BootstrapCluster ( configuration Configuration ) Future { bootstrapReq := & bootstrapFuture { } bootstrapReq . init ( ) bootstrapReq . configuration = configuration select { case <- r . shutdownCh : return errorFuture { ErrRaftShutdown } case r . bootstrapCh <- bootstrapReq : return bootstrapReq } }

// BootstrapCluster is equivalent to non-member BootstrapCluster but can be
// called on an un-bootstrapped Raft instance after it has been created. This
// should only be called at the beginning of time for the cluster, and you
// absolutely must make sure that you call it with the same configuration on all
// the Voter servers. There is no need to bootstrap Nonvoter and Staging
// servers.

BootstrapCluster is equivalent to non - member BootstrapCluster but can be called on an un - bootstrapped Raft instance after it has been created . This should only be called at the beginning of time for the cluster and you absolutely must make sure that you call it with the same configuration on all the Voter servers . There is no need to bootstrap Nonvoter and Staging servers .


func (r *Raft) Leader() ServerAddress {
	r.leaderLock.RLock()
	leader := r.leader
	r.leaderLock.RUnlock()
	return leader
}

func ( r * Raft ) Leader ( ) ServerAddress { r . leaderLock . RLock ( ) leader := r . leader r . leaderLock . RUnlock ( ) return leader }

// Leader is used to return the current leader of the cluster.
// It may return empty string if there is no current leader
// or the leader is unknown.

Leader is used to return the current leader of the cluster . It may return empty string if there is no current leader or the leader is unknown .


func (r *Raft) Apply(cmd []byte, timeout time.Duration) ApplyFuture {
	metrics.IncrCounter([]string{"raft", "apply"}, 1)
	var timer <-chan time.Time
	if timeout > 0 {
		timer = time.After(timeout)
	}

	// Create a log future, no index or term yet
	logFuture := &logFuture{
		log: Log{
			Type: LogCommand,
			Data: cmd,
		},
	}
	logFuture.init()

	select {
	case <-timer:
		return errorFuture{ErrEnqueueTimeout}
	case <-r.shutdownCh:
		return errorFuture{ErrRaftShutdown}
	case r.applyCh <- logFuture:
		return logFuture
	}
}

// Apply is used to apply a command to the FSM in a highly consistent
// manner. This returns a future that can be used to wait on the application.
// An optional timeout can be provided to limit the amount of time we wait
// for the command to be started. This must be run on the leader or it
// will fail.

Apply is used to apply a command to the FSM in a highly consistent manner . This returns a future that can be used to wait on the application . An optional timeout can be provided to limit the amount of time we wait for the command to be started . This must be run on the leader or it will fail .


func (r *Raft) Barrier(timeout time.Duration) Future {
	metrics.IncrCounter([]string{"raft", "barrier"}, 1)
	var timer <-chan time.Time
	if timeout > 0 {
		timer = time.After(timeout)
	}

	// Create a log future, no index or term yet
	logFuture := &logFuture{
		log: Log{
			Type: LogBarrier,
		},
	}
	logFuture.init()

	select {
	case <-timer:
		return errorFuture{ErrEnqueueTimeout}
	case <-r.shutdownCh:
		return errorFuture{ErrRaftShutdown}
	case r.applyCh <- logFuture:
		return logFuture
	}
}

// Barrier is used to issue a command that blocks until all preceeding
// operations have been applied to the FSM. It can be used to ensure the
// FSM reflects all queued writes. An optional timeout can be provided to
// limit the amount of time we wait for the command to be started. This
// must be run on the leader or it will fail.

Barrier is used to issue a command that blocks until all preceeding operations have been applied to the FSM . It can be used to ensure the FSM reflects all queued writes . An optional timeout can be provided to limit the amount of time we wait for the command to be started . This must be run on the leader or it will fail .


func (r *Raft) VerifyLeader() Future {
	metrics.IncrCounter([]string{"raft", "verify_leader"}, 1)
	verifyFuture := &verifyFuture{}
	verifyFuture.init()
	select {
	case <-r.shutdownCh:
		return errorFuture{ErrRaftShutdown}
	case r.verifyCh <- verifyFuture:
		return verifyFuture
	}
}

func ( r * Raft ) VerifyLeader ( ) Future { metrics . IncrCounter ( [ ] string { " " , " " } , 1 ) verifyFuture := & verifyFuture { } verifyFuture . init ( ) select { case <- r . shutdownCh : return errorFuture { ErrRaftShutdown } case r . verifyCh <- verifyFuture : return verifyFuture } }

// VerifyLeader is used to ensure the current node is still
// the leader. This can be done to prevent stale reads when a
// new leader has potentially been elected.

VerifyLeader is used to ensure the current node is still the leader . This can be done to prevent stale reads when a new leader has potentially been elected .


func (r *Raft) AddVoter(id ServerID, address ServerAddress, prevIndex uint64, timeout time.Duration) IndexFuture {
	if r.protocolVersion < 2 {
		return errorFuture{ErrUnsupportedProtocol}
	}

	return r.requestConfigChange(configurationChangeRequest{
		command:       AddStaging,
		serverID:      id,
		serverAddress: address,
		prevIndex:     prevIndex,
	}, timeout)
}

func ( r * Raft ) AddVoter ( id ServerID , address ServerAddress , prevIndex uint64 , timeout time . Duration ) IndexFuture { if r . protocolVersion < 2 { return errorFuture { ErrUnsupportedProtocol } } return r . requestConfigChange ( configurationChangeRequest { command : AddStaging , serverID : id , serverAddress : address , prevIndex : prevIndex , } , timeout ) }

// AddVoter will add the given server to the cluster as a staging server. If the
// server is already in the cluster as a voter, this updates the server's address.
// This must be run on the leader or it will fail. The leader will promote the
// staging server to a voter once that server is ready. If nonzero, prevIndex is
// the index of the only configuration upon which this change may be applied; if
// another configuration entry has been added in the meantime, this request will
// fail. If nonzero, timeout is how long this server should wait before the
// configuration change log entry is appended.

AddVoter will add the given server to the cluster as a staging server . If the server is already in the cluster as a voter this updates the server s address . This must be run on the leader or it will fail . The leader will promote the staging server to a voter once that server is ready . If nonzero prevIndex is the index of the only configuration upon which this change may be applied ; if another configuration entry has been added in the meantime this request will fail . If nonzero timeout is how long this server should wait before the configuration change log entry is appended .


func (r *Raft) RemoveServer(id ServerID, prevIndex uint64, timeout time.Duration) IndexFuture {
	if r.protocolVersion < 2 {
		return errorFuture{ErrUnsupportedProtocol}
	}

	return r.requestConfigChange(configurationChangeRequest{
		command:   RemoveServer,
		serverID:  id,
		prevIndex: prevIndex,
	}, timeout)
}

func ( r * Raft ) RemoveServer ( id ServerID , prevIndex uint64 , timeout time . Duration ) IndexFuture { if r . protocolVersion < 2 { return errorFuture { ErrUnsupportedProtocol } } return r . requestConfigChange ( configurationChangeRequest { command : RemoveServer , serverID : id , prevIndex : prevIndex , } , timeout ) }

// RemoveServer will remove the given server from the cluster. If the current
// leader is being removed, it will cause a new election to occur. This must be
// run on the leader or it will fail. For prevIndex and timeout, see AddVoter.

RemoveServer will remove the given server from the cluster . If the current leader is being removed it will cause a new election to occur . This must be run on the leader or it will fail . For prevIndex and timeout see AddVoter .


func (r *Raft) Shutdown() Future {
	r.shutdownLock.Lock()
	defer r.shutdownLock.Unlock()

	if !r.shutdown {
		close(r.shutdownCh)
		r.shutdown = true
		r.setState(Shutdown)
		return &shutdownFuture{r}
	}

	// avoid closing transport twice
	return &shutdownFuture{nil}
}

func
(
r
*
Raft
)
Shutdown
(
)
Future
{
r
.
shutdownLock
.
Lock
(
)
defer
r
.
shutdownLock
.
Unlock
(
)
if
!
r
.
shutdown
{
close
(
r
.
shutdownCh
)
r
.
shutdown
=
true
r
.
setState
(
Shutdown
)
return
&
shutdownFuture
{
r
}
}
// avoid closing transport twice
return
&
shutdownFuture
{
}
}

// Shutdown is used to stop the Raft background routines.
// This is not a graceful operation. Provides a future that
// can be used to block until all background routines have exited.

Shutdown is used to stop the Raft background routines . This is not a graceful operation . Provides a future that can be used to block until all background routines have exited .


func (r *Raft) Snapshot() SnapshotFuture {
	future := &userSnapshotFuture{}
	future.init()
	select {
	case r.userSnapshotCh <- future:
		return future
	case <-r.shutdownCh:
		future.respond(ErrRaftShutdown)
		return future
	}
}

func ( r * Raft ) Snapshot ( ) SnapshotFuture { future := & userSnapshotFuture { } future . init ( ) select { case r . userSnapshotCh <- future : return future case <- r . shutdownCh : future . respond ( ErrRaftShutdown ) return future } }

// Snapshot is used to manually force Raft to take a snapshot. Returns a future
// that can be used to block until complete, and that contains a function that
// can be used to open the snapshot.

Snapshot is used to manually force Raft to take a snapshot . Returns a future that can be used to block until complete and that contains a function that can be used to open the snapshot .


func (r *Raft) Restore(meta *SnapshotMeta, reader io.Reader, timeout time.Duration) error {
	metrics.IncrCounter([]string{"raft", "restore"}, 1)
	var timer <-chan time.Time
	if timeout > 0 {
		timer = time.After(timeout)
	}

	// Perform the restore.
	restore := &userRestoreFuture{
		meta:   meta,
		reader: reader,
	}
	restore.init()
	select {
	case <-timer:
		return ErrEnqueueTimeout
	case <-r.shutdownCh:
		return ErrRaftShutdown
	case r.userRestoreCh <- restore:
		// If the restore is ingested then wait for it to complete.
		if err := restore.Error(); err != nil {
			return err
		}
	}

	// Apply a no-op log entry. Waiting for this allows us to wait until the
	// followers have gotten the restore and replicated at least this new
	// entry, which shows that we've also faulted and installed the
	// snapshot with the contents of the restore.
	noop := &logFuture{
		log: Log{
			Type: LogNoop,
		},
	}
	noop.init()
	select {
	case <-timer:
		return ErrEnqueueTimeout
	case <-r.shutdownCh:
		return ErrRaftShutdown
	case r.applyCh <- noop:
		return noop.Error()
	}
}

// Restore is used to manually force Raft to consume an external snapshot, such
// as if restoring from a backup. We will use the current Raft configuration,
// not the one from the snapshot, so that we can restore into a new cluster. We
// will also use the higher of the index of the snapshot, or the current index,
// and then add 1 to that, so we force a new state with a hole in the Raft log,
// so that the snapshot will be sent to followers and used for any new joiners.
// This can only be run on the leader, and blocks until the restore is complete
// or an error occurs.
//
// WARNING! This operation has the leader take on the state of the snapshot and
// then sets itself up so that it replicates that to its followers though the
// install snapshot process. This involves a potentially dangerous period where
// the leader commits ahead of its followers, so should only be used for disaster
// recovery into a fresh cluster, and should not be used in normal operations.

Restore is used to manually force Raft to consume an external snapshot such as if restoring from a backup . We will use the current Raft configuration not the one from the snapshot so that we can restore into a new cluster . We will also use the higher of the index of the snapshot or the current index and then add 1 to that so we force a new state with a hole in the Raft log so that the snapshot will be sent to followers and used for any new joiners . This can only be run on the leader and blocks until the restore is complete or an error occurs . WARNING! This operation has the leader take on the state of the snapshot and then sets itself up so that it replicates that to its followers though the install snapshot process . This involves a potentially dangerous period where the leader commits ahead of its followers so should only be used for disaster recovery into a fresh cluster and should not be used in normal operations .


func (r *Raft) String() string {
	return fmt.Sprintf("Node at %s [%v]", r.localAddr, r.getState())
}

func ( r * Raft ) String ( ) string { return fmt . Sprintf ( " " , r . localAddr , r . getState ( ) ) }

// String returns a string representation of this Raft node.

String returns a string representation of this Raft node .


func (r *Raft) LastContact() time.Time {
	r.lastContactLock.RLock()
	last := r.lastContact
	r.lastContactLock.RUnlock()
	return last
}

func ( r * Raft ) LastContact ( ) time . Time { r . lastContactLock . RLock ( ) last := r . lastContact r . lastContactLock . RUnlock ( ) return last }

// LastContact returns the time of last contact by a leader.
// This only makes sense if we are currently a follower.

LastContact returns the time of last contact by a leader . This only makes sense if we are currently a follower .


func (a *LoggerAdapter) Logf(s string, v ...interface{}) {
	a.log.Printf(s, v...)
}

func ( a * LoggerAdapter ) Logf ( s string , v ... interface { } ) { a . log . Printf ( s , v ... ) }

// Logf will record a formatted message to the contained debug log


func containsNode(nodes []*raftNode, n *raftNode) bool {
	for _, rn := range nodes {
		if rn == n {
			return true
		}
	}
	return false
}

func containsNode ( nodes [ ] * raftNode , n * raftNode ) bool { for _ , rn := range nodes { if rn == n { return true } } return false }

// containsNode returns true if the slice 'nodes' contains 'n'

containsNode returns true if the slice nodes contains n


func (c *cluster) LeaderPlus(n int) []*raftNode {
	r := make([]*raftNode, 0, n+1)
	ldr := c.Leader(time.Second)
	if ldr != nil {
		r = append(r, ldr)
	}
	if len(r) >= n {
		return r
	}
	for _, node := range c.nodes {
		if !containsNode(r, node) {
			r = append(r, node)
			if len(r) >= n {
				return r
			}
		}
	}
	return r
}

func ( c * cluster ) LeaderPlus ( n int ) [ ] * raftNode { r := make ( [ ] * raftNode , 0 , n + 1 ) ldr := c . Leader ( time . Second ) if ldr != nil { r = append ( r , ldr ) } if len ( r ) >= n { return r } for _ , node := range c . nodes { if ! containsNode ( r , node ) { r = append ( r , node ) if len ( r ) >= n { return r } } } return r }

// LeaderPlus returns the leader + n additional nodes from the cluster
// the leader is always the first node in the returned slice.

LeaderPlus returns the leader + n additional nodes from the cluster the leader is always the first node in the returned slice .


func (c *cluster) WaitTilUptoDate(t *testing.T, maxWait time.Duration) {
	idx := c.lastApplySuccess.Index()
	start := time.Now()
	for true {
		allAtIdx := true
		for i := 0; i < len(c.nodes); i++ {
			nodeAppliedIdx := c.nodes[i].raft.AppliedIndex()
			if nodeAppliedIdx < idx {
				allAtIdx = false
				break
			} else if nodeAppliedIdx > idx {
				allAtIdx = false
				idx = nodeAppliedIdx
				break
			}
		}
		if allAtIdx {
			t.Logf("All nodes have appliedIndex=%d", idx)
			return
		}
		if time.Now().Sub(start) > maxWait {
			t.Fatalf("Gave up waiting for all nodes to reach raft Index %d, [currently at %v]", idx, c.appliedIndexes())
		}
		time.Sleep(time.Millisecond * 10)
	}
}

func ( c * cluster ) WaitTilUptoDate ( t * testing . T , maxWait time . Duration ) { idx := c . lastApplySuccess . Index ( ) start := time . Now ( ) for true { allAtIdx := true for i := 0 ; i < len ( c . nodes ) ; i ++ { nodeAppliedIdx := c . nodes [ i ] . raft . AppliedIndex ( ) if nodeAppliedIdx < idx { allAtIdx = false break } else if nodeAppliedIdx > idx { allAtIdx = false idx = nodeAppliedIdx break } } if allAtIdx { t . Logf ( " " , idx ) return } if time . Now ( ) . Sub ( start ) > maxWait { t . Fatalf ( " " , idx , c . appliedIndexes ( ) ) } time . Sleep ( time . Millisecond * 10 ) } }

// WaitTilUptoDate blocks until all nodes in the cluster have gotten their
// commitedIndex upto the Index from the last successful call to Apply

WaitTilUptoDate blocks until all nodes in the cluster have gotten their commitedIndex upto the Index from the last successful call to Apply


func assertLogEntryEqual(t *testing.T, node string, exp *raft.Log, act *raft.Log) bool {
	res := true
	if exp.Term != act.Term {
		t.Errorf("Log Entry at Index %d for node %v has mismatched terms %d/%d", exp.Index, node, exp.Term, act.Term)
		res = false
	}
	if exp.Index != act.Index {
		t.Errorf("Node %v, Log Entry should be Index %d,but is %d", node, exp.Index, act.Index)
		res = false
	}
	if exp.Type != act.Type {
		t.Errorf("Node %v, Log Entry at Index %d should have type %v but is %v", node, exp.Index, exp.Type, act.Type)
		res = false
	}
	if !bytes.Equal(exp.Data, act.Data) {
		t.Errorf("Node %v, Log Entry at Index %d should have data %v, but has %v", node, exp.Index, exp.Data, act.Data)
		res = false
	}
	return res
}

func assertLogEntryEqual ( t * testing . T , node string , exp * raft . Log , act * raft . Log ) bool { res := true if exp . Term != act . Term { t . Errorf ( " " , exp . Index , node , exp . Term , act . Term ) res = false } if exp . Index != act . Index { t . Errorf ( " " , node , exp . Index , act . Index ) res = false } if exp . Type != act . Type { t . Errorf ( " " , node , exp . Index , exp . Type , act . Type ) res = false } if ! bytes . Equal ( exp . Data , act . Data ) { t . Errorf ( " " , node , exp . Index , exp . Data , act . Data ) res = false } return res }

// assertLogEntryEqual compares the 2 raft Log entries and reports any differences to the supplied testing.T instance
// it return true if the 2 entries are equal, false otherwise.

assertLogEntryEqual compares the 2 raft Log entries and reports any differences to the supplied testing . T instance it return true if the 2 entries are equal false otherwise .


func (r *Raft) runFSM() {
	var lastIndex, lastTerm uint64

	commit := func(req *commitTuple) {
		// Apply the log if a command
		var resp interface{}
		if req.log.Type == LogCommand {
			start := time.Now()
			resp = r.fsm.Apply(req.log)
			metrics.MeasureSince([]string{"raft", "fsm", "apply"}, start)
		}

		// Update the indexes
		lastIndex = req.log.Index
		lastTerm = req.log.Term

		// Invoke the future if given
		if req.future != nil {
			req.future.response = resp
			req.future.respond(nil)
		}
	}

	restore := func(req *restoreFuture) {
		// Open the snapshot
		meta, source, err := r.snapshots.Open(req.ID)
		if err != nil {
			req.respond(fmt.Errorf("failed to open snapshot %v: %v", req.ID, err))
			return
		}

		// Attempt to restore
		start := time.Now()
		if err := r.fsm.Restore(source); err != nil {
			req.respond(fmt.Errorf("failed to restore snapshot %v: %v", req.ID, err))
			source.Close()
			return
		}
		source.Close()
		metrics.MeasureSince([]string{"raft", "fsm", "restore"}, start)

		// Update the last index and term
		lastIndex = meta.Index
		lastTerm = meta.Term
		req.respond(nil)
	}

	snapshot := func(req *reqSnapshotFuture) {
		// Is there something to snapshot?
		if lastIndex == 0 {
			req.respond(ErrNothingNewToSnapshot)
			return
		}

		// Start a snapshot
		start := time.Now()
		snap, err := r.fsm.Snapshot()
		metrics.MeasureSince([]string{"raft", "fsm", "snapshot"}, start)

		// Respond to the request
		req.index = lastIndex
		req.term = lastTerm
		req.snapshot = snap
		req.respond(err)
	}

	for {
		select {
		case ptr := <-r.fsmMutateCh:
			switch req := ptr.(type) {
			case *commitTuple:
				commit(req)

			case *restoreFuture:
				restore(req)

			default:
				panic(fmt.Errorf("bad type passed to fsmMutateCh: %#v", ptr))
			}

		case req := <-r.fsmSnapshotCh:
			snapshot(req)

		case <-r.shutdownCh:
			return
		}
	}
}

func
(
r
*
Raft
)
runFSM
(
)
{
var
lastIndex
,
lastTerm
uint64
commit
:=
func
(
req
*
commitTuple
)
{
// Apply the log if a command
var
resp
interface
{
}
if
req
.
log
.
Type
==
LogCommand
{
start
:=
time
.
Now
(
)
resp
=
r
.
fsm
.
Apply
(
req
.
log
)
metrics
.
MeasureSince
(
[
]
string
{
"
"
,
"
"
,
"
"
}
,
start
)
}
// Update the indexes
lastIndex
=
req
.
log
.
Index
lastTerm
=
req
.
log
.
Term
// Invoke the future if given
if
req
.
future
!=
{
req
.
future
.
response
=
resp
req
.
future
.
respond
(
)
}
}
restore
:=
func
(
req
*
restoreFuture
)
{
// Open the snapshot
meta
,
source
,
err
:=
r
.
snapshots
.
Open
(
req
.
ID
)
if
err
!=
{
req
.
respond
(
fmt
.
Errorf
(
"
"
,
req
.
ID
,
err
)
)
return
}
// Attempt to restore
start
:=
time
.
Now
(
)
if
err
:=
r
.
fsm
.
Restore
(
source
)
;
err
!=
{
req
.
respond
(
fmt
.
Errorf
(
"
"
,
req
.
ID
,
err
)
)
source
.
Close
(
)
return
}
source
.
Close
(
)
metrics
.
MeasureSince
(
[
]
string
{
"
"
,
"
"
,
"
"
}
,
start
)
// Update the last index and term
lastIndex
=
meta
.
Index
lastTerm
=
meta
.
Term
req
.
respond
(
)
}
snapshot
:=
func
(
req
*
reqSnapshotFuture
)
{
// Is there something to snapshot?
if
lastIndex
==
0
{
req
.
respond
(
ErrNothingNewToSnapshot
)
return
}
// Start a snapshot
start
:=
time
.
Now
(
)
snap
,
err
:=
r
.
fsm
.
Snapshot
(
)
metrics
.
MeasureSince
(
[
]
string
{
"
"
,
"
"
,
"
"
}
,
start
)
// Respond to the request
req
.
index
=
lastIndex
req
.
term
=
lastTerm
req
.
snapshot
=
snap
req
.
respond
(
err
)
}
for
{
select
{
case
ptr
:=
<-
r
.
fsmMutateCh
:
switch
req
:=
ptr
.
(
type
)
{
case
*
commitTuple
:
commit
(
req
)
case
*
restoreFuture
:
restore
(
req
)
default
:
panic
(
fmt
.
Errorf
(
"
"
,
ptr
)
)
}
case
req
:=
<-
r
.
fsmSnapshotCh
:
snapshot
(
req
)
case
<-
r
.
shutdownCh
:
return
}
}
}

// runFSM is a long running goroutine responsible for applying logs
// to the FSM. This is done async of other logs since we don't want
// the FSM to block our internal operations.

runFSM is a long running goroutine responsible for applying logs to the FSM . This is done async of other logs since we don t want the FSM to block our internal operations .


func (c *Configuration) Clone() (copy Configuration) {
	copy.Servers = append(copy.Servers, c.Servers...)
	return
}

func ( c * Configuration ) Clone ( ) ( copy Configuration ) { copy . Servers = append ( copy . Servers , c . Servers ... ) return }

// Clone makes a deep copy of a Configuration.

Clone makes a deep copy of a Configuration .


func (c *configurations) Clone() (copy configurations) {
	copy.committed = c.committed.Clone()
	copy.committedIndex = c.committedIndex
	copy.latest = c.latest.Clone()
	copy.latestIndex = c.latestIndex
	return
}

func ( c * configurations ) Clone ( ) ( copy configurations ) { copy . committed = c . committed . Clone ( ) copy . committedIndex = c . committedIndex copy . latest = c . latest . Clone ( ) copy . latestIndex = c . latestIndex return }

// Clone makes a deep copy of a configurations object.

Clone makes a deep copy of a configurations object .


func hasVote(configuration Configuration, id ServerID) bool {
	for _, server := range configuration.Servers {
		if server.ID == id {
			return server.Suffrage == Voter
		}
	}
	return false
}

func hasVote ( configuration Configuration , id ServerID ) bool { for _ , server := range configuration . Servers { if server . ID == id { return server . Suffrage == Voter } } return false }

// hasVote returns true if the server identified by 'id' is a Voter in the
// provided Configuration.

hasVote returns true if the server identified by id is a Voter in the provided Configuration .


func checkConfiguration(configuration Configuration) error {
	idSet := make(map[ServerID]bool)
	addressSet := make(map[ServerAddress]bool)
	var voters int
	for _, server := range configuration.Servers {
		if server.ID == "" {
			return fmt.Errorf("Empty ID in configuration: %v", configuration)
		}
		if server.Address == "" {
			return fmt.Errorf("Empty address in configuration: %v", server)
		}
		if idSet[server.ID] {
			return fmt.Errorf("Found duplicate ID in configuration: %v", server.ID)
		}
		idSet[server.ID] = true
		if addressSet[server.Address] {
			return fmt.Errorf("Found duplicate address in configuration: %v", server.Address)
		}
		addressSet[server.Address] = true
		if server.Suffrage == Voter {
			voters++
		}
	}
	if voters == 0 {
		return fmt.Errorf("Need at least one voter in configuration: %v", configuration)
	}
	return nil
}

func checkConfiguration ( configuration Configuration ) error { idSet := make ( map [ ServerID ] bool ) addressSet := make ( map [ ServerAddress ] bool ) var voters int for _ , server := range configuration . Servers { if server . ID == " " { return fmt . Errorf ( " " , configuration ) } if server . Address == " " { return fmt . Errorf ( " " , server ) } if idSet [ server . ID ] { return fmt . Errorf ( " " , server . ID ) } idSet [ server . ID ] = true if addressSet [ server . Address ] { return fmt . Errorf ( " " , server . Address ) } addressSet [ server . Address ] = true if server . Suffrage == Voter { voters ++ } } if voters == 0 { return fmt . Errorf ( " " , configuration ) } return nil }

// checkConfiguration tests a cluster membership configuration for common
// errors.

checkConfiguration tests a cluster membership configuration for common errors .


func nextConfiguration(current Configuration, currentIndex uint64, change configurationChangeRequest) (Configuration, error) {
	if change.prevIndex > 0 && change.prevIndex != currentIndex {
		return Configuration{}, fmt.Errorf("Configuration changed since %v (latest is %v)", change.prevIndex, currentIndex)
	}

	configuration := current.Clone()
	switch change.command {
	case AddStaging:
		// TODO: barf on new address?
		newServer := Server{
			// TODO: This should add the server as Staging, to be automatically
			// promoted to Voter later. However, the promotion to Voter is not yet
			// implemented, and doing so is not trivial with the way the leader loop
			// coordinates with the replication goroutines today. So, for now, the
			// server will have a vote right away, and the Promote case below is
			// unused.
			Suffrage: Voter,
			ID:       change.serverID,
			Address:  change.serverAddress,
		}
		found := false
		for i, server := range configuration.Servers {
			if server.ID == change.serverID {
				if server.Suffrage == Voter {
					configuration.Servers[i].Address = change.serverAddress
				} else {
					configuration.Servers[i] = newServer
				}
				found = true
				break
			}
		}
		if !found {
			configuration.Servers = append(configuration.Servers, newServer)
		}
	case AddNonvoter:
		newServer := Server{
			Suffrage: Nonvoter,
			ID:       change.serverID,
			Address:  change.serverAddress,
		}
		found := false
		for i, server := range configuration.Servers {
			if server.ID == change.serverID {
				if server.Suffrage != Nonvoter {
					configuration.Servers[i].Address = change.serverAddress
				} else {
					configuration.Servers[i] = newServer
				}
				found = true
				break
			}
		}
		if !found {
			configuration.Servers = append(configuration.Servers, newServer)
		}
	case DemoteVoter:
		for i, server := range configuration.Servers {
			if server.ID == change.serverID {
				configuration.Servers[i].Suffrage = Nonvoter
				break
			}
		}
	case RemoveServer:
		for i, server := range configuration.Servers {
			if server.ID == change.serverID {
				configuration.Servers = append(configuration.Servers[:i], configuration.Servers[i+1:]...)
				break
			}
		}
	case Promote:
		for i, server := range configuration.Servers {
			if server.ID == change.serverID && server.Suffrage == Staging {
				configuration.Servers[i].Suffrage = Voter
				break
			}
		}
	}

	// Make sure we didn't do something bad like remove the last voter
	if err := checkConfiguration(configuration); err != nil {
		return Configuration{}, err
	}

	return configuration, nil
}

// nextConfiguration generates a new Configuration from the current one and a
// configuration change request. It's split from appendConfigurationEntry so
// that it can be unit tested easily.

nextConfiguration generates a new Configuration from the current one and a configuration change request . It s split from appendConfigurationEntry so that it can be unit tested easily .


func encodePeers(configuration Configuration, trans Transport) []byte {
	// Gather up all the voters, other suffrage types are not supported by
	// this data format.
	var encPeers [][]byte
	for _, server := range configuration.Servers {
		if server.Suffrage == Voter {
			encPeers = append(encPeers, trans.EncodePeer(server.ID, server.Address))
		}
	}

	// Encode the entire array.
	buf, err := encodeMsgPack(encPeers)
	if err != nil {
		panic(fmt.Errorf("failed to encode peers: %v", err))
	}

	return buf.Bytes()
}

// encodePeers is used to serialize a Configuration into the old peers format.
// This is here for backwards compatibility when operating with a mix of old
// servers and should be removed once we deprecate support for protocol version 1.

encodePeers is used to serialize a Configuration into the old peers format . This is here for backwards compatibility when operating with a mix of old servers and should be removed once we deprecate support for protocol version 1 .


func decodePeers(buf []byte, trans Transport) Configuration {
	// Decode the buffer first.
	var encPeers [][]byte
	if err := decodeMsgPack(buf, &encPeers); err != nil {
		panic(fmt.Errorf("failed to decode peers: %v", err))
	}

	// Deserialize each peer.
	var servers []Server
	for _, enc := range encPeers {
		p := trans.DecodePeer(enc)
		servers = append(servers, Server{
			Suffrage: Voter,
			ID:       ServerID(p),
			Address:  ServerAddress(p),
		})
	}

	return Configuration{
		Servers: servers,
	}
}

func
decodePeers
(
buf
[
]
byte
,
trans
Transport
)
Configuration
{
// Decode the buffer first.
var
encPeers
[
]
[
]
byte
if
err
:=
decodeMsgPack
(
buf
,
&
encPeers
)
;
err
!=
{
panic
(
fmt
.
Errorf
(
"
"
,
err
)
)
}
// Deserialize each peer.
var
servers
[
]
Server
for
_
,
enc
:=
range
encPeers
{
p
:=
trans
.
DecodePeer
(
enc
)
servers
=
append
(
servers
,
Server
{
Suffrage
:
Voter
,
ID
:
ServerID
(
p
)
,
Address
:
ServerAddress
(
p
)
,
}
)
}
return
Configuration
{
Servers
:
servers
,
}
}

// decodePeers is used to deserialize an old list of peers into a Configuration.
// This is here for backwards compatibility with old log entries and snapshots;
// it should be removed eventually.

decodePeers is used to deserialize an old list of peers into a Configuration . This is here for backwards compatibility with old log entries and snapshots ; it should be removed eventually .


func encodeConfiguration(configuration Configuration) []byte {
	buf, err := encodeMsgPack(configuration)
	if err != nil {
		panic(fmt.Errorf("failed to encode configuration: %v", err))
	}
	return buf.Bytes()
}

func encodeConfiguration ( configuration Configuration ) [ ] byte { buf , err := encodeMsgPack ( configuration ) if err != nil { panic ( fmt . Errorf ( " " , err ) ) } return buf . Bytes ( ) }

// encodeConfiguration serializes a Configuration using MsgPack, or panics on
// errors.

encodeConfiguration serializes a Configuration using MsgPack or panics on errors .


func decodeConfiguration(buf []byte) Configuration {
	var configuration Configuration
	if err := decodeMsgPack(buf, &configuration); err != nil {
		panic(fmt.Errorf("failed to decode configuration: %v", err))
	}
	return configuration
}

func decodeConfiguration ( buf [ ] byte ) Configuration { var configuration Configuration if err := decodeMsgPack ( buf , & configuration ) ; err != nil { panic ( fmt . Errorf ( " " , err ) ) } return configuration }

// decodeConfiguration deserializes a Configuration using MsgPack, or panics on
// errors.

decodeConfiguration deserializes a Configuration using MsgPack or panics on errors .


func (r *raftState) goFunc(f func()) {
	r.routinesGroup.Add(1)
	go func() {
		defer r.routinesGroup.Done()
		f()
	}()
}

func ( r * raftState ) goFunc ( f func ( ) ) { r . routinesGroup . Add ( 1 ) go func ( ) { defer r . routinesGroup . Done ( ) f ( ) } ( ) }

// Start a goroutine and properly handle the race between a routine
// starting and incrementing, and exiting and decrementing.

Start a goroutine and properly handle the race between a routine starting and incrementing and exiting and decrementing .


func (r *raftState) getLastIndex() uint64 {
	r.lastLock.Lock()
	defer r.lastLock.Unlock()
	return max(r.lastLogIndex, r.lastSnapshotIndex)
}

func ( r * raftState ) getLastIndex ( ) uint64 { r . lastLock . Lock ( ) defer r . lastLock . Unlock ( ) return max ( r . lastLogIndex , r . lastSnapshotIndex ) }

// getLastIndex returns the last index in stable storage.
// Either from the last log or from the last snapshot.

getLastIndex returns the last index in stable storage . Either from the last log or from the last snapshot .


func (r *raftState) getLastEntry() (uint64, uint64) {
	r.lastLock.Lock()
	defer r.lastLock.Unlock()
	if r.lastLogIndex >= r.lastSnapshotIndex {
		return r.lastLogIndex, r.lastLogTerm
	}
	return r.lastSnapshotIndex, r.lastSnapshotTerm
}

func ( r * raftState ) getLastEntry ( ) ( uint64 , uint64 ) { r . lastLock . Lock ( ) defer r . lastLock . Unlock ( ) if r . lastLogIndex >= r . lastSnapshotIndex { return r . lastLogIndex , r . lastLogTerm } return r . lastSnapshotIndex , r . lastSnapshotTerm }

// getLastEntry returns the last index and term in stable storage.
// Either from the last log or from the last snapshot.

getLastEntry returns the last index and term in stable storage . Either from the last log or from the last snapshot .


func (r *Raft) checkRPCHeader(rpc RPC) error {
	// Get the header off the RPC message.
	wh, ok := rpc.Command.(WithRPCHeader)
	if !ok {
		return fmt.Errorf("RPC does not have a header")
	}
	header := wh.GetRPCHeader()

	// First check is to just make sure the code can understand the
	// protocol at all.
	if header.ProtocolVersion < ProtocolVersionMin ||
		header.ProtocolVersion > ProtocolVersionMax {
		return ErrUnsupportedProtocol
	}

	// Second check is whether we should support this message, given the
	// current protocol we are configured to run. This will drop support
	// for protocol version 0 starting at protocol version 2, which is
	// currently what we want, and in general support one version back. We
	// may need to revisit this policy depending on how future protocol
	// changes evolve.
	if header.ProtocolVersion < r.conf.ProtocolVersion-1 {
		return ErrUnsupportedProtocol
	}

	return nil
}

// checkRPCHeader houses logic about whether this instance of Raft can process
// the given RPC message.

checkRPCHeader houses logic about whether this instance of Raft can process the given RPC message .


func (r *Raft) setLeader(leader ServerAddress) {
	r.leaderLock.Lock()
	oldLeader := r.leader
	r.leader = leader
	r.leaderLock.Unlock()
	if oldLeader != leader {
		r.observe(LeaderObservation{leader: leader})
	}
}

func ( r * Raft ) setLeader ( leader ServerAddress ) { r . leaderLock . Lock ( ) oldLeader := r . leader r . leader = leader r . leaderLock . Unlock ( ) if oldLeader != leader { r . observe ( LeaderObservation { leader : leader } ) } }

// setLeader is used to modify the current leader of the cluster


func (r *Raft) requestConfigChange(req configurationChangeRequest, timeout time.Duration) IndexFuture {
	var timer <-chan time.Time
	if timeout > 0 {
		timer = time.After(timeout)
	}
	future := &configurationChangeFuture{
		req: req,
	}
	future.init()
	select {
	case <-timer:
		return errorFuture{ErrEnqueueTimeout}
	case r.configurationChangeCh <- future:
		return future
	case <-r.shutdownCh:
		return errorFuture{ErrRaftShutdown}
	}
}

func ( r * Raft ) requestConfigChange ( req configurationChangeRequest , timeout time . Duration ) IndexFuture { var timer <- chan time . Time if timeout > 0 { timer = time . After ( timeout ) } future := & configurationChangeFuture { req : req , } future . init ( ) select { case <- timer : return errorFuture { ErrEnqueueTimeout } case r . configurationChangeCh <- future : return future case <- r . shutdownCh : return errorFuture { ErrRaftShutdown } } }

// requestConfigChange is a helper for the above functions that make
// configuration change requests. 'req' describes the change. For timeout,
// see AddVoter.

requestConfigChange is a helper for the above functions that make configuration change requests . req describes the change . For timeout see AddVoter .


func (r *Raft) run() {
	for {
		// Check if we are doing a shutdown
		select {
		case <-r.shutdownCh:
			// Clear the leader to prevent forwarding
			r.setLeader("")
			return
		default:
		}

		// Enter into a sub-FSM
		switch r.getState() {
		case Follower:
			r.runFollower()
		case Candidate:
			r.runCandidate()
		case Leader:
			r.runLeader()
		}
	}
}

// run is a long running goroutine that runs the Raft FSM.

run is a long running goroutine that runs the Raft FSM .


func (r *Raft) runFollower() {
	didWarn := false
	r.logger.Info(fmt.Sprintf("%v entering Follower state (Leader: %q)", r, r.Leader()))
	metrics.IncrCounter([]string{"raft", "state", "follower"}, 1)
	heartbeatTimer := randomTimeout(r.conf.HeartbeatTimeout)
	for {
		select {
		case rpc := <-r.rpcCh:
			r.processRPC(rpc)

		case c := <-r.configurationChangeCh:
			// Reject any operations since we are not the leader
			c.respond(ErrNotLeader)

		case a := <-r.applyCh:
			// Reject any operations since we are not the leader
			a.respond(ErrNotLeader)

		case v := <-r.verifyCh:
			// Reject any operations since we are not the leader
			v.respond(ErrNotLeader)

		case r := <-r.userRestoreCh:
			// Reject any restores since we are not the leader
			r.respond(ErrNotLeader)

		case c := <-r.configurationsCh:
			c.configurations = r.configurations.Clone()
			c.respond(nil)

		case b := <-r.bootstrapCh:
			b.respond(r.liveBootstrap(b.configuration))

		case <-heartbeatTimer:
			// Restart the heartbeat timer
			heartbeatTimer = randomTimeout(r.conf.HeartbeatTimeout)

			// Check if we have had a successful contact
			lastContact := r.LastContact()
			if time.Now().Sub(lastContact) < r.conf.HeartbeatTimeout {
				continue
			}

			// Heartbeat failed! Transition to the candidate state
			lastLeader := r.Leader()
			r.setLeader("")

			if r.configurations.latestIndex == 0 {
				if !didWarn {
					r.logger.Warn("no known peers, aborting election")
					didWarn = true
				}
			} else if r.configurations.latestIndex == r.configurations.committedIndex &&
				!hasVote(r.configurations.latest, r.localID) {
				if !didWarn {
					r.logger.Warn("not part of stable configuration, aborting election")
					didWarn = true
				}
			} else {
				r.logger.Warn(fmt.Sprintf("Heartbeat timeout from %q reached, starting election", lastLeader))
				metrics.IncrCounter([]string{"raft", "transition", "heartbeat_timeout"}, 1)
				r.setState(Candidate)
				return
			}

		case <-r.shutdownCh:
			return
		}
	}
}

// runFollower runs the FSM for a follower.

runFollower runs the FSM for a follower .


func (r *Raft) liveBootstrap(configuration Configuration) error {
	// Use the pre-init API to make the static updates.
	err := BootstrapCluster(&r.conf, r.logs, r.stable, r.snapshots,
		r.trans, configuration)
	if err != nil {
		return err
	}

	// Make the configuration live.
	var entry Log
	if err := r.logs.GetLog(1, &entry); err != nil {
		panic(err)
	}
	r.setCurrentTerm(1)
	r.setLastLog(entry.Index, entry.Term)
	r.processConfigurationLogEntry(&entry)
	return nil
}

// liveBootstrap attempts to seed an initial configuration for the cluster. See
// the Raft object's member BootstrapCluster for more details. This must only be
// called on the main thread, and only makes sense in the follower state.

liveBootstrap attempts to seed an initial configuration for the cluster . See the Raft object s member BootstrapCluster for more details . This must only be called on the main thread and only makes sense in the follower state .


func (r *Raft) runCandidate() {
	r.logger.Info(fmt.Sprintf("%v entering Candidate state in term %v", r, r.getCurrentTerm()+1))
	metrics.IncrCounter([]string{"raft", "state", "candidate"}, 1)

	// Start vote for us, and set a timeout
	voteCh := r.electSelf()
	electionTimer := randomTimeout(r.conf.ElectionTimeout)

	// Tally the votes, need a simple majority
	grantedVotes := 0
	votesNeeded := r.quorumSize()
	r.logger.Debug(fmt.Sprintf("Votes needed: %d", votesNeeded))

	for r.getState() == Candidate {
		select {
		case rpc := <-r.rpcCh:
			r.processRPC(rpc)

		case vote := <-voteCh:
			// Check if the term is greater than ours, bail
			if vote.Term > r.getCurrentTerm() {
				r.logger.Debug("Newer term discovered, fallback to follower")
				r.setState(Follower)
				r.setCurrentTerm(vote.Term)
				return
			}

			// Check if the vote is granted
			if vote.Granted {
				grantedVotes++
				r.logger.Debug(fmt.Sprintf("Vote granted from %s in term %v. Tally: %d",
					vote.voterID, vote.Term, grantedVotes))
			}

			// Check if we've become the leader
			if grantedVotes >= votesNeeded {
				r.logger.Info(fmt.Sprintf("Election won. Tally: %d", grantedVotes))
				r.setState(Leader)
				r.setLeader(r.localAddr)
				return
			}

		case c := <-r.configurationChangeCh:
			// Reject any operations since we are not the leader
			c.respond(ErrNotLeader)

		case a := <-r.applyCh:
			// Reject any operations since we are not the leader
			a.respond(ErrNotLeader)

		case v := <-r.verifyCh:
			// Reject any operations since we are not the leader
			v.respond(ErrNotLeader)

		case r := <-r.userRestoreCh:
			// Reject any restores since we are not the leader
			r.respond(ErrNotLeader)

		case c := <-r.configurationsCh:
			c.configurations = r.configurations.Clone()
			c.respond(nil)

		case b := <-r.bootstrapCh:
			b.respond(ErrCantBootstrap)

		case <-electionTimer:
			// Election failed! Restart the election. We simply return,
			// which will kick us back into runCandidate
			r.logger.Warn("Election timeout reached, restarting election")
			return

		case <-r.shutdownCh:
			return
		}
	}
}

// runCandidate runs the FSM for a candidate.

runCandidate runs the FSM for a candidate .


func (r *Raft) runLeader() {
	r.logger.Info(fmt.Sprintf("%v entering Leader state", r))
	metrics.IncrCounter([]string{"raft", "state", "leader"}, 1)

	// Notify that we are the leader
	asyncNotifyBool(r.leaderCh, true)

	// Push to the notify channel if given
	if notify := r.conf.NotifyCh; notify != nil {
		select {
		case notify <- true:
		case <-r.shutdownCh:
		}
	}

	// Setup leader state
	r.leaderState.commitCh = make(chan struct{}, 1)
	r.leaderState.commitment = newCommitment(r.leaderState.commitCh,
		r.configurations.latest,
		r.getLastIndex()+1 /* first index that may be committed in this term */)
	r.leaderState.inflight = list.New()
	r.leaderState.replState = make(map[ServerID]*followerReplication)
	r.leaderState.notify = make(map[*verifyFuture]struct{})
	r.leaderState.stepDown = make(chan struct{}, 1)

	// Cleanup state on step down
	defer func() {
		// Since we were the leader previously, we update our
		// last contact time when we step down, so that we are not
		// reporting a last contact time from before we were the
		// leader. Otherwise, to a client it would seem our data
		// is extremely stale.
		r.setLastContact()

		// Stop replication
		for _, p := range r.leaderState.replState {
			close(p.stopCh)
		}

		// Respond to all inflight operations
		for e := r.leaderState.inflight.Front(); e != nil; e = e.Next() {
			e.Value.(*logFuture).respond(ErrLeadershipLost)
		}

		// Respond to any pending verify requests
		for future := range r.leaderState.notify {
			future.respond(ErrLeadershipLost)
		}

		// Clear all the state
		r.leaderState.commitCh = nil
		r.leaderState.commitment = nil
		r.leaderState.inflight = nil
		r.leaderState.replState = nil
		r.leaderState.notify = nil
		r.leaderState.stepDown = nil

		// If we are stepping down for some reason, no known leader.
		// We may have stepped down due to an RPC call, which would
		// provide the leader, so we cannot always blank this out.
		r.leaderLock.Lock()
		if r.leader == r.localAddr {
			r.leader = ""
		}
		r.leaderLock.Unlock()

		// Notify that we are not the leader
		asyncNotifyBool(r.leaderCh, false)

		// Push to the notify channel if given
		if notify := r.conf.NotifyCh; notify != nil {
			select {
			case notify <- false:
			case <-r.shutdownCh:
				// On shutdown, make a best effort but do not block
				select {
				case notify <- false:
				default:
				}
			}
		}
	}()

	// Start a replication routine for each peer
	r.startStopReplication()

	// Dispatch a no-op log entry first. This gets this leader up to the latest
	// possible commit index, even in the absence of client commands. This used
	// to append a configuration entry instead of a noop. However, that permits
	// an unbounded number of uncommitted configurations in the log. We now
	// maintain that there exists at most one uncommitted configuration entry in
	// any log, so we have to do proper no-ops here.
	noop := &logFuture{
		log: Log{
			Type: LogNoop,
		},
	}
	r.dispatchLogs([]*logFuture{noop})

	// Sit in the leader loop until we step down
	r.leaderLoop()
}

// runLeader runs the FSM for a leader. Do the setup here and drop into
// the leaderLoop for the hot loop.

runLeader runs the FSM for a leader . Do the setup here and drop into the leaderLoop for the hot loop .


func (r *Raft) startStopReplication() {
	inConfig := make(map[ServerID]bool, len(r.configurations.latest.Servers))
	lastIdx := r.getLastIndex()

	// Start replication goroutines that need starting
	for _, server := range r.configurations.latest.Servers {
		if server.ID == r.localID {
			continue
		}
		inConfig[server.ID] = true
		if _, ok := r.leaderState.replState[server.ID]; !ok {
			r.logger.Info(fmt.Sprintf("Added peer %v, starting replication", server.ID))
			s := &followerReplication{
				peer:        server,
				commitment:  r.leaderState.commitment,
				stopCh:      make(chan uint64, 1),
				triggerCh:   make(chan struct{}, 1),
				currentTerm: r.getCurrentTerm(),
				nextIndex:   lastIdx + 1,
				lastContact: time.Now(),
				notify:      make(map[*verifyFuture]struct{}),
				notifyCh:    make(chan struct{}, 1),
				stepDown:    r.leaderState.stepDown,
			}
			r.leaderState.replState[server.ID] = s
			r.goFunc(func() { r.replicate(s) })
			asyncNotifyCh(s.triggerCh)
		}
	}

	// Stop replication goroutines that need stopping
	for serverID, repl := range r.leaderState.replState {
		if inConfig[serverID] {
			continue
		}
		// Replicate up to lastIdx and stop
		r.logger.Info(fmt.Sprintf("Removed peer %v, stopping replication after %v", serverID, lastIdx))
		repl.stopCh <- lastIdx
		close(repl.stopCh)
		delete(r.leaderState.replState, serverID)
	}
}

// startStopReplication will set up state and start asynchronous replication to
// new peers, and stop replication to removed peers. Before removing a peer,
// it'll instruct the replication routines to try to replicate to the current
// index. This must only be called from the main thread.

startStopReplication will set up state and start asynchronous replication to new peers and stop replication to removed peers . Before removing a peer it ll instruct the replication routines to try to replicate to the current index . This must only be called from the main thread .


func (r *Raft) configurationChangeChIfStable() chan *configurationChangeFuture {
	// Have to wait until:
	// 1. The latest configuration is committed, and
	// 2. This leader has committed some entry (the noop) in this term
	//    https://groups.google.com/forum/#!msg/raft-dev/t4xj6dJTP6E/d2D9LrWRza8J
	if r.configurations.latestIndex == r.configurations.committedIndex &&
		r.getCommitIndex() >= r.leaderState.commitment.startIndex {
		return r.configurationChangeCh
	}
	return nil
}

// configurationChangeChIfStable returns r.configurationChangeCh if it's safe
// to process requests from it, or nil otherwise. This must only be called
// from the main thread.
//
// Note that if the conditions here were to change outside of leaderLoop to take
// this from nil to non-nil, we would need leaderLoop to be kicked.

configurationChangeChIfStable returns r . configurationChangeCh if it s safe to process requests from it or nil otherwise . This must only be called from the main thread . Note that if the conditions here were to change outside of leaderLoop to take this from nil to non - nil we would need leaderLoop to be kicked .


func (r *Raft) verifyLeader(v *verifyFuture) {
	// Current leader always votes for self
	v.votes = 1

	// Set the quorum size, hot-path for single node
	v.quorumSize = r.quorumSize()
	if v.quorumSize == 1 {
		v.respond(nil)
		return
	}

	// Track this request
	v.notifyCh = r.verifyCh
	r.leaderState.notify[v] = struct{}{}

	// Trigger immediate heartbeats
	for _, repl := range r.leaderState.replState {
		repl.notifyLock.Lock()
		repl.notify[v] = struct{}{}
		repl.notifyLock.Unlock()
		asyncNotifyCh(repl.notifyCh)
	}
}

// verifyLeader must be called from the main thread for safety.
// Causes the followers to attempt an immediate heartbeat.

verifyLeader must be called from the main thread for safety . Causes the followers to attempt an immediate heartbeat .


func (r *Raft) checkLeaderLease() time.Duration {
	// Track contacted nodes, we can always contact ourself
	contacted := 1

	// Check each follower
	var maxDiff time.Duration
	now := time.Now()
	for peer, f := range r.leaderState.replState {
		diff := now.Sub(f.LastContact())
		if diff <= r.conf.LeaderLeaseTimeout {
			contacted++
			if diff > maxDiff {
				maxDiff = diff
			}
		} else {
			// Log at least once at high value, then debug. Otherwise it gets very verbose.
			if diff <= 3*r.conf.LeaderLeaseTimeout {
				r.logger.Warn(fmt.Sprintf("Failed to contact %v in %v", peer, diff))
			} else {
				r.logger.Debug(fmt.Sprintf("Failed to contact %v in %v", peer, diff))
			}
		}
		metrics.AddSample([]string{"raft", "leader", "lastContact"}, float32(diff/time.Millisecond))
	}

	// Verify we can contact a quorum
	quorum := r.quorumSize()
	if contacted < quorum {
		r.logger.Warn("Failed to contact quorum of nodes, stepping down")
		r.setState(Follower)
		metrics.IncrCounter([]string{"raft", "transition", "leader_lease_timeout"}, 1)
	}
	return maxDiff
}

// checkLeaderLease is used to check if we can contact a quorum of nodes
// within the last leader lease interval. If not, we need to step down,
// as we may have lost connectivity. Returns the maximum duration without
// contact. This must only be called from the main thread.

checkLeaderLease is used to check if we can contact a quorum of nodes within the last leader lease interval . If not we need to step down as we may have lost connectivity . Returns the maximum duration without contact . This must only be called from the main thread .


func (r *Raft) restoreUserSnapshot(meta *SnapshotMeta, reader io.Reader) error {
	defer metrics.MeasureSince([]string{"raft", "restoreUserSnapshot"}, time.Now())

	// Sanity check the version.
	version := meta.Version
	if version < SnapshotVersionMin || version > SnapshotVersionMax {
		return fmt.Errorf("unsupported snapshot version %d", version)
	}

	// We don't support snapshots while there's a config change
	// outstanding since the snapshot doesn't have a means to
	// represent this state.
	committedIndex := r.configurations.committedIndex
	latestIndex := r.configurations.latestIndex
	if committedIndex != latestIndex {
		return fmt.Errorf("cannot restore snapshot now, wait until the configuration entry at %v has been applied (have applied %v)",
			latestIndex, committedIndex)
	}

	// Cancel any inflight requests.
	for {
		e := r.leaderState.inflight.Front()
		if e == nil {
			break
		}
		e.Value.(*logFuture).respond(ErrAbortedByRestore)
		r.leaderState.inflight.Remove(e)
	}

	// We will overwrite the snapshot metadata with the current term,
	// an index that's greater than the current index, or the last
	// index in the snapshot. It's important that we leave a hole in
	// the index so we know there's nothing in the Raft log there and
	// replication will fault and send the snapshot.
	term := r.getCurrentTerm()
	lastIndex := r.getLastIndex()
	if meta.Index > lastIndex {
		lastIndex = meta.Index
	}
	lastIndex++

	// Dump the snapshot. Note that we use the latest configuration,
	// not the one that came with the snapshot.
	sink, err := r.snapshots.Create(version, lastIndex, term,
		r.configurations.latest, r.configurations.latestIndex, r.trans)
	if err != nil {
		return fmt.Errorf("failed to create snapshot: %v", err)
	}
	n, err := io.Copy(sink, reader)
	if err != nil {
		sink.Cancel()
		return fmt.Errorf("failed to write snapshot: %v", err)
	}
	if n != meta.Size {
		sink.Cancel()
		return fmt.Errorf("failed to write snapshot, size didn't match (%d != %d)", n, meta.Size)
	}
	if err := sink.Close(); err != nil {
		return fmt.Errorf("failed to close snapshot: %v", err)
	}
	r.logger.Info(fmt.Sprintf("Copied %d bytes to local snapshot", n))

	// Restore the snapshot into the FSM. If this fails we are in a
	// bad state so we panic to take ourselves out.
	fsm := &restoreFuture{ID: sink.ID()}
	fsm.init()
	select {
	case r.fsmMutateCh <- fsm:
	case <-r.shutdownCh:
		return ErrRaftShutdown
	}
	if err := fsm.Error(); err != nil {
		panic(fmt.Errorf("failed to restore snapshot: %v", err))
	}

	// We set the last log so it looks like we've stored the empty
	// index we burned. The last applied is set because we made the
	// FSM take the snapshot state, and we store the last snapshot
	// in the stable store since we created a snapshot as part of
	// this process.
	r.setLastLog(lastIndex, term)
	r.setLastApplied(lastIndex)
	r.setLastSnapshot(lastIndex, term)

	r.logger.Info(fmt.Sprintf("Restored user snapshot (index %d)", lastIndex))
	return nil
}

// restoreUserSnapshot is used to manually consume an external snapshot, such
// as if restoring from a backup. We will use the current Raft configuration,
// not the one from the snapshot, so that we can restore into a new cluster. We
// will also use the higher of the index of the snapshot, or the current index,
// and then add 1 to that, so we force a new state with a hole in the Raft log,
// so that the snapshot will be sent to followers and used for any new joiners.
// This can only be run on the leader, and returns a future that can be used to
// block until complete.

restoreUserSnapshot is used to manually consume an external snapshot such as if restoring from a backup . We will use the current Raft configuration not the one from the snapshot so that we can restore into a new cluster . We will also use the higher of the index of the snapshot or the current index and then add 1 to that so we force a new state with a hole in the Raft log so that the snapshot will be sent to followers and used for any new joiners . This can only be run on the leader and returns a future that can be used to block until complete .


func (r *Raft) appendConfigurationEntry(future *configurationChangeFuture) {
	configuration, err := nextConfiguration(r.configurations.latest, r.configurations.latestIndex, future.req)
	if err != nil {
		future.respond(err)
		return
	}

	r.logger.Info(fmt.Sprintf("Updating configuration with %s (%v, %v) to %+v",
		future.req.command, future.req.serverID, future.req.serverAddress, configuration.Servers))

	// In pre-ID compatibility mode we translate all configuration changes
	// in to an old remove peer message, which can handle all supported
	// cases for peer changes in the pre-ID world (adding and removing
	// voters). Both add peer and remove peer log entries are handled
	// similarly on old Raft servers, but remove peer does extra checks to
	// see if a leader needs to step down. Since they both assert the full
	// configuration, then we can safely call remove peer for everything.
	if r.protocolVersion < 2 {
		future.log = Log{
			Type: LogRemovePeerDeprecated,
			Data: encodePeers(configuration, r.trans),
		}
	} else {
		future.log = Log{
			Type: LogConfiguration,
			Data: encodeConfiguration(configuration),
		}
	}

	r.dispatchLogs([]*logFuture{&future.logFuture})
	index := future.Index()
	r.configurations.latest = configuration
	r.configurations.latestIndex = index
	r.leaderState.commitment.setConfiguration(configuration)
	r.startStopReplication()
}

// appendConfigurationEntry changes the configuration and adds a new
// configuration entry to the log. This must only be called from the
// main thread.

appendConfigurationEntry changes the configuration and adds a new configuration entry to the log . This must only be called from the main thread .


func (r *Raft) dispatchLogs(applyLogs []*logFuture) {
	now := time.Now()
	defer metrics.MeasureSince([]string{"raft", "leader", "dispatchLog"}, now)

	term := r.getCurrentTerm()
	lastIndex := r.getLastIndex()

	n := len(applyLogs)
	logs := make([]*Log, n)
	metrics.SetGauge([]string{"raft", "leader", "dispatchNumLogs"}, float32(n))

	for idx, applyLog := range applyLogs {
		applyLog.dispatch = now
		lastIndex++
		applyLog.log.Index = lastIndex
		applyLog.log.Term = term
		logs[idx] = &applyLog.log
		r.leaderState.inflight.PushBack(applyLog)
	}

	// Write the log entry locally
	if err := r.logs.StoreLogs(logs); err != nil {
		r.logger.Error(fmt.Sprintf("Failed to commit logs: %v", err))
		for _, applyLog := range applyLogs {
			applyLog.respond(err)
		}
		r.setState(Follower)
		return
	}
	r.leaderState.commitment.match(r.localID, lastIndex)

	// Update the last log since it's on disk now
	r.setLastLog(lastIndex, term)

	// Notify the replicators of the new log
	for _, f := range r.leaderState.replState {
		asyncNotifyCh(f.triggerCh)
	}
}

// dispatchLog is called on the leader to push a log to disk, mark it
// as inflight and begin replication of it.

dispatchLog is called on the leader to push a log to disk mark it as inflight and begin replication of it .


func (r *Raft) processLogs(index uint64, future *logFuture) {
	// Reject logs we've applied already
	lastApplied := r.getLastApplied()
	if index <= lastApplied {
		r.logger.Warn(fmt.Sprintf("Skipping application of old log: %d", index))
		return
	}

	// Apply all the preceding logs
	for idx := r.getLastApplied() + 1; idx <= index; idx++ {
		// Get the log, either from the future or from our log store
		if future != nil && future.log.Index == idx {
			r.processLog(&future.log, future)
		} else {
			l := new(Log)
			if err := r.logs.GetLog(idx, l); err != nil {
				r.logger.Error(fmt.Sprintf("Failed to get log at %d: %v", idx, err))
				panic(err)
			}
			r.processLog(l, nil)
		}

		// Update the lastApplied index and term
		r.setLastApplied(idx)
	}
}

// processLogs is used to apply all the committed entries that haven't been
// applied up to the given index limit.
// This can be called from both leaders and followers.
// Followers call this from AppendEntries, for n entries at a time, and always
// pass future=nil.
// Leaders call this once per inflight when entries are committed. They pass
// the future from inflights.

processLogs is used to apply all the committed entries that haven t been applied up to the given index limit . This can be called from both leaders and followers . Followers call this from AppendEntries for n entries at a time and always pass future = nil . Leaders call this once per inflight when entries are committed . They pass the future from inflights .


func (r *Raft) processLog(l *Log, future *logFuture) {
	switch l.Type {
	case LogBarrier:
		// Barrier is handled by the FSM
		fallthrough

	case LogCommand:
		// Forward to the fsm handler
		select {
		case r.fsmMutateCh <- &commitTuple{l, future}:
		case <-r.shutdownCh:
			if future != nil {
				future.respond(ErrRaftShutdown)
			}
		}

		// Return so that the future is only responded to
		// by the FSM handler when the application is done
		return

	case LogConfiguration:
	case LogAddPeerDeprecated:
	case LogRemovePeerDeprecated:
	case LogNoop:
		// Ignore the no-op

	default:
		panic(fmt.Errorf("unrecognized log type: %#v", l))
	}

	// Invoke the future if given
	if future != nil {
		future.respond(nil)
	}
}

// processLog is invoked to process the application of a single committed log entry.

processLog is invoked to process the application of a single committed log entry .


func (r *Raft) processRPC(rpc RPC) {
	if err := r.checkRPCHeader(rpc); err != nil {
		rpc.Respond(nil, err)
		return
	}

	switch cmd := rpc.Command.(type) {
	case *AppendEntriesRequest:
		r.appendEntries(rpc, cmd)
	case *RequestVoteRequest:
		r.requestVote(rpc, cmd)
	case *InstallSnapshotRequest:
		r.installSnapshot(rpc, cmd)
	default:
		r.logger.Error(fmt.Sprintf("Got unexpected command: %#v", rpc.Command))
		rpc.Respond(nil, fmt.Errorf("unexpected command"))
	}
}

func ( r * Raft ) processRPC ( rpc RPC ) { if err := r . checkRPCHeader ( rpc ) ; err != nil { rpc . Respond ( nil , err ) return } switch cmd := rpc . Command . ( type ) { case * AppendEntriesRequest : r . appendEntries ( rpc , cmd ) case * RequestVoteRequest : r . requestVote ( rpc , cmd ) case * InstallSnapshotRequest : r . installSnapshot ( rpc , cmd ) default : r . logger . Error ( fmt . Sprintf ( " " , rpc . Command ) ) rpc . Respond ( nil , fmt . Errorf ( " " ) ) } }

// processRPC is called to handle an incoming RPC request. This must only be
// called from the main thread.

processRPC is called to handle an incoming RPC request . This must only be called from the main thread .


func (r *Raft) processHeartbeat(rpc RPC) {
	defer metrics.MeasureSince([]string{"raft", "rpc", "processHeartbeat"}, time.Now())

	// Check if we are shutdown, just ignore the RPC
	select {
	case <-r.shutdownCh:
		return
	default:
	}

	// Ensure we are only handling a heartbeat
	switch cmd := rpc.Command.(type) {
	case *AppendEntriesRequest:
		r.appendEntries(rpc, cmd)
	default:
		r.logger.Error(fmt.Sprintf("Expected heartbeat, got command: %#v", rpc.Command))
		rpc.Respond(nil, fmt.Errorf("unexpected command"))
	}
}

// processHeartbeat is a special handler used just for heartbeat requests
// so that they can be fast-pathed if a transport supports it. This must only
// be called from the main thread.

processHeartbeat is a special handler used just for heartbeat requests so that they can be fast - pathed if a transport supports it . This must only be called from the main thread .


func (r *Raft) setLastContact() {
	r.lastContactLock.Lock()
	r.lastContact = time.Now()
	r.lastContactLock.Unlock()
}

func ( r * Raft ) setLastContact ( ) { r . lastContactLock . Lock ( ) r . lastContact = time . Now ( ) r . lastContactLock . Unlock ( ) }

// setLastContact is used to set the last contact time to now


func (r *Raft) persistVote(term uint64, candidate []byte) error {
	if err := r.stable.SetUint64(keyLastVoteTerm, term); err != nil {
		return err
	}
	if err := r.stable.Set(keyLastVoteCand, candidate); err != nil {
		return err
	}
	return nil
}

func ( r * Raft ) persistVote ( term uint64 , candidate [ ] byte ) error { if err := r . stable . SetUint64 ( keyLastVoteTerm , term ) ; err != nil { return err } if err := r . stable . Set ( keyLastVoteCand , candidate ) ; err != nil { return err } return nil }

// persistVote is used to persist our vote for safety.

persistVote is used to persist our vote for safety .


func (r *Raft) setCurrentTerm(t uint64) {
	// Persist to disk first
	if err := r.stable.SetUint64(keyCurrentTerm, t); err != nil {
		panic(fmt.Errorf("failed to save current term: %v", err))
	}
	r.raftState.setCurrentTerm(t)
}

func
(
r
*
Raft
)
setCurrentTerm
(
t
uint64
)
{
// Persist to disk first
if
err
:=
r
.
stable
.
SetUint64
(
keyCurrentTerm
,
t
)
;
err
!=
{
panic
(
fmt
.
Errorf
(
"
"
,
err
)
)
}
r
.
raftState
.
setCurrentTerm
(
t
)
}

// setCurrentTerm is used to set the current term in a durable manner.

setCurrentTerm is used to set the current term in a durable manner .


func (r *Raft) setState(state RaftState) {
	r.setLeader("")
	oldState := r.raftState.getState()
	r.raftState.setState(state)
	if oldState != state {
		r.observe(state)
	}
}

func ( r * Raft ) setState ( state RaftState ) { r . setLeader ( " " ) oldState := r . raftState . getState ( ) r . raftState . setState ( state ) if oldState != state { r . observe ( state ) } }

// setState is used to update the current state. Any state
// transition causes the known leader to be cleared. This means
// that leader should be set only after updating the state.

setState is used to update the current state . Any state transition causes the known leader to be cleared . This means that leader should be set only after updating the state .


func (c *commitment) getCommitIndex() uint64 {
	c.Lock()
	defer c.Unlock()
	return c.commitIndex
}

func ( c * commitment ) getCommitIndex ( ) uint64 { c . Lock ( ) defer c . Unlock ( ) return c . commitIndex }

// Called by leader after commitCh is notified


func (c *commitment) recalculate() {
	if len(c.matchIndexes) == 0 {
		return
	}

	matched := make([]uint64, 0, len(c.matchIndexes))
	for _, idx := range c.matchIndexes {
		matched = append(matched, idx)
	}
	sort.Sort(uint64Slice(matched))
	quorumMatchIndex := matched[(len(matched)-1)/2]

	if quorumMatchIndex > c.commitIndex && quorumMatchIndex >= c.startIndex {
		c.commitIndex = quorumMatchIndex
		asyncNotifyCh(c.commitCh)
	}
}

func ( c * commitment ) recalculate ( ) { if len ( c . matchIndexes ) == 0 { return } matched := make ( [ ] uint64 , 0 , len ( c . matchIndexes ) ) for _ , idx := range c . matchIndexes { matched = append ( matched , idx ) } sort . Sort ( uint64Slice ( matched ) ) quorumMatchIndex := matched [ ( len ( matched ) - 1 ) / 2 ] if quorumMatchIndex > c . commitIndex && quorumMatchIndex >= c . startIndex { c . commitIndex = quorumMatchIndex asyncNotifyCh ( c . commitCh ) } }

// Internal helper to calculate new commitIndex from matchIndexes.
// Must be called with lock held.

Internal helper to calculate new commitIndex from matchIndexes . Must be called with lock held .


func randomTimeout(minVal time.Duration) <-chan time.Time {
	if minVal == 0 {
		return nil
	}
	extra := (time.Duration(rand.Int63()) % minVal)
	return time.After(minVal + extra)
}

func randomTimeout ( minVal time . Duration ) <- chan time . Time { if minVal == 0 { return nil } extra := ( time . Duration ( rand . Int63 ( ) ) % minVal ) return time . After ( minVal + extra ) }

// randomTimeout returns a value that is between the minVal and 2x minVal.

randomTimeout returns a value that is between the minVal and 2x minVal .


func generateUUID() string {
	buf := make([]byte, 16)
	if _, err := crand.Read(buf); err != nil {
		panic(fmt.Errorf("failed to read random bytes: %v", err))
	}

	return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x",
		buf[0:4],
		buf[4:6],
		buf[6:8],
		buf[8:10],
		buf[10:16])
}

func generateUUID ( ) string { buf := make ( [ ] byte , 16 ) if _ , err := crand . Read ( buf ) ; err != nil { panic ( fmt . Errorf ( " " , err ) ) } return fmt . Sprintf ( " " , buf [ 0 : 4 ] , buf [ 4 : 6 ] , buf [ 6 : 8 ] , buf [ 8 : 10 ] , buf [ 10 : 16 ] ) }

// generateUUID is used to generate a random UUID.

generateUUID is used to generate a random UUID .