func NewSTM(c *v3.Client, apply func(STM) error, so ...stmOption) (*v3.TxnResponse, error) {
	opts := &stmOptions{ctx: c.Ctx()}
	for _, f := range so {
		f(opts)
	}
	if len(opts.prefetch) != 0 {
		f := apply
		apply = func(s STM) error {
			s.Get(opts.prefetch...)
			return f(s)
		}
	}
	return runSTM(mkSTM(c, opts), apply)
}

func NewSTM ( c * v3 . Client , apply func ( STM ) error , so ... stmOption ) ( * v3 . TxnResponse , error ) { opts := & stmOptions { ctx : c . Ctx ( ) } for _ , f := range so { f ( opts ) } if len ( opts . prefetch ) != 0 { f := apply apply = func ( s STM ) error { s . Get ( opts . prefetch ... ) return f ( s ) } } return runSTM ( mkSTM ( c , opts ) , apply ) }

// NewSTM initiates a new STM instance, using serializable snapshot isolation by default.

NewSTM initiates a new STM instance using serializable snapshot isolation by default .


func (rs readSet) first() int64 {
	ret := int64(math.MaxInt64 - 1)
	for _, resp := range rs {
		if rev := resp.Header.Revision; rev < ret {
			ret = rev
		}
	}
	return ret
}

func ( rs readSet ) first ( ) int64 { ret := int64 ( math . MaxInt64 - 1 ) for _ , resp := range rs { if rev := resp . Header . Revision ; rev < ret { ret = rev } } return ret }

// first returns the store revision from the first fetch


func (ws writeSet) cmps(rev int64) []v3.Cmp {
	cmps := make([]v3.Cmp, 0, len(ws))
	for key := range ws {
		cmps = append(cmps, v3.Compare(v3.ModRevision(key), "<", rev))
	}
	return cmps
}

func ( ws writeSet ) cmps ( rev int64 ) [ ] v3 . Cmp { cmps := make ( [ ] v3 . Cmp , 0 , len ( ws ) ) for key := range ws { cmps = append ( cmps , v3 . Compare ( v3 . ModRevision ( key ) , " " , rev ) ) } return cmps }

// cmps returns a cmp list testing no writes have happened past rev


func NewSTMRepeatable(ctx context.Context, c *v3.Client, apply func(STM) error) (*v3.TxnResponse, error) {
	return NewSTM(c, apply, WithAbortContext(ctx), WithIsolation(RepeatableReads))
}

func NewSTMRepeatable ( ctx context . Context , c * v3 . Client , apply func ( STM ) error ) ( * v3 . TxnResponse , error ) { return NewSTM ( c , apply , WithAbortContext ( ctx ) , WithIsolation ( RepeatableReads ) ) }


func NewSTMSerializable(ctx context.Context, c *v3.Client, apply func(STM) error) (*v3.TxnResponse, error) {
	return NewSTM(c, apply, WithAbortContext(ctx), WithIsolation(Serializable))
}

func NewSTMSerializable ( ctx context . Context , c * v3 . Client , apply func ( STM ) error ) ( * v3 . TxnResponse , error ) { return NewSTM ( c , apply , WithAbortContext ( ctx ) , WithIsolation ( Serializable ) ) }


func NewSTMReadCommitted(ctx context.Context, c *v3.Client, apply func(STM) error) (*v3.TxnResponse, error) {
	return NewSTM(c, apply, WithAbortContext(ctx), WithIsolation(ReadCommitted))
}

func NewSTMReadCommitted ( ctx context . Context , c * v3 . Client , apply func ( STM ) error ) ( * v3 . TxnResponse , error ) { return NewSTM ( c , apply , WithAbortContext ( ctx ) , WithIsolation ( ReadCommitted ) ) }


func NewCertPool(CAFiles []string) (*x509.CertPool, error) {
	certPool := x509.NewCertPool()

	for _, CAFile := range CAFiles {
		pemByte, err := ioutil.ReadFile(CAFile)
		if err != nil {
			return nil, err
		}

		for {
			var block *pem.Block
			block, pemByte = pem.Decode(pemByte)
			if block == nil {
				break
			}
			cert, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				return nil, err
			}

			certPool.AddCert(cert)
		}
	}

	return certPool, nil
}

func NewCertPool ( CAFiles [ ] string ) ( * x509 . CertPool , error ) { certPool := x509 . NewCertPool ( ) for _ , CAFile := range CAFiles { pemByte , err := ioutil . ReadFile ( CAFile ) if err != nil { return nil , err } for { var block * pem . Block block , pemByte = pem . Decode ( pemByte ) if block == nil { break } cert , err := x509 . ParseCertificate ( block . Bytes ) if err != nil { return nil , err } certPool . AddCert ( cert ) } } return certPool , nil }

// NewCertPool creates x509 certPool with provided CA files.

NewCertPool creates x509 certPool with provided CA files .


func NewCert(certfile, keyfile string, parseFunc func([]byte, []byte) (tls.Certificate, error)) (*tls.Certificate, error) {
	cert, err := ioutil.ReadFile(certfile)
	if err != nil {
		return nil, err
	}

	key, err := ioutil.ReadFile(keyfile)
	if err != nil {
		return nil, err
	}

	if parseFunc == nil {
		parseFunc = tls.X509KeyPair
	}

	tlsCert, err := parseFunc(cert, key)
	if err != nil {
		return nil, err
	}
	return &tlsCert, nil
}

func NewCert ( certfile , keyfile string , parseFunc func ( [ ] byte , [ ] byte ) ( tls . Certificate , error ) ) ( * tls . Certificate , error ) { cert , err := ioutil . ReadFile ( certfile ) if err != nil { return nil , err } key , err := ioutil . ReadFile ( keyfile ) if err != nil { return nil , err } if parseFunc == nil { parseFunc = tls . X509KeyPair } tlsCert , err := parseFunc ( cert , key ) if err != nil { return nil , err } return & tlsCert , nil }

// NewCert generates TLS cert by using the given cert,key and parse function.

NewCert generates TLS cert by using the given cert key and parse function .


func (p *peer) Pause() {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.paused = true
	p.msgAppReader.pause()
	p.msgAppV2Reader.pause()
}

func ( p * peer ) Pause ( ) { p . mu . Lock ( ) defer p . mu . Unlock ( ) p . paused = true p . msgAppReader . pause ( ) p . msgAppV2Reader . pause ( ) }

// Pause pauses the peer. The peer will simply drops all incoming
// messages without returning an error.

Pause pauses the peer . The peer will simply drops all incoming messages without returning an error .


func (p *peer) Resume() {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.paused = false
	p.msgAppReader.resume()
	p.msgAppV2Reader.resume()
}

func ( p * peer ) Resume ( ) { p . mu . Lock ( ) defer p . mu . Unlock ( ) p . paused = false p . msgAppReader . resume ( ) p . msgAppV2Reader . resume ( ) }


func (p *peer) pick(m raftpb.Message) (writec chan<- raftpb.Message, picked string) {
	var ok bool
	// Considering MsgSnap may have a big size, e.g., 1G, and will block
	// stream for a long time, only use one of the N pipelines to send MsgSnap.
	if isMsgSnap(m) {
		return p.pipeline.msgc, pipelineMsg
	} else if writec, ok = p.msgAppV2Writer.writec(); ok && isMsgApp(m) {
		return writec, streamAppV2
	} else if writec, ok = p.writer.writec(); ok {
		return writec, streamMsg
	}
	return p.pipeline.msgc, pipelineMsg
}

// pick picks a chan for sending the given message. The picked chan and the picked chan
// string name are returned.

pick picks a chan for sending the given message . The picked chan and the picked chan string name are returned .


func (s *snapshotSender) post(req *http.Request) (err error) {
	ctx, cancel := context.WithCancel(context.Background())
	req = req.WithContext(ctx)
	defer cancel()

	type responseAndError struct {
		resp *http.Response
		body []byte
		err  error
	}
	result := make(chan responseAndError, 1)

	go func() {
		resp, err := s.tr.pipelineRt.RoundTrip(req)
		if err != nil {
			result <- responseAndError{resp, nil, err}
			return
		}

		// close the response body when timeouts.
		// prevents from reading the body forever when the other side dies right after
		// successfully receives the request body.
		time.AfterFunc(snapResponseReadTimeout, func() { httputil.GracefulClose(resp) })
		body, err := ioutil.ReadAll(resp.Body)
		result <- responseAndError{resp, body, err}
	}()

	select {
	case <-s.stopc:
		return errStopped
	case r := <-result:
		if r.err != nil {
			return r.err
		}
		return checkPostResponse(r.resp, r.body, req, s.to)
	}
}

// post posts the given request.
// It returns nil when request is sent out and processed successfully.

post posts the given request . It returns nil when request is sent out and processed successfully .


func newTxnResp(rt *pb.TxnRequest, txnPath []bool) (txnResp *pb.TxnResponse, txnCount int) {
	reqs := rt.Success
	if !txnPath[0] {
		reqs = rt.Failure
	}
	resps := make([]*pb.ResponseOp, len(reqs))
	txnResp = &pb.TxnResponse{
		Responses: resps,
		Succeeded: txnPath[0],
		Header:    &pb.ResponseHeader{},
	}
	for i, req := range reqs {
		switch tv := req.Request.(type) {
		case *pb.RequestOp_RequestRange:
			resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponseRange{}}
		case *pb.RequestOp_RequestPut:
			resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponsePut{}}
		case *pb.RequestOp_RequestDeleteRange:
			resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponseDeleteRange{}}
		case *pb.RequestOp_RequestTxn:
			resp, txns := newTxnResp(tv.RequestTxn, txnPath[1:])
			resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponseTxn{ResponseTxn: resp}}
			txnPath = txnPath[1+txns:]
			txnCount += txns + 1
		default:
		}
	}
	return txnResp, txnCount
}

func newTxnResp ( rt * pb . TxnRequest , txnPath [ ] bool ) ( txnResp * pb . TxnResponse , txnCount int ) { reqs := rt . Success if ! txnPath [ 0 ] { reqs = rt . Failure } resps := make ( [ ] * pb . ResponseOp , len ( reqs ) ) txnResp = & pb . TxnResponse { Responses : resps , Succeeded : txnPath [ 0 ] , Header : & pb . ResponseHeader { } , } for i , req := range reqs { switch tv := req . Request . ( type ) { case * pb . RequestOp_RequestRange : resps [ i ] = & pb . ResponseOp { Response : & pb . ResponseOp_ResponseRange { } } case * pb . RequestOp_RequestPut : resps [ i ] = & pb . ResponseOp { Response : & pb . ResponseOp_ResponsePut { } } case * pb . RequestOp_RequestDeleteRange : resps [ i ] = & pb . ResponseOp { Response : & pb . ResponseOp_ResponseDeleteRange { } } case * pb . RequestOp_RequestTxn : resp , txns := newTxnResp ( tv . RequestTxn , txnPath [ 1 : ] ) resps [ i ] = & pb . ResponseOp { Response : & pb . ResponseOp_ResponseTxn { ResponseTxn : resp } } txnPath = txnPath [ 1 + txns : ] txnCount += txns + 1 default : } } return txnResp , txnCount }

// newTxnResp allocates a txn response for a txn request given a path.

newTxnResp allocates a txn response for a txn request given a path .


func applyCompare(rv mvcc.ReadView, c *pb.Compare) bool {
	// TODO: possible optimizations
	// * chunk reads for large ranges to conserve memory
	// * rewrite rules for common patterns:
	//	ex. "[a, b) createrev > 0" => "limit 1 /\ kvs > 0"
	// * caching
	rr, err := rv.Range(c.Key, mkGteRange(c.RangeEnd), mvcc.RangeOptions{})
	if err != nil {
		return false
	}
	if len(rr.KVs) == 0 {
		if c.Target == pb.Compare_VALUE {
			// Always fail if comparing a value on a key/keys that doesn't exist;
			// nil == empty string in grpc; no way to represent missing value
			return false
		}
		return compareKV(c, mvccpb.KeyValue{})
	}
	for _, kv := range rr.KVs {
		if !compareKV(c, kv) {
			return false
		}
	}
	return true
}

// applyCompare applies the compare request.
// If the comparison succeeds, it returns true. Otherwise, returns false.

applyCompare applies the compare request . If the comparison succeeds it returns true . Otherwise returns false .


func OpCompact(rev int64, opts ...CompactOption) CompactOp {
	ret := CompactOp{revision: rev}
	ret.applyCompactOpts(opts)
	return ret
}

func OpCompact ( rev int64 , opts ... CompactOption ) CompactOp { ret := CompactOp { revision : rev } ret . applyCompactOpts ( opts ) return ret }

// OpCompact wraps slice CompactOption to create a CompactOp.

OpCompact wraps slice CompactOption to create a CompactOp .


func NewPriorityQueue(client *v3.Client, key string) *PriorityQueue {
	return &PriorityQueue{client, context.TODO(), key + "/"}
}

func NewPriorityQueue ( client * v3 . Client , key string ) * PriorityQueue { return & PriorityQueue { client , context . TODO ( ) , key + " " } }

// NewPriorityQueue creates an etcd priority queue.

NewPriorityQueue creates an etcd priority queue .


func (q *PriorityQueue) Enqueue(val string, pr uint16) error {
	prefix := fmt.Sprintf("%s%05d", q.key, pr)
	_, err := newSequentialKV(q.client, prefix, val)
	return err
}

func ( q * PriorityQueue ) Enqueue ( val string , pr uint16 ) error { prefix := fmt . Sprintf ( " " , q . key , pr ) _ , err := newSequentialKV ( q . client , prefix , val ) return err }

// Enqueue puts a value into a queue with a given priority.

Enqueue puts a value into a queue with a given priority .


func NewLeaderStats(id string) *LeaderStats {
	return &LeaderStats{
		leaderStats: leaderStats{
			Leader:    id,
			Followers: make(map[string]*FollowerStats),
		},
	}
}

func NewLeaderStats ( id string ) * LeaderStats { return & LeaderStats { leaderStats : leaderStats { Leader : id , Followers : make ( map [ string ] * FollowerStats ) , } , } }

// NewLeaderStats generates a new LeaderStats with the given id as leader


func (fs *FollowerStats) Succ(d time.Duration) {
	fs.Lock()
	defer fs.Unlock()

	total := float64(fs.Counts.Success) * fs.Latency.Average
	totalSquare := float64(fs.Counts.Success) * fs.Latency.averageSquare

	fs.Counts.Success++

	fs.Latency.Current = float64(d) / (1000000.0)

	if fs.Latency.Current > fs.Latency.Maximum {
		fs.Latency.Maximum = fs.Latency.Current
	}

	if fs.Latency.Current < fs.Latency.Minimum {
		fs.Latency.Minimum = fs.Latency.Current
	}

	fs.Latency.Average = (total + fs.Latency.Current) / float64(fs.Counts.Success)
	fs.Latency.averageSquare = (totalSquare + fs.Latency.Current*fs.Latency.Current) / float64(fs.Counts.Success)

	// sdv = sqrt(avg(x^2) - avg(x)^2)
	fs.Latency.StandardDeviation = math.Sqrt(fs.Latency.averageSquare - fs.Latency.Average*fs.Latency.Average)
}

func
(
fs
*
FollowerStats
)
Succ
(
d
time
.
Duration
)
{
fs
.
Lock
(
)
defer
fs
.
Unlock
(
)
total
:=
float64
(
fs
.
Counts
.
Success
)
*
fs
.
Latency
.
Average
totalSquare
:=
float64
(
fs
.
Counts
.
Success
)
*
fs
.
Latency
.
averageSquare
fs
.
Counts
.
Success
++
fs
.
Latency
.
Current
=
float64
(
d
)
/
(
1000000.0
)
if
fs
.
Latency
.
Current
>
fs
.
Latency
.
Maximum
{
fs
.
Latency
.
Maximum
=
fs
.
Latency
.
Current
}
if
fs
.
Latency
.
Current
<
fs
.
Latency
.
Minimum
{
fs
.
Latency
.
Minimum
=
fs
.
Latency
.
Current
}
fs
.
Latency
.
Average
=
(
total
+
fs
.
Latency
.
Current
)
/
float64
(
fs
.
Counts
.
Success
)
fs
.
Latency
.
averageSquare
=
(
totalSquare
+
fs
.
Latency
.
Current
*
fs
.
Latency
.
Current
)
/
float64
(
fs
.
Counts
.
Success
)
// sdv = sqrt(avg(x^2) - avg(x)^2)
fs
.
Latency
.
StandardDeviation
=
math
.
Sqrt
(
fs
.
Latency
.
averageSquare
fs
.
Latency
.
Average
*
fs
.
Latency
.
Average
)
}

// Succ updates the FollowerStats with a successful send


func (fs *FollowerStats) Fail() {
	fs.Lock()
	defer fs.Unlock()
	fs.Counts.Fail++
}

func ( fs * FollowerStats ) Fail ( ) { fs . Lock ( ) defer fs . Unlock ( ) fs . Counts . Fail ++ }

// Fail updates the FollowerStats with an unsuccessful send


func (wbs *watchBroadcasts) delete(w *watcher) int {
	wbs.mu.Lock()
	defer wbs.mu.Unlock()

	wb, ok := wbs.watchers[w]
	if !ok {
		panic("deleting missing watcher from broadcasts")
	}
	delete(wbs.watchers, w)
	wb.delete(w)
	if wb.empty() {
		delete(wbs.bcasts, wb)
		wb.stop()
	}
	return len(wbs.bcasts)
}

func ( wbs * watchBroadcasts ) delete ( w * watcher ) int { wbs . mu . Lock ( ) defer wbs . mu . Unlock ( ) wb , ok := wbs . watchers [ w ] if ! ok { panic ( " " ) } delete ( wbs . watchers , w ) wb . delete ( w ) if wb . empty ( ) { delete ( wbs . bcasts , wb ) wb . stop ( ) } return len ( wbs . bcasts ) }

// delete removes a watcher and returns the number of remaining watchers.

delete removes a watcher and returns the number of remaining watchers .


func startStreamWriter(lg *zap.Logger, local, id types.ID, status *peerStatus, fs *stats.FollowerStats, r Raft) *streamWriter {
	w := &streamWriter{
		lg: lg,

		localID: local,
		peerID:  id,

		status: status,
		fs:     fs,
		r:      r,
		msgc:   make(chan raftpb.Message, streamBufSize),
		connc:  make(chan *outgoingConn),
		stopc:  make(chan struct{}),
		done:   make(chan struct{}),
	}
	go w.run()
	return w
}

func startStreamWriter ( lg * zap . Logger , local , id types . ID , status * peerStatus , fs * stats . FollowerStats , r Raft ) * streamWriter { w := & streamWriter { lg : lg , localID : local , peerID : id , status : status , fs : fs , r : r , msgc : make ( chan raftpb . Message , streamBufSize ) , connc : make ( chan * outgoingConn ) , stopc : make ( chan struct { } ) , done : make ( chan struct { } ) , } go w . run ( ) return w }

// startStreamWriter creates a streamWrite and starts a long running go-routine that accepts
// messages and writes to the attached outgoing connection.

startStreamWriter creates a streamWrite and starts a long running go - routine that accepts messages and writes to the attached outgoing connection .


func checkStreamSupport(v *semver.Version, t streamType) bool {
	nv := &semver.Version{Major: v.Major, Minor: v.Minor}
	for _, s := range supportedStream[nv.String()] {
		if s == t {
			return true
		}
	}
	return false
}

func checkStreamSupport ( v * semver . Version , t streamType ) bool { nv := & semver . Version { Major : v . Major , Minor : v . Minor } for _ , s := range supportedStream [ nv . String ( ) ] { if s == t { return true } } return false }

// checkStreamSupport checks whether the stream type is supported in the
// given version.

checkStreamSupport checks whether the stream type is supported in the given version .


func (pr *Progress) maybeUpdate(n uint64) bool {
	var updated bool
	if pr.Match < n {
		pr.Match = n
		updated = true
		pr.resume()
	}
	if pr.Next < n+1 {
		pr.Next = n + 1
	}
	return updated
}

func ( pr * Progress ) maybeUpdate ( n uint64 ) bool { var updated bool if pr . Match < n { pr . Match = n updated = true pr . resume ( ) } if pr . Next < n + 1 { pr . Next = n + 1 } return updated }

// maybeUpdate returns false if the given n index comes from an outdated message.
// Otherwise it updates the progress and returns true.

maybeUpdate returns false if the given n index comes from an outdated message . Otherwise it updates the progress and returns true .


func (pr *Progress) IsPaused() bool {
	switch pr.State {
	case ProgressStateProbe:
		return pr.Paused
	case ProgressStateReplicate:
		return pr.ins.full()
	case ProgressStateSnapshot:
		return true
	default:
		panic("unexpected state")
	}
}

func ( pr * Progress ) IsPaused ( ) bool { switch pr . State { case ProgressStateProbe : return pr . Paused case ProgressStateReplicate : return pr . ins . full ( ) case ProgressStateSnapshot : return true default : panic ( " " ) } }

// IsPaused returns whether sending log entries to this node has been
// paused. A node may be paused because it has rejected recent
// MsgApps, is currently waiting for a snapshot, or has reached the
// MaxInflightMsgs limit.

IsPaused returns whether sending log entries to this node has been paused . A node may be paused because it has rejected recent MsgApps is currently waiting for a snapshot or has reached the MaxInflightMsgs limit .


func (pr *Progress) needSnapshotAbort() bool {
	return pr.State == ProgressStateSnapshot && pr.Match >= pr.PendingSnapshot
}

func ( pr * Progress ) needSnapshotAbort ( ) bool { return pr . State == ProgressStateSnapshot && pr . Match >= pr . PendingSnapshot }

// needSnapshotAbort returns true if snapshot progress's Match
// is equal or higher than the pendingSnapshot.

needSnapshotAbort returns true if snapshot progress s Match is equal or higher than the pendingSnapshot .


func (in *inflights) add(inflight uint64) {
	if in.full() {
		panic("cannot add into a full inflights")
	}
	next := in.start + in.count
	size := in.size
	if next >= size {
		next -= size
	}
	if next >= len(in.buffer) {
		in.growBuf()
	}
	in.buffer[next] = inflight
	in.count++
}

func ( in * inflights ) add ( inflight uint64 ) { if in . full ( ) { panic ( " " ) } next := in . start + in . count size := in . size if next >= size { next -= size } if next >= len ( in . buffer ) { in . growBuf ( ) } in . buffer [ next ] = inflight in . count ++ }


func (in *inflights) growBuf() {
	newSize := len(in.buffer) * 2
	if newSize == 0 {
		newSize = 1
	} else if newSize > in.size {
		newSize = in.size
	}
	newBuffer := make([]uint64, newSize)
	copy(newBuffer, in.buffer)
	in.buffer = newBuffer
}

func ( in * inflights ) growBuf ( ) { newSize := len ( in . buffer ) * 2 if newSize == 0 { newSize = 1 } else if newSize > in . size { newSize = in . size } newBuffer := make ( [ ] uint64 , newSize ) copy ( newBuffer , in . buffer ) in . buffer = newBuffer }

// grow the inflight buffer by doubling up to inflights.size. We grow on demand
// instead of preallocating to inflights.size to handle systems which have
// thousands of Raft groups per process.

grow the inflight buffer by doubling up to inflights . size . We grow on demand instead of preallocating to inflights . size to handle systems which have thousands of Raft groups per process .


func (in *inflights) freeTo(to uint64) {
	if in.count == 0 || to < in.buffer[in.start] {
		// out of the left side of the window
		return
	}

	idx := in.start
	var i int
	for i = 0; i < in.count; i++ {
		if to < in.buffer[idx] { // found the first large inflight
			break
		}

		// increase index and maybe rotate
		size := in.size
		if idx++; idx >= size {
			idx -= size
		}
	}
	// free i inflights and set new start index
	in.count -= i
	in.start = idx
	if in.count == 0 {
		// inflights is empty, reset the start index so that we don't grow the
		// buffer unnecessarily.
		in.start = 0
	}
}

// freeTo frees the inflights smaller or equal to the given `to` flight.

freeTo frees the inflights smaller or equal to the given to flight .


func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) {
	start := time.Now()

	f, err := ioutil.TempFile(s.dir, "tmp")
	if err != nil {
		return 0, err
	}
	var n int64
	n, err = io.Copy(f, r)
	if err == nil {
		fsyncStart := time.Now()
		err = fileutil.Fsync(f)
		snapDBFsyncSec.Observe(time.Since(fsyncStart).Seconds())
	}
	f.Close()
	if err != nil {
		os.Remove(f.Name())
		return n, err
	}
	fn := s.dbFilePath(id)
	if fileutil.Exist(fn) {
		os.Remove(f.Name())
		return n, nil
	}
	err = os.Rename(f.Name(), fn)
	if err != nil {
		os.Remove(f.Name())
		return n, err
	}

	if s.lg != nil {
		s.lg.Info(
			"saved database snapshot to disk",
			zap.String("path", fn),
			zap.Int64("bytes", n),
			zap.String("size", humanize.Bytes(uint64(n))),
		)
	} else {
		plog.Infof("saved database snapshot to disk [total bytes: %d]", n)
	}

	snapDBSaveSec.Observe(time.Since(start).Seconds())
	return n, nil
}

func ( s * Snapshotter ) SaveDBFrom ( r io . Reader , id uint64 ) ( int64 , error ) { start := time . Now ( ) f , err := ioutil . TempFile ( s . dir , " " ) if err != nil { return 0 , err } var n int64 n , err = io . Copy ( f , r ) if err == nil { fsyncStart := time . Now ( ) err = fileutil . Fsync ( f ) snapDBFsyncSec . Observe ( time . Since ( fsyncStart ) . Seconds ( ) ) } f . Close ( ) if err != nil { os . Remove ( f . Name ( ) ) return n , err } fn := s . dbFilePath ( id ) if fileutil . Exist ( fn ) { os . Remove ( f . Name ( ) ) return n , nil } err = os . Rename ( f . Name ( ) , fn ) if err != nil { os . Remove ( f . Name ( ) ) return n , err } if s . lg != nil { s . lg . Info ( " " , zap . String ( " " , fn ) , zap . Int64 ( " " , n ) , zap . String ( " " , humanize . Bytes ( uint64 ( n ) ) ) , ) } else { plog . Infof ( " " , n ) } snapDBSaveSec . Observe ( time . Since ( start ) . Seconds ( ) ) return n , nil }

// SaveDBFrom saves snapshot of the database from the given reader. It
// guarantees the save operation is atomic.

SaveDBFrom saves snapshot of the database from the given reader . It guarantees the save operation is atomic .


func (s *Snapshotter) DBFilePath(id uint64) (string, error) {
	if _, err := fileutil.ReadDir(s.dir); err != nil {
		return "", err
	}
	fn := s.dbFilePath(id)
	if fileutil.Exist(fn) {
		return fn, nil
	}
	if s.lg != nil {
		s.lg.Warn(
			"failed to find [SNAPSHOT-INDEX].snap.db",
			zap.Uint64("snapshot-index", id),
			zap.String("snapshot-file-path", fn),
			zap.Error(ErrNoDBSnapshot),
		)
	}
	return "", ErrNoDBSnapshot
}

func ( s * Snapshotter ) DBFilePath ( id uint64 ) ( string , error ) { if _ , err := fileutil . ReadDir ( s . dir ) ; err != nil { return " " , err } fn := s . dbFilePath ( id ) if fileutil . Exist ( fn ) { return fn , nil } if s . lg != nil { s . lg . Warn ( " " , zap . Uint64 ( " " , id ) , zap . String ( " " , fn ) , zap . Error ( ErrNoDBSnapshot ) , ) } return " " , ErrNoDBSnapshot }

// DBFilePath returns the file path for the snapshot of the database with
// given id. If the snapshot does not exist, it returns error.

DBFilePath returns the file path for the snapshot of the database with given id . If the snapshot does not exist it returns error .


func (us *UniqueStringsValue) Set(s string) error {
	us.Values = make(map[string]struct{})
	for _, v := range strings.Split(s, ",") {
		us.Values[v] = struct{}{}
	}
	return nil
}

func ( us * UniqueStringsValue ) Set ( s string ) error { us . Values = make ( map [ string ] struct { } ) for _ , v := range strings . Split ( s , " " ) { us . Values [ v ] = struct { } { } } return nil }

// Set parses a command line set of strings, separated by comma.
// Implements "flag.Value" interface.
// The values are set in order.

Set parses a command line set of strings separated by comma . Implements flag . Value interface . The values are set in order .


func NewUniqueStringsValue(s string) (us *UniqueStringsValue) {
	us = &UniqueStringsValue{Values: make(map[string]struct{})}
	if s == "" {
		return us
	}
	if err := us.Set(s); err != nil {
		plog.Panicf("new UniqueStringsValue should never fail: %v", err)
	}
	return us
}

func NewUniqueStringsValue ( s string ) ( us * UniqueStringsValue ) { us = & UniqueStringsValue { Values : make ( map [ string ] struct { } ) } if s == " " { return us } if err := us . Set ( s ) ; err != nil { plog . Panicf ( " " , err ) } return us }

// NewUniqueStringsValue implements string slice as "flag.Value" interface.
// Given value is to be separated by comma.
// The values are set in order.

NewUniqueStringsValue implements string slice as flag . Value interface . Given value is to be separated by comma . The values are set in order .


func UniqueStringsFromFlag(fs *flag.FlagSet, flagName string) []string {
	return (*fs.Lookup(flagName).Value.(*UniqueStringsValue)).stringSlice()
}

func UniqueStringsFromFlag ( fs * flag . FlagSet , flagName string ) [ ] string { return ( * fs . Lookup ( flagName ) . Value . ( * UniqueStringsValue ) ) . stringSlice ( ) }

// UniqueStringsFromFlag returns a string slice from the flag.

UniqueStringsFromFlag returns a string slice from the flag .


func UniqueStringsMapFromFlag(fs *flag.FlagSet, flagName string) map[string]struct{} {
	return (*fs.Lookup(flagName).Value.(*UniqueStringsValue)).Values
}

func UniqueStringsMapFromFlag ( fs * flag . FlagSet , flagName string ) map [ string ] struct { } { return ( * fs . Lookup ( flagName ) . Value . ( * UniqueStringsValue ) ) . Values }

// UniqueStringsMapFromFlag returns a map of strings from the flag.

UniqueStringsMapFromFlag returns a map of strings from the flag .


func Percentiles(nums []float64) (pcs []float64, data []float64) {
	return pctls, percentiles(nums)
}

func Percentiles ( nums [ ] float64 ) ( pcs [ ] float64 , data [ ] float64 ) { return pctls , percentiles ( nums ) }

// Percentiles returns percentile distribution of float64 slice.

Percentiles returns percentile distribution of float64 slice .


func (c *ServerConfig) VerifyBootstrap() error {
	if err := c.hasLocalMember(); err != nil {
		return err
	}
	if err := c.advertiseMatchesCluster(); err != nil {
		return err
	}
	if checkDuplicateURL(c.InitialPeerURLsMap) {
		return fmt.Errorf("initial cluster %s has duplicate url", c.InitialPeerURLsMap)
	}
	if c.InitialPeerURLsMap.String() == "" && c.DiscoveryURL == "" {
		return fmt.Errorf("initial cluster unset and no discovery URL found")
	}
	return nil
}

func ( c * ServerConfig ) VerifyBootstrap ( ) error { if err := c . hasLocalMember ( ) ; err != nil { return err } if err := c . advertiseMatchesCluster ( ) ; err != nil { return err } if checkDuplicateURL ( c . InitialPeerURLsMap ) { return fmt . Errorf ( " " , c . InitialPeerURLsMap ) } if c . InitialPeerURLsMap . String ( ) == " " && c . DiscoveryURL == " " { return fmt . Errorf ( " " ) } return nil }

// VerifyBootstrap sanity-checks the initial config for bootstrap case
// and returns an error for things that should never happen.

VerifyBootstrap sanity - checks the initial config for bootstrap case and returns an error for things that should never happen .


func (c *ServerConfig) VerifyJoinExisting() error {
	// The member has announced its peer urls to the cluster before starting; no need to
	// set the configuration again.
	if err := c.hasLocalMember(); err != nil {
		return err
	}
	if checkDuplicateURL(c.InitialPeerURLsMap) {
		return fmt.Errorf("initial cluster %s has duplicate url", c.InitialPeerURLsMap)
	}
	if c.DiscoveryURL != "" {
		return fmt.Errorf("discovery URL should not be set when joining existing initial cluster")
	}
	return nil
}

// VerifyJoinExisting sanity-checks the initial config for join existing cluster
// case and returns an error for things that should never happen.

VerifyJoinExisting sanity - checks the initial config for join existing cluster case and returns an error for things that should never happen .


func (c *ServerConfig) hasLocalMember() error {
	if urls := c.InitialPeerURLsMap[c.Name]; urls == nil {
		return fmt.Errorf("couldn't find local name %q in the initial cluster configuration", c.Name)
	}
	return nil
}

func ( c * ServerConfig ) hasLocalMember ( ) error { if urls := c . InitialPeerURLsMap [ c . Name ] ; urls == nil { return fmt . Errorf ( " " , c . Name ) } return nil }

// hasLocalMember checks that the cluster at least contains the local server.

hasLocalMember checks that the cluster at least contains the local server .


func (c *ServerConfig) advertiseMatchesCluster() error {
	urls, apurls := c.InitialPeerURLsMap[c.Name], c.PeerURLs.StringSlice()
	urls.Sort()
	sort.Strings(apurls)
	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
	defer cancel()
	ok, err := netutil.URLStringsEqual(ctx, c.Logger, apurls, urls.StringSlice())
	if ok {
		return nil
	}

	initMap, apMap := make(map[string]struct{}), make(map[string]struct{})
	for _, url := range c.PeerURLs {
		apMap[url.String()] = struct{}{}
	}
	for _, url := range c.InitialPeerURLsMap[c.Name] {
		initMap[url.String()] = struct{}{}
	}

	missing := []string{}
	for url := range initMap {
		if _, ok := apMap[url]; !ok {
			missing = append(missing, url)
		}
	}
	if len(missing) > 0 {
		for i := range missing {
			missing[i] = c.Name + "=" + missing[i]
		}
		mstr := strings.Join(missing, ",")
		apStr := strings.Join(apurls, ",")
		return fmt.Errorf("--initial-cluster has %s but missing from --initial-advertise-peer-urls=%s (%v)", mstr, apStr, err)
	}

	for url := range apMap {
		if _, ok := initMap[url]; !ok {
			missing = append(missing, url)
		}
	}
	if len(missing) > 0 {
		mstr := strings.Join(missing, ",")
		umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs})
		return fmt.Errorf("--initial-advertise-peer-urls has %s but missing from --initial-cluster=%s", mstr, umap.String())
	}

	// resolved URLs from "--initial-advertise-peer-urls" and "--initial-cluster" did not match or failed
	apStr := strings.Join(apurls, ",")
	umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs})
	return fmt.Errorf("failed to resolve %s to match --initial-cluster=%s (%v)", apStr, umap.String(), err)
}

// advertiseMatchesCluster confirms peer URLs match those in the cluster peer list.

advertiseMatchesCluster confirms peer URLs match those in the cluster peer list .


func (c *ServerConfig) ReqTimeout() time.Duration {
	// 5s for queue waiting, computation and disk IO delay
	// + 2 * election timeout for possible leader election
	return 5*time.Second + 2*time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond
}

// ReqTimeout returns timeout for request to finish.

ReqTimeout returns timeout for request to finish .


func getStatus(r *raft) Status {
	s := getStatusWithoutProgress(r)
	if s.RaftState == StateLeader {
		s.Progress = getProgressCopy(r)
	}
	return s
}

func getStatus ( r * raft ) Status { s := getStatusWithoutProgress ( r ) if s . RaftState == StateLeader { s . Progress = getProgressCopy ( r ) } return s }

// getStatus gets a copy of the current raft status.

getStatus gets a copy of the current raft status .


func GetDefaultHost() (string, error) {
	rmsgs, rerr := getDefaultRoutes()
	if rerr != nil {
		return "", rerr
	}

	// prioritize IPv4
	if rmsg, ok := rmsgs[syscall.AF_INET]; ok {
		if host, err := chooseHost(syscall.AF_INET, rmsg); host != "" || err != nil {
			return host, err
		}
		delete(rmsgs, syscall.AF_INET)
	}

	// sort so choice is deterministic
	var families []int
	for family := range rmsgs {
		families = append(families, int(family))
	}
	sort.Ints(families)

	for _, f := range families {
		family := uint8(f)
		if host, err := chooseHost(family, rmsgs[family]); host != "" || err != nil {
			return host, err
		}
	}

	return "", errNoDefaultHost
}

func
GetDefaultHost
(
)
(
string
,
error
)
{
rmsgs
,
rerr
:=
getDefaultRoutes
(
)
if
rerr
!=
{
return
"
"
,
rerr
}
// prioritize IPv4
if
rmsg
,
ok
:=
rmsgs
[
syscall
.
AF_INET
]
;
ok
{
if
host
,
err
:=
chooseHost
(
syscall
.
AF_INET
,
rmsg
)
;
host
!=
"
"
||
err
!=
{
return
host
,
err
}
delete
(
rmsgs
,
syscall
.
AF_INET
)
}
// sort so choice is deterministic
var
families
[
]
int
for
family
:=
range
rmsgs
{
families
=
append
(
families
,
int
(
family
)
)
}
sort
.
Ints
(
families
)
for
_
,
f
:=
range
families
{
family
:=
uint8
(
f
)
if
host
,
err
:=
chooseHost
(
family
,
rmsgs
[
family
]
)
;
host
!=
"
"
||
err
!=
{
return
host
,
err
}
}
return
"
"
,
errNoDefaultHost
}

// GetDefaultHost obtains the first IP address of machine from the routing table and returns the IP address as string.
// An IPv4 address is preferred to an IPv6 address for backward compatibility.

GetDefaultHost obtains the first IP address of machine from the routing table and returns the IP address as string . An IPv4 address is preferred to an IPv6 address for backward compatibility .


func getIfaceAddr(idx uint32, family uint8) (*syscall.NetlinkMessage, error) {
	dat, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, int(family))
	if err != nil {
		return nil, err
	}

	msgs, msgErr := syscall.ParseNetlinkMessage(dat)
	if msgErr != nil {
		return nil, msgErr
	}

	ifaddrmsg := syscall.IfAddrmsg{}
	for _, m := range msgs {
		if m.Header.Type != syscall.RTM_NEWADDR {
			continue
		}
		buf := bytes.NewBuffer(m.Data[:syscall.SizeofIfAddrmsg])
		if rerr := binary.Read(buf, cpuutil.ByteOrder(), &ifaddrmsg); rerr != nil {
			continue
		}
		if ifaddrmsg.Index == idx {
			return &m, nil
		}
	}

	return nil, fmt.Errorf("could not find address for interface index %v", idx)

}

func getIfaceAddr ( idx uint32 , family uint8 ) ( * syscall . NetlinkMessage , error ) { dat , err := syscall . NetlinkRIB ( syscall . RTM_GETADDR , int ( family ) ) if err != nil { return nil , err } msgs , msgErr := syscall . ParseNetlinkMessage ( dat ) if msgErr != nil { return nil , msgErr } ifaddrmsg := syscall . IfAddrmsg { } for _ , m := range msgs { if m . Header . Type != syscall . RTM_NEWADDR { continue } buf := bytes . NewBuffer ( m . Data [ : syscall . SizeofIfAddrmsg ] ) if rerr := binary . Read ( buf , cpuutil . ByteOrder ( ) , & ifaddrmsg ) ; rerr != nil { continue } if ifaddrmsg . Index == idx { return & m , nil } } return nil , fmt . Errorf ( " " , idx ) }


func getIfaceLink(idx uint32) (*syscall.NetlinkMessage, error) {
	dat, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC)
	if err != nil {
		return nil, err
	}

	msgs, msgErr := syscall.ParseNetlinkMessage(dat)
	if msgErr != nil {
		return nil, msgErr
	}

	ifinfomsg := syscall.IfInfomsg{}
	for _, m := range msgs {
		if m.Header.Type != syscall.RTM_NEWLINK {
			continue
		}
		buf := bytes.NewBuffer(m.Data[:syscall.SizeofIfInfomsg])
		if rerr := binary.Read(buf, cpuutil.ByteOrder(), &ifinfomsg); rerr != nil {
			continue
		}
		if ifinfomsg.Index == int32(idx) {
			return &m, nil
		}
	}

	return nil, fmt.Errorf("could not find link for interface index %v", idx)
}

func getIfaceLink ( idx uint32 ) ( * syscall . NetlinkMessage , error ) { dat , err := syscall . NetlinkRIB ( syscall . RTM_GETLINK , syscall . AF_UNSPEC ) if err != nil { return nil , err } msgs , msgErr := syscall . ParseNetlinkMessage ( dat ) if msgErr != nil { return nil , msgErr } ifinfomsg := syscall . IfInfomsg { } for _ , m := range msgs { if m . Header . Type != syscall . RTM_NEWLINK { continue } buf := bytes . NewBuffer ( m . Data [ : syscall . SizeofIfInfomsg ] ) if rerr := binary . Read ( buf , cpuutil . ByteOrder ( ) , & ifinfomsg ) ; rerr != nil { continue } if ifinfomsg . Index == int32 ( idx ) { return & m , nil } } return nil , fmt . Errorf ( " " , idx ) }


func lsCommandFunc(c *cli.Context, ki client.KeysAPI) {
	key := "/"
	if len(c.Args()) != 0 {
		key = c.Args()[0]
	}

	sort := c.Bool("sort")
	recursive := c.Bool("recursive")
	quorum := c.Bool("quorum")

	ctx, cancel := contextWithTotalTimeout(c)
	resp, err := ki.Get(ctx, key, &client.GetOptions{Sort: sort, Recursive: recursive, Quorum: quorum})
	cancel()
	if err != nil {
		handleError(c, ExitServerError, err)
	}

	printLs(c, resp)
}

func lsCommandFunc ( c * cli . Context , ki client . KeysAPI ) { key := " " if len ( c . Args ( ) ) != 0 { key = c . Args ( ) [ 0 ] } sort := c . Bool ( " " ) recursive := c . Bool ( " " ) quorum := c . Bool ( " " ) ctx , cancel := contextWithTotalTimeout ( c ) resp , err := ki . Get ( ctx , key , & client . GetOptions { Sort : sort , Recursive : recursive , Quorum : quorum } ) cancel ( ) if err != nil { handleError ( c , ExitServerError , err ) } printLs ( c , resp ) }

// lsCommandFunc executes the "ls" command.


func printLs(c *cli.Context, resp *client.Response) {
	if c.GlobalString("output") == "simple" {
		if !resp.Node.Dir {
			fmt.Println(resp.Node.Key)
		}
		for _, node := range resp.Node.Nodes {
			rPrint(c, node)
		}
	} else {
		// user wants JSON or extended output
		printResponseKey(resp, c.GlobalString("output"))
	}
}

func
printLs
(
c
*
cli
.
Context
,
resp
*
client
.
Response
)
{
if
c
.
GlobalString
(
"
"
)
==
"
"
{
if
!
resp
.
Node
.
Dir
{
fmt
.
Println
(
resp
.
Node
.
Key
)
}
for
_
,
node
:=
range
resp
.
Node
.
Nodes
{
rPrint
(
c
,
node
)
}
}
else
{
// user wants JSON or extended output
printResponseKey
(
resp
,
c
.
GlobalString
(
"
"
)
)
}
}

// printLs writes a response out in a manner similar to the `ls` command in unix.
// Non-empty directories list their contents and files list their name.

printLs writes a response out in a manner similar to the ls command in unix . Non - empty directories list their contents and files list their name .


func rPrint(c *cli.Context, n *client.Node) {
	if n.Dir && c.Bool("p") {
		fmt.Println(fmt.Sprintf("%v/", n.Key))
	} else {
		fmt.Println(n.Key)
	}

	for _, node := range n.Nodes {
		rPrint(c, node)
	}
}

func rPrint ( c * cli . Context , n * client . Node ) { if n . Dir && c . Bool ( " " ) { fmt . Println ( fmt . Sprintf ( " " , n . Key ) ) } else { fmt . Println ( n . Key ) } for _ , node := range n . Nodes { rPrint ( c , node ) } }

// rPrint recursively prints out the nodes in the node structure.

rPrint recursively prints out the nodes in the node structure .


func NewLeaseRenewerCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "lease-renewer",
		Short: "Performs lease renew operation",
		Run:   runLeaseRenewerFunc,
	}
	cmd.Flags().Int64Var(&leaseTTL, "ttl", 5, "lease's ttl")
	return cmd
}

func NewLeaseRenewerCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , Run : runLeaseRenewerFunc , } cmd . Flags ( ) . Int64Var ( & leaseTTL , " " , 5 , " " ) return cmd }

// NewLeaseRenewerCommand returns the cobra command for "lease-renewer runner".

NewLeaseRenewerCommand returns the cobra command for lease - renewer runner .


func Read(lg *zap.Logger, snapname string) (*raftpb.Snapshot, error) {
	b, err := ioutil.ReadFile(snapname)
	if err != nil {
		if lg != nil {
			lg.Warn("failed to read a snap file", zap.String("path", snapname), zap.Error(err))
		} else {
			plog.Errorf("cannot read file %v: %v", snapname, err)
		}
		return nil, err
	}

	if len(b) == 0 {
		if lg != nil {
			lg.Warn("failed to read empty snapshot file", zap.String("path", snapname))
		} else {
			plog.Errorf("unexpected empty snapshot")
		}
		return nil, ErrEmptySnapshot
	}

	var serializedSnap snappb.Snapshot
	if err = serializedSnap.Unmarshal(b); err != nil {
		if lg != nil {
			lg.Warn("failed to unmarshal snappb.Snapshot", zap.String("path", snapname), zap.Error(err))
		} else {
			plog.Errorf("corrupted snapshot file %v: %v", snapname, err)
		}
		return nil, err
	}

	if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
		if lg != nil {
			lg.Warn("failed to read empty snapshot data", zap.String("path", snapname))
		} else {
			plog.Errorf("unexpected empty snapshot")
		}
		return nil, ErrEmptySnapshot
	}

	crc := crc32.Update(0, crcTable, serializedSnap.Data)
	if crc != serializedSnap.Crc {
		if lg != nil {
			lg.Warn("snap file is corrupt",
				zap.String("path", snapname),
				zap.Uint32("prev-crc", serializedSnap.Crc),
				zap.Uint32("new-crc", crc),
			)
		} else {
			plog.Errorf("corrupted snapshot file %v: crc mismatch", snapname)
		}
		return nil, ErrCRCMismatch
	}

	var snap raftpb.Snapshot
	if err = snap.Unmarshal(serializedSnap.Data); err != nil {
		if lg != nil {
			lg.Warn("failed to unmarshal raftpb.Snapshot", zap.String("path", snapname), zap.Error(err))
		} else {
			plog.Errorf("corrupted snapshot file %v: %v", snapname, err)
		}
		return nil, err
	}
	return &snap, nil
}

func Read ( lg * zap . Logger , snapname string ) ( * raftpb . Snapshot , error ) { b , err := ioutil . ReadFile ( snapname ) if err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , snapname ) , zap . Error ( err ) ) } else { plog . Errorf ( " " , snapname , err ) } return nil , err } if len ( b ) == 0 { if lg != nil { lg . Warn ( " " , zap . String ( " " , snapname ) ) } else { plog . Errorf ( " " ) } return nil , ErrEmptySnapshot } var serializedSnap snappb . Snapshot if err = serializedSnap . Unmarshal ( b ) ; err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , snapname ) , zap . Error ( err ) ) } else { plog . Errorf ( " " , snapname , err ) } return nil , err } if len ( serializedSnap . Data ) == 0 || serializedSnap . Crc == 0 { if lg != nil { lg . Warn ( " " , zap . String ( " " , snapname ) ) } else { plog . Errorf ( " " ) } return nil , ErrEmptySnapshot } crc := crc32 . Update ( 0 , crcTable , serializedSnap . Data ) if crc != serializedSnap . Crc { if lg != nil { lg . Warn ( " " , zap . String ( " " , snapname ) , zap . Uint32 ( " " , serializedSnap . Crc ) , zap . Uint32 ( " " , crc ) , ) } else { plog . Errorf ( " " , snapname ) } return nil , ErrCRCMismatch } var snap raftpb . Snapshot if err = snap . Unmarshal ( serializedSnap . Data ) ; err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , snapname ) , zap . Error ( err ) ) } else { plog . Errorf ( " " , snapname , err ) } return nil , err } return & snap , nil }

// Read reads the snapshot named by snapname and returns the snapshot.

Read reads the snapshot named by snapname and returns the snapshot .


func GetCipherSuite(s string) (uint16, bool) {
	v, ok := cipherSuites[s]
	return v, ok
}

func GetCipherSuite ( s string ) ( uint16 , bool ) { v , ok := cipherSuites [ s ] return v , ok }

// GetCipherSuite returns the corresponding cipher suite,
// and boolean value if it is supported.

GetCipherSuite returns the corresponding cipher suite and boolean value if it is supported .


func (p *pipeline) post(data []byte) (err error) {
	u := p.picker.pick()
	req := createPostRequest(u, RaftPrefix, bytes.NewBuffer(data), "application/protobuf", p.tr.URLs, p.tr.ID, p.tr.ClusterID)

	done := make(chan struct{}, 1)
	ctx, cancel := context.WithCancel(context.Background())
	req = req.WithContext(ctx)
	go func() {
		select {
		case <-done:
		case <-p.stopc:
			waitSchedule()
			cancel()
		}
	}()

	resp, err := p.tr.pipelineRt.RoundTrip(req)
	done <- struct{}{}
	if err != nil {
		p.picker.unreachable(u)
		return err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		p.picker.unreachable(u)
		return err
	}

	err = checkPostResponse(resp, b, req, p.peerID)
	if err != nil {
		p.picker.unreachable(u)
		// errMemberRemoved is a critical error since a removed member should
		// always be stopped. So we use reportCriticalError to report it to errorc.
		if err == errMemberRemoved {
			reportCriticalError(err, p.errorc)
		}
		return err
	}

	return nil
}

// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.

post POSTs a data payload to a url . Returns nil if the POST succeeds error on any failure .


func (r *raft) send(m pb.Message) {
	m.From = r.id
	if m.Type == pb.MsgVote || m.Type == pb.MsgVoteResp || m.Type == pb.MsgPreVote || m.Type == pb.MsgPreVoteResp {
		if m.Term == 0 {
			// All {pre-,}campaign messages need to have the term set when
			// sending.
			// - MsgVote: m.Term is the term the node is campaigning for,
			//   non-zero as we increment the term when campaigning.
			// - MsgVoteResp: m.Term is the new r.Term if the MsgVote was
			//   granted, non-zero for the same reason MsgVote is
			// - MsgPreVote: m.Term is the term the node will campaign,
			//   non-zero as we use m.Term to indicate the next term we'll be
			//   campaigning for
			// - MsgPreVoteResp: m.Term is the term received in the original
			//   MsgPreVote if the pre-vote was granted, non-zero for the
			//   same reasons MsgPreVote is
			panic(fmt.Sprintf("term should be set when sending %s", m.Type))
		}
	} else {
		if m.Term != 0 {
			panic(fmt.Sprintf("term should not be set when sending %s (was %d)", m.Type, m.Term))
		}
		// do not attach term to MsgProp, MsgReadIndex
		// proposals are a way to forward to the leader and
		// should be treated as local message.
		// MsgReadIndex is also forwarded to leader.
		if m.Type != pb.MsgProp && m.Type != pb.MsgReadIndex {
			m.Term = r.Term
		}
	}
	r.msgs = append(r.msgs, m)
}

// send persists state to stable storage and then sends to its mailbox.

send persists state to stable storage and then sends to its mailbox .


func (r *raft) sendHeartbeat(to uint64, ctx []byte) {
	// Attach the commit as min(to.matched, r.committed).
	// When the leader sends out heartbeat message,
	// the receiver(follower) might not be matched with the leader
	// or it might not have all the committed entries.
	// The leader MUST NOT forward the follower's commit to
	// an unmatched index.
	commit := min(r.getProgress(to).Match, r.raftLog.committed)
	m := pb.Message{
		To:      to,
		Type:    pb.MsgHeartbeat,
		Commit:  commit,
		Context: ctx,
	}

	r.send(m)
}

// sendHeartbeat sends a heartbeat RPC to the given peer.

sendHeartbeat sends a heartbeat RPC to the given peer .


func (r *raft) bcastAppend() {
	r.forEachProgress(func(id uint64, _ *Progress) {
		if id == r.id {
			return
		}

		r.sendAppend(id)
	})
}

func ( r * raft ) bcastAppend ( ) { r . forEachProgress ( func ( id uint64 , _ * Progress ) { if id == r . id { return } r . sendAppend ( id ) } ) }

// bcastAppend sends RPC, with entries to all peers that are not up-to-date
// according to the progress recorded in r.prs.

bcastAppend sends RPC with entries to all peers that are not up - to - date according to the progress recorded in r . prs .


func (r *raft) bcastHeartbeat() {
	lastCtx := r.readOnly.lastPendingRequestCtx()
	if len(lastCtx) == 0 {
		r.bcastHeartbeatWithCtx(nil)
	} else {
		r.bcastHeartbeatWithCtx([]byte(lastCtx))
	}
}

func ( r * raft ) bcastHeartbeat ( ) { lastCtx := r . readOnly . lastPendingRequestCtx ( ) if len ( lastCtx ) == 0 { r . bcastHeartbeatWithCtx ( nil ) } else { r . bcastHeartbeatWithCtx ( [ ] byte ( lastCtx ) ) } }

// bcastHeartbeat sends RPC, without entries to all the peers.

bcastHeartbeat sends RPC without entries to all the peers .


func (r *raft) tickElection() {
	r.electionElapsed++

	if r.promotable() && r.pastElectionTimeout() {
		r.electionElapsed = 0
		r.Step(pb.Message{From: r.id, Type: pb.MsgHup})
	}
}

func ( r * raft ) tickElection ( ) { r . electionElapsed ++ if r . promotable ( ) && r . pastElectionTimeout ( ) { r . electionElapsed = 0 r . Step ( pb . Message { From : r . id , Type : pb . MsgHup } ) } }

// tickElection is run by followers and candidates after r.electionTimeout.

tickElection is run by followers and candidates after r . electionTimeout .


func (r *raft) tickHeartbeat() {
	r.heartbeatElapsed++
	r.electionElapsed++

	if r.electionElapsed >= r.electionTimeout {
		r.electionElapsed = 0
		if r.checkQuorum {
			r.Step(pb.Message{From: r.id, Type: pb.MsgCheckQuorum})
		}
		// If current leader cannot transfer leadership in electionTimeout, it becomes leader again.
		if r.state == StateLeader && r.leadTransferee != None {
			r.abortLeaderTransfer()
		}
	}

	if r.state != StateLeader {
		return
	}

	if r.heartbeatElapsed >= r.heartbeatTimeout {
		r.heartbeatElapsed = 0
		r.Step(pb.Message{From: r.id, Type: pb.MsgBeat})
	}
}

// tickHeartbeat is run by leaders to send a MsgBeat after r.heartbeatTimeout.

tickHeartbeat is run by leaders to send a MsgBeat after r . heartbeatTimeout .


func stepCandidate(r *raft, m pb.Message) error {
	// Only handle vote responses corresponding to our candidacy (while in
	// StateCandidate, we may get stale MsgPreVoteResp messages in this term from
	// our pre-candidate state).
	var myVoteRespType pb.MessageType
	if r.state == StatePreCandidate {
		myVoteRespType = pb.MsgPreVoteResp
	} else {
		myVoteRespType = pb.MsgVoteResp
	}
	switch m.Type {
	case pb.MsgProp:
		r.logger.Infof("%x no leader at term %d; dropping proposal", r.id, r.Term)
		return ErrProposalDropped
	case pb.MsgApp:
		r.becomeFollower(m.Term, m.From) // always m.Term == r.Term
		r.handleAppendEntries(m)
	case pb.MsgHeartbeat:
		r.becomeFollower(m.Term, m.From) // always m.Term == r.Term
		r.handleHeartbeat(m)
	case pb.MsgSnap:
		r.becomeFollower(m.Term, m.From) // always m.Term == r.Term
		r.handleSnapshot(m)
	case myVoteRespType:
		gr := r.poll(m.From, m.Type, !m.Reject)
		r.logger.Infof("%x [quorum:%d] has received %d %s votes and %d vote rejections", r.id, r.quorum(), gr, m.Type, len(r.votes)-gr)
		switch r.quorum() {
		case gr:
			if r.state == StatePreCandidate {
				r.campaign(campaignElection)
			} else {
				r.becomeLeader()
				r.bcastAppend()
			}
		case len(r.votes) - gr:
			// pb.MsgPreVoteResp contains future term of pre-candidate
			// m.Term > r.Term; reuse r.Term
			r.becomeFollower(r.Term, None)
		}
	case pb.MsgTimeoutNow:
		r.logger.Debugf("%x [term %d state %v] ignored MsgTimeoutNow from %x", r.id, r.Term, r.state, m.From)
	}
	return nil
}

// stepCandidate is shared by StateCandidate and StatePreCandidate; the difference is
// whether they respond to MsgVoteResp or MsgPreVoteResp.

stepCandidate is shared by StateCandidate and StatePreCandidate ; the difference is whether they respond to MsgVoteResp or MsgPreVoteResp .


func (r *raft) restore(s pb.Snapshot) bool {
	if s.Metadata.Index <= r.raftLog.committed {
		return false
	}
	if r.raftLog.matchTerm(s.Metadata.Index, s.Metadata.Term) {
		r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] fast-forwarded commit to snapshot [index: %d, term: %d]",
			r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term)
		r.raftLog.commitTo(s.Metadata.Index)
		return false
	}

	// The normal peer can't become learner.
	if !r.isLearner {
		for _, id := range s.Metadata.ConfState.Learners {
			if id == r.id {
				r.logger.Errorf("%x can't become learner when restores snapshot [index: %d, term: %d]", r.id, s.Metadata.Index, s.Metadata.Term)
				return false
			}
		}
	}

	r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] starts to restore snapshot [index: %d, term: %d]",
		r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term)

	r.raftLog.restore(s)
	r.prs = make(map[uint64]*Progress)
	r.learnerPrs = make(map[uint64]*Progress)
	r.restoreNode(s.Metadata.ConfState.Nodes, false)
	r.restoreNode(s.Metadata.ConfState.Learners, true)
	return true
}

// restore recovers the state machine from a snapshot. It restores the log and the
// configuration of state machine.

restore recovers the state machine from a snapshot . It restores the log and the configuration of state machine .


func (r *raft) promotable() bool {
	_, ok := r.prs[r.id]
	return ok
}

func ( r * raft ) promotable ( ) bool { _ , ok := r . prs [ r . id ] return ok }

// promotable indicates whether state machine can be promoted to leader,
// which is true when its own id is in progress list.

promotable indicates whether state machine can be promoted to leader which is true when its own id is in progress list .


func (r *raft) checkQuorumActive() bool {
	var act int

	r.forEachProgress(func(id uint64, pr *Progress) {
		if id == r.id { // self is always active
			act++
			return
		}

		if pr.RecentActive && !pr.IsLearner {
			act++
		}

		pr.RecentActive = false
	})

	return act >= r.quorum()
}

func
(
r
*
raft
)
checkQuorumActive
(
)
bool
{
var
act
int
r
.
forEachProgress
(
func
(
id
uint64
,
pr
*
Progress
)
{
if
id
==
r
.
id
{
// self is always active
act
++
return
}
if
pr
.
RecentActive
&&
!
pr
.
IsLearner
{
act
++
}
pr
.
RecentActive
=
false
}
)
return
act
>=
r
.
quorum
(
)
}

// checkQuorumActive returns true if the quorum is active from
// the view of the local raft state machine. Otherwise, it returns
// false.
// checkQuorumActive also resets all RecentActive to false.

checkQuorumActive returns true if the quorum is active from the view of the local raft state machine . Otherwise it returns false . checkQuorumActive also resets all RecentActive to false .


func (r *raft) increaseUncommittedSize(ents []pb.Entry) bool {
	var s uint64
	for _, e := range ents {
		s += uint64(PayloadSize(e))
	}

	if r.uncommittedSize > 0 && r.uncommittedSize+s > r.maxUncommittedSize {
		// If the uncommitted tail of the Raft log is empty, allow any size
		// proposal. Otherwise, limit the size of the uncommitted tail of the
		// log and drop any proposal that would push the size over the limit.
		return false
	}
	r.uncommittedSize += s
	return true
}

// increaseUncommittedSize computes the size of the proposed entries and
// determines whether they would push leader over its maxUncommittedSize limit.
// If the new entries would exceed the limit, the method returns false. If not,
// the increase in uncommitted entry size is recorded and the method returns
// true.

increaseUncommittedSize computes the size of the proposed entries and determines whether they would push leader over its maxUncommittedSize limit . If the new entries would exceed the limit the method returns false . If not the increase in uncommitted entry size is recorded and the method returns true .


func (r *raft) reduceUncommittedSize(ents []pb.Entry) {
	if r.uncommittedSize == 0 {
		// Fast-path for followers, who do not track or enforce the limit.
		return
	}

	var s uint64
	for _, e := range ents {
		s += uint64(PayloadSize(e))
	}
	if s > r.uncommittedSize {
		// uncommittedSize may underestimate the size of the uncommitted Raft
		// log tail but will never overestimate it. Saturate at 0 instead of
		// allowing overflow.
		r.uncommittedSize = 0
	} else {
		r.uncommittedSize -= s
	}
}

// reduceUncommittedSize accounts for the newly committed entries by decreasing
// the uncommitted entry size limit.

reduceUncommittedSize accounts for the newly committed entries by decreasing the uncommitted entry size limit .


func newPeriodic(lg *zap.Logger, clock clockwork.Clock, h time.Duration, rg RevGetter, c Compactable) *Periodic {
	pc := &Periodic{
		lg:     lg,
		clock:  clock,
		period: h,
		rg:     rg,
		c:      c,
		revs:   make([]int64, 0),
	}
	pc.ctx, pc.cancel = context.WithCancel(context.Background())
	return pc
}

func newPeriodic ( lg * zap . Logger , clock clockwork . Clock , h time . Duration , rg RevGetter , c Compactable ) * Periodic { pc := & Periodic { lg : lg , clock : clock , period : h , rg : rg , c : c , revs : make ( [ ] int64 , 0 ) , } pc . ctx , pc . cancel = context . WithCancel ( context . Background ( ) ) return pc }

// newPeriodic creates a new instance of Periodic compactor that purges
// the log older than h Duration.

newPeriodic creates a new instance of Periodic compactor that purges the log older than h Duration .


func (pc *Periodic) Pause() {
	pc.mu.Lock()
	pc.paused = true
	pc.mu.Unlock()
}

func ( pc * Periodic ) Pause ( ) { pc . mu . Lock ( ) pc . paused = true pc . mu . Unlock ( ) }


func (pc *Periodic) Resume() {
	pc.mu.Lock()
	pc.paused = false
	pc.mu.Unlock()
}

func ( pc * Periodic ) Resume ( ) { pc . mu . Lock ( ) pc . paused = false pc . mu . Unlock ( ) }


func (m *Mutex) Lock(ctx context.Context) error {
	s := m.s
	client := m.s.Client()

	m.myKey = fmt.Sprintf("%s%x", m.pfx, s.Lease())
	cmp := v3.Compare(v3.CreateRevision(m.myKey), "=", 0)
	// put self in lock waiters via myKey; oldest waiter holds lock
	put := v3.OpPut(m.myKey, "", v3.WithLease(s.Lease()))
	// reuse key in case this session already holds the lock
	get := v3.OpGet(m.myKey)
	// fetch current holder to complete uncontended path with only one RPC
	getOwner := v3.OpGet(m.pfx, v3.WithFirstCreate()...)
	resp, err := client.Txn(ctx).If(cmp).Then(put, getOwner).Else(get, getOwner).Commit()
	if err != nil {
		return err
	}
	m.myRev = resp.Header.Revision
	if !resp.Succeeded {
		m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision
	}
	// if no key on prefix / the minimum rev is key, already hold the lock
	ownerKey := resp.Responses[1].GetResponseRange().Kvs
	if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev {
		m.hdr = resp.Header
		return nil
	}

	// wait for deletion revisions prior to myKey
	hdr, werr := waitDeletes(ctx, client, m.pfx, m.myRev-1)
	// release lock key if wait failed
	if werr != nil {
		m.Unlock(client.Ctx())
	} else {
		m.hdr = hdr
	}
	return werr
}

// Lock locks the mutex with a cancelable context. If the context is canceled
// while trying to acquire the lock, the mutex tries to clean its stale lock entry.

Lock locks the mutex with a cancelable context . If the context is canceled while trying to acquire the lock the mutex tries to clean its stale lock entry .


func NewLocker(s *Session, pfx string) sync.Locker {
	return &lockerMutex{NewMutex(s, pfx)}
}

func NewLocker ( s * Session , pfx string ) sync . Locker { return & lockerMutex { NewMutex ( s , pfx ) } }

// NewLocker creates a sync.Locker backed by an etcd mutex.

NewLocker creates a sync . Locker backed by an etcd mutex .


func NewFIFOScheduler() Scheduler {
	f := &fifo{
		resume: make(chan struct{}, 1),
		donec:  make(chan struct{}, 1),
	}
	f.finishCond = sync.NewCond(&f.mu)
	f.ctx, f.cancel = context.WithCancel(context.Background())
	go f.run()
	return f
}

func NewFIFOScheduler ( ) Scheduler { f := & fifo { resume : make ( chan struct { } , 1 ) , donec : make ( chan struct { } , 1 ) , } f . finishCond = sync . NewCond ( & f . mu ) f . ctx , f . cancel = context . WithCancel ( context . Background ( ) ) go f . run ( ) return f }

// NewFIFOScheduler returns a Scheduler that schedules jobs in FIFO
// order sequentially

NewFIFOScheduler returns a Scheduler that schedules jobs in FIFO order sequentially


func (f *fifo) Schedule(j Job) {
	f.mu.Lock()
	defer f.mu.Unlock()

	if f.cancel == nil {
		panic("schedule: schedule to stopped scheduler")
	}

	if len(f.pendings) == 0 {
		select {
		case f.resume <- struct{}{}:
		default:
		}
	}
	f.pendings = append(f.pendings, j)
}

func ( f * fifo ) Schedule ( j Job ) { f . mu . Lock ( ) defer f . mu . Unlock ( ) if f . cancel == nil { panic ( " " ) } if len ( f . pendings ) == 0 { select { case f . resume <- struct { } { } : default : } } f . pendings = append ( f . pendings , j ) }

// Schedule schedules a job that will be ran in FIFO order sequentially.

Schedule schedules a job that will be ran in FIFO order sequentially .


func (f *fifo) Stop() {
	f.mu.Lock()
	f.cancel()
	f.cancel = nil
	f.mu.Unlock()
	<-f.donec
}

func ( f * fifo ) Stop ( ) { f . mu . Lock ( ) f . cancel ( ) f . cancel = nil f . mu . Unlock ( ) <- f . donec }

// Stop stops the scheduler and cancels all pending jobs.

Stop stops the scheduler and cancels all pending jobs .


func NewServer(
	lg *zap.Logger,
	network string,
	address string,
) *Server {
	return &Server{
		lg:                         lg,
		network:                    network,
		address:                    address,
		last:                       rpcpb.Operation_NOT_STARTED,
		advertiseClientPortToProxy: make(map[int]proxy.Server),
		advertisePeerPortToProxy:   make(map[int]proxy.Server),
	}
}

func NewServer ( lg * zap . Logger , network string , address string , ) * Server { return & Server { lg : lg , network : network , address : address , last : rpcpb . Operation_NOT_STARTED , advertiseClientPortToProxy : make ( map [ int ] proxy . Server ) , advertisePeerPortToProxy : make ( map [ int ] proxy . Server ) , } }

// NewServer returns a new agent server.


func (srv *Server) StartServe() error {
	var err error
	srv.ln, err = net.Listen(srv.network, srv.address)
	if err != nil {
		return err
	}

	var opts []grpc.ServerOption
	opts = append(opts, grpc.MaxRecvMsgSize(int(maxRequestBytes+grpcOverheadBytes)))
	opts = append(opts, grpc.MaxSendMsgSize(maxSendBytes))
	opts = append(opts, grpc.MaxConcurrentStreams(maxStreams))
	srv.grpcServer = grpc.NewServer(opts...)

	rpcpb.RegisterTransportServer(srv.grpcServer, srv)

	srv.lg.Info(
		"gRPC server started",
		zap.String("address", srv.address),
		zap.String("listener-address", srv.ln.Addr().String()),
	)
	err = srv.grpcServer.Serve(srv.ln)
	if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
		srv.lg.Info(
			"gRPC server is shut down",
			zap.String("address", srv.address),
			zap.Error(err),
		)
	} else {
		srv.lg.Warn(
			"gRPC server returned with error",
			zap.String("address", srv.address),
			zap.Error(err),
		)
	}
	return err
}

func ( srv * Server ) StartServe ( ) error { var err error srv . ln , err = net . Listen ( srv . network , srv . address ) if err != nil { return err } var opts [ ] grpc . ServerOption opts = append ( opts , grpc . MaxRecvMsgSize ( int ( maxRequestBytes + grpcOverheadBytes ) ) ) opts = append ( opts , grpc . MaxSendMsgSize ( maxSendBytes ) ) opts = append ( opts , grpc . MaxConcurrentStreams ( maxStreams ) ) srv . grpcServer = grpc . NewServer ( opts ... ) rpcpb . RegisterTransportServer ( srv . grpcServer , srv ) srv . lg . Info ( " " , zap . String ( " " , srv . address ) , zap . String ( " " , srv . ln . Addr ( ) . String ( ) ) , ) err = srv . grpcServer . Serve ( srv . ln ) if err != nil && strings . Contains ( err . Error ( ) , " " ) { srv . lg . Info ( " " , zap . String ( " " , srv . address ) , zap . Error ( err ) , ) } else { srv . lg . Warn ( " " , zap . String ( " " , srv . address ) , zap . Error ( err ) , ) } return err }

// StartServe starts serving agent server.

StartServe starts serving agent server .


func (srv *Server) Stop() {
	srv.lg.Info("gRPC server stopping", zap.String("address", srv.address))
	srv.grpcServer.Stop()
	srv.lg.Info("gRPC server stopped", zap.String("address", srv.address))
}

func ( srv * Server ) Stop ( ) { srv . lg . Info ( " " , zap . String ( " " , srv . address ) ) srv . grpcServer . Stop ( ) srv . lg . Info ( " " , zap . String ( " " , srv . address ) ) }


func (srv *Server) Transport(stream rpcpb.Transport_TransportServer) (err error) {
	errc := make(chan error)
	go func() {
		for {
			var req *rpcpb.Request
			req, err = stream.Recv()
			if err != nil {
				errc <- err
				// TODO: handle error and retry
				return
			}
			if req.Member != nil {
				srv.Member = req.Member
			}
			if req.Tester != nil {
				srv.Tester = req.Tester
			}

			var resp *rpcpb.Response
			resp, err = srv.handleTesterRequest(req)
			if err != nil {
				errc <- err
				// TODO: handle error and retry
				return
			}

			if err = stream.Send(resp); err != nil {
				errc <- err
				// TODO: handle error and retry
				return
			}
		}
	}()

	select {
	case err = <-errc:
	case <-stream.Context().Done():
		err = stream.Context().Err()
	}
	return err
}

func
(
srv
*
Server
)
Transport
(
stream
rpcpb
.
Transport_TransportServer
)
(
err
error
)
{
errc
:=
make
(
chan
error
)
go
func
(
)
{
for
{
var
req
*
rpcpb
.
Request
req
,
err
=
stream
.
Recv
(
)
if
err
!=
{
errc
<-
err
// TODO: handle error and retry
return
}
if
req
.
Member
!=
{
srv
.
Member
=
req
.
Member
}
if
req
.
Tester
!=
{
srv
.
Tester
=
req
.
Tester
}
var
resp
*
rpcpb
.
Response
resp
,
err
=
srv
.
handleTesterRequest
(
req
)
if
err
!=
{
errc
<-
err
// TODO: handle error and retry
return
}
if
err
=
stream
.
Send
(
resp
)
;
err
!=
{
errc
<-
err
// TODO: handle error and retry
return
}
}
}
(
)
select
{
case
err
=
<-
errc
:
case
<-
stream
.
Context
(
)
.
Done
(
)
:
err
=
stream
.
Context
(
)
.
Err
(
)
}
return
err
}

// Transport communicates with etcd tester.

Transport communicates with etcd tester .


func RegisterInterruptHandler(h InterruptHandler) {
	interruptRegisterMu.Lock()
	defer interruptRegisterMu.Unlock()
	interruptHandlers = append(interruptHandlers, h)
}

func RegisterInterruptHandler ( h InterruptHandler ) { interruptRegisterMu . Lock ( ) defer interruptRegisterMu . Unlock ( ) interruptHandlers = append ( interruptHandlers , h ) }

// RegisterInterruptHandler registers a new InterruptHandler. Handlers registered
// after interrupt handing was initiated will not be executed.

RegisterInterruptHandler registers a new InterruptHandler . Handlers registered after interrupt handing was initiated will not be executed .


func HandleInterrupts(lg *zap.Logger) {
	notifier := make(chan os.Signal, 1)
	signal.Notify(notifier, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		sig := <-notifier

		interruptRegisterMu.Lock()
		ihs := make([]InterruptHandler, len(interruptHandlers))
		copy(ihs, interruptHandlers)
		interruptRegisterMu.Unlock()

		interruptExitMu.Lock()

		if lg != nil {
			lg.Info("received signal; shutting down", zap.String("signal", sig.String()))
		} else {
			plog.Noticef("received %v signal, shutting down...", sig)
		}

		for _, h := range ihs {
			h()
		}
		signal.Stop(notifier)
		pid := syscall.Getpid()
		// exit directly if it is the "init" process, since the kernel will not help to kill pid 1.
		if pid == 1 {
			os.Exit(0)
		}
		setDflSignal(sig.(syscall.Signal))
		syscall.Kill(pid, sig.(syscall.Signal))
	}()
}

// HandleInterrupts calls the handler functions on receiving a SIGINT or SIGTERM.

HandleInterrupts calls the handler functions on receiving a SIGINT or SIGTERM .


func OpGet(key string, opts ...OpOption) Op {
	// WithPrefix and WithFromKey are not supported together
	if isWithPrefix(opts) && isWithFromKey(opts) {
		panic("`WithPrefix` and `WithFromKey` cannot be set at the same time, choose one")
	}
	ret := Op{t: tRange, key: []byte(key)}
	ret.applyOpts(opts)
	return ret
}

// OpGet returns "get" operation based on given key and operation options.

OpGet returns get operation based on given key and operation options .


func OpDelete(key string, opts ...OpOption) Op {
	// WithPrefix and WithFromKey are not supported together
	if isWithPrefix(opts) && isWithFromKey(opts) {
		panic("`WithPrefix` and `WithFromKey` cannot be set at the same time, choose one")
	}
	ret := Op{t: tDeleteRange, key: []byte(key)}
	ret.applyOpts(opts)
	switch {
	case ret.leaseID != 0:
		panic("unexpected lease in delete")
	case ret.limit != 0:
		panic("unexpected limit in delete")
	case ret.rev != 0:
		panic("unexpected revision in delete")
	case ret.sort != nil:
		panic("unexpected sort in delete")
	case ret.serializable:
		panic("unexpected serializable in delete")
	case ret.countOnly:
		panic("unexpected countOnly in delete")
	case ret.minModRev != 0, ret.maxModRev != 0:
		panic("unexpected mod revision filter in delete")
	case ret.minCreateRev != 0, ret.maxCreateRev != 0:
		panic("unexpected create revision filter in delete")
	case ret.filterDelete, ret.filterPut:
		panic("unexpected filter in delete")
	case ret.createdNotify:
		panic("unexpected createdNotify in delete")
	}
	return ret
}

// OpDelete returns "delete" operation based on given key and operation options.

OpDelete returns delete operation based on given key and operation options .


func OpPut(key, val string, opts ...OpOption) Op {
	ret := Op{t: tPut, key: []byte(key), val: []byte(val)}
	ret.applyOpts(opts)
	switch {
	case ret.end != nil:
		panic("unexpected range in put")
	case ret.limit != 0:
		panic("unexpected limit in put")
	case ret.rev != 0:
		panic("unexpected revision in put")
	case ret.sort != nil:
		panic("unexpected sort in put")
	case ret.serializable:
		panic("unexpected serializable in put")
	case ret.countOnly:
		panic("unexpected countOnly in put")
	case ret.minModRev != 0, ret.maxModRev != 0:
		panic("unexpected mod revision filter in put")
	case ret.minCreateRev != 0, ret.maxCreateRev != 0:
		panic("unexpected create revision filter in put")
	case ret.filterDelete, ret.filterPut:
		panic("unexpected filter in put")
	case ret.createdNotify:
		panic("unexpected createdNotify in put")
	}
	return ret
}

func OpPut ( key , val string , opts ... OpOption ) Op { ret := Op { t : tPut , key : [ ] byte ( key ) , val : [ ] byte ( val ) } ret . applyOpts ( opts ) switch { case ret . end != nil : panic ( " " ) case ret . limit != 0 : panic ( " " ) case ret . rev != 0 : panic ( " " ) case ret . sort != nil : panic ( " " ) case ret . serializable : panic ( " " ) case ret . countOnly : panic ( " " ) case ret . minModRev != 0 , ret . maxModRev != 0 : panic ( " " ) case ret . minCreateRev != 0 , ret . maxCreateRev != 0 : panic ( " " ) case ret . filterDelete , ret . filterPut : panic ( " " ) case ret . createdNotify : panic ( " " ) } return ret }

// OpPut returns "put" operation based on given key-value and operation options.

OpPut returns put operation based on given key - value and operation options .


func OpTxn(cmps []Cmp, thenOps []Op, elseOps []Op) Op {
	return Op{t: tTxn, cmps: cmps, thenOps: thenOps, elseOps: elseOps}
}

func OpTxn ( cmps [ ] Cmp , thenOps [ ] Op , elseOps [ ] Op ) Op { return Op { t : tTxn , cmps : cmps , thenOps : thenOps , elseOps : elseOps } }

// OpTxn returns "txn" operation based on given transaction conditions.

OpTxn returns txn operation based on given transaction conditions .


func WithFromKey() OpOption {
	return func(op *Op) {
		if len(op.key) == 0 {
			op.key = []byte{0}
		}
		op.end = []byte("\x00")
	}
}

func WithFromKey ( ) OpOption { return func ( op * Op ) { if len ( op . key ) == 0 { op . key = [ ] byte { 0 } } op . end = [ ] byte ( " \x00 " ) } }

// WithFromKey specifies the range of 'Get', 'Delete', 'Watch' requests
// to be equal or greater than the key in the argument.

WithFromKey specifies the range of Get Delete Watch requests to be equal or greater than the key in the argument .


func withTop(target SortTarget, order SortOrder) []OpOption {
	return []OpOption{WithPrefix(), WithSort(target, order), WithLimit(1)}
}

func withTop ( target SortTarget , order SortOrder ) [ ] OpOption { return [ ] OpOption { WithPrefix ( ) , WithSort ( target , order ) , WithLimit ( 1 ) } }

// withTop gets the first key over the get's prefix given a sort order

withTop gets the first key over the get s prefix given a sort order


func Exist(dir string) bool {
	names, err := fileutil.ReadDir(dir, fileutil.WithExt(".wal"))
	if err != nil {
		return false
	}
	return len(names) != 0
}

func Exist ( dir string ) bool { names , err := fileutil . ReadDir ( dir , fileutil . WithExt ( " " ) ) if err != nil { return false } return len ( names ) != 0 }

// Exist returns true if there are any files in a given directory.

Exist returns true if there are any files in a given directory .


func searchIndex(lg *zap.Logger, names []string, index uint64) (int, bool) {
	for i := len(names) - 1; i >= 0; i-- {
		name := names[i]
		_, curIndex, err := parseWALName(name)
		if err != nil {
			if lg != nil {
				lg.Panic("failed to parse WAL file name", zap.String("path", name), zap.Error(err))
			} else {
				plog.Panicf("parse correct name should never fail: %v", err)
			}
		}
		if index >= curIndex {
			return i, true
		}
	}
	return -1, false
}

func searchIndex ( lg * zap . Logger , names [ ] string , index uint64 ) ( int , bool ) { for i := len ( names ) - 1 ; i >= 0 ; i -- { name := names [ i ] _ , curIndex , err := parseWALName ( name ) if err != nil { if lg != nil { lg . Panic ( " " , zap . String ( " " , name ) , zap . Error ( err ) ) } else { plog . Panicf ( " " , err ) } } if index >= curIndex { return i , true } } return - 1 , false }

// searchIndex returns the last array index of names whose raft index section is
// equal to or smaller than the given index.
// The given names MUST be sorted.

searchIndex returns the last array index of names whose raft index section is equal to or smaller than the given index . The given names MUST be sorted .


func isValidSeq(lg *zap.Logger, names []string) bool {
	var lastSeq uint64
	for _, name := range names {
		curSeq, _, err := parseWALName(name)
		if err != nil {
			if lg != nil {
				lg.Panic("failed to parse WAL file name", zap.String("path", name), zap.Error(err))
			} else {
				plog.Panicf("parse correct name should never fail: %v", err)
			}
		}
		if lastSeq != 0 && lastSeq != curSeq-1 {
			return false
		}
		lastSeq = curSeq
	}
	return true
}

func isValidSeq ( lg * zap . Logger , names [ ] string ) bool { var lastSeq uint64 for _ , name := range names { curSeq , _ , err := parseWALName ( name ) if err != nil { if lg != nil { lg . Panic ( " " , zap . String ( " " , name ) , zap . Error ( err ) ) } else { plog . Panicf ( " " , err ) } } if lastSeq != 0 && lastSeq != curSeq - 1 { return false } lastSeq = curSeq } return true }

// names should have been sorted based on sequence number.
// isValidSeq checks whether seq increases continuously.

names should have been sorted based on sequence number . isValidSeq checks whether seq increases continuously .


func NewListener(addr, scheme string, tlsinfo *TLSInfo) (l net.Listener, err error) {
	if l, err = newListener(addr, scheme); err != nil {
		return nil, err
	}
	return wrapTLS(scheme, tlsinfo, l)
}

func NewListener ( addr , scheme string , tlsinfo * TLSInfo ) ( l net . Listener , err error ) { if l , err = newListener ( addr , scheme ) ; err != nil { return nil , err } return wrapTLS ( scheme , tlsinfo , l ) }


func (info TLSInfo) cafiles() []string {
	cs := make([]string, 0)
	if info.TrustedCAFile != "" {
		cs = append(cs, info.TrustedCAFile)
	}
	return cs
}

func ( info TLSInfo ) cafiles ( ) [ ] string { cs := make ( [ ] string , 0 ) if info . TrustedCAFile != " " { cs = append ( cs , info . TrustedCAFile ) } return cs }

// cafiles returns a list of CA file paths.

cafiles returns a list of CA file paths .


func (info TLSInfo) ServerConfig() (*tls.Config, error) {
	cfg, err := info.baseConfig()
	if err != nil {
		return nil, err
	}

	cfg.ClientAuth = tls.NoClientCert
	if info.TrustedCAFile != "" || info.ClientCertAuth {
		cfg.ClientAuth = tls.RequireAndVerifyClientCert
	}

	cs := info.cafiles()
	if len(cs) > 0 {
		cp, err := tlsutil.NewCertPool(cs)
		if err != nil {
			return nil, err
		}
		cfg.ClientCAs = cp
	}

	// "h2" NextProtos is necessary for enabling HTTP2 for go's HTTP server
	cfg.NextProtos = []string{"h2"}

	return cfg, nil
}

// ServerConfig generates a tls.Config object for use by an HTTP server.

ServerConfig generates a tls . Config object for use by an HTTP server .


func (info TLSInfo) ClientConfig() (*tls.Config, error) {
	var cfg *tls.Config
	var err error

	if !info.Empty() {
		cfg, err = info.baseConfig()
		if err != nil {
			return nil, err
		}
	} else {
		cfg = &tls.Config{ServerName: info.ServerName}
	}
	cfg.InsecureSkipVerify = info.InsecureSkipVerify

	cs := info.cafiles()
	if len(cs) > 0 {
		cfg.RootCAs, err = tlsutil.NewCertPool(cs)
		if err != nil {
			return nil, err
		}
	}

	if info.selfCert {
		cfg.InsecureSkipVerify = true
	}

	if info.EmptyCN {
		hasNonEmptyCN := false
		cn := ""
		tlsutil.NewCert(info.CertFile, info.KeyFile, func(certPEMBlock []byte, keyPEMBlock []byte) (tls.Certificate, error) {
			var block *pem.Block
			block, _ = pem.Decode(certPEMBlock)
			cert, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				return tls.Certificate{}, err
			}
			if len(cert.Subject.CommonName) != 0 {
				hasNonEmptyCN = true
				cn = cert.Subject.CommonName
			}
			return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
		})
		if hasNonEmptyCN {
			return nil, fmt.Errorf("cert has non empty Common Name (%s)", cn)
		}
	}

	return cfg, nil
}

func ( info TLSInfo ) ClientConfig ( ) ( * tls . Config , error ) { var cfg * tls . Config var err error if ! info . Empty ( ) { cfg , err = info . baseConfig ( ) if err != nil { return nil , err } } else { cfg = & tls . Config { ServerName : info . ServerName } } cfg . InsecureSkipVerify = info . InsecureSkipVerify cs := info . cafiles ( ) if len ( cs ) > 0 { cfg . RootCAs , err = tlsutil . NewCertPool ( cs ) if err != nil { return nil , err } } if info . selfCert { cfg . InsecureSkipVerify = true } if info . EmptyCN { hasNonEmptyCN := false cn := " " tlsutil . NewCert ( info . CertFile , info . KeyFile , func ( certPEMBlock [ ] byte , keyPEMBlock [ ] byte ) ( tls . Certificate , error ) { var block * pem . Block block , _ = pem . Decode ( certPEMBlock ) cert , err := x509 . ParseCertificate ( block . Bytes ) if err != nil { return tls . Certificate { } , err } if len ( cert . Subject . CommonName ) != 0 { hasNonEmptyCN = true cn = cert . Subject . CommonName } return tls . X509KeyPair ( certPEMBlock , keyPEMBlock ) } ) if hasNonEmptyCN { return nil , fmt . Errorf ( " " , cn ) } } return cfg , nil }

// ClientConfig generates a tls.Config object for use by an HTTP client.

ClientConfig generates a tls . Config object for use by an HTTP client .


func newTLSKeepaliveListener(inner net.Listener, config *tls.Config) net.Listener {
	l := &tlsKeepaliveListener{}
	l.Listener = inner
	l.config = config
	return l
}

func newTLSKeepaliveListener ( inner net . Listener , config * tls . Config ) net . Listener { l := & tlsKeepaliveListener { } l . Listener = inner l . config = config return l }

// NewListener creates a Listener which accepts connections from an inner
// Listener and wraps each connection with Server.
// The configuration config must be non-nil and must have
// at least one certificate.

NewListener creates a Listener which accepts connections from an inner Listener and wraps each connection with Server . The configuration config must be non - nil and must have at least one certificate .


func (s *EtcdServer) applyV2Request(r *RequestV2) Response {
	defer warnOfExpensiveRequest(s.getLogger(), time.Now(), r, nil, nil)

	switch r.Method {
	case "POST":
		return s.applyV2.Post(r)
	case "PUT":
		return s.applyV2.Put(r)
	case "DELETE":
		return s.applyV2.Delete(r)
	case "QGET":
		return s.applyV2.QGet(r)
	case "SYNC":
		return s.applyV2.Sync(r)
	default:
		// This should never be reached, but just in case:
		return Response{Err: ErrUnknownMethod}
	}
}

// applyV2Request interprets r as a call to v2store.X
// and returns a Response interpreted from v2store.Event

applyV2Request interprets r as a call to v2store . X and returns a Response interpreted from v2store . Event


func NewRoleCommand() *cobra.Command {
	ac := &cobra.Command{
		Use:   "role <subcommand>",
		Short: "Role related commands",
	}

	ac.AddCommand(newRoleAddCommand())
	ac.AddCommand(newRoleDeleteCommand())
	ac.AddCommand(newRoleGetCommand())
	ac.AddCommand(newRoleListCommand())
	ac.AddCommand(newRoleGrantPermissionCommand())
	ac.AddCommand(newRoleRevokePermissionCommand())

	return ac
}

func NewRoleCommand ( ) * cobra . Command { ac := & cobra . Command { Use : " " , Short : " " , } ac . AddCommand ( newRoleAddCommand ( ) ) ac . AddCommand ( newRoleDeleteCommand ( ) ) ac . AddCommand ( newRoleGetCommand ( ) ) ac . AddCommand ( newRoleListCommand ( ) ) ac . AddCommand ( newRoleGrantPermissionCommand ( ) ) ac . AddCommand ( newRoleRevokePermissionCommand ( ) ) return ac }

// NewRoleCommand returns the cobra command for "role".

NewRoleCommand returns the cobra command for role .


func roleAddCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, fmt.Errorf("role add command requires role name as its argument"))
	}

	resp, err := mustClientFromCmd(cmd).Auth.RoleAdd(context.TODO(), args[0])
	if err != nil {
		ExitWithError(ExitError, err)
	}

	display.RoleAdd(args[0], *resp)
}

func roleAddCommandFunc ( cmd * cobra . Command , args [ ] string ) { if len ( args ) != 1 { ExitWithError ( ExitBadArgs , fmt . Errorf ( " " ) ) } resp , err := mustClientFromCmd ( cmd ) . Auth . RoleAdd ( context . TODO ( ) , args [ 0 ] ) if err != nil { ExitWithError ( ExitError , err ) } display . RoleAdd ( args [ 0 ] , * resp ) }

// roleAddCommandFunc executes the "role add" command.

roleAddCommandFunc executes the role add command .


func roleGetCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, fmt.Errorf("role get command requires role name as its argument"))
	}

	name := args[0]
	resp, err := mustClientFromCmd(cmd).Auth.RoleGet(context.TODO(), name)
	if err != nil {
		ExitWithError(ExitError, err)
	}

	display.RoleGet(name, *resp)
}

func roleGetCommandFunc ( cmd * cobra . Command , args [ ] string ) { if len ( args ) != 1 { ExitWithError ( ExitBadArgs , fmt . Errorf ( " " ) ) } name := args [ 0 ] resp , err := mustClientFromCmd ( cmd ) . Auth . RoleGet ( context . TODO ( ) , name ) if err != nil { ExitWithError ( ExitError , err ) } display . RoleGet ( name , * resp ) }

// roleGetCommandFunc executes the "role get" command.

roleGetCommandFunc executes the role get command .


func roleGrantPermissionCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) < 3 {
		ExitWithError(ExitBadArgs, fmt.Errorf("role grant command requires role name, permission type, and key [endkey] as its argument"))
	}

	perm, err := clientv3.StrToPermissionType(args[1])
	if err != nil {
		ExitWithError(ExitBadArgs, err)
	}

	key, rangeEnd := permRange(args[2:])
	resp, err := mustClientFromCmd(cmd).Auth.RoleGrantPermission(context.TODO(), args[0], key, rangeEnd, perm)
	if err != nil {
		ExitWithError(ExitError, err)
	}

	display.RoleGrantPermission(args[0], *resp)
}

func roleGrantPermissionCommandFunc ( cmd * cobra . Command , args [ ] string ) { if len ( args ) < 3 { ExitWithError ( ExitBadArgs , fmt . Errorf ( " " ) ) } perm , err := clientv3 . StrToPermissionType ( args [ 1 ] ) if err != nil { ExitWithError ( ExitBadArgs , err ) } key , rangeEnd := permRange ( args [ 2 : ] ) resp , err := mustClientFromCmd ( cmd ) . Auth . RoleGrantPermission ( context . TODO ( ) , args [ 0 ] , key , rangeEnd , perm ) if err != nil { ExitWithError ( ExitError , err ) } display . RoleGrantPermission ( args [ 0 ] , * resp ) }

// roleGrantPermissionCommandFunc executes the "role grant-permission" command.

roleGrantPermissionCommandFunc executes the role grant - permission command .


func roleRevokePermissionCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		ExitWithError(ExitBadArgs, fmt.Errorf("role revoke-permission command requires role name and key [endkey] as its argument"))
	}

	key, rangeEnd := permRange(args[1:])
	resp, err := mustClientFromCmd(cmd).Auth.RoleRevokePermission(context.TODO(), args[0], key, rangeEnd)
	if err != nil {
		ExitWithError(ExitError, err)
	}
	display.RoleRevokePermission(args[0], args[1], rangeEnd, *resp)
}

func roleRevokePermissionCommandFunc ( cmd * cobra . Command , args [ ] string ) { if len ( args ) < 2 { ExitWithError ( ExitBadArgs , fmt . Errorf ( " " ) ) } key , rangeEnd := permRange ( args [ 1 : ] ) resp , err := mustClientFromCmd ( cmd ) . Auth . RoleRevokePermission ( context . TODO ( ) , args [ 0 ] , key , rangeEnd ) if err != nil { ExitWithError ( ExitError , err ) } display . RoleRevokePermission ( args [ 0 ] , args [ 1 ] , rangeEnd , * resp ) }

// roleRevokePermissionCommandFunc executes the "role revoke-permission" command.

roleRevokePermissionCommandFunc executes the role revoke - permission command .


func NewCluster(t testing.TB, size int) *cluster {
	return newCluster(t, &ClusterConfig{Size: size})
}

func NewCluster ( t testing . TB , size int ) * cluster { return newCluster ( t , & ClusterConfig { Size : size } ) }

// NewCluster returns an unlaunched cluster of the given size which has been
// set to use static bootstrap.

NewCluster returns an unlaunched cluster of the given size which has been set to use static bootstrap .


func NewClusterByConfig(t testing.TB, cfg *ClusterConfig) *cluster {
	return newCluster(t, cfg)
}

func NewClusterByConfig ( t testing . TB , cfg * ClusterConfig ) * cluster { return newCluster ( t , cfg ) }

// NewClusterByConfig returns an unlaunched cluster defined by a cluster configuration


func (c *cluster) HTTPMembers() []client.Member {
	ms := []client.Member{}
	for _, m := range c.Members {
		pScheme := schemeFromTLSInfo(m.PeerTLSInfo)
		cScheme := schemeFromTLSInfo(m.ClientTLSInfo)
		cm := client.Member{Name: m.Name}
		for _, ln := range m.PeerListeners {
			cm.PeerURLs = append(cm.PeerURLs, pScheme+"://"+ln.Addr().String())
		}
		for _, ln := range m.ClientListeners {
			cm.ClientURLs = append(cm.ClientURLs, cScheme+"://"+ln.Addr().String())
		}
		ms = append(ms, cm)
	}
	return ms
}

func ( c * cluster ) HTTPMembers ( ) [ ] client . Member { ms := [ ] client . Member { } for _ , m := range c . Members { pScheme := schemeFromTLSInfo ( m . PeerTLSInfo ) cScheme := schemeFromTLSInfo ( m . ClientTLSInfo ) cm := client . Member { Name : m . Name } for _ , ln := range m . PeerListeners { cm . PeerURLs = append ( cm . PeerURLs , pScheme + " " + ln . Addr ( ) . String ( ) ) } for _ , ln := range m . ClientListeners { cm . ClientURLs = append ( cm . ClientURLs , cScheme + " " + ln . Addr ( ) . String ( ) ) } ms = append ( ms , cm ) } return ms }

// HTTPMembers returns a list of all active members as client.Members

HTTPMembers returns a list of all active members as client . Members


func (c *cluster) waitLeader(t testing.TB, membs []*member) int {
	possibleLead := make(map[uint64]bool)
	var lead uint64
	for _, m := range membs {
		possibleLead[uint64(m.s.ID())] = true
	}
	cc := MustNewHTTPClient(t, getMembersURLs(membs), nil)
	kapi := client.NewKeysAPI(cc)

	// ensure leader is up via linearizable get
	for {
		ctx, cancel := context.WithTimeout(context.Background(), 10*tickDuration+time.Second)
		_, err := kapi.Get(ctx, "0", &client.GetOptions{Quorum: true})
		cancel()
		if err == nil || strings.Contains(err.Error(), "Key not found") {
			break
		}
	}

	for lead == 0 || !possibleLead[lead] {
		lead = 0
		for _, m := range membs {
			select {
			case <-m.s.StopNotify():
				continue
			default:
			}
			if lead != 0 && lead != m.s.Lead() {
				lead = 0
				time.Sleep(10 * tickDuration)
				break
			}
			lead = m.s.Lead()
		}
	}

	for i, m := range membs {
		if uint64(m.s.ID()) == lead {
			return i
		}
	}

	return -1
}

// waitLeader waits until given members agree on the same leader.

waitLeader waits until given members agree on the same leader .


func (c *cluster) waitNoLeader(membs []*member) {
	noLeader := false
	for !noLeader {
		noLeader = true
		for _, m := range membs {
			select {
			case <-m.s.StopNotify():
				continue
			default:
			}
			if m.s.Lead() != 0 {
				noLeader = false
				time.Sleep(10 * tickDuration)
				break
			}
		}
	}
}

func ( c * cluster ) waitNoLeader ( membs [ ] * member ) { noLeader := false for ! noLeader { noLeader = true for _ , m := range membs { select { case <- m . s . StopNotify ( ) : continue default : } if m . s . Lead ( ) != 0 { noLeader = false time . Sleep ( 10 * tickDuration ) break } } } }

// waitNoLeader waits until given members lose leader.

waitNoLeader waits until given members lose leader .


func isMembersEqual(membs []client.Member, wmembs []client.Member) bool {
	sort.Sort(SortableMemberSliceByPeerURLs(membs))
	sort.Sort(SortableMemberSliceByPeerURLs(wmembs))
	for i := range membs {
		membs[i].ID = ""
	}
	return reflect.DeepEqual(membs, wmembs)
}

func isMembersEqual ( membs [ ] client . Member , wmembs [ ] client . Member ) bool { sort . Sort ( SortableMemberSliceByPeerURLs ( membs ) ) sort . Sort ( SortableMemberSliceByPeerURLs ( wmembs ) ) for i := range membs { membs [ i ] . ID = " " } return reflect . DeepEqual ( membs , wmembs ) }

// isMembersEqual checks whether two members equal except ID field.
// The given wmembs should always set ID field to empty string.

isMembersEqual checks whether two members equal except ID field . The given wmembs should always set ID field to empty string .


func (m *member) listenGRPC() error {
	// prefix with localhost so cert has right domain
	m.grpcAddr = "localhost:" + m.Name
	if m.useIP { // for IP-only TLS certs
		m.grpcAddr = "127.0.0.1:" + m.Name
	}
	l, err := transport.NewUnixListener(m.grpcAddr)
	if err != nil {
		return fmt.Errorf("listen failed on grpc socket %s (%v)", m.grpcAddr, err)
	}
	m.grpcBridge, err = newBridge(m.grpcAddr)
	if err != nil {
		l.Close()
		return err
	}
	m.grpcAddr = schemeFromTLSInfo(m.ClientTLSInfo) + "://" + m.grpcBridge.inaddr
	m.grpcListener = l
	return nil
}

// listenGRPC starts a grpc server over a unix domain socket on the member


func NewClientV3(m *member) (*clientv3.Client, error) {
	if m.grpcAddr == "" {
		return nil, fmt.Errorf("member not configured for grpc")
	}

	cfg := clientv3.Config{
		Endpoints:          []string{m.grpcAddr},
		DialTimeout:        5 * time.Second,
		DialOptions:        []grpc.DialOption{grpc.WithBlock()},
		MaxCallSendMsgSize: m.clientMaxCallSendMsgSize,
		MaxCallRecvMsgSize: m.clientMaxCallRecvMsgSize,
	}

	if m.ClientTLSInfo != nil {
		tls, err := m.ClientTLSInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		cfg.TLS = tls
	}
	if m.DialOptions != nil {
		cfg.DialOptions = append(cfg.DialOptions, m.DialOptions...)
	}
	return newClientV3(cfg)
}

func NewClientV3 ( m * member ) ( * clientv3 . Client , error ) { if m . grpcAddr == " " { return nil , fmt . Errorf ( " " ) } cfg := clientv3 . Config { Endpoints : [ ] string { m . grpcAddr } , DialTimeout : 5 * time . Second , DialOptions : [ ] grpc . DialOption { grpc . WithBlock ( ) } , MaxCallSendMsgSize : m . clientMaxCallSendMsgSize , MaxCallRecvMsgSize : m . clientMaxCallRecvMsgSize , } if m . ClientTLSInfo != nil { tls , err := m . ClientTLSInfo . ClientConfig ( ) if err != nil { return nil , err } cfg . TLS = tls } if m . DialOptions != nil { cfg . DialOptions = append ( cfg . DialOptions , m . DialOptions ... ) } return newClientV3 ( cfg ) }

// NewClientV3 creates a new grpc client connection to the member


func (m *member) Clone(t testing.TB) *member {
	mm := &member{}
	mm.ServerConfig = m.ServerConfig

	var err error
	clientURLStrs := m.ClientURLs.StringSlice()
	mm.ClientURLs, err = types.NewURLs(clientURLStrs)
	if err != nil {
		// this should never fail
		panic(err)
	}
	peerURLStrs := m.PeerURLs.StringSlice()
	mm.PeerURLs, err = types.NewURLs(peerURLStrs)
	if err != nil {
		// this should never fail
		panic(err)
	}
	clusterStr := m.InitialPeerURLsMap.String()
	mm.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr)
	if err != nil {
		// this should never fail
		panic(err)
	}
	mm.InitialClusterToken = m.InitialClusterToken
	mm.ElectionTicks = m.ElectionTicks
	mm.PeerTLSInfo = m.PeerTLSInfo
	mm.ClientTLSInfo = m.ClientTLSInfo
	return mm
}

func
(
m
*
member
)
Clone
(
t
testing
.
TB
)
*
member
{
mm
:=
&
member
{
}
mm
.
ServerConfig
=
m
.
ServerConfig
var
err
error
clientURLStrs
:=
m
.
ClientURLs
.
StringSlice
(
)
mm
.
ClientURLs
,
err
=
types
.
NewURLs
(
clientURLStrs
)
if
err
!=
{
// this should never fail
panic
(
err
)
}
peerURLStrs
:=
m
.
PeerURLs
.
StringSlice
(
)
mm
.
PeerURLs
,
err
=
types
.
NewURLs
(
peerURLStrs
)
if
err
!=
{
// this should never fail
panic
(
err
)
}
clusterStr
:=
m
.
InitialPeerURLsMap
.
String
(
)
mm
.
InitialPeerURLsMap
,
err
=
types
.
NewURLsMap
(
clusterStr
)
if
err
!=
{
// this should never fail
panic
(
err
)
}
mm
.
InitialClusterToken
=
m
.
InitialClusterToken
mm
.
ElectionTicks
=
m
.
ElectionTicks
mm
.
PeerTLSInfo
=
m
.
PeerTLSInfo
mm
.
ClientTLSInfo
=
m
.
ClientTLSInfo
return
mm
}

// Clone returns a member with the same server configuration. The returned
// member will not set PeerListeners and ClientListeners.

Clone returns a member with the same server configuration . The returned member will not set PeerListeners and ClientListeners .


func (m *member) Close() {
	if m.grpcBridge != nil {
		m.grpcBridge.Close()
		m.grpcBridge = nil
	}
	if m.serverClient != nil {
		m.serverClient.Close()
		m.serverClient = nil
	}
	if m.grpcServer != nil {
		m.grpcServer.Stop()
		m.grpcServer.GracefulStop()
		m.grpcServer = nil
		m.grpcServerPeer.Stop()
		m.grpcServerPeer.GracefulStop()
		m.grpcServerPeer = nil
	}
	m.s.HardStop()
	for _, f := range m.serverClosers {
		f()
	}
}

func ( m * member ) Close ( ) { if m . grpcBridge != nil { m . grpcBridge . Close ( ) m . grpcBridge = nil } if m . serverClient != nil { m . serverClient . Close ( ) m . serverClient = nil } if m . grpcServer != nil { m . grpcServer . Stop ( ) m . grpcServer . GracefulStop ( ) m . grpcServer = nil m . grpcServerPeer . Stop ( ) m . grpcServerPeer . GracefulStop ( ) m . grpcServerPeer = nil } m . s . HardStop ( ) for _ , f := range m . serverClosers { f ( ) } }

// Close stops the member's etcdserver and closes its connections

Close stops the member s etcdserver and closes its connections


func (m *member) Stop(t testing.TB) {
	lg.Info(
		"stopping a member",
		zap.String("name", m.Name),
		zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()),
		zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()),
		zap.String("grpc-address", m.grpcAddr),
	)
	m.Close()
	m.serverClosers = nil
	lg.Info(
		"stopped a member",
		zap.String("name", m.Name),
		zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()),
		zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()),
		zap.String("grpc-address", m.grpcAddr),
	)
}

func ( m * member ) Stop ( t testing . TB ) { lg . Info ( " " , zap . String ( " " , m . Name ) , zap . Strings ( " " , m . PeerURLs . StringSlice ( ) ) , zap . Strings ( " " , m . ClientURLs . StringSlice ( ) ) , zap . String ( " " , m . grpcAddr ) , ) m . Close ( ) m . serverClosers = nil lg . Info ( " " , zap . String ( " " , m . Name ) , zap . Strings ( " " , m . PeerURLs . StringSlice ( ) ) , zap . Strings ( " " , m . ClientURLs . StringSlice ( ) ) , zap . String ( " " , m . grpcAddr ) , ) }

// Stop stops the member, but the data dir of the member is preserved.

Stop stops the member but the data dir of the member is preserved .


func checkLeaderTransition(m *member, oldLead uint64) uint64 {
	interval := time.Duration(m.s.Cfg.TickMs) * time.Millisecond
	for m.s.Lead() == 0 || (m.s.Lead() == oldLead) {
		time.Sleep(interval)
	}
	return m.s.Lead()
}

func checkLeaderTransition ( m * member , oldLead uint64 ) uint64 { interval := time . Duration ( m . s . Cfg . TickMs ) * time . Millisecond for m . s . Lead ( ) == 0 || ( m . s . Lead ( ) == oldLead ) { time . Sleep ( interval ) } return m . s . Lead ( ) }

// checkLeaderTransition waits for leader transition, returning the new leader ID.

checkLeaderTransition waits for leader transition returning the new leader ID .


func (m *member) Restart(t testing.TB) error {
	lg.Info(
		"restarting a member",
		zap.String("name", m.Name),
		zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()),
		zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()),
		zap.String("grpc-address", m.grpcAddr),
	)
	newPeerListeners := make([]net.Listener, 0)
	for _, ln := range m.PeerListeners {
		newPeerListeners = append(newPeerListeners, NewListenerWithAddr(t, ln.Addr().String()))
	}
	m.PeerListeners = newPeerListeners
	newClientListeners := make([]net.Listener, 0)
	for _, ln := range m.ClientListeners {
		newClientListeners = append(newClientListeners, NewListenerWithAddr(t, ln.Addr().String()))
	}
	m.ClientListeners = newClientListeners

	if m.grpcListener != nil {
		if err := m.listenGRPC(); err != nil {
			t.Fatal(err)
		}
	}

	err := m.Launch()
	lg.Info(
		"restarted a member",
		zap.String("name", m.Name),
		zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()),
		zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()),
		zap.String("grpc-address", m.grpcAddr),
		zap.Error(err),
	)
	return err
}

func ( m * member ) Restart ( t testing . TB ) error { lg . Info ( " " , zap . String ( " " , m . Name ) , zap . Strings ( " " , m . PeerURLs . StringSlice ( ) ) , zap . Strings ( " " , m . ClientURLs . StringSlice ( ) ) , zap . String ( " " , m . grpcAddr ) , ) newPeerListeners := make ( [ ] net . Listener , 0 ) for _ , ln := range m . PeerListeners { newPeerListeners = append ( newPeerListeners , NewListenerWithAddr ( t , ln . Addr ( ) . String ( ) ) ) } m . PeerListeners = newPeerListeners newClientListeners := make ( [ ] net . Listener , 0 ) for _ , ln := range m . ClientListeners { newClientListeners = append ( newClientListeners , NewListenerWithAddr ( t , ln . Addr ( ) . String ( ) ) ) } m . ClientListeners = newClientListeners if m . grpcListener != nil { if err := m . listenGRPC ( ) ; err != nil { t . Fatal ( err ) } } err := m . Launch ( ) lg . Info ( " " , zap . String ( " " , m . Name ) , zap . Strings ( " " , m . PeerURLs . StringSlice ( ) ) , zap . Strings ( " " , m . ClientURLs . StringSlice ( ) ) , zap . String ( " " , m . grpcAddr ) , zap . Error ( err ) , ) return err }

// Restart starts the member using the preserved data dir.

Restart starts the member using the preserved data dir .


func (m *member) Terminate(t testing.TB) {
	lg.Info(
		"terminating a member",
		zap.String("name", m.Name),
		zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()),
		zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()),
		zap.String("grpc-address", m.grpcAddr),
	)
	m.Close()
	if !m.keepDataDirTerminate {
		if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
			t.Fatal(err)
		}
	}
	lg.Info(
		"terminated a member",
		zap.String("name", m.Name),
		zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()),
		zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()),
		zap.String("grpc-address", m.grpcAddr),
	)
}

func ( m * member ) Terminate ( t testing . TB ) { lg . Info ( " " , zap . String ( " " , m . Name ) , zap . Strings ( " " , m . PeerURLs . StringSlice ( ) ) , zap . Strings ( " " , m . ClientURLs . StringSlice ( ) ) , zap . String ( " " , m . grpcAddr ) , ) m . Close ( ) if ! m . keepDataDirTerminate { if err := os . RemoveAll ( m . ServerConfig . DataDir ) ; err != nil { t . Fatal ( err ) } } lg . Info ( " " , zap . String ( " " , m . Name ) , zap . Strings ( " " , m . PeerURLs . StringSlice ( ) ) , zap . Strings ( " " , m . ClientURLs . StringSlice ( ) ) , zap . String ( " " , m . grpcAddr ) , ) }

// Terminate stops the member and removes the data dir.

Terminate stops the member and removes the data dir .