func isMemberBootstrapped(lg *zap.Logger, cl *membership.RaftCluster, member string, rt http.RoundTripper, timeout time.Duration) bool {
	rcl, err := getClusterFromRemotePeers(lg, getRemotePeerURLs(cl, member), timeout, false, rt)
	if err != nil {
		return false
	}
	id := cl.MemberByName(member).ID
	m := rcl.Member(id)
	if m == nil {
		return false
	}
	if len(m.ClientURLs) > 0 {
		return true
	}
	return false
}

func isMemberBootstrapped ( lg * zap . Logger , cl * membership . RaftCluster , member string , rt http . RoundTripper , timeout time . Duration ) bool { rcl , err := getClusterFromRemotePeers ( lg , getRemotePeerURLs ( cl , member ) , timeout , false , rt ) if err != nil { return false } id := cl . MemberByName ( member ) . ID m := rcl . Member ( id ) if m == nil { return false } if len ( m . ClientURLs ) > 0 { return true } return false }

// isMemberBootstrapped tries to check if the given member has been bootstrapped
// in the given cluster.

isMemberBootstrapped tries to check if the given member has been bootstrapped in the given cluster .


func GetClusterFromRemotePeers(lg *zap.Logger, urls []string, rt http.RoundTripper) (*membership.RaftCluster, error) {
	return getClusterFromRemotePeers(lg, urls, 10*time.Second, true, rt)
}

func GetClusterFromRemotePeers ( lg * zap . Logger , urls [ ] string , rt http . RoundTripper ) ( * membership . RaftCluster , error ) { return getClusterFromRemotePeers ( lg , urls , 10 * time . Second , true , rt ) }

// GetClusterFromRemotePeers takes a set of URLs representing etcd peers, and
// attempts to construct a Cluster by accessing the members endpoint on one of
// these URLs. The first URL to provide a response is used. If no URLs provide
// a response, or a Cluster cannot be successfully created from a received
// response, an error is returned.
// Each request has a 10-second timeout. Because the upper limit of TTL is 5s,
// 10 second is enough for building connection and finishing request.

GetClusterFromRemotePeers takes a set of URLs representing etcd peers and attempts to construct a Cluster by accessing the members endpoint on one of these URLs . The first URL to provide a response is used . If no URLs provide a response or a Cluster cannot be successfully created from a received response an error is returned . Each request has a 10 - second timeout . Because the upper limit of TTL is 5s 10 second is enough for building connection and finishing request .


func getClusterFromRemotePeers(lg *zap.Logger, urls []string, timeout time.Duration, logerr bool, rt http.RoundTripper) (*membership.RaftCluster, error) {
	cc := &http.Client{
		Transport: rt,
		Timeout:   timeout,
	}
	for _, u := range urls {
		addr := u + "/members"
		resp, err := cc.Get(addr)
		if err != nil {
			if logerr {
				if lg != nil {
					lg.Warn("failed to get cluster response", zap.String("address", addr), zap.Error(err))
				} else {
					plog.Warningf("could not get cluster response from %s: %v", u, err)
				}
			}
			continue
		}
		b, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			if logerr {
				if lg != nil {
					lg.Warn("failed to read body of cluster response", zap.String("address", addr), zap.Error(err))
				} else {
					plog.Warningf("could not read the body of cluster response: %v", err)
				}
			}
			continue
		}
		var membs []*membership.Member
		if err = json.Unmarshal(b, &membs); err != nil {
			if logerr {
				if lg != nil {
					lg.Warn("failed to unmarshal cluster response", zap.String("address", addr), zap.Error(err))
				} else {
					plog.Warningf("could not unmarshal cluster response: %v", err)
				}
			}
			continue
		}
		id, err := types.IDFromString(resp.Header.Get("X-Etcd-Cluster-ID"))
		if err != nil {
			if logerr {
				if lg != nil {
					lg.Warn(
						"failed to parse cluster ID",
						zap.String("address", addr),
						zap.String("header", resp.Header.Get("X-Etcd-Cluster-ID")),
						zap.Error(err),
					)
				} else {
					plog.Warningf("could not parse the cluster ID from cluster res: %v", err)
				}
			}
			continue
		}

		// check the length of membership members
		// if the membership members are present then prepare and return raft cluster
		// if membership members are not present then the raft cluster formed will be
		// an invalid empty cluster hence return failed to get raft cluster member(s) from the given urls error
		if len(membs) > 0 {
			return membership.NewClusterFromMembers(lg, "", id, membs), nil
		}
		return nil, fmt.Errorf("failed to get raft cluster member(s) from the given URLs")
	}
	return nil, fmt.Errorf("could not retrieve cluster information from the given URLs")
}

// If logerr is true, it prints out more error messages.

If logerr is true it prints out more error messages .


func getRemotePeerURLs(cl *membership.RaftCluster, local string) []string {
	us := make([]string, 0)
	for _, m := range cl.Members() {
		if m.Name == local {
			continue
		}
		us = append(us, m.PeerURLs...)
	}
	sort.Strings(us)
	return us
}

func getRemotePeerURLs ( cl * membership . RaftCluster , local string ) [ ] string { us := make ( [ ] string , 0 ) for _ , m := range cl . Members ( ) { if m . Name == local { continue } us = append ( us , m . PeerURLs ... ) } sort . Strings ( us ) return us }

// getRemotePeerURLs returns peer urls of remote members in the cluster. The
// returned list is sorted in ascending lexicographical order.

getRemotePeerURLs returns peer urls of remote members in the cluster . The returned list is sorted in ascending lexicographical order .


func getVersions(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) map[string]*version.Versions {
	members := cl.Members()
	vers := make(map[string]*version.Versions)
	for _, m := range members {
		if m.ID == local {
			cv := "not_decided"
			if cl.Version() != nil {
				cv = cl.Version().String()
			}
			vers[m.ID.String()] = &version.Versions{Server: version.Version, Cluster: cv}
			continue
		}
		ver, err := getVersion(lg, m, rt)
		if err != nil {
			if lg != nil {
				lg.Warn("failed to get version", zap.String("remote-member-id", m.ID.String()), zap.Error(err))
			} else {
				plog.Warningf("cannot get the version of member %s (%v)", m.ID, err)
			}
			vers[m.ID.String()] = nil
		} else {
			vers[m.ID.String()] = ver
		}
	}
	return vers
}

func getVersions ( lg * zap . Logger , cl * membership . RaftCluster , local types . ID , rt http . RoundTripper ) map [ string ] * version . Versions { members := cl . Members ( ) vers := make ( map [ string ] * version . Versions ) for _ , m := range members { if m . ID == local { cv := " " if cl . Version ( ) != nil { cv = cl . Version ( ) . String ( ) } vers [ m . ID . String ( ) ] = & version . Versions { Server : version . Version , Cluster : cv } continue } ver , err := getVersion ( lg , m , rt ) if err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , m . ID . String ( ) ) , zap . Error ( err ) ) } else { plog . Warningf ( " " , m . ID , err ) } vers [ m . ID . String ( ) ] = nil } else { vers [ m . ID . String ( ) ] = ver } } return vers }

// getVersions returns the versions of the members in the given cluster.
// The key of the returned map is the member's ID. The value of the returned map
// is the semver versions string, including server and cluster.
// If it fails to get the version of a member, the key will be nil.

getVersions returns the versions of the members in the given cluster . The key of the returned map is the member s ID . The value of the returned map is the semver versions string including server and cluster . If it fails to get the version of a member the key will be nil .


func decideClusterVersion(lg *zap.Logger, vers map[string]*version.Versions) *semver.Version {
	var cv *semver.Version
	lv := semver.Must(semver.NewVersion(version.Version))

	for mid, ver := range vers {
		if ver == nil {
			return nil
		}
		v, err := semver.NewVersion(ver.Server)
		if err != nil {
			if lg != nil {
				lg.Warn(
					"failed to parse server version of remote member",
					zap.String("remote-member-id", mid),
					zap.String("remote-member-version", ver.Server),
					zap.Error(err),
				)
			} else {
				plog.Errorf("cannot understand the version of member %s (%v)", mid, err)
			}
			return nil
		}
		if lv.LessThan(*v) {
			if lg != nil {
				lg.Warn(
					"leader found higher-versioned member",
					zap.String("local-member-version", lv.String()),
					zap.String("remote-member-id", mid),
					zap.String("remote-member-version", ver.Server),
				)
			} else {
				plog.Warningf("the local etcd version %s is not up-to-date", lv.String())
				plog.Warningf("member %s has a higher version %s", mid, ver.Server)
			}
		}
		if cv == nil {
			cv = v
		} else if v.LessThan(*cv) {
			cv = v
		}
	}
	return cv
}

func decideClusterVersion ( lg * zap . Logger , vers map [ string ] * version . Versions ) * semver . Version { var cv * semver . Version lv := semver . Must ( semver . NewVersion ( version . Version ) ) for mid , ver := range vers { if ver == nil { return nil } v , err := semver . NewVersion ( ver . Server ) if err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , mid ) , zap . String ( " " , ver . Server ) , zap . Error ( err ) , ) } else { plog . Errorf ( " " , mid , err ) } return nil } if lv . LessThan ( * v ) { if lg != nil { lg . Warn ( " " , zap . String ( " " , lv . String ( ) ) , zap . String ( " " , mid ) , zap . String ( " " , ver . Server ) , ) } else { plog . Warningf ( " " , lv . String ( ) ) plog . Warningf ( " " , mid , ver . Server ) } } if cv == nil { cv = v } else if v . LessThan ( * cv ) { cv = v } } return cv }

// decideClusterVersion decides the cluster version based on the versions map.
// The returned version is the min server version in the map, or nil if the min
// version in unknown.

decideClusterVersion decides the cluster version based on the versions map . The returned version is the min server version in the map or nil if the min version in unknown .


func getVersion(lg *zap.Logger, m *membership.Member, rt http.RoundTripper) (*version.Versions, error) {
	cc := &http.Client{
		Transport: rt,
	}
	var (
		err  error
		resp *http.Response
	)

	for _, u := range m.PeerURLs {
		addr := u + "/version"
		resp, err = cc.Get(addr)
		if err != nil {
			if lg != nil {
				lg.Warn(
					"failed to reach the peer URL",
					zap.String("address", addr),
					zap.String("remote-member-id", m.ID.String()),
					zap.Error(err),
				)
			} else {
				plog.Warningf("failed to reach the peerURL(%s) of member %s (%v)", u, m.ID, err)
			}
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			if lg != nil {
				lg.Warn(
					"failed to read body of response",
					zap.String("address", addr),
					zap.String("remote-member-id", m.ID.String()),
					zap.Error(err),
				)
			} else {
				plog.Warningf("failed to read out the response body from the peerURL(%s) of member %s (%v)", u, m.ID, err)
			}
			continue
		}
		var vers version.Versions
		if err = json.Unmarshal(b, &vers); err != nil {
			if lg != nil {
				lg.Warn(
					"failed to unmarshal response",
					zap.String("address", addr),
					zap.String("remote-member-id", m.ID.String()),
					zap.Error(err),
				)
			} else {
				plog.Warningf("failed to unmarshal the response body got from the peerURL(%s) of member %s (%v)", u, m.ID, err)
			}
			continue
		}
		return &vers, nil
	}
	return nil, err
}

func getVersion ( lg * zap . Logger , m * membership . Member , rt http . RoundTripper ) ( * version . Versions , error ) { cc := & http . Client { Transport : rt , } var ( err error resp * http . Response ) for _ , u := range m . PeerURLs { addr := u + " " resp , err = cc . Get ( addr ) if err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , addr ) , zap . String ( " " , m . ID . String ( ) ) , zap . Error ( err ) , ) } else { plog . Warningf ( " " , u , m . ID , err ) } continue } var b [ ] byte b , err = ioutil . ReadAll ( resp . Body ) resp . Body . Close ( ) if err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , addr ) , zap . String ( " " , m . ID . String ( ) ) , zap . Error ( err ) , ) } else { plog . Warningf ( " " , u , m . ID , err ) } continue } var vers version . Versions if err = json . Unmarshal ( b , & vers ) ; err != nil { if lg != nil { lg . Warn ( " " , zap . String ( " " , addr ) , zap . String ( " " , m . ID . String ( ) ) , zap . Error ( err ) , ) } else { plog . Warningf ( " " , u , m . ID , err ) } continue } return & vers , nil } return nil , err }

// getVersion returns the Versions of the given member via its
// peerURLs. Returns the last error if it fails to get the version.

getVersion returns the Versions of the given member via its peerURLs . Returns the last error if it fails to get the version .


func NewTimeoutDetector(maxDuration time.Duration) *TimeoutDetector {
	return &TimeoutDetector{
		maxDuration: maxDuration,
		records:     make(map[uint64]time.Time),
	}
}

func NewTimeoutDetector ( maxDuration time . Duration ) * TimeoutDetector { return & TimeoutDetector { maxDuration : maxDuration , records : make ( map [ uint64 ] time . Time ) , } }

// NewTimeoutDetector creates the TimeoutDetector.

NewTimeoutDetector creates the TimeoutDetector .


func (td *TimeoutDetector) Reset() {
	td.mu.Lock()
	defer td.mu.Unlock()

	td.records = make(map[uint64]time.Time)
}

func ( td * TimeoutDetector ) Reset ( ) { td . mu . Lock ( ) defer td . mu . Unlock ( ) td . records = make ( map [ uint64 ] time . Time ) }


func (td *TimeoutDetector) Observe(which uint64) (bool, time.Duration) {
	td.mu.Lock()
	defer td.mu.Unlock()

	ok := true
	now := time.Now()
	exceed := time.Duration(0)

	if pt, found := td.records[which]; found {
		exceed = now.Sub(pt) - td.maxDuration
		if exceed > 0 {
			ok = false
		}
	}
	td.records[which] = now
	return ok, exceed
}

func ( td * TimeoutDetector ) Observe ( which uint64 ) ( bool , time . Duration ) { td . mu . Lock ( ) defer td . mu . Unlock ( ) ok := true now := time . Now ( ) exceed := time . Duration ( 0 ) if pt , found := td . records [ which ] ; found { exceed = now . Sub ( pt ) - td . maxDuration if exceed > 0 { ok = false } } td . records [ which ] = now return ok , exceed }

// Observe observes an event for given id. It returns false and exceeded duration
// if the interval is longer than the expectation.

Observe observes an event for given id . It returns false and exceeded duration if the interval is longer than the expectation .


func NewPeerHandler(lg *zap.Logger, s etcdserver.ServerPeer) http.Handler {
	return newPeerHandler(lg, s.Cluster(), s.RaftHandler(), s.LeaseHandler())
}

func NewPeerHandler ( lg * zap . Logger , s etcdserver . ServerPeer ) http . Handler { return newPeerHandler ( lg , s . Cluster ( ) , s . RaftHandler ( ) , s . LeaseHandler ( ) ) }

// NewPeerHandler generates an http.Handler to handle etcd peer requests.

NewPeerHandler generates an http . Handler to handle etcd peer requests .


func (ki *keyIndex) put(lg *zap.Logger, main int64, sub int64) {
	rev := revision{main: main, sub: sub}

	if !rev.GreaterThan(ki.modified) {
		if lg != nil {
			lg.Panic(
				"'put' with an unexpected smaller revision",
				zap.Int64("given-revision-main", rev.main),
				zap.Int64("given-revision-sub", rev.sub),
				zap.Int64("modified-revision-main", ki.modified.main),
				zap.Int64("modified-revision-sub", ki.modified.sub),
			)
		} else {
			plog.Panicf("store.keyindex: put with unexpected smaller revision [%v / %v]", rev, ki.modified)
		}
	}
	if len(ki.generations) == 0 {
		ki.generations = append(ki.generations, generation{})
	}
	g := &ki.generations[len(ki.generations)-1]
	if len(g.revs) == 0 { // create a new key
		keysGauge.Inc()
		g.created = rev
	}
	g.revs = append(g.revs, rev)
	g.ver++
	ki.modified = rev
}

func
(
ki
*
keyIndex
)
put
(
lg
*
zap
.
Logger
,
main
int64
,
sub
int64
)
{
rev
:=
revision
{
main
:
main
,
sub
:
sub
}
if
!
rev
.
GreaterThan
(
ki
.
modified
)
{
if
lg
!=
{
lg
.
Panic
(
"
"
,
zap
.
Int64
(
"
"
,
rev
.
main
)
,
zap
.
Int64
(
"
"
,
rev
.
sub
)
,
zap
.
Int64
(
"
"
,
ki
.
modified
.
main
)
,
zap
.
Int64
(
"
"
,
ki
.
modified
.
sub
)
,
)
}
else
{
plog
.
Panicf
(
"
"
,
rev
,
ki
.
modified
)
}
}
if
len
(
ki
.
generations
)
==
0
{
ki
.
generations
=
append
(
ki
.
generations
,
generation
{
}
)
}
g
:=
&
ki
.
generations
[
len
(
ki
.
generations
)
1
]
if
len
(
g
.
revs
)
==
0
{
// create a new key
keysGauge
.
Inc
(
)
g
.
created
=
rev
}
g
.
revs
=
append
(
g
.
revs
,
rev
)
g
.
ver
++
ki
.
modified
=
rev
}


func (ki *keyIndex) tombstone(lg *zap.Logger, main int64, sub int64) error {
	if ki.isEmpty() {
		if lg != nil {
			lg.Panic(
				"'tombstone' got an unexpected empty keyIndex",
				zap.String("key", string(ki.key)),
			)
		} else {
			plog.Panicf("store.keyindex: unexpected tombstone on empty keyIndex %s", string(ki.key))
		}
	}
	if ki.generations[len(ki.generations)-1].isEmpty() {
		return ErrRevisionNotFound
	}
	ki.put(lg, main, sub)
	ki.generations = append(ki.generations, generation{})
	keysGauge.Dec()
	return nil
}

func ( ki * keyIndex ) tombstone ( lg * zap . Logger , main int64 , sub int64 ) error { if ki . isEmpty ( ) { if lg != nil { lg . Panic ( " " , zap . String ( " " , string ( ki . key ) ) , ) } else { plog . Panicf ( " " , string ( ki . key ) ) } } if ki . generations [ len ( ki . generations ) - 1 ] . isEmpty ( ) { return ErrRevisionNotFound } ki . put ( lg , main , sub ) ki . generations = append ( ki . generations , generation { } ) keysGauge . Dec ( ) return nil }

// tombstone puts a revision, pointing to a tombstone, to the keyIndex.
// It also creates a new empty generation in the keyIndex.
// It returns ErrRevisionNotFound when tombstone on an empty generation.

tombstone puts a revision pointing to a tombstone to the keyIndex . It also creates a new empty generation in the keyIndex . It returns ErrRevisionNotFound when tombstone on an empty generation .


func (ki *keyIndex) get(lg *zap.Logger, atRev int64) (modified, created revision, ver int64, err error) {
	if ki.isEmpty() {
		if lg != nil {
			lg.Panic(
				"'get' got an unexpected empty keyIndex",
				zap.String("key", string(ki.key)),
			)
		} else {
			plog.Panicf("store.keyindex: unexpected get on empty keyIndex %s", string(ki.key))
		}
	}
	g := ki.findGeneration(atRev)
	if g.isEmpty() {
		return revision{}, revision{}, 0, ErrRevisionNotFound
	}

	n := g.walk(func(rev revision) bool { return rev.main > atRev })
	if n != -1 {
		return g.revs[n], g.created, g.ver - int64(len(g.revs)-n-1), nil
	}

	return revision{}, revision{}, 0, ErrRevisionNotFound
}

func ( ki * keyIndex ) get ( lg * zap . Logger , atRev int64 ) ( modified , created revision , ver int64 , err error ) { if ki . isEmpty ( ) { if lg != nil { lg . Panic ( " " , zap . String ( " " , string ( ki . key ) ) , ) } else { plog . Panicf ( " " , string ( ki . key ) ) } } g := ki . findGeneration ( atRev ) if g . isEmpty ( ) { return revision { } , revision { } , 0 , ErrRevisionNotFound } n := g . walk ( func ( rev revision ) bool { return rev . main > atRev } ) if n != - 1 { return g . revs [ n ] , g . created , g . ver - int64 ( len ( g . revs ) - n - 1 ) , nil } return revision { } , revision { } , 0 , ErrRevisionNotFound }

// get gets the modified, created revision and version of the key that satisfies the given atRev.
// Rev must be higher than or equal to the given atRev.

get gets the modified created revision and version of the key that satisfies the given atRev . Rev must be higher than or equal to the given atRev .


func (ki *keyIndex) since(lg *zap.Logger, rev int64) []revision {
	if ki.isEmpty() {
		if lg != nil {
			lg.Panic(
				"'since' got an unexpected empty keyIndex",
				zap.String("key", string(ki.key)),
			)
		} else {
			plog.Panicf("store.keyindex: unexpected get on empty keyIndex %s", string(ki.key))
		}
	}
	since := revision{rev, 0}
	var gi int
	// find the generations to start checking
	for gi = len(ki.generations) - 1; gi > 0; gi-- {
		g := ki.generations[gi]
		if g.isEmpty() {
			continue
		}
		if since.GreaterThan(g.created) {
			break
		}
	}

	var revs []revision
	var last int64
	for ; gi < len(ki.generations); gi++ {
		for _, r := range ki.generations[gi].revs {
			if since.GreaterThan(r) {
				continue
			}
			if r.main == last {
				// replace the revision with a new one that has higher sub value,
				// because the original one should not be seen by external
				revs[len(revs)-1] = r
				continue
			}
			revs = append(revs, r)
			last = r.main
		}
	}
	return revs
}

// since returns revisions since the given rev. Only the revision with the
// largest sub revision will be returned if multiple revisions have the same
// main revision.

since returns revisions since the given rev . Only the revision with the largest sub revision will be returned if multiple revisions have the same main revision .


func (ki *keyIndex) keep(atRev int64, available map[revision]struct{}) {
	if ki.isEmpty() {
		return
	}

	genIdx, revIndex := ki.doCompact(atRev, available)
	g := &ki.generations[genIdx]
	if !g.isEmpty() {
		// remove any tombstone
		if revIndex == len(g.revs)-1 && genIdx != len(ki.generations)-1 {
			delete(available, g.revs[revIndex])
		}
	}
}

func
(
ki
*
keyIndex
)
keep
(
atRev
int64
,
available
map
[
revision
]
struct
{
}
)
{
if
ki
.
isEmpty
(
)
{
return
}
genIdx
,
revIndex
:=
ki
.
doCompact
(
atRev
,
available
)
g
:=
&
ki
.
generations
[
genIdx
]
if
!
g
.
isEmpty
(
)
{
// remove any tombstone
if
revIndex
==
len
(
g
.
revs
)
1
&&
genIdx
!=
len
(
ki
.
generations
)
1
{
delete
(
available
,
g
.
revs
[
revIndex
]
)
}
}
}

// keep finds the revision to be kept if compact is called at given atRev.

keep finds the revision to be kept if compact is called at given atRev .


func (ki *keyIndex) findGeneration(rev int64) *generation {
	lastg := len(ki.generations) - 1
	cg := lastg

	for cg >= 0 {
		if len(ki.generations[cg].revs) == 0 {
			cg--
			continue
		}
		g := ki.generations[cg]
		if cg != lastg {
			if tomb := g.revs[len(g.revs)-1].main; tomb <= rev {
				return nil
			}
		}
		if g.revs[0].main <= rev {
			return &ki.generations[cg]
		}
		cg--
	}
	return nil
}

func ( ki * keyIndex ) findGeneration ( rev int64 ) * generation { lastg := len ( ki . generations ) - 1 cg := lastg for cg >= 0 { if len ( ki . generations [ cg ] . revs ) == 0 { cg -- continue } g := ki . generations [ cg ] if cg != lastg { if tomb := g . revs [ len ( g . revs ) - 1 ] . main ; tomb <= rev { return nil } } if g . revs [ 0 ] . main <= rev { return & ki . generations [ cg ] } cg -- } return nil }

// findGeneration finds out the generation of the keyIndex that the
// given rev belongs to. If the given rev is at the gap of two generations,
// which means that the key does not exist at the given rev, it returns nil.

findGeneration finds out the generation of the keyIndex that the given rev belongs to . If the given rev is at the gap of two generations which means that the key does not exist at the given rev it returns nil .


func (s *watchableStore) cancelWatcher(wa *watcher) {
	for {
		s.mu.Lock()
		if s.unsynced.delete(wa) {
			slowWatcherGauge.Dec()
			break
		} else if s.synced.delete(wa) {
			break
		} else if wa.compacted {
			break
		} else if wa.ch == nil {
			// already canceled (e.g., cancel/close race)
			break
		}

		if !wa.victim {
			panic("watcher not victim but not in watch groups")
		}

		var victimBatch watcherBatch
		for _, wb := range s.victims {
			if wb[wa] != nil {
				victimBatch = wb
				break
			}
		}
		if victimBatch != nil {
			slowWatcherGauge.Dec()
			delete(victimBatch, wa)
			break
		}

		// victim being processed so not accessible; retry
		s.mu.Unlock()
		time.Sleep(time.Millisecond)
	}

	watcherGauge.Dec()
	wa.ch = nil
	s.mu.Unlock()
}

// cancelWatcher removes references of the watcher from the watchableStore


func (s *watchableStore) syncWatchersLoop() {
	defer s.wg.Done()

	for {
		s.mu.RLock()
		st := time.Now()
		lastUnsyncedWatchers := s.unsynced.size()
		s.mu.RUnlock()

		unsyncedWatchers := 0
		if lastUnsyncedWatchers > 0 {
			unsyncedWatchers = s.syncWatchers()
		}
		syncDuration := time.Since(st)

		waitDuration := 100 * time.Millisecond
		// more work pending?
		if unsyncedWatchers != 0 && lastUnsyncedWatchers > unsyncedWatchers {
			// be fair to other store operations by yielding time taken
			waitDuration = syncDuration
		}

		select {
		case <-time.After(waitDuration):
		case <-s.stopc:
			return
		}
	}
}

// syncWatchersLoop syncs the watcher in the unsynced map every 100ms.

syncWatchersLoop syncs the watcher in the unsynced map every 100ms .


func (s *watchableStore) syncVictimsLoop() {
	defer s.wg.Done()

	for {
		for s.moveVictims() != 0 {
			// try to update all victim watchers
		}
		s.mu.RLock()
		isEmpty := len(s.victims) == 0
		s.mu.RUnlock()

		var tickc <-chan time.Time
		if !isEmpty {
			tickc = time.After(10 * time.Millisecond)
		}

		select {
		case <-tickc:
		case <-s.victimc:
		case <-s.stopc:
			return
		}
	}
}

func
(
s
*
watchableStore
)
syncVictimsLoop
(
)
{
defer
s
.
wg
.
Done
(
)
for
{
for
s
.
moveVictims
(
)
!=
0
{
// try to update all victim watchers
}
s
.
mu
.
RLock
(
)
isEmpty
:=
len
(
s
.
victims
)
==
0
s
.
mu
.
RUnlock
(
)
var
tickc
<-
chan
time
.
Time
if
!
isEmpty
{
tickc
=
time
.
After
(
10
*
time
.
Millisecond
)
}
select
{
case
<-
tickc
:
case
<-
s
.
victimc
:
case
<-
s
.
stopc
:
return
}
}
}

// syncVictimsLoop tries to write precomputed watcher responses to
// watchers that had a blocked watcher channel

syncVictimsLoop tries to write precomputed watcher responses to watchers that had a blocked watcher channel


func (s *watchableStore) moveVictims() (moved int) {
	s.mu.Lock()
	victims := s.victims
	s.victims = nil
	s.mu.Unlock()

	var newVictim watcherBatch
	for _, wb := range victims {
		// try to send responses again
		for w, eb := range wb {
			// watcher has observed the store up to, but not including, w.minRev
			rev := w.minRev - 1
			if w.send(WatchResponse{WatchID: w.id, Events: eb.evs, Revision: rev}) {
				pendingEventsGauge.Add(float64(len(eb.evs)))
			} else {
				if newVictim == nil {
					newVictim = make(watcherBatch)
				}
				newVictim[w] = eb
				continue
			}
			moved++
		}

		// assign completed victim watchers to unsync/sync
		s.mu.Lock()
		s.store.revMu.RLock()
		curRev := s.store.currentRev
		for w, eb := range wb {
			if newVictim != nil && newVictim[w] != nil {
				// couldn't send watch response; stays victim
				continue
			}
			w.victim = false
			if eb.moreRev != 0 {
				w.minRev = eb.moreRev
			}
			if w.minRev <= curRev {
				s.unsynced.add(w)
			} else {
				slowWatcherGauge.Dec()
				s.synced.add(w)
			}
		}
		s.store.revMu.RUnlock()
		s.mu.Unlock()
	}

	if len(newVictim) > 0 {
		s.mu.Lock()
		s.victims = append(s.victims, newVictim)
		s.mu.Unlock()
	}

	return moved
}

// moveVictims tries to update watches with already pending event data


func kvsToEvents(lg *zap.Logger, wg *watcherGroup, revs, vals [][]byte) (evs []mvccpb.Event) {
	for i, v := range vals {
		var kv mvccpb.KeyValue
		if err := kv.Unmarshal(v); err != nil {
			if lg != nil {
				lg.Panic("failed to unmarshal mvccpb.KeyValue", zap.Error(err))
			} else {
				plog.Panicf("cannot unmarshal event: %v", err)
			}
		}

		if !wg.contains(string(kv.Key)) {
			continue
		}

		ty := mvccpb.PUT
		if isTombstone(revs[i]) {
			ty = mvccpb.DELETE
			// patch in mod revision so watchers won't skip
			kv.ModRevision = bytesToRev(revs[i]).main
		}
		evs = append(evs, mvccpb.Event{Kv: &kv, Type: ty})
	}
	return evs
}

// kvsToEvents gets all events for the watchers from all key-value pairs

kvsToEvents gets all events for the watchers from all key - value pairs


func (s *watchableStore) notify(rev int64, evs []mvccpb.Event) {
	var victim watcherBatch
	for w, eb := range newWatcherBatch(&s.synced, evs) {
		if eb.revs != 1 {
			if s.store != nil && s.store.lg != nil {
				s.store.lg.Panic(
					"unexpected multiple revisions in watch notification",
					zap.Int("number-of-revisions", eb.revs),
				)
			} else {
				plog.Panicf("unexpected multiple revisions in notification")
			}
		}
		if w.send(WatchResponse{WatchID: w.id, Events: eb.evs, Revision: rev}) {
			pendingEventsGauge.Add(float64(len(eb.evs)))
		} else {
			// move slow watcher to victims
			w.minRev = rev + 1
			if victim == nil {
				victim = make(watcherBatch)
			}
			w.victim = true
			victim[w] = eb
			s.synced.delete(w)
			slowWatcherGauge.Inc()
		}
	}
	s.addVictim(victim)
}

func
(
s
*
watchableStore
)
notify
(
rev
int64
,
evs
[
]
mvccpb
.
Event
)
{
var
victim
watcherBatch
for
w
,
eb
:=
range
newWatcherBatch
(
&
s
.
synced
,
evs
)
{
if
eb
.
revs
!=
1
{
if
s
.
store
!=
&&
s
.
store
.
lg
!=
{
s
.
store
.
lg
.
Panic
(
"
"
,
zap
.
Int
(
"
"
,
eb
.
revs
)
,
)
}
else
{
plog
.
Panicf
(
"
"
)
}
}
if
w
.
send
(
WatchResponse
{
WatchID
:
w
.
id
,
Events
:
eb
.
evs
,
Revision
:
rev
}
)
{
pendingEventsGauge
.
Add
(
float64
(
len
(
eb
.
evs
)
)
)
}
else
{
// move slow watcher to victims
w
.
minRev
=
rev
+
1
if
victim
==
{
victim
=
make
(
watcherBatch
)
}
w
.
victim
=
true
victim
[
w
]
=
eb
s
.
synced
.
delete
(
w
)
slowWatcherGauge
.
Inc
(
)
}
}
s
.
addVictim
(
victim
)
}

// notify notifies the fact that given event at the given rev just happened to
// watchers that watch on the key of the event.

notify notifies the fact that given event at the given rev just happened to watchers that watch on the key of the event .


func isOpFuncCalled(op string, opts []OpOption) bool {
	for _, opt := range opts {
		v := reflect.ValueOf(opt)
		if v.Kind() == reflect.Func {
			if opFunc := runtime.FuncForPC(v.Pointer()); opFunc != nil {
				if strings.Contains(opFunc.Name(), op) {
					return true
				}
			}
		}
	}
	return false
}

func isOpFuncCalled ( op string , opts [ ] OpOption ) bool { for _ , opt := range opts { v := reflect . ValueOf ( opt ) if v . Kind ( ) == reflect . Func { if opFunc := runtime . FuncForPC ( v . Pointer ( ) ) ; opFunc != nil { if strings . Contains ( opFunc . Name ( ) , op ) { return true } } } } return false }

// Check if the provided function is being called in the op options.

Check if the provided function is being called in the op options .


func (t *batchTx) UnsafePut(bucketName []byte, key []byte, value []byte) {
	t.unsafePut(bucketName, key, value, false)
}

func ( t * batchTx ) UnsafePut ( bucketName [ ] byte , key [ ] byte , value [ ] byte ) { t . unsafePut ( bucketName , key , value , false ) }

// UnsafePut must be called holding the lock on the tx.

UnsafePut must be called holding the lock on the tx .


func (t *batchTx) UnsafeSeqPut(bucketName []byte, key []byte, value []byte) {
	t.unsafePut(bucketName, key, value, true)
}

func ( t * batchTx ) UnsafeSeqPut ( bucketName [ ] byte , key [ ] byte , value [ ] byte ) { t . unsafePut ( bucketName , key , value , true ) }

// UnsafeSeqPut must be called holding the lock on the tx.

UnsafeSeqPut must be called holding the lock on the tx .


func (t *batchTx) UnsafeRange(bucketName, key, endKey []byte, limit int64) ([][]byte, [][]byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		if t.backend.lg != nil {
			t.backend.lg.Fatal(
				"failed to find a bucket",
				zap.String("bucket-name", string(bucketName)),
			)
		} else {
			plog.Fatalf("bucket %s does not exist", bucketName)
		}
	}
	return unsafeRange(bucket.Cursor(), key, endKey, limit)
}

func ( t * batchTx ) UnsafeRange ( bucketName , key , endKey [ ] byte , limit int64 ) ( [ ] [ ] byte , [ ] [ ] byte ) { bucket := t . tx . Bucket ( bucketName ) if bucket == nil { if t . backend . lg != nil { t . backend . lg . Fatal ( " " , zap . String ( " " , string ( bucketName ) ) , ) } else { plog . Fatalf ( " " , bucketName ) } } return unsafeRange ( bucket . Cursor ( ) , key , endKey , limit ) }

// UnsafeRange must be called holding the lock on the tx.

UnsafeRange must be called holding the lock on the tx .


func (t *batchTx) UnsafeDelete(bucketName []byte, key []byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		if t.backend.lg != nil {
			t.backend.lg.Fatal(
				"failed to find a bucket",
				zap.String("bucket-name", string(bucketName)),
			)
		} else {
			plog.Fatalf("bucket %s does not exist", bucketName)
		}
	}
	err := bucket.Delete(key)
	if err != nil {
		if t.backend.lg != nil {
			t.backend.lg.Fatal(
				"failed to delete a key",
				zap.String("bucket-name", string(bucketName)),
				zap.Error(err),
			)
		} else {
			plog.Fatalf("cannot delete key from bucket (%v)", err)
		}
	}
	t.pending++
}

func ( t * batchTx ) UnsafeDelete ( bucketName [ ] byte , key [ ] byte ) { bucket := t . tx . Bucket ( bucketName ) if bucket == nil { if t . backend . lg != nil { t . backend . lg . Fatal ( " " , zap . String ( " " , string ( bucketName ) ) , ) } else { plog . Fatalf ( " " , bucketName ) } } err := bucket . Delete ( key ) if err != nil { if t . backend . lg != nil { t . backend . lg . Fatal ( " " , zap . String ( " " , string ( bucketName ) ) , zap . Error ( err ) , ) } else { plog . Fatalf ( " " , err ) } } t . pending ++ }

// UnsafeDelete must be called holding the lock on the tx.

UnsafeDelete must be called holding the lock on the tx .


func (t *batchTx) UnsafeForEach(bucketName []byte, visitor func(k, v []byte) error) error {
	return unsafeForEach(t.tx, bucketName, visitor)
}

func ( t * batchTx ) UnsafeForEach ( bucketName [ ] byte , visitor func ( k , v [ ] byte ) error ) error { return unsafeForEach ( t . tx , bucketName , visitor ) }

// UnsafeForEach must be called holding the lock on the tx.

UnsafeForEach must be called holding the lock on the tx .


func (t *batchTx) Commit() {
	t.Lock()
	t.commit(false)
	t.Unlock()
}

func ( t * batchTx ) Commit ( ) { t . Lock ( ) t . commit ( false ) t . Unlock ( ) }

// Commit commits a previous tx and begins a new writable one.

Commit commits a previous tx and begins a new writable one .


func (t *batchTx) CommitAndStop() {
	t.Lock()
	t.commit(true)
	t.Unlock()
}

func ( t * batchTx ) CommitAndStop ( ) { t . Lock ( ) t . commit ( true ) t . Unlock ( ) }

// CommitAndStop commits the previous tx and does not create a new one.

CommitAndStop commits the previous tx and does not create a new one .


func (le *lessor) Renew(id LeaseID) (int64, error) {
	le.mu.RLock()
	if !le.isPrimary() {
		// forward renew request to primary instead of returning error.
		le.mu.RUnlock()
		return -1, ErrNotPrimary
	}

	demotec := le.demotec

	l := le.leaseMap[id]
	if l == nil {
		le.mu.RUnlock()
		return -1, ErrLeaseNotFound
	}
	// Clear remaining TTL when we renew if it is set
	clearRemainingTTL := le.cp != nil && l.remainingTTL > 0

	le.mu.RUnlock()
	if l.expired() {
		select {
		// A expired lease might be pending for revoking or going through
		// quorum to be revoked. To be accurate, renew request must wait for the
		// deletion to complete.
		case <-l.revokec:
			return -1, ErrLeaseNotFound
		// The expired lease might fail to be revoked if the primary changes.
		// The caller will retry on ErrNotPrimary.
		case <-demotec:
			return -1, ErrNotPrimary
		case <-le.stopC:
			return -1, ErrNotPrimary
		}
	}

	// Clear remaining TTL when we renew if it is set
	// By applying a RAFT entry only when the remainingTTL is already set, we limit the number
	// of RAFT entries written per lease to a max of 2 per checkpoint interval.
	if clearRemainingTTL {
		le.cp(context.Background(), &pb.LeaseCheckpointRequest{Checkpoints: []*pb.LeaseCheckpoint{{ID: int64(l.ID), Remaining_TTL: 0}}})
	}

	le.mu.Lock()
	l.refresh(0)
	item := &LeaseWithTime{id: l.ID, time: l.expiry.UnixNano()}
	heap.Push(&le.leaseHeap, item)
	le.mu.Unlock()

	leaseRenewed.Inc()
	return l.ttl, nil
}

// Renew renews an existing lease. If the given lease does not exist or
// has expired, an error will be returned.

Renew renews an existing lease . If the given lease does not exist or has expired an error will be returned .


func (le *lessor) Attach(id LeaseID, items []LeaseItem) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return ErrLeaseNotFound
	}

	l.mu.Lock()
	for _, it := range items {
		l.itemSet[it] = struct{}{}
		le.itemMap[it] = id
	}
	l.mu.Unlock()
	return nil
}

func ( le * lessor ) Attach ( id LeaseID , items [ ] LeaseItem ) error { le . mu . Lock ( ) defer le . mu . Unlock ( ) l := le . leaseMap [ id ] if l == nil { return ErrLeaseNotFound } l . mu . Lock ( ) for _ , it := range items { l . itemSet [ it ] = struct { } { } le . itemMap [ it ] = id } l . mu . Unlock ( ) return nil }

// Attach attaches items to the lease with given ID. When the lease
// expires, the attached items will be automatically removed.
// If the given lease does not exist, an error will be returned.

Attach attaches items to the lease with given ID . When the lease expires the attached items will be automatically removed . If the given lease does not exist an error will be returned .


func (le *lessor) revokeExpiredLeases() {
	var ls []*Lease

	// rate limit
	revokeLimit := leaseRevokeRate / 2

	le.mu.RLock()
	if le.isPrimary() {
		ls = le.findExpiredLeases(revokeLimit)
	}
	le.mu.RUnlock()

	if len(ls) != 0 {
		select {
		case <-le.stopC:
			return
		case le.expiredC <- ls:
		default:
			// the receiver of expiredC is probably busy handling
			// other stuff
			// let's try this next time after 500ms
		}
	}
}

// revokeExpiredLeases finds all leases past their expiry and sends them to epxired channel for
// to be revoked.

revokeExpiredLeases finds all leases past their expiry and sends them to epxired channel for to be revoked .


func (le *lessor) checkpointScheduledLeases() {
	var cps []*pb.LeaseCheckpoint

	// rate limit
	for i := 0; i < leaseCheckpointRate/2; i++ {
		le.mu.Lock()
		if le.isPrimary() {
			cps = le.findDueScheduledCheckpoints(maxLeaseCheckpointBatchSize)
		}
		le.mu.Unlock()

		if len(cps) != 0 {
			le.cp(context.Background(), &pb.LeaseCheckpointRequest{Checkpoints: cps})
		}
		if len(cps) < maxLeaseCheckpointBatchSize {
			return
		}
	}
}

func
(
le
*
lessor
)
checkpointScheduledLeases
(
)
{
var
cps
[
]
*
pb
.
LeaseCheckpoint
// rate limit
for
i
:=
0
;
i
<
leaseCheckpointRate
/
2
;
i
++
{
le
.
mu
.
Lock
(
)
if
le
.
isPrimary
(
)
{
cps
=
le
.
findDueScheduledCheckpoints
(
maxLeaseCheckpointBatchSize
)
}
le
.
mu
.
Unlock
(
)
if
len
(
cps
)
!=
0
{
le
.
cp
(
context
.
Background
(
)
,
&
pb
.
LeaseCheckpointRequest
{
Checkpoints
:
cps
}
)
}
if
len
(
cps
)
<
maxLeaseCheckpointBatchSize
{
return
}
}
}

// checkpointScheduledLeases finds all scheduled lease checkpoints that are due and
// submits them to the checkpointer to persist them to the consensus log.

checkpointScheduledLeases finds all scheduled lease checkpoints that are due and submits them to the checkpointer to persist them to the consensus log .


func (le *lessor) expireExists() (l *Lease, ok bool, next bool) {
	if le.leaseHeap.Len() == 0 {
		return nil, false, false
	}

	item := le.leaseHeap[0]
	l = le.leaseMap[item.id]
	if l == nil {
		// lease has expired or been revoked
		// no need to revoke (nothing is expiry)
		heap.Pop(&le.leaseHeap) // O(log N)
		return nil, false, true
	}

	if time.Now().UnixNano() < item.time /* expiration time */ {
		// Candidate expirations are caught up, reinsert this item
		// and no need to revoke (nothing is expiry)
		return l, false, false
	}
	// if the lease is actually expired, add to the removal list. If it is not expired, we can ignore it because another entry will have been inserted into the heap

	heap.Pop(&le.leaseHeap) // O(log N)
	return l, true, false
}

// expireExists returns true if expiry items exist.
// It pops only when expiry item exists.
// "next" is true, to indicate that it may exist in next attempt.

expireExists returns true if expiry items exist . It pops only when expiry item exists . next is true to indicate that it may exist in next attempt .


func (le *lessor) findExpiredLeases(limit int) []*Lease {
	leases := make([]*Lease, 0, 16)

	for {
		l, ok, next := le.expireExists()
		if !ok && !next {
			break
		}
		if !ok {
			continue
		}
		if next {
			continue
		}

		if l.expired() {
			leases = append(leases, l)

			// reach expired limit
			if len(leases) == limit {
				break
			}
		}
	}

	return leases
}

func
(
le
*
lessor
)
findExpiredLeases
(
limit
int
)
[
]
*
Lease
{
leases
:=
make
(
[
]
*
Lease
,
0
,
16
)
for
{
l
,
ok
,
next
:=
le
.
expireExists
(
)
if
!
ok
&&
!
next
{
break
}
if
!
ok
{
continue
}
if
next
{
continue
}
if
l
.
expired
(
)
{
leases
=
append
(
leases
,
l
)
// reach expired limit
if
len
(
leases
)
==
limit
{
break
}
}
}
return
leases
}

// findExpiredLeases loops leases in the leaseMap until reaching expired limit
// and returns the expired leases that needed to be revoked.

findExpiredLeases loops leases in the leaseMap until reaching expired limit and returns the expired leases that needed to be revoked .


func (l *Lease) refresh(extend time.Duration) {
	newExpiry := time.Now().Add(extend + time.Duration(l.RemainingTTL())*time.Second)
	l.expiryMu.Lock()
	defer l.expiryMu.Unlock()
	l.expiry = newExpiry
}

func ( l * Lease ) refresh ( extend time . Duration ) { newExpiry := time . Now ( ) . Add ( extend + time . Duration ( l . RemainingTTL ( ) ) * time . Second ) l . expiryMu . Lock ( ) defer l . expiryMu . Unlock ( ) l . expiry = newExpiry }

// refresh refreshes the expiry of the lease.

refresh refreshes the expiry of the lease .


func (l *Lease) forever() {
	l.expiryMu.Lock()
	defer l.expiryMu.Unlock()
	l.expiry = forever
}

func ( l * Lease ) forever ( ) { l . expiryMu . Lock ( ) defer l . expiryMu . Unlock ( ) l . expiry = forever }

// forever sets the expiry of lease to be forever.

forever sets the expiry of lease to be forever .


func (l *Lease) Keys() []string {
	l.mu.RLock()
	keys := make([]string, 0, len(l.itemSet))
	for k := range l.itemSet {
		keys = append(keys, k.Key)
	}
	l.mu.RUnlock()
	return keys
}

func ( l * Lease ) Keys ( ) [ ] string { l . mu . RLock ( ) keys := make ( [ ] string , 0 , len ( l . itemSet ) ) for k := range l . itemSet { keys = append ( keys , k . Key ) } l . mu . RUnlock ( ) return keys }

// Keys returns all the keys attached to the lease.

Keys returns all the keys attached to the lease .


func (l *Lease) Remaining() time.Duration {
	l.expiryMu.RLock()
	defer l.expiryMu.RUnlock()
	if l.expiry.IsZero() {
		return time.Duration(math.MaxInt64)
	}
	return time.Until(l.expiry)
}

func ( l * Lease ) Remaining ( ) time . Duration { l . expiryMu . RLock ( ) defer l . expiryMu . RUnlock ( ) if l . expiry . IsZero ( ) { return time . Duration ( math . MaxInt64 ) } return time . Until ( l . expiry ) }

// Remaining returns the remaining time of the lease.

Remaining returns the remaining time of the lease .


func NewCompactionCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "compaction [options] <revision>",
		Short: "Compacts the event history in etcd",
		Run:   compactionCommandFunc,
	}
	cmd.Flags().BoolVar(&compactPhysical, "physical", false, "'true' to wait for compaction to physically remove all old revisions")
	return cmd
}

func NewCompactionCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , Run : compactionCommandFunc , } cmd . Flags ( ) . BoolVar ( & compactPhysical , " " , false , " " ) return cmd }

// NewCompactionCommand returns the cobra command for "compaction".

NewCompactionCommand returns the cobra command for compaction .


func compactionCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, fmt.Errorf("compaction command needs 1 argument"))
	}

	rev, err := strconv.ParseInt(args[0], 10, 64)
	if err != nil {
		ExitWithError(ExitError, err)
	}

	var opts []clientv3.CompactOption
	if compactPhysical {
		opts = append(opts, clientv3.WithCompactPhysical())
	}

	c := mustClientFromCmd(cmd)
	ctx, cancel := commandCtx(cmd)
	_, cerr := c.Compact(ctx, rev, opts...)
	cancel()
	if cerr != nil {
		ExitWithError(ExitError, cerr)
	}
	fmt.Println("compacted revision", rev)
}

func compactionCommandFunc ( cmd * cobra . Command , args [ ] string ) { if len ( args ) != 1 { ExitWithError ( ExitBadArgs , fmt . Errorf ( " " ) ) } rev , err := strconv . ParseInt ( args [ 0 ] , 10 , 64 ) if err != nil { ExitWithError ( ExitError , err ) } var opts [ ] clientv3 . CompactOption if compactPhysical { opts = append ( opts , clientv3 . WithCompactPhysical ( ) ) } c := mustClientFromCmd ( cmd ) ctx , cancel := commandCtx ( cmd ) _ , cerr := c . Compact ( ctx , rev , opts ... ) cancel ( ) if cerr != nil { ExitWithError ( ExitError , cerr ) } fmt . Println ( " " , rev ) }

// compactionCommandFunc executes the "compaction" command.

compactionCommandFunc executes the compaction command .


func NewPutCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "put [options] <key> <value> (<value> can also be given from stdin)",
		Short: "Puts the given key into the store",
		Long: `
Puts the given key into the store.

When <value> begins with '-', <value> is interpreted as a flag.
Insert '--' for workaround:

$ put <key> -- <value>
$ put -- <key> <value>

If <value> isn't given as a command line argument and '--ignore-value' is not specified,
this command tries to read the value from standard input.

If <lease> isn't given as a command line argument and '--ignore-lease' is not specified,
this command tries to read the value from standard input.

For example,
$ cat file | put <key>
will store the content of the file to <key>.
`,
		Run: putCommandFunc,
	}
	cmd.Flags().StringVar(&leaseStr, "lease", "0", "lease ID (in hexadecimal) to attach to the key")
	cmd.Flags().BoolVar(&putPrevKV, "prev-kv", false, "return the previous key-value pair before modification")
	cmd.Flags().BoolVar(&putIgnoreVal, "ignore-value", false, "updates the key using its current value")
	cmd.Flags().BoolVar(&putIgnoreLease, "ignore-lease", false, "updates the key using its current lease")
	return cmd
}

// NewPutCommand returns the cobra command for "put".

NewPutCommand returns the cobra command for put .


func putCommandFunc(cmd *cobra.Command, args []string) {
	key, value, opts := getPutOp(args)

	ctx, cancel := commandCtx(cmd)
	resp, err := mustClientFromCmd(cmd).Put(ctx, key, value, opts...)
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	display.Put(*resp)
}

func putCommandFunc ( cmd * cobra . Command , args [ ] string ) { key , value , opts := getPutOp ( args ) ctx , cancel := commandCtx ( cmd ) resp , err := mustClientFromCmd ( cmd ) . Put ( ctx , key , value , opts ... ) cancel ( ) if err != nil { ExitWithError ( ExitError , err ) } display . Put ( * resp ) }

// putCommandFunc executes the "put" command.

putCommandFunc executes the put command .


func NewHandler(t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler {
	if t.TLSClientConfig != nil {
		// Enable http2, see Issue 5033.
		err := http2.ConfigureTransport(t)
		if err != nil {
			plog.Infof("Error enabling Transport HTTP/2 support: %v", err)
		}
	}

	p := &reverseProxy{
		director:  newDirector(urlsFunc, failureWait, refreshInterval),
		transport: t,
	}

	mux := http.NewServeMux()
	mux.Handle("/", p)
	mux.HandleFunc("/v2/config/local/proxy", p.configHandler)

	return mux
}

func
NewHandler
(
t
*
http
.
Transport
,
urlsFunc
GetProxyURLs
,
failureWait
time
.
Duration
,
refreshInterval
time
.
Duration
)
http
.
Handler
{
if
t
.
TLSClientConfig
!=
{
// Enable http2, see Issue 5033.
err
:=
http2
.
ConfigureTransport
(
t
)
if
err
!=
{
plog
.
Infof
(
"
"
,
err
)
}
}
p
:=
&
reverseProxy
{
director
:
newDirector
(
urlsFunc
,
failureWait
,
refreshInterval
)
,
transport
:
t
,
}
mux
:=
http
.
NewServeMux
(
)
mux
.
Handle
(
"
"
,
p
)
mux
.
HandleFunc
(
"
"
,
p
.
configHandler
)
return
mux
}

// NewHandler creates a new HTTP handler, listening on the given transport,
// which will proxy requests to an etcd cluster.
// The handler will periodically update its view of the cluster.

NewHandler creates a new HTTP handler listening on the given transport which will proxy requests to an etcd cluster . The handler will periodically update its view of the cluster .


func NewReadonlyHandler(hdlr http.Handler) http.Handler {
	readonly := readonlyHandlerFunc(hdlr)
	return http.HandlerFunc(readonly)
}

func NewReadonlyHandler ( hdlr http . Handler ) http . Handler { readonly := readonlyHandlerFunc ( hdlr ) return http . HandlerFunc ( readonly ) }

// NewReadonlyHandler wraps the given HTTP handler to allow only GET requests


func NewSetCommand() cli.Command {
	return cli.Command{
		Name:      "set",
		Usage:     "set the value of a key",
		ArgsUsage: "<key> <value>",
		Description: `Set sets the value of a key.

   When <value> begins with '-', <value> is interpreted as a flag.
   Insert '--' for workaround:

   $ set -- <key> <value>`,
		Flags: []cli.Flag{
			cli.IntFlag{Name: "ttl", Value: 0, Usage: "key time-to-live in seconds"},
			cli.StringFlag{Name: "swap-with-value", Value: "", Usage: "previous value"},
			cli.IntFlag{Name: "swap-with-index", Value: 0, Usage: "previous index"},
		},
		Action: func(c *cli.Context) error {
			setCommandFunc(c, mustNewKeyAPI(c))
			return nil
		},
	}
}

// NewSetCommand returns the CLI command for "set".

NewSetCommand returns the CLI command for set .


func setCommandFunc(c *cli.Context, ki client.KeysAPI) {
	if len(c.Args()) == 0 {
		handleError(c, ExitBadArgs, errors.New("key required"))
	}
	key := c.Args()[0]
	value, err := argOrStdin(c.Args(), os.Stdin, 1)
	if err != nil {
		handleError(c, ExitBadArgs, errors.New("value required"))
	}

	ttl := c.Int("ttl")
	prevValue := c.String("swap-with-value")
	prevIndex := c.Int("swap-with-index")

	ctx, cancel := contextWithTotalTimeout(c)
	resp, err := ki.Set(ctx, key, value, &client.SetOptions{TTL: time.Duration(ttl) * time.Second, PrevIndex: uint64(prevIndex), PrevValue: prevValue})
	cancel()
	if err != nil {
		handleError(c, ExitServerError, err)
	}

	printResponseKey(resp, c.GlobalString("output"))
}

func setCommandFunc ( c * cli . Context , ki client . KeysAPI ) { if len ( c . Args ( ) ) == 0 { handleError ( c , ExitBadArgs , errors . New ( " " ) ) } key := c . Args ( ) [ 0 ] value , err := argOrStdin ( c . Args ( ) , os . Stdin , 1 ) if err != nil { handleError ( c , ExitBadArgs , errors . New ( " " ) ) } ttl := c . Int ( " " ) prevValue := c . String ( " " ) prevIndex := c . Int ( " " ) ctx , cancel := contextWithTotalTimeout ( c ) resp , err := ki . Set ( ctx , key , value , & client . SetOptions { TTL : time . Duration ( ttl ) * time . Second , PrevIndex : uint64 ( prevIndex ) , PrevValue : prevValue } ) cancel ( ) if err != nil { handleError ( c , ExitServerError , err ) } printResponseKey ( resp , c . GlobalString ( " " ) ) }

// setCommandFunc executes the "set" command.

setCommandFunc executes the set command .


func (rwm *RWMutex) waitOnLastRev(pfx string) (bool, error) {
	client := rwm.s.Client()
	// get key that's blocking myKey
	opts := append(v3.WithLastRev(), v3.WithMaxModRev(rwm.myKey.Revision()-1))
	lastKey, err := client.Get(rwm.ctx, pfx, opts...)
	if err != nil {
		return false, err
	}
	if len(lastKey.Kvs) == 0 {
		return true, nil
	}
	// wait for release on blocking key
	_, err = WaitEvents(
		client,
		string(lastKey.Kvs[0].Key),
		rwm.myKey.Revision(),
		[]mvccpb.Event_EventType{mvccpb.DELETE})
	return false, err
}

func
(
rwm
*
RWMutex
)
waitOnLastRev
(
pfx
string
)
(
bool
,
error
)
{
client
:=
rwm
.
s
.
Client
(
)
// get key that's blocking myKey
opts
:=
append
(
v3
.
WithLastRev
(
)
,
v3
.
WithMaxModRev
(
rwm
.
myKey
.
Revision
(
)
1
)
)
lastKey
,
err
:=
client
.
Get
(
rwm
.
ctx
,
pfx
,
opts
...
)
if
err
!=
{
return
false
,
err
}
if
len
(
lastKey
.
Kvs
)
==
0
{
return
true
,
}
// wait for release on blocking key
_
,
err
=
WaitEvents
(
client
,
string
(
lastKey
.
Kvs
[
0
]
.
Key
)
,
rwm
.
myKey
.
Revision
(
)
,
[
]
mvccpb
.
Event_EventType
{
mvccpb
.
DELETE
}
)
return
false
,
err
}

// waitOnLowest will wait on the last key with a revision < rwm.myKey.Revision with a
// given prefix. If there are no keys left to wait on, return true.

waitOnLowest will wait on the last key with a revision < rwm . myKey . Revision with a given prefix . If there are no keys left to wait on return true .


func GetDefaultInterfaces() (map[string]uint8, error) {
	return nil, fmt.Errorf("default host not supported on %s_%s", runtime.GOOS, runtime.GOARCH)
}

func GetDefaultInterfaces ( ) ( map [ string ] uint8 , error ) { return nil , fmt . Errorf ( " " , runtime . GOOS , runtime . GOARCH ) }

// GetDefaultInterfaces fetches the device name of default routable interface.

GetDefaultInterfaces fetches the device name of default routable interface .


func NewSnapshotCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "snapshot <subcommand>",
		Short: "Manages etcd node snapshots",
	}
	cmd.AddCommand(NewSnapshotSaveCommand())
	cmd.AddCommand(NewSnapshotRestoreCommand())
	cmd.AddCommand(newSnapshotStatusCommand())
	return cmd
}

func NewSnapshotCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , } cmd . AddCommand ( NewSnapshotSaveCommand ( ) ) cmd . AddCommand ( NewSnapshotRestoreCommand ( ) ) cmd . AddCommand ( newSnapshotStatusCommand ( ) ) return cmd }

// NewSnapshotCommand returns the cobra command for "snapshot".

NewSnapshotCommand returns the cobra command for snapshot .


func NewMoveLeaderCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "move-leader <transferee-member-id>",
		Short: "Transfers leadership to another etcd cluster member.",
		Run:   transferLeadershipCommandFunc,
	}
	return cmd
}

func NewMoveLeaderCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , Run : transferLeadershipCommandFunc , } return cmd }

// NewMoveLeaderCommand returns the cobra command for "move-leader".

NewMoveLeaderCommand returns the cobra command for move - leader .


func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, fmt.Errorf("move-leader command needs 1 argument"))
	}
	target, err := strconv.ParseUint(args[0], 16, 64)
	if err != nil {
		ExitWithError(ExitBadArgs, err)
	}

	c := mustClientFromCmd(cmd)
	eps := c.Endpoints()
	c.Close()

	ctx, cancel := commandCtx(cmd)

	// find current leader
	var leaderCli *clientv3.Client
	var leaderID uint64
	for _, ep := range eps {
		cfg := clientConfigFromCmd(cmd)
		cfg.endpoints = []string{ep}
		cli := cfg.mustClient()
		resp, serr := cli.Status(ctx, ep)
		if serr != nil {
			ExitWithError(ExitError, serr)
		}

		if resp.Header.GetMemberId() == resp.Leader {
			leaderCli = cli
			leaderID = resp.Leader
			break
		}
		cli.Close()
	}
	if leaderCli == nil {
		ExitWithError(ExitBadArgs, fmt.Errorf("no leader endpoint given at %v", eps))
	}

	var resp *clientv3.MoveLeaderResponse
	resp, err = leaderCli.MoveLeader(ctx, target)
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	display.MoveLeader(leaderID, target, *resp)
}

func
transferLeadershipCommandFunc
(
cmd
*
cobra
.
Command
,
args
[
]
string
)
{
if
len
(
args
)
!=
1
{
ExitWithError
(
ExitBadArgs
,
fmt
.
Errorf
(
"
"
)
)
}
target
,
err
:=
strconv
.
ParseUint
(
args
[
0
]
,
16
,
64
)
if
err
!=
{
ExitWithError
(
ExitBadArgs
,
err
)
}
c
:=
mustClientFromCmd
(
cmd
)
eps
:=
c
.
Endpoints
(
)
c
.
Close
(
)
ctx
,
cancel
:=
commandCtx
(
cmd
)
// find current leader
var
leaderCli
*
clientv3
.
Client
var
leaderID
uint64
for
_
,
ep
:=
range
eps
{
cfg
:=
clientConfigFromCmd
(
cmd
)
cfg
.
endpoints
=
[
]
string
{
ep
}
cli
:=
cfg
.
mustClient
(
)
resp
,
serr
:=
cli
.
Status
(
ctx
,
ep
)
if
serr
!=
{
ExitWithError
(
ExitError
,
serr
)
}
if
resp
.
Header
.
GetMemberId
(
)
==
resp
.
Leader
{
leaderCli
=
cli
leaderID
=
resp
.
Leader
break
}
cli
.
Close
(
)
}
if
leaderCli
==
{
ExitWithError
(
ExitBadArgs
,
fmt
.
Errorf
(
"
"
,
eps
)
)
}
var
resp
*
clientv3
.
MoveLeaderResponse
resp
,
err
=
leaderCli
.
MoveLeader
(
ctx
,
target
)
cancel
(
)
if
err
!=
{
ExitWithError
(
ExitError
,
err
)
}
display
.
MoveLeader
(
leaderID
,
target
,
*
resp
)
}

// transferLeadershipCommandFunc executes the "compaction" command.

transferLeadershipCommandFunc executes the compaction command .


func OpenDir(path string) (*os.File, error) {
	fd, err := openDir(path)
	if err != nil {
		return nil, err
	}
	return os.NewFile(uintptr(fd), path), nil
}

func OpenDir ( path string ) ( * os . File , error ) { fd , err := openDir ( path ) if err != nil { return nil , err } return os . NewFile ( uintptr ( fd ) , path ) , nil }

// OpenDir opens a directory in windows with write access for syncing.

OpenDir opens a directory in windows with write access for syncing .


func NewRemoveDirCommand() cli.Command {
	return cli.Command{
		Name:      "rmdir",
		Usage:     "removes the key if it is an empty directory or a key-value pair",
		ArgsUsage: "<key>",
		Action: func(c *cli.Context) error {
			rmdirCommandFunc(c, mustNewKeyAPI(c))
			return nil
		},
	}
}

func NewRemoveDirCommand ( ) cli . Command { return cli . Command { Name : " " , Usage : " " , ArgsUsage : " " , Action : func ( c * cli . Context ) error { rmdirCommandFunc ( c , mustNewKeyAPI ( c ) ) return nil } , } }

// NewRemoveDirCommand returns the CLI command for "rmdir".

NewRemoveDirCommand returns the CLI command for rmdir .


func rmdirCommandFunc(c *cli.Context, ki client.KeysAPI) {
	if len(c.Args()) == 0 {
		handleError(c, ExitBadArgs, errors.New("key required"))
	}
	key := c.Args()[0]

	ctx, cancel := contextWithTotalTimeout(c)
	resp, err := ki.Delete(ctx, key, &client.DeleteOptions{Dir: true})
	cancel()
	if err != nil {
		handleError(c, ExitServerError, err)
	}

	if !resp.Node.Dir || c.GlobalString("output") != "simple" {
		printResponseKey(resp, c.GlobalString("output"))
	}
}

func rmdirCommandFunc ( c * cli . Context , ki client . KeysAPI ) { if len ( c . Args ( ) ) == 0 { handleError ( c , ExitBadArgs , errors . New ( " " ) ) } key := c . Args ( ) [ 0 ] ctx , cancel := contextWithTotalTimeout ( c ) resp , err := ki . Delete ( ctx , key , & client . DeleteOptions { Dir : true } ) cancel ( ) if err != nil { handleError ( c , ExitServerError , err ) } if ! resp . Node . Dir || c . GlobalString ( " " ) != " " { printResponseKey ( resp , c . GlobalString ( " " ) ) } }

// rmdirCommandFunc executes the "rmdir" command.

rmdirCommandFunc executes the rmdir command .


func NewDelCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "del [options] <key> [range_end]",
		Short: "Removes the specified key or range of keys [key, range_end)",
		Run:   delCommandFunc,
	}

	cmd.Flags().BoolVar(&delPrefix, "prefix", false, "delete keys with matching prefix")
	cmd.Flags().BoolVar(&delPrevKV, "prev-kv", false, "return deleted key-value pairs")
	cmd.Flags().BoolVar(&delFromKey, "from-key", false, "delete keys that are greater than or equal to the given key using byte compare")
	return cmd
}

func NewDelCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , Run : delCommandFunc , } cmd . Flags ( ) . BoolVar ( & delPrefix , " " , false , " " ) cmd . Flags ( ) . BoolVar ( & delPrevKV , " " , false , " " ) cmd . Flags ( ) . BoolVar ( & delFromKey , " " , false , " " ) return cmd }

// NewDelCommand returns the cobra command for "del".

NewDelCommand returns the cobra command for del .


func delCommandFunc(cmd *cobra.Command, args []string) {
	key, opts := getDelOp(args)
	ctx, cancel := commandCtx(cmd)
	resp, err := mustClientFromCmd(cmd).Delete(ctx, key, opts...)
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	display.Del(*resp)
}

func delCommandFunc ( cmd * cobra . Command , args [ ] string ) { key , opts := getDelOp ( args ) ctx , cancel := commandCtx ( cmd ) resp , err := mustClientFromCmd ( cmd ) . Delete ( ctx , key , opts ... ) cancel ( ) if err != nil { ExitWithError ( ExitError , err ) } display . Del ( * resp ) }

// delCommandFunc executes the "del" command.

delCommandFunc executes the del command .


func NewExpect(name string, arg ...string) (ep *ExpectProcess, err error) {
	// if env[] is nil, use current system env
	return NewExpectWithEnv(name, arg, nil)
}

// NewExpect creates a new process for expect testing.

NewExpect creates a new process for expect testing .


func NewExpectWithEnv(name string, args []string, env []string) (ep *ExpectProcess, err error) {
	cmd := exec.Command(name, args...)
	cmd.Env = env
	ep = &ExpectProcess{
		cmd:        cmd,
		StopSignal: syscall.SIGKILL,
	}
	ep.cond = sync.NewCond(&ep.mu)
	ep.cmd.Stderr = ep.cmd.Stdout
	ep.cmd.Stdin = nil

	if ep.fpty, err = pty.Start(ep.cmd); err != nil {
		return nil, err
	}

	ep.wg.Add(1)
	go ep.read()
	return ep, nil
}

func NewExpectWithEnv ( name string , args [ ] string , env [ ] string ) ( ep * ExpectProcess , err error ) { cmd := exec . Command ( name , args ... ) cmd . Env = env ep = & ExpectProcess { cmd : cmd , StopSignal : syscall . SIGKILL , } ep . cond = sync . NewCond ( & ep . mu ) ep . cmd . Stderr = ep . cmd . Stdout ep . cmd . Stdin = nil if ep . fpty , err = pty . Start ( ep . cmd ) ; err != nil { return nil , err } ep . wg . Add ( 1 ) go ep . read ( ) return ep , nil }

// NewExpectWithEnv creates a new process with user defined env variables for expect testing.

NewExpectWithEnv creates a new process with user defined env variables for expect testing .


func (ep *ExpectProcess) ExpectFunc(f func(string) bool) (string, error) {
	ep.mu.Lock()
	for {
		for len(ep.lines) == 0 && ep.err == nil {
			ep.cond.Wait()
		}
		if len(ep.lines) == 0 {
			break
		}
		l := ep.lines[0]
		ep.lines = ep.lines[1:]
		if f(l) {
			ep.mu.Unlock()
			return l, nil
		}
	}
	ep.mu.Unlock()
	return "", ep.err
}

func ( ep * ExpectProcess ) ExpectFunc ( f func ( string ) bool ) ( string , error ) { ep . mu . Lock ( ) for { for len ( ep . lines ) == 0 && ep . err == nil { ep . cond . Wait ( ) } if len ( ep . lines ) == 0 { break } l := ep . lines [ 0 ] ep . lines = ep . lines [ 1 : ] if f ( l ) { ep . mu . Unlock ( ) return l , nil } } ep . mu . Unlock ( ) return " " , ep . err }

// ExpectFunc returns the first line satisfying the function f.

ExpectFunc returns the first line satisfying the function f .


func (ep *ExpectProcess) Expect(s string) (string, error) {
	return ep.ExpectFunc(func(txt string) bool { return strings.Contains(txt, s) })
}

func ( ep * ExpectProcess ) Expect ( s string ) ( string , error ) { return ep . ExpectFunc ( func ( txt string ) bool { return strings . Contains ( txt , s ) } ) }

// Expect returns the first line containing the given string.

Expect returns the first line containing the given string .


func (ep *ExpectProcess) LineCount() int {
	ep.mu.Lock()
	defer ep.mu.Unlock()
	return ep.count
}

func ( ep * ExpectProcess ) LineCount ( ) int { ep . mu . Lock ( ) defer ep . mu . Unlock ( ) return ep . count }

// LineCount returns the number of recorded lines since
// the beginning of the process.

LineCount returns the number of recorded lines since the beginning of the process .


func (ep *ExpectProcess) Signal(sig os.Signal) error {
	return ep.cmd.Process.Signal(sig)
}

func ( ep * ExpectProcess ) Signal ( sig os . Signal ) error { return ep . cmd . Process . Signal ( sig ) }

// Signal sends a signal to the expect process


func keyFunc(req *pb.RangeRequest) string {
	// TODO: use marshalTo to reduce allocation
	b, err := req.Marshal()
	if err != nil {
		panic(err)
	}
	return string(b)
}

// keyFunc returns the key of a request, which is used to look up its caching response in the cache.

keyFunc returns the key of a request which is used to look up its caching response in the cache .


func (c *cache) Add(req *pb.RangeRequest, resp *pb.RangeResponse) {
	key := keyFunc(req)

	c.mu.Lock()
	defer c.mu.Unlock()

	if req.Revision > c.compactedRev {
		c.lru.Add(key, resp)
	}
	// we do not need to invalidate a request with a revision specified.
	// so we do not need to add it into the reverse index.
	if req.Revision != 0 {
		return
	}

	var (
		iv  *adt.IntervalValue
		ivl adt.Interval
	)
	if len(req.RangeEnd) != 0 {
		ivl = adt.NewStringAffineInterval(string(req.Key), string(req.RangeEnd))
	} else {
		ivl = adt.NewStringAffinePoint(string(req.Key))
	}

	iv = c.cachedRanges.Find(ivl)

	if iv == nil {
		val := map[string]struct{}{key: {}}
		c.cachedRanges.Insert(ivl, val)
	} else {
		val := iv.Val.(map[string]struct{})
		val[key] = struct{}{}
		iv.Val = val
	}
}

// Add adds the response of a request to the cache if its revision is larger than the compacted revision of the cache.

Add adds the response of a request to the cache if its revision is larger than the compacted revision of the cache .


func (c *cache) Get(req *pb.RangeRequest) (*pb.RangeResponse, error) {
	key := keyFunc(req)

	c.mu.Lock()
	defer c.mu.Unlock()

	if req.Revision > 0 && req.Revision < c.compactedRev {
		c.lru.Remove(key)
		return nil, ErrCompacted
	}

	if resp, ok := c.lru.Get(key); ok {
		return resp.(*pb.RangeResponse), nil
	}
	return nil, errors.New("not exist")
}

func ( c * cache ) Get ( req * pb . RangeRequest ) ( * pb . RangeResponse , error ) { key := keyFunc ( req ) c . mu . Lock ( ) defer c . mu . Unlock ( ) if req . Revision > 0 && req . Revision < c . compactedRev { c . lru . Remove ( key ) return nil , ErrCompacted } if resp , ok := c . lru . Get ( key ) ; ok { return resp . ( * pb . RangeResponse ) , nil } return nil , errors . New ( " " ) }

// Get looks up the caching response for a given request.
// Get is also responsible for lazy eviction when accessing compacted entries.

Get looks up the caching response for a given request . Get is also responsible for lazy eviction when accessing compacted entries .


func (c *cache) Invalidate(key, endkey []byte) {
	c.mu.Lock()
	defer c.mu.Unlock()

	var (
		ivs []*adt.IntervalValue
		ivl adt.Interval
	)
	if len(endkey) == 0 {
		ivl = adt.NewStringAffinePoint(string(key))
	} else {
		ivl = adt.NewStringAffineInterval(string(key), string(endkey))
	}

	ivs = c.cachedRanges.Stab(ivl)
	for _, iv := range ivs {
		keys := iv.Val.(map[string]struct{})
		for key := range keys {
			c.lru.Remove(key)
		}
	}
	// delete after removing all keys since it is destructive to 'ivs'
	c.cachedRanges.Delete(ivl)
}

// Invalidate invalidates the cache entries that intersecting with the given range from key to endkey.

Invalidate invalidates the cache entries that intersecting with the given range from key to endkey .


func (c *cache) Compact(revision int64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if revision > c.compactedRev {
		c.compactedRev = revision
	}
}

func ( c * cache ) Compact ( revision int64 ) { c . mu . Lock ( ) defer c . mu . Unlock ( ) if revision > c . compactedRev { c . compactedRev = revision } }

// Compact invalidate all caching response before the given rev.
// Replace with the invalidation is lazy. The actual removal happens when the entries is accessed.

Compact invalidate all caching response before the given rev . Replace with the invalidation is lazy . The actual removal happens when the entries is accessed .


func NewUniqueURLsWithExceptions(s string, exceptions ...string) *UniqueURLs {
	us := &UniqueURLs{Values: make(map[string]struct{}), Allowed: make(map[string]struct{})}
	for _, v := range exceptions {
		us.Allowed[v] = struct{}{}
	}
	if s == "" {
		return us
	}
	if err := us.Set(s); err != nil {
		plog.Panicf("new UniqueURLs should never fail: %v", err)
	}
	return us
}

func NewUniqueURLsWithExceptions ( s string , exceptions ... string ) * UniqueURLs { us := & UniqueURLs { Values : make ( map [ string ] struct { } ) , Allowed : make ( map [ string ] struct { } ) } for _ , v := range exceptions { us . Allowed [ v ] = struct { } { } } if s == " " { return us } if err := us . Set ( s ) ; err != nil { plog . Panicf ( " " , err ) } return us }

// NewUniqueURLsWithExceptions implements "url.URL" slice as flag.Value interface.
// Given value is to be separated by comma.

NewUniqueURLsWithExceptions implements url . URL slice as flag . Value interface . Given value is to be separated by comma .


func UniqueURLsFromFlag(fs *flag.FlagSet, urlsFlagName string) []url.URL {
	return (*fs.Lookup(urlsFlagName).Value.(*UniqueURLs)).uss
}

func UniqueURLsFromFlag ( fs * flag . FlagSet , urlsFlagName string ) [ ] url . URL { return ( * fs . Lookup ( urlsFlagName ) . Value . ( * UniqueURLs ) ) . uss }

// UniqueURLsFromFlag returns a slice from urls got from the flag.

UniqueURLsFromFlag returns a slice from urls got from the flag .


func UniqueURLsMapFromFlag(fs *flag.FlagSet, urlsFlagName string) map[string]struct{} {
	return (*fs.Lookup(urlsFlagName).Value.(*UniqueURLs)).Values
}

func UniqueURLsMapFromFlag ( fs * flag . FlagSet , urlsFlagName string ) map [ string ] struct { } { return ( * fs . Lookup ( urlsFlagName ) . Value . ( * UniqueURLs ) ) . Values }

// UniqueURLsMapFromFlag returns a map from url strings got from the flag.

UniqueURLsMapFromFlag returns a map from url strings got from the flag .


func (b *Barrier) Hold() error {
	_, err := newKey(b.client, b.key, v3.NoLease)
	return err
}

func ( b * Barrier ) Hold ( ) error { _ , err := newKey ( b . client , b . key , v3 . NoLease ) return err }

// Hold creates the barrier key causing processes to block on Wait.

Hold creates the barrier key causing processes to block on Wait .


func (b *Barrier) Release() error {
	_, err := b.client.Delete(b.ctx, b.key)
	return err
}

func ( b * Barrier ) Release ( ) error { _ , err := b . client . Delete ( b . ctx , b . key ) return err }

// Release deletes the barrier key to unblock all waiting processes.

Release deletes the barrier key to unblock all waiting processes .


func (b *Barrier) Wait() error {
	resp, err := b.client.Get(b.ctx, b.key, v3.WithFirstKey()...)
	if err != nil {
		return err
	}
	if len(resp.Kvs) == 0 {
		// key already removed
		return nil
	}
	_, err = WaitEvents(
		b.client,
		b.key,
		resp.Header.Revision,
		[]mvccpb.Event_EventType{mvccpb.PUT, mvccpb.DELETE})
	return err
}

func
(
b
*
Barrier
)
Wait
(
)
error
{
resp
,
err
:=
b
.
client
.
Get
(
b
.
ctx
,
b
.
key
,
v3
.
WithFirstKey
(
)
...
)
if
err
!=
{
return
err
}
if
len
(
resp
.
Kvs
)
==
0
{
// key already removed
return
}
_
,
err
=
WaitEvents
(
b
.
client
,
b
.
key
,
resp
.
Header
.
Revision
,
[
]
mvccpb
.
Event_EventType
{
mvccpb
.
PUT
,
mvccpb
.
DELETE
}
)
return
err
}

// Wait blocks on the barrier key until it is deleted. If there is no key, Wait
// assumes Release has already been called and returns immediately.

Wait blocks on the barrier key until it is deleted . If there is no key Wait assumes Release has already been called and returns immediately .


func NewLockRacerCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "lock-racer [name of lock (defaults to 'racers')]",
		Short: "Performs lock race operation",
		Run:   runRacerFunc,
	}
	cmd.Flags().IntVar(&totalClientConnections, "total-client-connections", 10, "total number of client connections")
	return cmd
}

func NewLockRacerCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , Run : runRacerFunc , } cmd . Flags ( ) . IntVar ( & totalClientConnections , " " , 10 , " " ) return cmd }

// NewLockRacerCommand returns the cobra command for "lock-racer runner".

NewLockRacerCommand returns the cobra command for lock - racer runner .


func (m *Member) ElectionTimeout() time.Duration {
	return time.Duration(m.Etcd.ElectionTimeoutMs) * time.Millisecond
}

func ( m * Member ) ElectionTimeout ( ) time . Duration { return time . Duration ( m . Etcd . ElectionTimeoutMs ) * time . Millisecond }

// ElectionTimeout returns an election timeout duration.

ElectionTimeout returns an election timeout duration .


func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	dialOpts := []grpc.DialOption{
		grpc.WithTimeout(5 * time.Second),
		grpc.WithBlock(),
	}

	secure := false
	for _, cu := range m.Etcd.AdvertiseClientURLs {
		u, err := url.Parse(cu)
		if err != nil {
			return nil, err
		}
		if u.Scheme == "https" { // TODO: handle unix
			secure = true
		}
	}

	if secure {
		// assume save TLS assets are already stord on disk
		tlsInfo := transport.TLSInfo{
			CertFile:      m.ClientCertPath,
			KeyFile:       m.ClientKeyPath,
			TrustedCAFile: m.ClientTrustedCAPath,

			// TODO: remove this with generated certs
			// only need it for auto TLS
			InsecureSkipVerify: true,
		}
		tlsConfig, err := tlsInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		creds := credentials.NewTLS(tlsConfig)
		dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
	} else {
		dialOpts = append(dialOpts, grpc.WithInsecure())
	}
	dialOpts = append(dialOpts, opts...)
	return grpc.Dial(m.EtcdClientEndpoint, dialOpts...)
}

// DialEtcdGRPCServer creates a raw gRPC connection to an etcd member.

DialEtcdGRPCServer creates a raw gRPC connection to an etcd member .


func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3.Config, err error) {
	secure := false
	for _, cu := range m.Etcd.AdvertiseClientURLs {
		var u *url.URL
		u, err = url.Parse(cu)
		if err != nil {
			return nil, err
		}
		if u.Scheme == "https" { // TODO: handle unix
			secure = true
		}
	}

	cfg = &clientv3.Config{
		Endpoints:   []string{m.EtcdClientEndpoint},
		DialTimeout: 10 * time.Second,
		DialOptions: opts,
	}
	if secure {
		// assume save TLS assets are already stord on disk
		tlsInfo := transport.TLSInfo{
			CertFile:      m.ClientCertPath,
			KeyFile:       m.ClientKeyPath,
			TrustedCAFile: m.ClientTrustedCAPath,

			// TODO: remove this with generated certs
			// only need it for auto TLS
			InsecureSkipVerify: true,
		}
		var tlsConfig *tls.Config
		tlsConfig, err = tlsInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		cfg.TLS = tlsConfig
	}
	return cfg, err
}

// CreateEtcdClientConfig creates a client configuration from member.

CreateEtcdClientConfig creates a client configuration from member .


func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) {
	cfg, err := m.CreateEtcdClientConfig(opts...)
	if err != nil {
		return nil, err
	}
	return clientv3.New(*cfg)
}

func ( m * Member ) CreateEtcdClient ( opts ... grpc . DialOption ) ( * clientv3 . Client , error ) { cfg , err := m . CreateEtcdClientConfig ( opts ... ) if err != nil { return nil , err } return clientv3 . New ( * cfg ) }

// CreateEtcdClient creates a client from member.

CreateEtcdClient creates a client from member .


func (m *Member) CheckCompact(rev int64) error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	wch := cli.Watch(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithRev(rev-1))
	wr, ok := <-wch
	cancel()

	if !ok {
		return fmt.Errorf("watch channel terminated (endpoint %q)", m.EtcdClientEndpoint)
	}
	if wr.CompactRevision != rev {
		return fmt.Errorf("got compact revision %v, wanted %v (endpoint %q)", wr.CompactRevision, rev, m.EtcdClientEndpoint)
	}

	return nil
}

func ( m * Member ) CheckCompact ( rev int64 ) error { cli , err := m . CreateEtcdClient ( ) if err != nil { return fmt . Errorf ( " " , err , m . EtcdClientEndpoint ) } defer cli . Close ( ) ctx , cancel := context . WithTimeout ( context . Background ( ) , 5 * time . Second ) wch := cli . Watch ( ctx , " \x00 " , clientv3 . WithFromKey ( ) , clientv3 . WithRev ( rev - 1 ) ) wr , ok := <- wch cancel ( ) if ! ok { return fmt . Errorf ( " " , m . EtcdClientEndpoint ) } if wr . CompactRevision != rev { return fmt . Errorf ( " " , wr . CompactRevision , rev , m . EtcdClientEndpoint ) } return nil }

// CheckCompact ensures that historical data before given revision has been compacted.

CheckCompact ensures that historical data before given revision has been compacted .


func (m *Member) Defrag() error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	_, err = cli.Defragment(ctx, m.EtcdClientEndpoint)
	cancel()
	return err
}

func ( m * Member ) Defrag ( ) error { cli , err := m . CreateEtcdClient ( ) if err != nil { return fmt . Errorf ( " " , err , m . EtcdClientEndpoint ) } defer cli . Close ( ) ctx , cancel := context . WithTimeout ( context . Background ( ) , 5 * time . Minute ) _ , err = cli . Defragment ( ctx , m . EtcdClientEndpoint ) cancel ( ) return err }

// Defrag runs defragmentation on this member.

Defrag runs defragmentation on this member .


func (m *Member) RevHash() (int64, int64, error) {
	conn, err := m.DialEtcdGRPCServer()
	if err != nil {
		return 0, 0, err
	}
	defer conn.Close()

	mt := pb.NewMaintenanceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	resp, err := mt.Hash(ctx, &pb.HashRequest{}, grpc.FailFast(false))
	cancel()

	if err != nil {
		return 0, 0, err
	}

	return resp.Header.Revision, int64(resp.Hash), nil
}

func ( m * Member ) RevHash ( ) ( int64 , int64 , error ) { conn , err := m . DialEtcdGRPCServer ( ) if err != nil { return 0 , 0 , err } defer conn . Close ( ) mt := pb . NewMaintenanceClient ( conn ) ctx , cancel := context . WithTimeout ( context . Background ( ) , 5 * time . Second ) resp , err := mt . Hash ( ctx , & pb . HashRequest { } , grpc . FailFast ( false ) ) cancel ( ) if err != nil { return 0 , 0 , err } return resp . Header . Revision , int64 ( resp . Hash ) , nil }

// RevHash fetches current revision and hash on this member.

RevHash fetches current revision and hash on this member .


func (m *Member) Rev(ctx context.Context) (int64, error) {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return 0, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	resp, err := cli.Status(ctx, m.EtcdClientEndpoint)
	if err != nil {
		return 0, err
	}
	return resp.Header.Revision, nil
}

func ( m * Member ) Rev ( ctx context . Context ) ( int64 , error ) { cli , err := m . CreateEtcdClient ( ) if err != nil { return 0 , fmt . Errorf ( " " , err , m . EtcdClientEndpoint ) } defer cli . Close ( ) resp , err := cli . Status ( ctx , m . EtcdClientEndpoint ) if err != nil { return 0 , err } return resp . Header . Revision , nil }

// Rev fetches current revision on this member.

Rev fetches current revision on this member .


func (m *Member) Compact(rev int64, timeout time.Duration) error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	_, err = cli.Compact(ctx, rev, clientv3.WithCompactPhysical())
	cancel()
	return err
}

func ( m * Member ) Compact ( rev int64 , timeout time . Duration ) error { cli , err := m . CreateEtcdClient ( ) if err != nil { return fmt . Errorf ( " " , err , m . EtcdClientEndpoint ) } defer cli . Close ( ) ctx , cancel := context . WithTimeout ( context . Background ( ) , timeout ) _ , err = cli . Compact ( ctx , rev , clientv3 . WithCompactPhysical ( ) ) cancel ( ) return err }

// Compact compacts member storage with given revision.
// It blocks until it's physically done.

Compact compacts member storage with given revision . It blocks until it s physically done .


func (m *Member) IsLeader() (bool, error) {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return false, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	resp, err := cli.Status(context.Background(), m.EtcdClientEndpoint)
	if err != nil {
		return false, err
	}
	return resp.Header.MemberId == resp.Leader, nil
}

func ( m * Member ) IsLeader ( ) ( bool , error ) { cli , err := m . CreateEtcdClient ( ) if err != nil { return false , fmt . Errorf ( " " , err , m . EtcdClientEndpoint ) } defer cli . Close ( ) resp , err := cli . Status ( context . Background ( ) , m . EtcdClientEndpoint ) if err != nil { return false , err } return resp . Header . MemberId == resp . Leader , nil }

// IsLeader returns true if this member is the current cluster leader.

IsLeader returns true if this member is the current cluster leader .


func (m *Member) WriteHealthKey() error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	// give enough time-out in case expensive requests (range/delete) are pending
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	_, err = cli.Put(ctx, "health", "good")
	cancel()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	return nil
}

// WriteHealthKey writes a health key to this member.

WriteHealthKey writes a health key to this member .


func (m *Member) SaveSnapshot(lg *zap.Logger) (err error) {
	// remove existing snapshot first
	if err = os.RemoveAll(m.SnapshotPath); err != nil {
		return err
	}

	var ccfg *clientv3.Config
	ccfg, err = m.CreateEtcdClientConfig()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}

	lg.Info(
		"snapshot save START",
		zap.String("member-name", m.Etcd.Name),
		zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
	)
	now := time.Now()
	mgr := snapshot.NewV3(lg)
	if err = mgr.Save(context.Background(), *ccfg, m.SnapshotPath); err != nil {
		return err
	}
	took := time.Since(now)

	var fi os.FileInfo
	fi, err = os.Stat(m.SnapshotPath)
	if err != nil {
		return err
	}
	var st snapshot.Status
	st, err = mgr.Status(m.SnapshotPath)
	if err != nil {
		return err
	}
	m.SnapshotInfo = &SnapshotInfo{
		MemberName:        m.Etcd.Name,
		MemberClientURLs:  m.Etcd.AdvertiseClientURLs,
		SnapshotPath:      m.SnapshotPath,
		SnapshotFileSize:  humanize.Bytes(uint64(fi.Size())),
		SnapshotTotalSize: humanize.Bytes(uint64(st.TotalSize)),
		SnapshotTotalKey:  int64(st.TotalKey),
		SnapshotHash:      int64(st.Hash),
		SnapshotRevision:  st.Revision,
		Took:              fmt.Sprintf("%v", took),
	}
	lg.Info(
		"snapshot save END",
		zap.String("member-name", m.SnapshotInfo.MemberName),
		zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
		zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
		zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
		zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
		zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
		zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
		zap.String("took", m.SnapshotInfo.Took),
	)
	return nil
}

func
(
m
*
Member
)
SaveSnapshot
(
lg
*
zap
.
Logger
)
(
err
error
)
{
// remove existing snapshot first
if
err
=
os
.
RemoveAll
(
m
.
SnapshotPath
)
;
err
!=
{
return
err
}
var
ccfg
*
clientv3
.
Config
ccfg
,
err
=
m
.
CreateEtcdClientConfig
(
)
if
err
!=
{
return
fmt
.
Errorf
(
"
"
,
err
,
m
.
EtcdClientEndpoint
)
}
lg
.
Info
(
"
"
,
zap
.
String
(
"
"
,
m
.
Etcd
.
Name
)
,
zap
.
Strings
(
"
"
,
m
.
Etcd
.
AdvertiseClientURLs
)
,
zap
.
String
(
"
"
,
m
.
SnapshotPath
)
,
)
now
:=
time
.
Now
(
)
mgr
:=
snapshot
.
NewV3
(
lg
)
if
err
=
mgr
.
Save
(
context
.
Background
(
)
,
*
ccfg
,
m
.
SnapshotPath
)
;
err
!=
{
return
err
}
took
:=
time
.
Since
(
now
)
var
fi
os
.
FileInfo
fi
,
err
=
os
.
Stat
(
m
.
SnapshotPath
)
if
err
!=
{
return
err
}
var
st
snapshot
.
Status
st
,
err
=
mgr
.
Status
(
m
.
SnapshotPath
)
if
err
!=
{
return
err
}
m
.
SnapshotInfo
=
&
SnapshotInfo
{
MemberName
:
m
.
Etcd
.
Name
,
MemberClientURLs
:
m
.
Etcd
.
AdvertiseClientURLs
,
SnapshotPath
:
m
.
SnapshotPath
,
SnapshotFileSize
:
humanize
.
Bytes
(
uint64
(
fi
.
Size
(
)
)
)
,
SnapshotTotalSize
:
humanize
.
Bytes
(
uint64
(
st
.
TotalSize
)
)
,
SnapshotTotalKey
:
int64
(
st
.
TotalKey
)
,
SnapshotHash
:
int64
(
st
.
Hash
)
,
SnapshotRevision
:
st
.
Revision
,
Took
:
fmt
.
Sprintf
(
"
"
,
took
)
,
}
lg
.
Info
(
"
"
,
zap
.
String
(
"
"
,
m
.
SnapshotInfo
.
MemberName
)
,
zap
.
Strings
(
"
"
,
m
.
SnapshotInfo
.
MemberClientURLs
)
,
zap
.
String
(
"
"
,
m
.
SnapshotPath
)
,
zap
.
String
(
"
"
,
m
.
SnapshotInfo
.
SnapshotFileSize
)
,
zap
.
String
(
"
"
,
m
.
SnapshotInfo
.
SnapshotTotalSize
)
,
zap
.
Int64
(
"
"
,
m
.
SnapshotInfo
.
SnapshotTotalKey
)
,
zap
.
Int64
(
"
"
,
m
.
SnapshotInfo
.
SnapshotHash
)
,
zap
.
Int64
(
"
"
,
m
.
SnapshotInfo
.
SnapshotRevision
)
,
zap
.
String
(
"
"
,
m
.
SnapshotInfo
.
Took
)
,
)
return
}

// SaveSnapshot downloads a snapshot file from this member, locally.
// It's meant to requested remotely, so that local member can store
// snapshot file on local disk.

SaveSnapshot downloads a snapshot file from this member locally . It s meant to requested remotely so that local member can store snapshot file on local disk .


func (m *Member) RestoreSnapshot(lg *zap.Logger) (err error) {
	if err = os.RemoveAll(m.EtcdOnSnapshotRestore.DataDir); err != nil {
		return err
	}
	if err = os.RemoveAll(m.EtcdOnSnapshotRestore.WALDir); err != nil {
		return err
	}

	lg.Info(
		"snapshot restore START",
		zap.String("member-name", m.Etcd.Name),
		zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
	)
	now := time.Now()
	mgr := snapshot.NewV3(lg)
	err = mgr.Restore(snapshot.RestoreConfig{
		SnapshotPath:        m.SnapshotInfo.SnapshotPath,
		Name:                m.EtcdOnSnapshotRestore.Name,
		OutputDataDir:       m.EtcdOnSnapshotRestore.DataDir,
		OutputWALDir:        m.EtcdOnSnapshotRestore.WALDir,
		PeerURLs:            m.EtcdOnSnapshotRestore.AdvertisePeerURLs,
		InitialCluster:      m.EtcdOnSnapshotRestore.InitialCluster,
		InitialClusterToken: m.EtcdOnSnapshotRestore.InitialClusterToken,
		SkipHashCheck:       false,
		// TODO: set SkipHashCheck it true, to recover from existing db file
	})
	took := time.Since(now)
	lg.Info(
		"snapshot restore END",
		zap.String("member-name", m.SnapshotInfo.MemberName),
		zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
		zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
		zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
		zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
		zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
		zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
		zap.String("took", took.String()),
		zap.Error(err),
	)
	return err
}

// RestoreSnapshot restores a cluster from a given snapshot file on disk.
// It's meant to requested remotely, so that local member can load the
// snapshot file from local disk.

RestoreSnapshot restores a cluster from a given snapshot file on disk . It s meant to requested remotely so that local member can load the snapshot file from local disk .


func NewWatcher(w clientv3.Watcher, prefix string) clientv3.Watcher {
	return &watcherPrefix{Watcher: w, pfx: prefix, stopc: make(chan struct{})}
}

func NewWatcher ( w clientv3 . Watcher , prefix string ) clientv3 . Watcher { return & watcherPrefix { Watcher : w , pfx : prefix , stopc : make ( chan struct { } ) } }

// NewWatcher wraps a Watcher instance so that all Watch requests
// are prefixed with a given string and all Watch responses have
// the prefix removed.

NewWatcher wraps a Watcher instance so that all Watch requests are prefixed with a given string and all Watch responses have the prefix removed .


func NewRawNode(config *Config, peers []Peer) (*RawNode, error) {
	if config.ID == 0 {
		panic("config.ID must not be zero")
	}
	r := newRaft(config)
	rn := &RawNode{
		raft: r,
	}
	lastIndex, err := config.Storage.LastIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	// If the log is empty, this is a new RawNode (like StartNode); otherwise it's
	// restoring an existing RawNode (like RestartNode).
	// TODO(bdarnell): rethink RawNode initialization and whether the application needs
	// to be able to tell us when it expects the RawNode to exist.
	if lastIndex == 0 {
		r.becomeFollower(1, None)
		ents := make([]pb.Entry, len(peers))
		for i, peer := range peers {
			cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context}
			data, err := cc.Marshal()
			if err != nil {
				panic("unexpected marshal error")
			}

			ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data}
		}
		r.raftLog.append(ents...)
		r.raftLog.committed = uint64(len(ents))
		for _, peer := range peers {
			r.addNode(peer.ID)
		}
	}

	// Set the initial hard and soft states after performing all initialization.
	rn.prevSoftSt = r.softState()
	if lastIndex == 0 {
		rn.prevHardSt = emptyState
	} else {
		rn.prevHardSt = r.hardState()
	}

	return rn, nil
}

// NewRawNode returns a new RawNode given configuration and a list of raft peers.

NewRawNode returns a new RawNode given configuration and a list of raft peers .


func (rn *RawNode) Campaign() error {
	return rn.raft.Step(pb.Message{
		Type: pb.MsgHup,
	})
}

func ( rn * RawNode ) Campaign ( ) error { return rn . raft . Step ( pb . Message { Type : pb . MsgHup , } ) }

// Campaign causes this RawNode to transition to candidate state.

Campaign causes this RawNode to transition to candidate state .


func (rn *RawNode) Propose(data []byte) error {
	return rn.raft.Step(pb.Message{
		Type: pb.MsgProp,
		From: rn.raft.id,
		Entries: []pb.Entry{
			{Data: data},
		}})
}

func ( rn * RawNode ) Propose ( data [ ] byte ) error { return rn . raft . Step ( pb . Message { Type : pb . MsgProp , From : rn . raft . id , Entries : [ ] pb . Entry { { Data : data } , } } ) }

// Propose proposes data be appended to the raft log.

Propose proposes data be appended to the raft log .


func (rn *RawNode) ProposeConfChange(cc pb.ConfChange) error {
	data, err := cc.Marshal()
	if err != nil {
		return err
	}
	return rn.raft.Step(pb.Message{
		Type: pb.MsgProp,
		Entries: []pb.Entry{
			{Type: pb.EntryConfChange, Data: data},
		},
	})
}

func ( rn * RawNode ) ProposeConfChange ( cc pb . ConfChange ) error { data , err := cc . Marshal ( ) if err != nil { return err } return rn . raft . Step ( pb . Message { Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange , Data : data } , } , } ) }

// ProposeConfChange proposes a config change.

ProposeConfChange proposes a config change .


func (rn *RawNode) ApplyConfChange(cc pb.ConfChange) *pb.ConfState {
	if cc.NodeID == None {
		return &pb.ConfState{Nodes: rn.raft.nodes(), Learners: rn.raft.learnerNodes()}
	}
	switch cc.Type {
	case pb.ConfChangeAddNode:
		rn.raft.addNode(cc.NodeID)
	case pb.ConfChangeAddLearnerNode:
		rn.raft.addLearner(cc.NodeID)
	case pb.ConfChangeRemoveNode:
		rn.raft.removeNode(cc.NodeID)
	case pb.ConfChangeUpdateNode:
	default:
		panic("unexpected conf type")
	}
	return &pb.ConfState{Nodes: rn.raft.nodes(), Learners: rn.raft.learnerNodes()}
}

func ( rn * RawNode ) ApplyConfChange ( cc pb . ConfChange ) * pb . ConfState { if cc . NodeID == None { return & pb . ConfState { Nodes : rn . raft . nodes ( ) , Learners : rn . raft . learnerNodes ( ) } } switch cc . Type { case pb . ConfChangeAddNode : rn . raft . addNode ( cc . NodeID ) case pb . ConfChangeAddLearnerNode : rn . raft . addLearner ( cc . NodeID ) case pb . ConfChangeRemoveNode : rn . raft . removeNode ( cc . NodeID ) case pb . ConfChangeUpdateNode : default : panic ( " " ) } return & pb . ConfState { Nodes : rn . raft . nodes ( ) , Learners : rn . raft . learnerNodes ( ) } }

// ApplyConfChange applies a config change to the local node.

ApplyConfChange applies a config change to the local node .


func (rn *RawNode) Step(m pb.Message) error {
	// ignore unexpected local messages receiving over network
	if IsLocalMsg(m.Type) {
		return ErrStepLocalMsg
	}
	if pr := rn.raft.getProgress(m.From); pr != nil || !IsResponseMsg(m.Type) {
		return rn.raft.Step(m)
	}
	return ErrStepPeerNotFound
}

// Step advances the state machine using the given message.

Step advances the state machine using the given message .


func (rn *RawNode) Ready() Ready {
	rd := rn.newReady()
	rn.raft.msgs = nil
	rn.raft.reduceUncommittedSize(rd.CommittedEntries)
	return rd
}

func ( rn * RawNode ) Ready ( ) Ready { rd := rn . newReady ( ) rn . raft . msgs = nil rn . raft . reduceUncommittedSize ( rd . CommittedEntries ) return rd }

// Ready returns the current point-in-time state of this RawNode.

Ready returns the current point - in - time state of this RawNode .


func (rn *RawNode) WithProgress(visitor func(id uint64, typ ProgressType, pr Progress)) {
	for id, pr := range rn.raft.prs {
		pr := *pr
		pr.ins = nil
		visitor(id, ProgressTypePeer, pr)
	}
	for id, pr := range rn.raft.learnerPrs {
		pr := *pr
		pr.ins = nil
		visitor(id, ProgressTypeLearner, pr)
	}
}

func ( rn * RawNode ) WithProgress ( visitor func ( id uint64 , typ ProgressType , pr Progress ) ) { for id , pr := range rn . raft . prs { pr := * pr pr . ins = nil visitor ( id , ProgressTypePeer , pr ) } for id , pr := range rn . raft . learnerPrs { pr := * pr pr . ins = nil visitor ( id , ProgressTypeLearner , pr ) } }

// WithProgress is a helper to introspect the Progress for this node and its
// peers.

WithProgress is a helper to introspect the Progress for this node and its peers .


func (rn *RawNode) ReportUnreachable(id uint64) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgUnreachable, From: id})
}

func ( rn * RawNode ) ReportUnreachable ( id uint64 ) { _ = rn . raft . Step ( pb . Message { Type : pb . MsgUnreachable , From : id } ) }

// ReportUnreachable reports the given node is not reachable for the last send.

ReportUnreachable reports the given node is not reachable for the last send .


func (rn *RawNode) ReportSnapshot(id uint64, status SnapshotStatus) {
	rej := status == SnapshotFailure

	_ = rn.raft.Step(pb.Message{Type: pb.MsgSnapStatus, From: id, Reject: rej})
}

func ( rn * RawNode ) ReportSnapshot ( id uint64 , status SnapshotStatus ) { rej := status == SnapshotFailure _ = rn . raft . Step ( pb . Message { Type : pb . MsgSnapStatus , From : id , Reject : rej } ) }

// ReportSnapshot reports the status of the sent snapshot.

ReportSnapshot reports the status of the sent snapshot .


func (rn *RawNode) TransferLeader(transferee uint64) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgTransferLeader, From: transferee})
}

func ( rn * RawNode ) TransferLeader ( transferee uint64 ) { _ = rn . raft . Step ( pb . Message { Type : pb . MsgTransferLeader , From : transferee } ) }

// TransferLeader tries to transfer leadership to the given transferee.

TransferLeader tries to transfer leadership to the given transferee .


func (rn *RawNode) ReadIndex(rctx []byte) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgReadIndex, Entries: []pb.Entry{{Data: rctx}}})
}

func ( rn * RawNode ) ReadIndex ( rctx [ ] byte ) { _ = rn . raft . Step ( pb . Message { Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : rctx } } } ) }

// ReadIndex requests a read state. The read state will be set in ready.
// Read State has a read index. Once the application advances further than the read
// index, any linearizable read requests issued before the read request can be
// processed safely. The read state will have the same rctx attached.

ReadIndex requests a read state . The read state will be set in ready . Read State has a read index . Once the application advances further than the read index any linearizable read requests issued before the read request can be processed safely . The read state will have the same rctx attached .


func printcURL(req *http.Request) error {
	if !cURLDebug {
		return nil
	}
	var (
		command string
		b       []byte
		err     error
	)

	if req.URL != nil {
		command = fmt.Sprintf("curl -X %s %s", req.Method, req.URL.String())
	}

	if req.Body != nil {
		b, err = ioutil.ReadAll(req.Body)
		if err != nil {
			return err
		}
		command += fmt.Sprintf(" -d %q", string(b))
	}

	fmt.Fprintf(os.Stderr, "cURL Command: %s\n", command)

	// reset body
	body := bytes.NewBuffer(b)
	req.Body = ioutil.NopCloser(body)

	return nil
}

func
printcURL
(
req
*
http
.
Request
)
error
{
if
!
cURLDebug
{
return
}
var
(
command
string
b
[
]
byte
err
error
)
if
req
.
URL
!=
{
command
=
fmt
.
Sprintf
(
"
"
,
req
.
Method
,
req
.
URL
.
String
(
)
)
}
if
req
.
Body
!=
{
b
,
err
=
ioutil
.
ReadAll
(
req
.
Body
)
if
err
!=
{
return
err
}
command
+=
fmt
.
Sprintf
(
"
"
,
string
(
b
)
)
}
fmt
.
Fprintf
(
os
.
Stderr
,
"
\n
"
,
command
)
// reset body
body
:=
bytes
.
NewBuffer
(
b
)
req
.
Body
=
ioutil
.
NopCloser
(
body
)
return
}

// printcURL prints the cURL equivalent request to stderr.
// It returns an error if the body of the request cannot
// be read.
// The caller MUST cancel the request if there is an error.

printcURL prints the cURL equivalent request to stderr . It returns an error if the body of the request cannot be read . The caller MUST cancel the request if there is an error .


func StartNode(c *Config, peers []Peer) Node {
	r := newRaft(c)
	// become the follower at term 1 and apply initial configuration
	// entries of term 1
	r.becomeFollower(1, None)
	for _, peer := range peers {
		cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context}
		d, err := cc.Marshal()
		if err != nil {
			panic("unexpected marshal error")
		}
		e := pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: r.raftLog.lastIndex() + 1, Data: d}
		r.raftLog.append(e)
	}
	// Mark these initial entries as committed.
	// TODO(bdarnell): These entries are still unstable; do we need to preserve
	// the invariant that committed < unstable?
	r.raftLog.committed = r.raftLog.lastIndex()
	// Now apply them, mainly so that the application can call Campaign
	// immediately after StartNode in tests. Note that these nodes will
	// be added to raft twice: here and when the application's Ready
	// loop calls ApplyConfChange. The calls to addNode must come after
	// all calls to raftLog.append so progress.next is set after these
	// bootstrapping entries (it is an error if we try to append these
	// entries since they have already been committed).
	// We do not set raftLog.applied so the application will be able
	// to observe all conf changes via Ready.CommittedEntries.
	for _, peer := range peers {
		r.addNode(peer.ID)
	}

	n := newNode()
	n.logger = c.Logger
	go n.run(r)
	return &n
}

// StartNode returns a new Node given configuration and a list of raft peers.
// It appends a ConfChangeAddNode entry for each given peer to the initial log.

StartNode returns a new Node given configuration and a list of raft peers . It appends a ConfChangeAddNode entry for each given peer to the initial log .


func RestartNode(c *Config) Node {
	r := newRaft(c)

	n := newNode()
	n.logger = c.Logger
	go n.run(r)
	return &n
}

func RestartNode ( c * Config ) Node { r := newRaft ( c ) n := newNode ( ) n . logger = c . Logger go n . run ( r ) return & n }

// RestartNode is similar to StartNode but does not take a list of peers.
// The current membership of the cluster will be restored from the Storage.
// If the caller has an existing state machine, pass in the last log index that
// has been applied to it; otherwise use zero.

RestartNode is similar to StartNode but does not take a list of peers . The current membership of the cluster will be restored from the Storage . If the caller has an existing state machine pass in the last log index that has been applied to it ; otherwise use zero .