func (n *node) Tick() {
	select {
	case n.tickc <- struct{}{}:
	case <-n.done:
	default:
		n.logger.Warningf("A tick missed to fire. Node blocks too long!")
	}
}

func ( n * node ) Tick ( ) { select { case n . tickc <- struct { } { } : case <- n . done : default : n . logger . Warningf ( " " ) } }

// Tick increments the internal logical clock for this Node. Election timeouts
// and heartbeat timeouts are in units of ticks.

Tick increments the internal logical clock for this Node . Election timeouts and heartbeat timeouts are in units of ticks .


func MustSync(st, prevst pb.HardState, entsnum int) bool {
	// Persistent state on all servers:
	// (Updated on stable storage before responding to RPCs)
	// currentTerm
	// votedFor
	// log entries[]
	return entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term
}

// MustSync returns true if the hard state and count of Raft entries indicate
// that a synchronous write to persistent storage is required.

MustSync returns true if the hard state and count of Raft entries indicate that a synchronous write to persistent storage is required .


func NewGRPC17Health(
	eps []string,
	timeout time.Duration,
	dialFunc DialFunc,
) *GRPC17Health {
	notifyCh := make(chan []grpc.Address)
	addrs := eps2addrs(eps)
	hb := &GRPC17Health{
		addrs:              addrs,
		eps:                eps,
		notifyCh:           notifyCh,
		readyc:             make(chan struct{}),
		healthCheck:        func(ep string) (bool, error) { return grpcHealthCheck(ep, dialFunc) },
		unhealthyHostPorts: make(map[string]time.Time),
		upc:                make(chan struct{}),
		stopc:              make(chan struct{}),
		downc:              make(chan struct{}),
		donec:              make(chan struct{}),
		updateAddrsC:       make(chan NotifyMsg),
		hostPort2ep:        getHostPort2ep(eps),
	}
	if timeout < minHealthRetryDuration {
		timeout = minHealthRetryDuration
	}
	hb.healthCheckTimeout = timeout

	close(hb.downc)
	go hb.updateNotifyLoop()
	hb.wg.Add(1)
	go func() {
		defer hb.wg.Done()
		hb.updateUnhealthy()
	}()
	return hb
}

func NewGRPC17Health ( eps [ ] string , timeout time . Duration , dialFunc DialFunc , ) * GRPC17Health { notifyCh := make ( chan [ ] grpc . Address ) addrs := eps2addrs ( eps ) hb := & GRPC17Health { addrs : addrs , eps : eps , notifyCh : notifyCh , readyc : make ( chan struct { } ) , healthCheck : func ( ep string ) ( bool , error ) { return grpcHealthCheck ( ep , dialFunc ) } , unhealthyHostPorts : make ( map [ string ] time . Time ) , upc : make ( chan struct { } ) , stopc : make ( chan struct { } ) , downc : make ( chan struct { } ) , donec : make ( chan struct { } ) , updateAddrsC : make ( chan NotifyMsg ) , hostPort2ep : getHostPort2ep ( eps ) , } if timeout < minHealthRetryDuration { timeout = minHealthRetryDuration } hb . healthCheckTimeout = timeout close ( hb . downc ) go hb . updateNotifyLoop ( ) hb . wg . Add ( 1 ) go func ( ) { defer hb . wg . Done ( ) hb . updateUnhealthy ( ) } ( ) return hb }

// NewGRPC17Health returns a new health balancer with gRPC v1.7.

NewGRPC17Health returns a new health balancer with gRPC v1 . 7 .


func (b *GRPC17Health) NeedUpdate() bool {
	// updating notifyCh can trigger new connections,
	// need update addrs if all connections are down
	// or addrs does not include pinAddr.
	b.mu.RLock()
	update := !hasAddr(b.addrs, b.pinAddr)
	b.mu.RUnlock()
	return update
}

// NeedUpdate returns true if all connections are down or
// addresses do not include current pinned address.

NeedUpdate returns true if all connections are down or addresses do not include current pinned address .


func dflSignal(sig syscall.Signal) {
	// clearing out the sigact sets the signal to SIG_DFL
	var sigactBuf [32]uint64
	ptr := unsafe.Pointer(&sigactBuf)
	syscall.Syscall6(uintptr(syscall.SYS_RT_SIGACTION), uintptr(sig), uintptr(ptr), 0, 8, 0, 0)
}

// dflSignal sets the given signal to SIG_DFL


func New(namespaces ...string) Store {
	s := newStore(namespaces...)
	s.clock = clockwork.NewRealClock()
	return s
}

func New ( namespaces ... string ) Store { s := newStore ( namespaces ... ) s . clock = clockwork . NewRealClock ( ) return s }

// New creates a store where the given namespaces will be created as initial directories.

New creates a store where the given namespaces will be created as initial directories .


func (s *store) Index() uint64 {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()
	return s.CurrentIndex
}

func ( s * store ) Index ( ) uint64 { s . worldLock . RLock ( ) defer s . worldLock . RUnlock ( ) return s . CurrentIndex }

// Index retrieves the current index of the store.

Index retrieves the current index of the store .


func (s *store) Get(nodePath string, recursive, sorted bool) (*Event, error) {
	var err *v2error.Error

	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	defer func() {
		if err == nil {
			s.Stats.Inc(GetSuccess)
			if recursive {
				reportReadSuccess(GetRecursive)
			} else {
				reportReadSuccess(Get)
			}
			return
		}

		s.Stats.Inc(GetFail)
		if recursive {
			reportReadFailure(GetRecursive)
		} else {
			reportReadFailure(Get)
		}
	}()

	n, err := s.internalGet(nodePath)
	if err != nil {
		return nil, err
	}

	e := newEvent(Get, nodePath, n.ModifiedIndex, n.CreatedIndex)
	e.EtcdIndex = s.CurrentIndex
	e.Node.loadInternalNode(n, recursive, sorted, s.clock)

	return e, nil
}

func ( s * store ) Get ( nodePath string , recursive , sorted bool ) ( * Event , error ) { var err * v2error . Error s . worldLock . RLock ( ) defer s . worldLock . RUnlock ( ) defer func ( ) { if err == nil { s . Stats . Inc ( GetSuccess ) if recursive { reportReadSuccess ( GetRecursive ) } else { reportReadSuccess ( Get ) } return } s . Stats . Inc ( GetFail ) if recursive { reportReadFailure ( GetRecursive ) } else { reportReadFailure ( Get ) } } ( ) n , err := s . internalGet ( nodePath ) if err != nil { return nil , err } e := newEvent ( Get , nodePath , n . ModifiedIndex , n . CreatedIndex ) e . EtcdIndex = s . CurrentIndex e . Node . loadInternalNode ( n , recursive , sorted , s . clock ) return e , nil }

// Get returns a get event.
// If recursive is true, it will return all the content under the node path.
// If sorted is true, it will sort the content by keys.

Get returns a get event . If recursive is true it will return all the content under the node path . If sorted is true it will sort the content by keys .


func (s *store) Create(nodePath string, dir bool, value string, unique bool, expireOpts TTLOptionSet) (*Event, error) {
	var err *v2error.Error

	s.worldLock.Lock()
	defer s.worldLock.Unlock()

	defer func() {
		if err == nil {
			s.Stats.Inc(CreateSuccess)
			reportWriteSuccess(Create)
			return
		}

		s.Stats.Inc(CreateFail)
		reportWriteFailure(Create)
	}()

	e, err := s.internalCreate(nodePath, dir, value, unique, false, expireOpts.ExpireTime, Create)
	if err != nil {
		return nil, err
	}

	e.EtcdIndex = s.CurrentIndex
	s.WatcherHub.notify(e)

	return e, nil
}

func ( s * store ) Create ( nodePath string , dir bool , value string , unique bool , expireOpts TTLOptionSet ) ( * Event , error ) { var err * v2error . Error s . worldLock . Lock ( ) defer s . worldLock . Unlock ( ) defer func ( ) { if err == nil { s . Stats . Inc ( CreateSuccess ) reportWriteSuccess ( Create ) return } s . Stats . Inc ( CreateFail ) reportWriteFailure ( Create ) } ( ) e , err := s . internalCreate ( nodePath , dir , value , unique , false , expireOpts . ExpireTime , Create ) if err != nil { return nil , err } e . EtcdIndex = s . CurrentIndex s . WatcherHub . notify ( e ) return e , nil }

// Create creates the node at nodePath. Create will help to create intermediate directories with no ttl.
// If the node has already existed, create will fail.
// If any node on the path is a file, create will fail.

Create creates the node at nodePath . Create will help to create intermediate directories with no ttl . If the node has already existed create will fail . If any node on the path is a file create will fail .


func (s *store) Set(nodePath string, dir bool, value string, expireOpts TTLOptionSet) (*Event, error) {
	var err *v2error.Error

	s.worldLock.Lock()
	defer s.worldLock.Unlock()

	defer func() {
		if err == nil {
			s.Stats.Inc(SetSuccess)
			reportWriteSuccess(Set)
			return
		}

		s.Stats.Inc(SetFail)
		reportWriteFailure(Set)
	}()

	// Get prevNode value
	n, getErr := s.internalGet(nodePath)
	if getErr != nil && getErr.ErrorCode != v2error.EcodeKeyNotFound {
		err = getErr
		return nil, err
	}

	if expireOpts.Refresh {
		if getErr != nil {
			err = getErr
			return nil, err
		}
		value = n.Value
	}

	// Set new value
	e, err := s.internalCreate(nodePath, dir, value, false, true, expireOpts.ExpireTime, Set)
	if err != nil {
		return nil, err
	}
	e.EtcdIndex = s.CurrentIndex

	// Put prevNode into event
	if getErr == nil {
		prev := newEvent(Get, nodePath, n.ModifiedIndex, n.CreatedIndex)
		prev.Node.loadInternalNode(n, false, false, s.clock)
		e.PrevNode = prev.Node
	}

	if !expireOpts.Refresh {
		s.WatcherHub.notify(e)
	} else {
		e.SetRefresh()
		s.WatcherHub.add(e)
	}

	return e, nil
}

func
(
s
*
store
)
Set
(
nodePath
string
,
dir
bool
,
value
string
,
expireOpts
TTLOptionSet
)
(
*
Event
,
error
)
{
var
err
*
v2error
.
Error
s
.
worldLock
.
Lock
(
)
defer
s
.
worldLock
.
Unlock
(
)
defer
func
(
)
{
if
err
==
{
s
.
Stats
.
Inc
(
SetSuccess
)
reportWriteSuccess
(
Set
)
return
}
s
.
Stats
.
Inc
(
SetFail
)
reportWriteFailure
(
Set
)
}
(
)
// Get prevNode value
n
,
getErr
:=
s
.
internalGet
(
nodePath
)
if
getErr
!=
&&
getErr
.
ErrorCode
!=
v2error
.
EcodeKeyNotFound
{
err
=
getErr
return
,
err
}
if
expireOpts
.
Refresh
{
if
getErr
!=
{
err
=
getErr
return
,
err
}
value
=
n
.
Value
}
// Set new value
e
,
err
:=
s
.
internalCreate
(
nodePath
,
dir
,
value
,
false
,
true
,
expireOpts
.
ExpireTime
,
Set
)
if
err
!=
{
return
,
err
}
e
.
EtcdIndex
=
s
.
CurrentIndex
// Put prevNode into event
if
getErr
==
{
prev
:=
newEvent
(
Get
,
nodePath
,
n
.
ModifiedIndex
,
n
.
CreatedIndex
)
prev
.
Node
.
loadInternalNode
(
n
,
false
,
false
,
s
.
clock
)
e
.
PrevNode
=
prev
.
Node
}
if
!
expireOpts
.
Refresh
{
s
.
WatcherHub
.
notify
(
e
)
}
else
{
e
.
SetRefresh
(
)
s
.
WatcherHub
.
add
(
e
)
}
return
e
,
}

// Set creates or replace the node at nodePath.

Set creates or replace the node at nodePath .


func getCompareFailCause(n *node, which int, prevValue string, prevIndex uint64) string {
	switch which {
	case CompareIndexNotMatch:
		return fmt.Sprintf("[%v != %v]", prevIndex, n.ModifiedIndex)
	case CompareValueNotMatch:
		return fmt.Sprintf("[%v != %v]", prevValue, n.Value)
	default:
		return fmt.Sprintf("[%v != %v] [%v != %v]", prevValue, n.Value, prevIndex, n.ModifiedIndex)
	}
}

func getCompareFailCause ( n * node , which int , prevValue string , prevIndex uint64 ) string { switch which { case CompareIndexNotMatch : return fmt . Sprintf ( " " , prevIndex , n . ModifiedIndex ) case CompareValueNotMatch : return fmt . Sprintf ( " " , prevValue , n . Value ) default : return fmt . Sprintf ( " " , prevValue , n . Value , prevIndex , n . ModifiedIndex ) } }

// returns user-readable cause of failed comparison

returns user - readable cause of failed comparison


func (s *store) Delete(nodePath string, dir, recursive bool) (*Event, error) {
	var err *v2error.Error

	s.worldLock.Lock()
	defer s.worldLock.Unlock()

	defer func() {
		if err == nil {
			s.Stats.Inc(DeleteSuccess)
			reportWriteSuccess(Delete)
			return
		}

		s.Stats.Inc(DeleteFail)
		reportWriteFailure(Delete)
	}()

	nodePath = path.Clean(path.Join("/", nodePath))
	// we do not allow the user to change "/"
	if s.readonlySet.Contains(nodePath) {
		return nil, v2error.NewError(v2error.EcodeRootROnly, "/", s.CurrentIndex)
	}

	// recursive implies dir
	if recursive {
		dir = true
	}

	n, err := s.internalGet(nodePath)
	if err != nil { // if the node does not exist, return error
		return nil, err
	}

	nextIndex := s.CurrentIndex + 1
	e := newEvent(Delete, nodePath, nextIndex, n.CreatedIndex)
	e.EtcdIndex = nextIndex
	e.PrevNode = n.Repr(false, false, s.clock)
	eNode := e.Node

	if n.IsDir() {
		eNode.Dir = true
	}

	callback := func(path string) { // notify function
		// notify the watchers with deleted set true
		s.WatcherHub.notifyWatchers(e, path, true)
	}

	err = n.Remove(dir, recursive, callback)
	if err != nil {
		return nil, err
	}

	// update etcd index
	s.CurrentIndex++

	s.WatcherHub.notify(e)

	return e, nil
}

// Delete deletes the node at the given path.
// If the node is a directory, recursive must be true to delete it.

Delete deletes the node at the given path . If the node is a directory recursive must be true to delete it .


func (s *store) walk(nodePath string, walkFunc func(prev *node, component string) (*node, *v2error.Error)) (*node, *v2error.Error) {
	components := strings.Split(nodePath, "/")

	curr := s.Root
	var err *v2error.Error

	for i := 1; i < len(components); i++ {
		if len(components[i]) == 0 { // ignore empty string
			return curr, nil
		}

		curr, err = walkFunc(curr, components[i])
		if err != nil {
			return nil, err
		}
	}

	return curr, nil
}

func
(
s
*
store
)
walk
(
nodePath
string
,
walkFunc
func
(
prev
*
node
,
component
string
)
(
*
node
,
*
v2error
.
Error
)
)
(
*
node
,
*
v2error
.
Error
)
{
components
:=
strings
.
Split
(
nodePath
,
"
"
)
curr
:=
s
.
Root
var
err
*
v2error
.
Error
for
i
:=
1
;
i
<
len
(
components
)
;
i
++
{
if
len
(
components
[
i
]
)
==
0
{
// ignore empty string
return
curr
,
}
curr
,
err
=
walkFunc
(
curr
,
components
[
i
]
)
if
err
!=
{
return
,
err
}
}
return
curr
,
}

// walk walks all the nodePath and apply the walkFunc on each directory


func (s *store) internalGet(nodePath string) (*node, *v2error.Error) {
	nodePath = path.Clean(path.Join("/", nodePath))

	walkFunc := func(parent *node, name string) (*node, *v2error.Error) {

		if !parent.IsDir() {
			err := v2error.NewError(v2error.EcodeNotDir, parent.Path, s.CurrentIndex)
			return nil, err
		}

		child, ok := parent.Children[name]
		if ok {
			return child, nil
		}

		return nil, v2error.NewError(v2error.EcodeKeyNotFound, path.Join(parent.Path, name), s.CurrentIndex)
	}

	f, err := s.walk(nodePath, walkFunc)

	if err != nil {
		return nil, err
	}
	return f, nil
}

func ( s * store ) internalGet ( nodePath string ) ( * node , * v2error . Error ) { nodePath = path . Clean ( path . Join ( " " , nodePath ) ) walkFunc := func ( parent * node , name string ) ( * node , * v2error . Error ) { if ! parent . IsDir ( ) { err := v2error . NewError ( v2error . EcodeNotDir , parent . Path , s . CurrentIndex ) return nil , err } child , ok := parent . Children [ name ] if ok { return child , nil } return nil , v2error . NewError ( v2error . EcodeKeyNotFound , path . Join ( parent . Path , name ) , s . CurrentIndex ) } f , err := s . walk ( nodePath , walkFunc ) if err != nil { return nil , err } return f , nil }

// InternalGet gets the node of the given nodePath.

InternalGet gets the node of the given nodePath .


func (s *store) DeleteExpiredKeys(cutoff time.Time) {
	s.worldLock.Lock()
	defer s.worldLock.Unlock()

	for {
		node := s.ttlKeyHeap.top()
		if node == nil || node.ExpireTime.After(cutoff) {
			break
		}

		s.CurrentIndex++
		e := newEvent(Expire, node.Path, s.CurrentIndex, node.CreatedIndex)
		e.EtcdIndex = s.CurrentIndex
		e.PrevNode = node.Repr(false, false, s.clock)
		if node.IsDir() {
			e.Node.Dir = true
		}

		callback := func(path string) { // notify function
			// notify the watchers with deleted set true
			s.WatcherHub.notifyWatchers(e, path, true)
		}

		s.ttlKeyHeap.pop()
		node.Remove(true, true, callback)

		reportExpiredKey()
		s.Stats.Inc(ExpireCount)

		s.WatcherHub.notify(e)
	}

}

// DeleteExpiredKeys will delete all expired keys


func (s *store) checkDir(parent *node, dirName string) (*node, *v2error.Error) {
	node, ok := parent.Children[dirName]

	if ok {
		if node.IsDir() {
			return node, nil
		}

		return nil, v2error.NewError(v2error.EcodeNotDir, node.Path, s.CurrentIndex)
	}

	n := newDir(s, path.Join(parent.Path, dirName), s.CurrentIndex+1, parent, Permanent)

	parent.Children[dirName] = n

	return n, nil
}

func ( s * store ) checkDir ( parent * node , dirName string ) ( * node , * v2error . Error ) { node , ok := parent . Children [ dirName ] if ok { if node . IsDir ( ) { return node , nil } return nil , v2error . NewError ( v2error . EcodeNotDir , node . Path , s . CurrentIndex ) } n := newDir ( s , path . Join ( parent . Path , dirName ) , s . CurrentIndex + 1 , parent , Permanent ) parent . Children [ dirName ] = n return n , nil }

// checkDir will check whether the component is a directory under parent node.
// If it is a directory, this function will return the pointer to that node.
// If it does not exist, this function will create a new directory and return the pointer to that node.
// If it is a file, this function will return error.

checkDir will check whether the component is a directory under parent node . If it is a directory this function will return the pointer to that node . If it does not exist this function will create a new directory and return the pointer to that node . If it is a file this function will return error .


func (s *store) Save() ([]byte, error) {
	b, err := json.Marshal(s.Clone())
	if err != nil {
		return nil, err
	}

	return b, nil
}

func ( s * store ) Save ( ) ( [ ] byte , error ) { b , err := json . Marshal ( s . Clone ( ) ) if err != nil { return nil , err } return b , nil }

// Save saves the static state of the store system.
// It will not be able to save the state of watchers.
// It will not save the parent field of the node. Or there will
// be cyclic dependencies issue for the json package.

Save saves the static state of the store system . It will not be able to save the state of watchers . It will not save the parent field of the node . Or there will be cyclic dependencies issue for the json package .


func (s *store) Recovery(state []byte) error {
	s.worldLock.Lock()
	defer s.worldLock.Unlock()
	err := json.Unmarshal(state, s)

	if err != nil {
		return err
	}

	s.ttlKeyHeap = newTtlKeyHeap()

	s.Root.recoverAndclean()
	return nil
}

func ( s * store ) Recovery ( state [ ] byte ) error { s . worldLock . Lock ( ) defer s . worldLock . Unlock ( ) err := json . Unmarshal ( state , s ) if err != nil { return err } s . ttlKeyHeap = newTtlKeyHeap ( ) s . Root . recoverAndclean ( ) return nil }

// Recovery recovers the store system from a static state
// It needs to recover the parent field of the nodes.
// It needs to delete the expired nodes since the saved time and also
// needs to create monitoring go routines.

Recovery recovers the store system from a static state It needs to recover the parent field of the nodes . It needs to delete the expired nodes since the saved time and also needs to create monitoring go routines .


func (g *Generator) Next() uint64 {
	suffix := atomic.AddUint64(&g.suffix, 1)
	id := g.prefix | lowbit(suffix, suffixLen)
	return id
}

func ( g * Generator ) Next ( ) uint64 { suffix := atomic . AddUint64 ( & g . suffix , 1 ) id := g . prefix | lowbit ( suffix , suffixLen ) return id }


func NewMakeMirrorCommand() *cobra.Command {
	c := &cobra.Command{
		Use:   "make-mirror [options] <destination>",
		Short: "Makes a mirror at the destination etcd cluster",
		Run:   makeMirrorCommandFunc,
	}

	c.Flags().StringVar(&mmprefix, "prefix", "", "Key-value prefix to mirror")
	c.Flags().StringVar(&mmdestprefix, "dest-prefix", "", "destination prefix to mirror a prefix to a different prefix in the destination cluster")
	c.Flags().BoolVar(&mmnodestprefix, "no-dest-prefix", false, "mirror key-values to the root of the destination cluster")
	c.Flags().StringVar(&mmcert, "dest-cert", "", "Identify secure client using this TLS certificate file for the destination cluster")
	c.Flags().StringVar(&mmkey, "dest-key", "", "Identify secure client using this TLS key file")
	c.Flags().StringVar(&mmcacert, "dest-cacert", "", "Verify certificates of TLS enabled secure servers using this CA bundle")
	// TODO: secure by default when etcd enables secure gRPC by default.
	c.Flags().BoolVar(&mminsecureTr, "dest-insecure-transport", true, "Disable transport security for client connections")

	return c
}

// NewMakeMirrorCommand returns the cobra command for "makeMirror".

NewMakeMirrorCommand returns the cobra command for makeMirror .


func NewZapCoreLoggerBuilder(lg *zap.Logger, cr zapcore.Core, syncer zapcore.WriteSyncer) func(*Config) error {
	return func(cfg *Config) error {
		cfg.loggerMu.Lock()
		defer cfg.loggerMu.Unlock()
		cfg.logger = lg
		cfg.loggerConfig = nil
		cfg.loggerCore = cr
		cfg.loggerWriteSyncer = syncer

		grpcLogOnce.Do(func() {
			grpclog.SetLoggerV2(logutil.NewGRPCLoggerV2FromZapCore(cr, syncer))
		})
		return nil
	}
}

func NewZapCoreLoggerBuilder ( lg * zap . Logger , cr zapcore . Core , syncer zapcore . WriteSyncer ) func ( * Config ) error { return func ( cfg * Config ) error { cfg . loggerMu . Lock ( ) defer cfg . loggerMu . Unlock ( ) cfg . logger = lg cfg . loggerConfig = nil cfg . loggerCore = cr cfg . loggerWriteSyncer = syncer grpcLogOnce . Do ( func ( ) { grpclog . SetLoggerV2 ( logutil . NewGRPCLoggerV2FromZapCore ( cr , syncer ) ) } ) return nil } }

// NewZapCoreLoggerBuilder generates a zap core logger builder.

NewZapCoreLoggerBuilder generates a zap core logger builder .


func NewSyncer(c *clientv3.Client, prefix string, rev int64) Syncer {
	return &syncer{c: c, prefix: prefix, rev: rev}
}

func NewSyncer ( c * clientv3 . Client , prefix string , rev int64 ) Syncer { return & syncer { c : c , prefix : prefix , rev : rev } }


func DropPort(port int) error {
	cmdStr := fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --destination-port %d -j DROP", port)
	if _, err := exec.Command("/bin/sh", "-c", cmdStr).Output(); err != nil {
		return err
	}
	cmdStr = fmt.Sprintf("sudo iptables -A INPUT -p tcp --destination-port %d -j DROP", port)
	_, err := exec.Command("/bin/sh", "-c", cmdStr).Output()
	return err
}

func DropPort ( port int ) error { cmdStr := fmt . Sprintf ( " " , port ) if _ , err := exec . Command ( " " , " " , cmdStr ) . Output ( ) ; err != nil { return err } cmdStr = fmt . Sprintf ( " " , port ) _ , err := exec . Command ( " " , " " , cmdStr ) . Output ( ) return err }

// DropPort drops all tcp packets that are received from the given port and sent to the given port.

DropPort drops all tcp packets that are received from the given port and sent to the given port .


func SetLatency(ms, rv int) error {
	ifces, err := GetDefaultInterfaces()
	if err != nil {
		return err
	}

	if rv > ms {
		rv = 1
	}
	for ifce := range ifces {
		cmdStr := fmt.Sprintf("sudo tc qdisc add dev %s root netem delay %dms %dms distribution normal", ifce, ms, rv)
		_, err = exec.Command("/bin/sh", "-c", cmdStr).Output()
		if err != nil {
			// the rule has already been added. Overwrite it.
			cmdStr = fmt.Sprintf("sudo tc qdisc change dev %s root netem delay %dms %dms distribution normal", ifce, ms, rv)
			_, err = exec.Command("/bin/sh", "-c", cmdStr).Output()
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// SetLatency adds latency in millisecond scale with random variations.

SetLatency adds latency in millisecond scale with random variations .


func RemoveLatency() error {
	ifces, err := GetDefaultInterfaces()
	if err != nil {
		return err
	}
	for ifce := range ifces {
		_, err = exec.Command("/bin/sh", "-c", fmt.Sprintf("sudo tc qdisc del dev %s root netem", ifce)).Output()
		if err != nil {
			return err
		}
	}
	return nil
}

func RemoveLatency ( ) error { ifces , err := GetDefaultInterfaces ( ) if err != nil { return err } for ifce := range ifces { _ , err = exec . Command ( " " , " " , fmt . Sprintf ( " " , ifce ) ) . Output ( ) if err != nil { return err } } return nil }

// RemoveLatency resets latency configurations.

RemoveLatency resets latency configurations .


func NewTxnCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "txn [options]",
		Short: "Txn processes all the requests in one transaction",
		Run:   txnCommandFunc,
	}
	cmd.Flags().BoolVarP(&txnInteractive, "interactive", "i", false, "Input transaction in interactive mode")
	return cmd
}

func NewTxnCommand ( ) * cobra . Command { cmd := & cobra . Command { Use : " " , Short : " " , Run : txnCommandFunc , } cmd . Flags ( ) . BoolVarP ( & txnInteractive , " " , " " , false , " " ) return cmd }

// NewTxnCommand returns the cobra command for "txn".

NewTxnCommand returns the cobra command for txn .


func txnCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		ExitWithError(ExitBadArgs, fmt.Errorf("txn command does not accept argument"))
	}

	reader := bufio.NewReader(os.Stdin)

	txn := mustClientFromCmd(cmd).Txn(context.Background())
	promptInteractive("compares:")
	txn.If(readCompares(reader)...)
	promptInteractive("success requests (get, put, del):")
	txn.Then(readOps(reader)...)
	promptInteractive("failure requests (get, put, del):")
	txn.Else(readOps(reader)...)

	resp, err := txn.Commit()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	display.Txn(*resp)
}

func txnCommandFunc ( cmd * cobra . Command , args [ ] string ) { if len ( args ) != 0 { ExitWithError ( ExitBadArgs , fmt . Errorf ( " " ) ) } reader := bufio . NewReader ( os . Stdin ) txn := mustClientFromCmd ( cmd ) . Txn ( context . Background ( ) ) promptInteractive ( " " ) txn . If ( readCompares ( reader ) ... ) promptInteractive ( " " ) txn . Then ( readOps ( reader ) ... ) promptInteractive ( " " ) txn . Else ( readOps ( reader ) ... ) resp , err := txn . Commit ( ) if err != nil { ExitWithError ( ExitError , err ) } display . Txn ( * resp ) }

// txnCommandFunc executes the "txn" command.

txnCommandFunc executes the txn command .


func New(
	lg *zap.Logger,
	mode string,
	retention time.Duration,
	rg RevGetter,
	c Compactable,
) (Compactor, error) {
	switch mode {
	case ModePeriodic:
		return newPeriodic(lg, clockwork.NewRealClock(), retention, rg, c), nil
	case ModeRevision:
		return newRevision(lg, clockwork.NewRealClock(), int64(retention), rg, c), nil
	default:
		return nil, fmt.Errorf("unsupported compaction mode %s", mode)
	}
}

func New ( lg * zap . Logger , mode string , retention time . Duration , rg RevGetter , c Compactable , ) ( Compactor , error ) { switch mode { case ModePeriodic : return newPeriodic ( lg , clockwork . NewRealClock ( ) , retention , rg , c ) , nil case ModeRevision : return newRevision ( lg , clockwork . NewRealClock ( ) , int64 ( retention ) , rg , c ) , nil default : return nil , fmt . Errorf ( " " , mode ) } }

// New returns a new Compactor based on given "mode".

New returns a new Compactor based on given mode .


func printResponseKey(resp *client.Response, format string) {
	// Format the result.
	switch format {
	case "simple":
		if resp.Action != "delete" {
			fmt.Println(resp.Node.Value)
		} else {
			fmt.Println("PrevNode.Value:", resp.PrevNode.Value)
		}
	case "extended":
		// Extended prints in a rfc2822 style format
		fmt.Println("Key:", resp.Node.Key)
		fmt.Println("Created-Index:", resp.Node.CreatedIndex)
		fmt.Println("Modified-Index:", resp.Node.ModifiedIndex)

		if resp.PrevNode != nil {
			fmt.Println("PrevNode.Value:", resp.PrevNode.Value)
		}

		fmt.Println("TTL:", resp.Node.TTL)
		fmt.Println("Index:", resp.Index)
		if resp.Action != "delete" {
			fmt.Println("")
			fmt.Println(resp.Node.Value)
		}
	case "json":
		b, err := json.Marshal(resp)
		if err != nil {
			panic(err)
		}
		fmt.Println(string(b))
	default:
		fmt.Fprintln(os.Stderr, "Unsupported output format:", format)
	}
}

// printResponseKey only supports to print key correctly.

printResponseKey only supports to print key correctly .


func RegisterLockHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error {
	return RegisterLockHandlerClient(ctx, mux, v3lockpb.NewLockClient(conn))
}

func RegisterLockHandler ( ctx context . Context , mux * runtime . ServeMux , conn * grpc . ClientConn ) error { return RegisterLockHandlerClient ( ctx , mux , v3lockpb . NewLockClient ( conn ) ) }

// RegisterLockHandler registers the http handlers for service Lock to "mux".
// The handlers forward requests to the grpc endpoint over "conn".

RegisterLockHandler registers the http handlers for service Lock to mux . The handlers forward requests to the grpc endpoint over conn .


func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler {
	if otherHandler == nil {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			grpcServer.ServeHTTP(w, r)
		})
	}
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") {
			grpcServer.ServeHTTP(w, r)
		} else {
			otherHandler.ServeHTTP(w, r)
		}
	})
}

func grpcHandlerFunc ( grpcServer * grpc . Server , otherHandler http . Handler ) http . Handler { if otherHandler == nil { return http . HandlerFunc ( func ( w http . ResponseWriter , r * http . Request ) { grpcServer . ServeHTTP ( w , r ) } ) } return http . HandlerFunc ( func ( w http . ResponseWriter , r * http . Request ) { if r . ProtoMajor == 2 && strings . Contains ( r . Header . Get ( " " ) , " " ) { grpcServer . ServeHTTP ( w , r ) } else { otherHandler . ServeHTTP ( w , r ) } } ) }

// grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC
// connections or otherHandler otherwise. Given in gRPC docs.

grpcHandlerFunc returns an http . Handler that delegates to grpcServer on incoming gRPC connections or otherHandler otherwise . Given in gRPC docs .


func addCORSHeader(w http.ResponseWriter, origin string) {
	w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Add("Access-Control-Allow-Origin", origin)
	w.Header().Add("Access-Control-Allow-Headers", "accept, content-type, authorization")
}

func addCORSHeader ( w http . ResponseWriter , origin string ) { w . Header ( ) . Add ( " " , " " ) w . Header ( ) . Add ( " " , origin ) w . Header ( ) . Add ( " " , " " ) }

// addCORSHeader adds the correct cors headers given an origin


func (txn *txnLeasing) fallback(ops []v3.Op) (fbOps []v3.Op) {
	for _, op := range ops {
		if op.IsGet() {
			continue
		}
		lkey, lend := txn.lkv.pfx+string(op.KeyBytes()), ""
		if len(op.RangeBytes()) > 0 {
			lend = txn.lkv.pfx + string(op.RangeBytes())
		}
		fbOps = append(fbOps, v3.OpGet(lkey, v3.WithRange(lend)))
	}
	return fbOps
}

func ( txn * txnLeasing ) fallback ( ops [ ] v3 . Op ) ( fbOps [ ] v3 . Op ) { for _ , op := range ops { if op . IsGet ( ) { continue } lkey , lend := txn . lkv . pfx + string ( op . KeyBytes ( ) ) , " " if len ( op . RangeBytes ( ) ) > 0 { lend = txn . lkv . pfx + string ( op . RangeBytes ( ) ) } fbOps = append ( fbOps , v3 . OpGet ( lkey , v3 . WithRange ( lend ) ) ) } return fbOps }

// fallback computes the ops to fetch all possible conflicting
// leasing keys for a list of ops.

fallback computes the ops to fetch all possible conflicting leasing keys for a list of ops .


func IDFromString(s string) (ID, error) {
	i, err := strconv.ParseUint(s, 16, 64)
	return ID(i), err
}

func IDFromString ( s string ) ( ID , error ) { i , err := strconv . ParseUint ( s , 16 , 64 ) return ID ( i ) , err }

// IDFromString attempts to create an ID from a base-16 string.

IDFromString attempts to create an ID from a base - 16 string .


func (guard *directoryLockGuard) release() error {
	var err error
	if !guard.readOnly {
		// It's important that we remove the pid file first.
		err = os.Remove(guard.path)
	}

	if closeErr := guard.f.Close(); err == nil {
		err = closeErr
	}
	guard.path = ""
	guard.f = nil

	return err
}

// Release deletes the pid file and releases our lock on the directory.

Release deletes the pid file and releases our lock on the directory .


func (v *ValueStruct) EncodedSize() uint16 {
	sz := len(v.Value) + 2 // meta, usermeta.
	if v.ExpiresAt == 0 {
		return uint16(sz + 1)
	}

	enc := sizeVarint(v.ExpiresAt)
	return uint16(sz + enc)
}

func
(
v
*
ValueStruct
)
EncodedSize
(
)
uint16
{
sz
:=
len
(
v
.
Value
)
+
2
// meta, usermeta.
if
v
.
ExpiresAt
==
0
{
return
uint16
(
sz
+
1
)
}
enc
:=
sizeVarint
(
v
.
ExpiresAt
)
return
uint16
(
sz
+
enc
)
}

// EncodedSize is the size of the ValueStruct when encoded


func (v *ValueStruct) Decode(b []byte) {
	v.Meta = b[0]
	v.UserMeta = b[1]
	var sz int
	v.ExpiresAt, sz = binary.Uvarint(b[2:])
	v.Value = b[2+sz:]
}

func ( v * ValueStruct ) Decode ( b [ ] byte ) { v . Meta = b [ 0 ] v . UserMeta = b [ 1 ] var sz int v . ExpiresAt , sz = binary . Uvarint ( b [ 2 : ] ) v . Value = b [ 2 + sz : ] }

// Decode uses the length of the slice to infer the length of the Value field.

Decode uses the length of the slice to infer the length of the Value field .


func NewMergeIterator(iters []Iterator, reversed bool) *MergeIterator {
	m := &MergeIterator{all: iters, reversed: reversed}
	m.h = make(elemHeap, 0, len(iters))
	m.initHeap()
	return m
}

func NewMergeIterator ( iters [ ] Iterator , reversed bool ) * MergeIterator { m := & MergeIterator { all : iters , reversed : reversed } m . h = make ( elemHeap , 0 , len ( iters ) ) m . initHeap ( ) return m }

// NewMergeIterator returns a new MergeIterator from a list of Iterators.

NewMergeIterator returns a new MergeIterator from a list of Iterators .


func (s *MergeIterator) initHeap() {
	s.h = s.h[:0]
	for idx, itr := range s.all {
		if !itr.Valid() {
			continue
		}
		e := &elem{itr: itr, nice: idx, reversed: s.reversed}
		s.h = append(s.h, e)
	}
	heap.Init(&s.h)
	for len(s.h) > 0 {
		it := s.h[0].itr
		if it == nil || !it.Valid() {
			heap.Pop(&s.h)
			continue
		}
		s.storeKey(s.h[0].itr)
		break
	}
}

func ( s * MergeIterator ) initHeap ( ) { s . h = s . h [ : 0 ] for idx , itr := range s . all { if ! itr . Valid ( ) { continue } e := & elem { itr : itr , nice : idx , reversed : s . reversed } s . h = append ( s . h , e ) } heap . Init ( & s . h ) for len ( s . h ) > 0 { it := s . h [ 0 ] . itr if it == nil || ! it . Valid ( ) { heap . Pop ( & s . h ) continue } s . storeKey ( s . h [ 0 ] . itr ) break } }

// initHeap checks all iterators and initializes our heap and array of keys.
// Whenever we reverse direction, we need to run this.

initHeap checks all iterators and initializes our heap and array of keys . Whenever we reverse direction we need to run this .


func (s *MergeIterator) Valid() bool {
	if s == nil {
		return false
	}
	if len(s.h) == 0 {
		return false
	}
	return s.h[0].itr.Valid()
}

func ( s * MergeIterator ) Valid ( ) bool { if s == nil { return false } if len ( s . h ) == 0 { return false } return s . h [ 0 ] . itr . Valid ( ) }

// Valid returns whether the MergeIterator is at a valid element.

Valid returns whether the MergeIterator is at a valid element .


func (s *MergeIterator) Key() []byte {
	if len(s.h) == 0 {
		return nil
	}
	return s.h[0].itr.Key()
}

func ( s * MergeIterator ) Key ( ) [ ] byte { if len ( s . h ) == 0 { return nil } return s . h [ 0 ] . itr . Key ( ) }

// Key returns the key associated with the current iterator


func (s *MergeIterator) Value() ValueStruct {
	if len(s.h) == 0 {
		return ValueStruct{}
	}
	return s.h[0].itr.Value()
}

func ( s * MergeIterator ) Value ( ) ValueStruct { if len ( s . h ) == 0 { return ValueStruct { } } return s . h [ 0 ] . itr . Value ( ) }

// Value returns the value associated with the iterator.

Value returns the value associated with the iterator .


func (s *MergeIterator) Next() {
	if len(s.h) == 0 {
		return
	}

	smallest := s.h[0].itr
	smallest.Next()

	for len(s.h) > 0 {
		smallest = s.h[0].itr
		if !smallest.Valid() {
			heap.Pop(&s.h)
			continue
		}

		heap.Fix(&s.h, 0)
		smallest = s.h[0].itr
		if smallest.Valid() {
			if !bytes.Equal(smallest.Key(), s.curKey) {
				break
			}
			smallest.Next()
		}
	}
	if !smallest.Valid() {
		return
	}
	s.storeKey(smallest)
}

func ( s * MergeIterator ) Next ( ) { if len ( s . h ) == 0 { return } smallest := s . h [ 0 ] . itr smallest . Next ( ) for len ( s . h ) > 0 { smallest = s . h [ 0 ] . itr if ! smallest . Valid ( ) { heap . Pop ( & s . h ) continue } heap . Fix ( & s . h , 0 ) smallest = s . h [ 0 ] . itr if smallest . Valid ( ) { if ! bytes . Equal ( smallest . Key ( ) , s . curKey ) { break } smallest . Next ( ) } } if ! smallest . Valid ( ) { return } s . storeKey ( smallest ) }

// Next returns the next element. If it is the same as the current key, ignore it.

Next returns the next element . If it is the same as the current key ignore it .


func (s *MergeIterator) Seek(key []byte) {
	for _, itr := range s.all {
		itr.Seek(key)
	}
	s.initHeap()
}

func ( s * MergeIterator ) Seek ( key [ ] byte ) { for _ , itr := range s . all { itr . Seek ( key ) } s . initHeap ( ) }

// Seek brings us to element with key >= given key.

Seek brings us to element with key > = given key .


func (s *MergeIterator) Close() error {
	for _, itr := range s.all {
		if err := itr.Close(); err != nil {
			return errors.Wrap(err, "MergeIterator")
		}
	}
	return nil
}

func ( s * MergeIterator ) Close ( ) error { for _ , itr := range s . all { if err := itr . Close ( ) ; err != nil { return errors . Wrap ( err , " " ) } } return nil }


func (p valuePointer) Encode(b []byte) []byte {
	binary.BigEndian.PutUint32(b[:4], p.Fid)
	binary.BigEndian.PutUint32(b[4:8], p.Len)
	binary.BigEndian.PutUint32(b[8:12], p.Offset)
	return b[:vptrSize]
}

func ( p valuePointer ) Encode ( b [ ] byte ) [ ] byte { binary . BigEndian . PutUint32 ( b [ : 4 ] , p . Fid ) binary . BigEndian . PutUint32 ( b [ 4 : 8 ] , p . Len ) binary . BigEndian . PutUint32 ( b [ 8 : 12 ] , p . Offset ) return b [ : vptrSize ] }

// Encode encodes Pointer into byte buffer.

Encode encodes Pointer into byte buffer .


func (h *header) Decode(buf []byte) {
	h.klen = binary.BigEndian.Uint32(buf[0:4])
	h.vlen = binary.BigEndian.Uint32(buf[4:8])
	h.expiresAt = binary.BigEndian.Uint64(buf[8:16])
	h.meta = buf[16]
	h.userMeta = buf[17]
}

func ( h * header ) Decode ( buf [ ] byte ) { h . klen = binary . BigEndian . Uint32 ( buf [ 0 : 4 ] ) h . vlen = binary . BigEndian . Uint32 ( buf [ 4 : 8 ] ) h . expiresAt = binary . BigEndian . Uint64 ( buf [ 8 : 16 ] ) h . meta = buf [ 16 ] h . userMeta = buf [ 17 ] }


func encodeEntry(e *Entry, buf *bytes.Buffer) (int, error) {
	h := header{
		klen:      uint32(len(e.Key)),
		vlen:      uint32(len(e.Value)),
		expiresAt: e.ExpiresAt,
		meta:      e.meta,
		userMeta:  e.UserMeta,
	}

	var headerEnc [headerBufSize]byte
	h.Encode(headerEnc[:])

	hash := crc32.New(y.CastagnoliCrcTable)

	buf.Write(headerEnc[:])
	hash.Write(headerEnc[:])

	buf.Write(e.Key)
	hash.Write(e.Key)

	buf.Write(e.Value)
	hash.Write(e.Value)

	var crcBuf [crc32.Size]byte
	binary.BigEndian.PutUint32(crcBuf[:], hash.Sum32())
	buf.Write(crcBuf[:])

	return len(headerEnc) + len(e.Key) + len(e.Value) + len(crcBuf), nil
}

func encodeEntry ( e * Entry , buf * bytes . Buffer ) ( int , error ) { h := header { klen : uint32 ( len ( e . Key ) ) , vlen : uint32 ( len ( e . Value ) ) , expiresAt : e . ExpiresAt , meta : e . meta , userMeta : e . UserMeta , } var headerEnc [ headerBufSize ] byte h . Encode ( headerEnc [ : ] ) hash := crc32 . New ( y . CastagnoliCrcTable ) buf . Write ( headerEnc [ : ] ) hash . Write ( headerEnc [ : ] ) buf . Write ( e . Key ) hash . Write ( e . Key ) buf . Write ( e . Value ) hash . Write ( e . Value ) var crcBuf [ crc32 . Size ] byte binary . BigEndian . PutUint32 ( crcBuf [ : ] , hash . Sum32 ( ) ) buf . Write ( crcBuf [ : ] ) return len ( headerEnc ) + len ( e . Key ) + len ( e . Value ) + len ( crcBuf ) , nil }

// Encodes e to buf. Returns number of bytes written.

Encodes e to buf . Returns number of bytes written .


func (wb *WriteBatch) SetEntry(e *Entry) error {
	wb.Lock()
	defer wb.Unlock()

	if err := wb.txn.SetEntry(e); err != ErrTxnTooBig {
		return err
	}
	// Txn has reached it's zenith. Commit now.
	if cerr := wb.commit(); cerr != nil {
		return cerr
	}
	// This time the error must not be ErrTxnTooBig, otherwise, we make the
	// error permanent.
	if err := wb.txn.SetEntry(e); err != nil {
		wb.err = err
		return err
	}
	return nil
}

// SetEntry is the equivalent of Txn.SetEntry.

SetEntry is the equivalent of Txn . SetEntry .


func (wb *WriteBatch) Set(k, v []byte, meta byte) error {
	e := &Entry{Key: k, Value: v, UserMeta: meta}
	return wb.SetEntry(e)
}

func ( wb * WriteBatch ) Set ( k , v [ ] byte , meta byte ) error { e := & Entry { Key : k , Value : v , UserMeta : meta } return wb . SetEntry ( e ) }

// Set is equivalent of Txn.SetWithMeta.

Set is equivalent of Txn . SetWithMeta .


func (wb *WriteBatch) SetWithTTL(key, val []byte, dur time.Duration) error {
	expire := time.Now().Add(dur).Unix()
	e := &Entry{Key: key, Value: val, ExpiresAt: uint64(expire)}
	return wb.SetEntry(e)
}

func ( wb * WriteBatch ) SetWithTTL ( key , val [ ] byte , dur time . Duration ) error { expire := time . Now ( ) . Add ( dur ) . Unix ( ) e := & Entry { Key : key , Value : val , ExpiresAt : uint64 ( expire ) } return wb . SetEntry ( e ) }

// SetWithTTL is equivalent of Txn.SetWithTTL.

SetWithTTL is equivalent of Txn . SetWithTTL .


func (wb *WriteBatch) Delete(k []byte) error {
	wb.Lock()
	defer wb.Unlock()

	if err := wb.txn.Delete(k); err != ErrTxnTooBig {
		return err
	}
	if err := wb.commit(); err != nil {
		return err
	}
	if err := wb.txn.Delete(k); err != nil {
		wb.err = err
		return err
	}
	return nil
}

func ( wb * WriteBatch ) Delete ( k [ ] byte ) error { wb . Lock ( ) defer wb . Unlock ( ) if err := wb . txn . Delete ( k ) ; err != ErrTxnTooBig { return err } if err := wb . commit ( ) ; err != nil { return err } if err := wb . txn . Delete ( k ) ; err != nil { wb . err = err return err } return nil }


func (wb *WriteBatch) commit() error {
	if wb.err != nil {
		return wb.err
	}
	// Get a new txn before we commit this one. So, the new txn doesn't need
	// to wait for this one to commit.
	wb.wg.Add(1)
	wb.txn.CommitWith(wb.callback)
	wb.txn = wb.db.newTransaction(true, true)
	// See comment about readTs in NewWriteBatch.
	wb.txn.readTs = wb.db.orc.readMark.DoneUntil()
	return wb.err
}

// Caller to commit must hold a write lock.

Caller to commit must hold a write lock .


func (wb *WriteBatch) Flush() error {
	wb.Lock()
	_ = wb.commit()
	wb.txn.Discard()
	wb.Unlock()

	wb.wg.Wait()
	// Safe to access error without any synchronization here.
	return wb.err
}

// Flush must be called at the end to ensure that any pending writes get committed to Badger. Flush
// returns any error stored by WriteBatch.

Flush must be called at the end to ensure that any pending writes get committed to Badger . Flush returns any error stored by WriteBatch .


func (wb *WriteBatch) Error() error {
	wb.Lock()
	defer wb.Unlock()
	return wb.err
}

func ( wb * WriteBatch ) Error ( ) error { wb . Lock ( ) defer wb . Unlock ( ) return wb . err }

// Error returns any errors encountered so far. No commits would be run once an error is detected.

Error returns any errors encountered so far . No commits would be run once an error is detected .


func (db *DB) getMemTables() ([]*skl.Skiplist, func()) {
	db.RLock()
	defer db.RUnlock()

	tables := make([]*skl.Skiplist, len(db.imm)+1)

	// Get mutable memtable.
	tables[0] = db.mt
	tables[0].IncrRef()

	// Get immutable memtables.
	last := len(db.imm) - 1
	for i := range db.imm {
		tables[i+1] = db.imm[last-i]
		tables[i+1].IncrRef()
	}
	return tables, func() {
		for _, tbl := range tables {
			tbl.DecrRef()
		}
	}
}

func
(
db
*
DB
)
getMemTables
(
)
(
[
]
*
skl
.
Skiplist
,
func
(
)
)
{
db
.
RLock
(
)
defer
db
.
RUnlock
(
)
tables
:=
make
(
[
]
*
skl
.
Skiplist
,
len
(
db
.
imm
)
+
1
)
// Get mutable memtable.
tables
[
0
]
=
db
.
mt
tables
[
0
]
.
IncrRef
(
)
// Get immutable memtables.
last
:=
len
(
db
.
imm
)
1
for
i
:=
range
db
.
imm
{
tables
[
i
+
1
]
=
db
.
imm
[
last
i
]
tables
[
i
+
1
]
.
IncrRef
(
)
}
return
tables
,
func
(
)
{
for
_
,
tbl
:=
range
tables
{
tbl
.
DecrRef
(
)
}
}
}

// getMemtables returns the current memtables and get references.

getMemtables returns the current memtables and get references .


func (db *DB) writeRequests(reqs []*request) error {
	if len(reqs) == 0 {
		return nil
	}

	done := func(err error) {
		for _, r := range reqs {
			r.Err = err
			r.Wg.Done()
		}
	}
	db.elog.Printf("writeRequests called. Writing to value log")

	err := db.vlog.write(reqs)
	if err != nil {
		done(err)
		return err
	}

	db.elog.Printf("Writing to memtable")
	var count int
	for _, b := range reqs {
		if len(b.Entries) == 0 {
			continue
		}
		count += len(b.Entries)
		var i uint64
		for err = db.ensureRoomForWrite(); err == errNoRoom; err = db.ensureRoomForWrite() {
			i++
			if i%100 == 0 {
				db.elog.Printf("Making room for writes")
			}
			// We need to poll a bit because both hasRoomForWrite and the flusher need access to s.imm.
			// When flushChan is full and you are blocked there, and the flusher is trying to update s.imm,
			// you will get a deadlock.
			time.Sleep(10 * time.Millisecond)
		}
		if err != nil {
			done(err)
			return errors.Wrap(err, "writeRequests")
		}
		if err := db.writeToLSM(b); err != nil {
			done(err)
			return errors.Wrap(err, "writeRequests")
		}
		db.updateHead(b.Ptrs)
	}
	done(nil)
	db.elog.Printf("%d entries written", count)
	return nil
}

// writeRequests is called serially by only one goroutine.

writeRequests is called serially by only one goroutine .


func (db *DB) ensureRoomForWrite() error {
	var err error
	db.Lock()
	defer db.Unlock()
	if db.mt.MemSize() < db.opt.MaxTableSize {
		return nil
	}

	y.AssertTrue(db.mt != nil) // A nil mt indicates that DB is being closed.
	select {
	case db.flushChan <- flushTask{mt: db.mt, vptr: db.vhead}:
		db.elog.Printf("Flushing value log to disk if async mode.")
		// Ensure value log is synced to disk so this memtable's contents wouldn't be lost.
		err = db.vlog.sync(db.vhead.Fid)
		if err != nil {
			return err
		}

		db.elog.Printf("Flushing memtable, mt.size=%d size of flushChan: %d\n",
			db.mt.MemSize(), len(db.flushChan))
		// We manage to push this task. Let's modify imm.
		db.imm = append(db.imm, db.mt)
		db.mt = skl.NewSkiplist(arenaSize(db.opt))
		// New memtable is empty. We certainly have room.
		return nil
	default:
		// We need to do this to unlock and allow the flusher to modify imm.
		return errNoRoom
	}
}

// ensureRoomForWrite is always called serially.

ensureRoomForWrite is always called serially .


func writeLevel0Table(ft flushTask, f io.Writer) error {
	iter := ft.mt.NewIterator()
	defer iter.Close()
	b := table.NewTableBuilder()
	defer b.Close()
	for iter.SeekToFirst(); iter.Valid(); iter.Next() {
		if len(ft.dropPrefix) > 0 && bytes.HasPrefix(iter.Key(), ft.dropPrefix) {
			continue
		}
		if err := b.Add(iter.Key(), iter.Value()); err != nil {
			return err
		}
	}
	_, err := f.Write(b.Finish())
	return err
}

func writeLevel0Table ( ft flushTask , f io . Writer ) error { iter := ft . mt . NewIterator ( ) defer iter . Close ( ) b := table . NewTableBuilder ( ) defer b . Close ( ) for iter . SeekToFirst ( ) ; iter . Valid ( ) ; iter . Next ( ) { if len ( ft . dropPrefix ) > 0 && bytes . HasPrefix ( iter . Key ( ) , ft . dropPrefix ) { continue } if err := b . Add ( iter . Key ( ) , iter . Value ( ) ) ; err != nil { return err } } _ , err := f . Write ( b . Finish ( ) ) return err }


func (db *DB) handleFlushTask(ft flushTask) error {
	if !ft.mt.Empty() {
		// Store badger head even if vptr is zero, need it for readTs
		db.opt.Debugf("Storing value log head: %+v\n", ft.vptr)
		db.elog.Printf("Storing offset: %+v\n", ft.vptr)
		offset := make([]byte, vptrSize)
		ft.vptr.Encode(offset)

		// Pick the max commit ts, so in case of crash, our read ts would be higher than all the
		// commits.
		headTs := y.KeyWithTs(head, db.orc.nextTs())
		ft.mt.Put(headTs, y.ValueStruct{Value: offset})

		// Also store lfDiscardStats before flushing memtables
		discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, 1)
		ft.mt.Put(discardStatsKey, y.ValueStruct{Value: db.vlog.encodedDiscardStats()})
	}

	fileID := db.lc.reserveFileID()
	fd, err := y.CreateSyncedFile(table.NewFilename(fileID, db.opt.Dir), true)
	if err != nil {
		return y.Wrap(err)
	}

	// Don't block just to sync the directory entry.
	dirSyncCh := make(chan error)
	go func() { dirSyncCh <- syncDir(db.opt.Dir) }()

	err = writeLevel0Table(ft, fd)
	dirSyncErr := <-dirSyncCh

	if err != nil {
		db.elog.Errorf("ERROR while writing to level 0: %v", err)
		return err
	}
	if dirSyncErr != nil {
		// Do dir sync as best effort. No need to return due to an error there.
		db.elog.Errorf("ERROR while syncing level directory: %v", dirSyncErr)
	}

	tbl, err := table.OpenTable(fd, db.opt.TableLoadingMode, nil)
	if err != nil {
		db.elog.Printf("ERROR while opening table: %v", err)
		return err
	}
	// We own a ref on tbl.
	err = db.lc.addLevel0Table(tbl) // This will incrRef (if we don't error, sure)
	tbl.DecrRef()                   // Releases our ref.
	return err
}

// handleFlushTask must be run serially.


func (db *DB) flushMemtable(lc *y.Closer) error {
	defer lc.Done()

	for ft := range db.flushChan {
		if ft.mt == nil {
			// We close db.flushChan now, instead of sending a nil ft.mt.
			continue
		}
		for {
			err := db.handleFlushTask(ft)
			if err == nil {
				// Update s.imm. Need a lock.
				db.Lock()
				// This is a single-threaded operation. ft.mt corresponds to the head of
				// db.imm list. Once we flush it, we advance db.imm. The next ft.mt
				// which would arrive here would match db.imm[0], because we acquire a
				// lock over DB when pushing to flushChan.
				// TODO: This logic is dirty AF. Any change and this could easily break.
				y.AssertTrue(ft.mt == db.imm[0])
				db.imm = db.imm[1:]
				ft.mt.DecrRef() // Return memory.
				db.Unlock()

				break
			}
			// Encountered error. Retry indefinitely.
			db.opt.Errorf("Failure while flushing memtable to disk: %v. Retrying...\n", err)
			time.Sleep(time.Second)
		}
	}
	return nil
}

// flushMemtable must keep running until we send it an empty flushTask. If there
// are errors during handling the flush task, we'll retry indefinitely.

flushMemtable must keep running until we send it an empty flushTask . If there are errors during handling the flush task we ll retry indefinitely .


func (db *DB) calculateSize() {
	newInt := func(val int64) *expvar.Int {
		v := new(expvar.Int)
		v.Add(val)
		return v
	}

	totalSize := func(dir string) (int64, int64) {
		var lsmSize, vlogSize int64
		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			ext := filepath.Ext(path)
			if ext == ".sst" {
				lsmSize += info.Size()
			} else if ext == ".vlog" {
				vlogSize += info.Size()
			}
			return nil
		})
		if err != nil {
			db.elog.Printf("Got error while calculating total size of directory: %s", dir)
		}
		return lsmSize, vlogSize
	}

	lsmSize, vlogSize := totalSize(db.opt.Dir)
	y.LSMSize.Set(db.opt.Dir, newInt(lsmSize))
	// If valueDir is different from dir, we'd have to do another walk.
	if db.opt.ValueDir != db.opt.Dir {
		_, vlogSize = totalSize(db.opt.ValueDir)
	}
	y.VlogSize.Set(db.opt.Dir, newInt(vlogSize))
}

// This function does a filewalk, calculates the size of vlog and sst files and stores it in
// y.LSMSize and y.VlogSize.

This function does a filewalk calculates the size of vlog and sst files and stores it in y . LSMSize and y . VlogSize .


func (db *DB) Size() (lsm, vlog int64) {
	if y.LSMSize.Get(db.opt.Dir) == nil {
		lsm, vlog = 0, 0
		return
	}
	lsm = y.LSMSize.Get(db.opt.Dir).(*expvar.Int).Value()
	vlog = y.VlogSize.Get(db.opt.Dir).(*expvar.Int).Value()
	return
}

func ( db * DB ) Size ( ) ( lsm , vlog int64 ) { if y . LSMSize . Get ( db . opt . Dir ) == nil { lsm , vlog = 0 , 0 return } lsm = y . LSMSize . Get ( db . opt . Dir ) . ( * expvar . Int ) . Value ( ) vlog = y . VlogSize . Get ( db . opt . Dir ) . ( * expvar . Int ) . Value ( ) return }

// Size returns the size of lsm and value log files in bytes. It can be used to decide how often to
// call RunValueLogGC.

Size returns the size of lsm and value log files in bytes . It can be used to decide how often to call RunValueLogGC .


func (seq *Sequence) Next() (uint64, error) {
	seq.Lock()
	defer seq.Unlock()
	if seq.next >= seq.leased {
		if err := seq.updateLease(); err != nil {
			return 0, err
		}
	}
	val := seq.next
	seq.next++
	return val, nil
}

func ( seq * Sequence ) Next ( ) ( uint64 , error ) { seq . Lock ( ) defer seq . Unlock ( ) if seq . next >= seq . leased { if err := seq . updateLease ( ) ; err != nil { return 0 , err } } val := seq . next seq . next ++ return val , nil }

// Next would return the next integer in the sequence, updating the lease by running a transaction
// if needed.

Next would return the next integer in the sequence updating the lease by running a transaction if needed .


func (seq *Sequence) Release() error {
	seq.Lock()
	defer seq.Unlock()
	err := seq.db.Update(func(txn *Txn) error {
		var buf [8]byte
		binary.BigEndian.PutUint64(buf[:], seq.next)
		return txn.Set(seq.key, buf[:])
	})
	if err != nil {
		return err
	}
	seq.leased = seq.next
	return nil
}

func ( seq * Sequence ) Release ( ) error { seq . Lock ( ) defer seq . Unlock ( ) err := seq . db . Update ( func ( txn * Txn ) error { var buf [ 8 ] byte binary . BigEndian . PutUint64 ( buf [ : ] , seq . next ) return txn . Set ( seq . key , buf [ : ] ) } ) if err != nil { return err } seq . leased = seq . next return nil }

// Release the leased sequence to avoid wasted integers. This should be done right
// before closing the associated DB. However it is valid to use the sequence after
// it was released, causing a new lease with full bandwidth.

Release the leased sequence to avoid wasted integers . This should be done right before closing the associated DB . However it is valid to use the sequence after it was released causing a new lease with full bandwidth .


func (db *DB) KeySplits(prefix []byte) []string {
	var splits []string
	for _, ti := range db.Tables() {
		// We don't use ti.Left, because that has a tendency to store !badger
		// keys.
		if bytes.HasPrefix(ti.Right, prefix) {
			splits = append(splits, string(ti.Right))
		}
	}
	sort.Strings(splits)
	return splits
}

// KeySplits can be used to get rough key ranges to divide up iteration over
// the DB.

KeySplits can be used to get rough key ranges to divide up iteration over the DB .


func (db *DB) Flatten(workers int) error {
	db.stopCompactions()
	defer db.startCompactions()

	compactAway := func(cp compactionPriority) error {
		db.opt.Infof("Attempting to compact with %+v\n", cp)
		errCh := make(chan error, 1)
		for i := 0; i < workers; i++ {
			go func() {
				errCh <- db.lc.doCompact(cp)
			}()
		}
		var success int
		var rerr error
		for i := 0; i < workers; i++ {
			err := <-errCh
			if err != nil {
				rerr = err
				db.opt.Warningf("While running doCompact with %+v. Error: %v\n", cp, err)
			} else {
				success++
			}
		}
		if success == 0 {
			return rerr
		}
		// We could do at least one successful compaction. So, we'll consider this a success.
		db.opt.Infof("%d compactor(s) succeeded. One or more tables from level %d compacted.\n",
			success, cp.level)
		return nil
	}

	hbytes := func(sz int64) string {
		return humanize.Bytes(uint64(sz))
	}

	for {
		db.opt.Infof("\n")
		var levels []int
		for i, l := range db.lc.levels {
			sz := l.getTotalSize()
			db.opt.Infof("Level: %d. %8s Size. %8s Max.\n",
				i, hbytes(l.getTotalSize()), hbytes(l.maxTotalSize))
			if sz > 0 {
				levels = append(levels, i)
			}
		}
		if len(levels) <= 1 {
			prios := db.lc.pickCompactLevels()
			if len(prios) == 0 || prios[0].score <= 1.0 {
				db.opt.Infof("All tables consolidated into one level. Flattening done.\n")
				return nil
			}
			if err := compactAway(prios[0]); err != nil {
				return err
			}
			continue
		}
		// Create an artificial compaction priority, to ensure that we compact the level.
		cp := compactionPriority{level: levels[0], score: 1.71}
		if err := compactAway(cp); err != nil {
			return err
		}
	}
}

// Flatten can be used to force compactions on the LSM tree so all the tables fall on the same
// level. This ensures that all the versions of keys are colocated and not split across multiple
// levels, which is necessary after a restore from backup. During Flatten, live compactions are
// stopped. Ideally, no writes are going on during Flatten. Otherwise, it would create competition
// between flattening the tree and new tables being created at level zero.

Flatten can be used to force compactions on the LSM tree so all the tables fall on the same level . This ensures that all the versions of keys are colocated and not split across multiple levels which is necessary after a restore from backup . During Flatten live compactions are stopped . Ideally no writes are going on during Flatten . Otherwise it would create competition between flattening the tree and new tables being created at level zero .


func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
	mtype := unix.PROT_READ
	if writable {
		mtype |= unix.PROT_WRITE
	}
	return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
}

func Mmap ( fd * os . File , writable bool , size int64 ) ( [ ] byte , error ) { mtype := unix . PROT_READ if writable { mtype |= unix . PROT_WRITE } return unix . Mmap ( int ( fd . Fd ( ) ) , 0 , int ( size ) , mtype , unix . MAP_SHARED ) }

// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.

Mmap uses the mmap system call to memory - map a file . If writable is true memory protection of the pages is set so that they may be written to as well .


func Madvise(b []byte, readahead bool) error {
	flags := unix.MADV_NORMAL
	if !readahead {
		flags = unix.MADV_RANDOM
	}
	return madvise(b, flags)
}

func Madvise ( b [ ] byte , readahead bool ) error { flags := unix . MADV_NORMAL if ! readahead { flags = unix . MADV_RANDOM } return madvise ( b , flags ) }

// Madvise uses the madvise system call to give advise about the use of memory
// when using a slice that is memory-mapped to a file. Set the readahead flag to
// false if page references are expected in random order.

Madvise uses the madvise system call to give advise about the use of memory when using a slice that is memory - mapped to a file . Set the readahead flag to false if page references are expected in random order .


func (o *oracle) setDiscardTs(ts uint64) {
	o.Lock()
	defer o.Unlock()
	o.discardTs = ts
}

func ( o * oracle ) setDiscardTs ( ts uint64 ) { o . Lock ( ) defer o . Unlock ( ) o . discardTs = ts }

// Any deleted or invalid versions at or below ts would be discarded during
// compaction to reclaim disk space in LSM tree and thence value log.

Any deleted or invalid versions at or below ts would be discarded during compaction to reclaim disk space in LSM tree and thence value log .


func (o *oracle) hasConflict(txn *Txn) bool {
	if len(txn.reads) == 0 {
		return false
	}
	for _, ro := range txn.reads {
		// A commit at the read timestamp is expected.
		// But, any commit after the read timestamp should cause a conflict.
		if ts, has := o.commits[ro]; has && ts > txn.readTs {
			return true
		}
	}
	return false
}

// hasConflict must be called while having a lock.

hasConflict must be called while having a lock .


func (txn *Txn) Set(key, val []byte) error {
	e := &Entry{
		Key:   key,
		Value: val,
	}
	return txn.SetEntry(e)
}

func ( txn * Txn ) Set ( key , val [ ] byte ) error { e := & Entry { Key : key , Value : val , } return txn . SetEntry ( e ) }

// Set adds a key-value pair to the database.
//
// It will return ErrReadOnlyTxn if update flag was set to false when creating the
// transaction.
//
// The current transaction keeps a reference to the key and val byte slice
// arguments. Users must not modify key and val until the end of the transaction.

Set adds a key - value pair to the database . It will return ErrReadOnlyTxn if update flag was set to false when creating the transaction . The current transaction keeps a reference to the key and val byte slice arguments . Users must not modify key and val until the end of the transaction .


func (txn *Txn) SetWithMeta(key, val []byte, meta byte) error {
	e := &Entry{Key: key, Value: val, UserMeta: meta}
	return txn.SetEntry(e)
}

func ( txn * Txn ) SetWithMeta ( key , val [ ] byte , meta byte ) error { e := & Entry { Key : key , Value : val , UserMeta : meta } return txn . SetEntry ( e ) }

// SetWithMeta adds a key-value pair to the database, along with a metadata
// byte.
//
// This byte is stored alongside the key, and can be used as an aid to
// interpret the value or store other contextual bits corresponding to the
// key-value pair.
//
// The current transaction keeps a reference to the key and val byte slice
// arguments. Users must not modify key and val until the end of the transaction.

SetWithMeta adds a key - value pair to the database along with a metadata byte . This byte is stored alongside the key and can be used as an aid to interpret the value or store other contextual bits corresponding to the key - value pair . The current transaction keeps a reference to the key and val byte slice arguments . Users must not modify key and val until the end of the transaction .


func (txn *Txn) Delete(key []byte) error {
	e := &Entry{
		Key:  key,
		meta: bitDelete,
	}
	return txn.modify(e)
}

func ( txn * Txn ) Delete ( key [ ] byte ) error { e := & Entry { Key : key , meta : bitDelete , } return txn . modify ( e ) }

// Delete deletes a key.
//
// This is done by adding a delete marker for the key at commit timestamp.  Any
// reads happening before this timestamp would be unaffected. Any reads after
// this commit would see the deletion.
//
// The current transaction keeps a reference to the key byte slice argument.
// Users must not modify the key until the end of the transaction.

Delete deletes a key . This is done by adding a delete marker for the key at commit timestamp . Any reads happening before this timestamp would be unaffected . Any reads after this commit would see the deletion . The current transaction keeps a reference to the key byte slice argument . Users must not modify the key until the end of the transaction .


func (txn *Txn) Get(key []byte) (item *Item, rerr error) {
	if len(key) == 0 {
		return nil, ErrEmptyKey
	} else if txn.discarded {
		return nil, ErrDiscardedTxn
	}

	item = new(Item)
	if txn.update {
		if e, has := txn.pendingWrites[string(key)]; has && bytes.Equal(key, e.Key) {
			if isDeletedOrExpired(e.meta, e.ExpiresAt) {
				return nil, ErrKeyNotFound
			}
			// Fulfill from cache.
			item.meta = e.meta
			item.val = e.Value
			item.userMeta = e.UserMeta
			item.key = key
			item.status = prefetched
			item.version = txn.readTs
			item.expiresAt = e.ExpiresAt
			// We probably don't need to set db on item here.
			return item, nil
		}
		// Only track reads if this is update txn. No need to track read if txn serviced it
		// internally.
		txn.addReadKey(key)
	}

	seek := y.KeyWithTs(key, txn.readTs)
	vs, err := txn.db.get(seek)
	if err != nil {
		return nil, errors.Wrapf(err, "DB::Get key: %q", key)
	}
	if vs.Value == nil && vs.Meta == 0 {
		return nil, ErrKeyNotFound
	}
	if isDeletedOrExpired(vs.Meta, vs.ExpiresAt) {
		return nil, ErrKeyNotFound
	}

	item.key = key
	item.version = vs.Version
	item.meta = vs.Meta
	item.userMeta = vs.UserMeta
	item.db = txn.db
	item.vptr = vs.Value // TODO: Do we need to copy this over?
	item.txn = txn
	item.expiresAt = vs.ExpiresAt
	return item, nil
}

// Get looks for key and returns corresponding Item.
// If key is not found, ErrKeyNotFound is returned.

Get looks for key and returns corresponding Item . If key is not found ErrKeyNotFound is returned .


func (txn *Txn) CommitWith(cb func(error)) {
	txn.commitPrecheck() // Precheck before discarding txn.
	defer txn.Discard()

	if cb == nil {
		panic("Nil callback provided to CommitWith")
	}

	if len(txn.writes) == 0 {
		// Do not run these callbacks from here, because the CommitWith and the
		// callback might be acquiring the same locks. Instead run the callback
		// from another goroutine.
		go runTxnCallback(&txnCb{user: cb, err: nil})
		return
	}

	commitCb, err := txn.commitAndSend()
	if err != nil {
		go runTxnCallback(&txnCb{user: cb, err: err})
		return
	}

	go runTxnCallback(&txnCb{user: cb, commit: commitCb})
}

// CommitWith acts like Commit, but takes a callback, which gets run via a
// goroutine to avoid blocking this function. The callback is guaranteed to run,
// so it is safe to increment sync.WaitGroup before calling CommitWith, and
// decrementing it in the callback; to block until all callbacks are run.

CommitWith acts like Commit but takes a callback which gets run via a goroutine to avoid blocking this function . The callback is guaranteed to run so it is safe to increment sync . WaitGroup before calling CommitWith and decrementing it in the callback ; to block until all callbacks are run .


func (db *DB) View(fn func(txn *Txn) error) error {
	var txn *Txn
	if db.opt.managedTxns {
		txn = db.NewTransactionAt(math.MaxUint64, false)
	} else {
		txn = db.NewTransaction(false)
	}
	defer txn.Discard()

	return fn(txn)
}

func ( db * DB ) View ( fn func ( txn * Txn ) error ) error { var txn * Txn if db . opt . managedTxns { txn = db . NewTransactionAt ( math . MaxUint64 , false ) } else { txn = db . NewTransaction ( false ) } defer txn . Discard ( ) return fn ( txn ) }

// View executes a function creating and managing a read-only transaction for the user. Error
// returned by the function is relayed by the View method.
// If View is used with managed transactions, it would assume a read timestamp of MaxUint64.

View executes a function creating and managing a read - only transaction for the user . Error returned by the function is relayed by the View method . If View is used with managed transactions it would assume a read timestamp of MaxUint64 .


func (db *DB) Update(fn func(txn *Txn) error) error {
	if db.opt.managedTxns {
		panic("Update can only be used with managedDB=false.")
	}
	txn := db.NewTransaction(true)
	defer txn.Discard()

	if err := fn(txn); err != nil {
		return err
	}

	return txn.Commit()
}

func ( db * DB ) Update ( fn func ( txn * Txn ) error ) error { if db . opt . managedTxns { panic ( " " ) } txn := db . NewTransaction ( true ) defer txn . Discard ( ) if err := fn ( txn ) ; err != nil { return err } return txn . Commit ( ) }

// Update executes a function, creating and managing a read-write transaction
// for the user. Error returned by the function is relayed by the Update method.
// Update cannot be used with managed transactions.

Update executes a function creating and managing a read - write transaction for the user . Error returned by the function is relayed by the Update method . Update cannot be used with managed transactions .


func (itr *blockIterator) Seek(key []byte, whence int) {
	itr.err = nil

	switch whence {
	case origin:
		itr.Reset()
	case current:
	}

	var done bool
	for itr.Init(); itr.Valid(); itr.Next() {
		k := itr.Key()
		if y.CompareKeys(k, key) >= 0 {
			// We are done as k is >= key.
			done = true
			break
		}
	}
	if !done {
		itr.err = io.EOF
	}
}

func
(
itr
*
blockIterator
)
Seek
(
key
[
]
byte
,
whence
int
)
{
itr
.
err
=
switch
whence
{
case
origin
:
itr
.
Reset
(
)
case
current
:
}
var
done
bool
for
itr
.
Init
(
)
;
itr
.
Valid
(
)
;
itr
.
Next
(
)
{
k
:=
itr
.
Key
(
)
if
y
.
CompareKeys
(
k
,
key
)
>=
0
{
// We are done as k is >= key.
done
=
true
break
}
}
if
!
done
{
itr
.
err
=
io
.
EOF
}
}

// Seek brings us to the first block element that is >= input key.

Seek brings us to the first block element that is > = input key .


func (itr *blockIterator) SeekToLast() {
	itr.err = nil
	for itr.Init(); itr.Valid(); itr.Next() {
	}
	itr.Prev()
}

func ( itr * blockIterator ) SeekToLast ( ) { itr . err = nil for itr . Init ( ) ; itr . Valid ( ) ; itr . Next ( ) { } itr . Prev ( ) }

// SeekToLast brings us to the last element. Valid should return true.

SeekToLast brings us to the last element . Valid should return true .


func (itr *blockIterator) parseKV(h header) {
	if cap(itr.key) < int(h.plen+h.klen) {
		sz := int(h.plen) + int(h.klen) // Convert to int before adding to avoid uint16 overflow.
		itr.key = make([]byte, 2*sz)
	}
	itr.key = itr.key[:h.plen+h.klen]
	copy(itr.key, itr.baseKey[:h.plen])
	copy(itr.key[h.plen:], itr.data[itr.pos:itr.pos+uint32(h.klen)])
	itr.pos += uint32(h.klen)

	if itr.pos+uint32(h.vlen) > uint32(len(itr.data)) {
		itr.err = errors.Errorf("Value exceeded size of block: %d %d %d %d %v",
			itr.pos, h.klen, h.vlen, len(itr.data), h)
		return
	}
	itr.val = y.SafeCopy(itr.val, itr.data[itr.pos:itr.pos+uint32(h.vlen)])
	itr.pos += uint32(h.vlen)
}

// parseKV would allocate a new byte slice for key and for value.

parseKV would allocate a new byte slice for key and for value .


func (t *Table) NewIterator(reversed bool) *Iterator {
	t.IncrRef() // Important.
	ti := &Iterator{t: t, reversed: reversed}
	ti.next()
	return ti
}

func
(
t
*
Table
)
NewIterator
(
reversed
bool
)
*
Iterator
{
t
.
IncrRef
(
)
// Important.
ti
:=
&
Iterator
{
t
:
t
,
reversed
:
reversed
}
ti
.
next
(
)
return
ti
}

// NewIterator returns a new iterator of the Table


func (itr *Iterator) seekFrom(key []byte, whence int) {
	itr.err = nil
	switch whence {
	case origin:
		itr.reset()
	case current:
	}

	idx := sort.Search(len(itr.t.blockIndex), func(idx int) bool {
		ko := itr.t.blockIndex[idx]
		return y.CompareKeys(ko.key, key) > 0
	})
	if idx == 0 {
		// The smallest key in our table is already strictly > key. We can return that.
		// This is like a SeekToFirst.
		itr.seekHelper(0, key)
		return
	}

	// block[idx].smallest is > key.
	// Since idx>0, we know block[idx-1].smallest is <= key.
	// There are two cases.
	// 1) Everything in block[idx-1] is strictly < key. In this case, we should go to the first
	//    element of block[idx].
	// 2) Some element in block[idx-1] is >= key. We should go to that element.
	itr.seekHelper(idx-1, key)
	if itr.err == io.EOF {
		// Case 1. Need to visit block[idx].
		if idx == len(itr.t.blockIndex) {
			// If idx == len(itr.t.blockIndex), then input key is greater than ANY element of table.
			// There's nothing we can do. Valid() should return false as we seek to end of table.
			return
		}
		// Since block[idx].smallest is > key. This is essentially a block[idx].SeekToFirst.
		itr.seekHelper(idx, key)
	}
	// Case 2: No need to do anything. We already did the seek in block[idx-1].
}

// seekFrom brings us to a key that is >= input key.

seekFrom brings us to a key that is > = input key .


func (itr *Iterator) seekForPrev(key []byte) {
	// TODO: Optimize this. We shouldn't have to take a Prev step.
	itr.seekFrom(key, origin)
	if !bytes.Equal(itr.Key(), key) {
		itr.prev()
	}
}

// seekForPrev will reset iterator and seek to <= key.

seekForPrev will reset iterator and seek to < = key .


func (itr *Iterator) Value() (ret y.ValueStruct) {
	ret.Decode(itr.bi.Value())
	return
}

func ( itr * Iterator ) Value ( ) ( ret y . ValueStruct ) { ret . Decode ( itr . bi . Value ( ) ) return }

// Value follows the y.Iterator interface

Value follows the y . Iterator interface


func (itr *Iterator) Seek(key []byte) {
	if !itr.reversed {
		itr.seek(key)
	} else {
		itr.seekForPrev(key)
	}
}

func ( itr * Iterator ) Seek ( key [ ] byte ) { if ! itr . reversed { itr . seek ( key ) } else { itr . seekForPrev ( key ) } }

// Seek follows the y.Iterator interface


func NewConcatIterator(tbls []*Table, reversed bool) *ConcatIterator {
	iters := make([]*Iterator, len(tbls))
	for i := 0; i < len(tbls); i++ {
		iters[i] = tbls[i].NewIterator(reversed)
	}
	return &ConcatIterator{
		reversed: reversed,
		iters:    iters,
		tables:   tbls,
		idx:      -1, // Not really necessary because s.it.Valid()=false, but good to have.
	}
}

// NewConcatIterator creates a new concatenated iterator


func (s *ConcatIterator) Valid() bool {
	return s.cur != nil && s.cur.Valid()
}

func ( s * ConcatIterator ) Valid ( ) bool { return s . cur != nil && s . cur . Valid ( ) }


func (s *ConcatIterator) Next() {
	s.cur.Next()
	if s.cur.Valid() {
		// Nothing to do. Just stay with the current table.
		return
	}
	for { // In case there are empty tables.
		if !s.reversed {
			s.setIdx(s.idx + 1)
		} else {
			s.setIdx(s.idx - 1)
		}
		if s.cur == nil {
			// End of list. Valid will become false.
			return
		}
		s.cur.Rewind()
		if s.cur.Valid() {
			break
		}
	}
}


func (s *ConcatIterator) Close() error {
	for _, it := range s.iters {
		if err := it.Close(); err != nil {
			return errors.Wrap(err, "ConcatIterator")
		}
	}
	return nil
}

func ( s * ConcatIterator ) Close ( ) error { for _ , it := range s . iters { if err := it . Close ( ) ; err != nil { return errors . Wrap ( err , " " ) } } return nil }


func OpenExistingFile(filename string, flags uint32) (*os.File, error) {
	openFlags := os.O_RDWR
	if flags&ReadOnly != 0 {
		openFlags = os.O_RDONLY
	}

	if flags&Sync != 0 {
		openFlags |= datasyncFileFlag
	}
	return os.OpenFile(filename, openFlags, 0)
}

func OpenExistingFile ( filename string , flags uint32 ) ( * os . File , error ) { openFlags := os . O_RDWR if flags & ReadOnly != 0 { openFlags = os . O_RDONLY } if flags & Sync != 0 { openFlags |= datasyncFileFlag } return os . OpenFile ( filename , openFlags , 0 ) }

// OpenExistingFile opens an existing file, errors if it doesn't exist.

OpenExistingFile opens an existing file errors if it doesn t exist .


func Copy(a []byte) []byte {
	b := make([]byte, len(a))
	copy(b, a)
	return b
}

func Copy ( a [ ] byte ) [ ] byte { b := make ( [ ] byte , len ( a ) ) copy ( b , a ) return b }

// Copy copies a byte slice and returns the copied slice.

Copy copies a byte slice and returns the copied slice .


func KeyWithTs(key []byte, ts uint64) []byte {
	out := make([]byte, len(key)+8)
	copy(out, key)
	binary.BigEndian.PutUint64(out[len(key):], math.MaxUint64-ts)
	return out
}

func KeyWithTs ( key [ ] byte , ts uint64 ) [ ] byte { out := make ( [ ] byte , len ( key ) + 8 ) copy ( out , key ) binary . BigEndian . PutUint64 ( out [ len ( key ) : ] , math . MaxUint64 - ts ) return out }

// KeyWithTs generates a new key by appending ts to key.

KeyWithTs generates a new key by appending ts to key .


func ParseTs(key []byte) uint64 {
	if len(key) <= 8 {
		return 0
	}
	return math.MaxUint64 - binary.BigEndian.Uint64(key[len(key)-8:])
}

func ParseTs ( key [ ] byte ) uint64 { if len ( key ) <= 8 { return 0 } return math . MaxUint64 - binary . BigEndian . Uint64 ( key [ len ( key ) - 8 : ] ) }

// ParseTs parses the timestamp from the key bytes.

ParseTs parses the timestamp from the key bytes .


func ParseKey(key []byte) []byte {
	if key == nil {
		return nil
	}

	AssertTrue(len(key) > 8)
	return key[:len(key)-8]
}

func ParseKey ( key [ ] byte ) [ ] byte { if key == nil { return nil } AssertTrue ( len ( key ) > 8 ) return key [ : len ( key ) - 8 ] }

// ParseKey parses the actual key from the key bytes.

ParseKey parses the actual key from the key bytes .


func SameKey(src, dst []byte) bool {
	if len(src) != len(dst) {
		return false
	}
	return bytes.Equal(ParseKey(src), ParseKey(dst))
}

func SameKey ( src , dst [ ] byte ) bool { if len ( src ) != len ( dst ) { return false } return bytes . Equal ( ParseKey ( src ) , ParseKey ( dst ) ) }

// SameKey checks for key equality ignoring the version timestamp suffix.

SameKey checks for key equality ignoring the version timestamp suffix .


func FixedDuration(d time.Duration) string {
	str := fmt.Sprintf("%02ds", int(d.Seconds())%60)
	if d >= time.Minute {
		str = fmt.Sprintf("%02dm", int(d.Minutes())%60) + str
	}
	if d >= time.Hour {
		str = fmt.Sprintf("%02dh", int(d.Hours())) + str
	}
	return str
}

func FixedDuration ( d time . Duration ) string { str := fmt . Sprintf ( " " , int ( d . Seconds ( ) ) % 60 ) if d >= time . Minute { str = fmt . Sprintf ( " " , int ( d . Minutes ( ) ) % 60 ) + str } if d >= time . Hour { str = fmt . Sprintf ( " " , int ( d . Hours ( ) ) ) + str } return str }

// FixedDuration returns a string representation of the given duration with the
// hours, minutes, and seconds.

FixedDuration returns a string representation of the given duration with the hours minutes and seconds .


func NewCloser(initial int) *Closer {
	ret := &Closer{closed: make(chan struct{})}
	ret.waiting.Add(initial)
	return ret
}

func NewCloser ( initial int ) * Closer { ret := & Closer { closed : make ( chan struct { } ) } ret . waiting . Add ( initial ) return ret }

// NewCloser constructs a new Closer, with an initial count on the WaitGroup.

NewCloser constructs a new Closer with an initial count on the WaitGroup .


func NewThrottle(max int) *Throttle {
	return &Throttle{
		ch:    make(chan struct{}, max),
		errCh: make(chan error, max),
	}
}

func NewThrottle ( max int ) * Throttle { return & Throttle { ch : make ( chan struct { } , max ) , errCh : make ( chan error , max ) , } }

// NewThrottle creates a new throttle with a max number of workers.

NewThrottle creates a new throttle with a max number of workers .


func (t *Throttle) Do() error {
	for {
		select {
		case t.ch <- struct{}{}:
			t.wg.Add(1)
			return nil
		case err := <-t.errCh:
			if err != nil {
				return err
			}
		}
	}
}

func ( t * Throttle ) Do ( ) error { for { select { case t . ch <- struct { } { } : t . wg . Add ( 1 ) return nil case err := <- t . errCh : if err != nil { return err } } } }

// Do should be called by workers before they start working. It blocks if there
// are already maximum number of workers working. If it detects an error from
// previously Done workers, it would return it.

Do should be called by workers before they start working . It blocks if there are already maximum number of workers working . If it detects an error from previously Done workers it would return it .


func (t *Throttle) Done(err error) {
	if err != nil {
		t.errCh <- err
	}
	select {
	case <-t.ch:
	default:
		panic("Throttle Do Done mismatch")
	}
	t.wg.Done()
}

func ( t * Throttle ) Done ( err error ) { if err != nil { t . errCh <- err } select { case <- t . ch : default : panic ( " " ) } t . wg . Done ( ) }

// Done should be called by workers when they finish working. They can also
// pass the error status of work done.

Done should be called by workers when they finish working . They can also pass the error status of work done .


func (t *Throttle) Finish() error {
	t.wg.Wait()
	close(t.ch)
	close(t.errCh)
	for err := range t.errCh {
		if err != nil {
			return err
		}
	}
	return nil
}

func ( t * Throttle ) Finish ( ) error { t . wg . Wait ( ) close ( t . ch ) close ( t . errCh ) for err := range t . errCh { if err != nil { return err } } return nil }

// Finish waits until all workers have finished working. It would return any
// error passed by Done.

Finish waits until all workers have finished working . It would return any error passed by Done .


func (db *DB) SetDiscardTs(ts uint64) {
	if !db.opt.managedTxns {
		panic("Cannot use SetDiscardTs with managedDB=false.")
	}
	db.orc.setDiscardTs(ts)
}

func ( db * DB ) SetDiscardTs ( ts uint64 ) { if ! db . opt . managedTxns { panic ( " " ) } db . orc . setDiscardTs ( ts ) }

// SetDiscardTs sets a timestamp at or below which, any invalid or deleted
// versions can be discarded from the LSM tree, and thence from the value log to
// reclaim disk space. Can only be used with managed transactions.

SetDiscardTs sets a timestamp at or below which any invalid or deleted versions can be discarded from the LSM tree and thence from the value log to reclaim disk space . Can only be used with managed transactions .


func (lf *logFile) openReadOnly() error {
	var err error
	lf.fd, err = os.OpenFile(lf.path, os.O_RDONLY, 0666)
	if err != nil {
		return errors.Wrapf(err, "Unable to open %q as RDONLY.", lf.path)
	}

	fi, err := lf.fd.Stat()
	if err != nil {
		return errors.Wrapf(err, "Unable to check stat for %q", lf.path)
	}
	y.AssertTrue(fi.Size() <= math.MaxUint32)
	lf.size = uint32(fi.Size())

	if err = lf.mmap(fi.Size()); err != nil {
		_ = lf.fd.Close()
		return y.Wrapf(err, "Unable to map file")
	}

	return nil
}

func ( lf * logFile ) openReadOnly ( ) error { var err error lf . fd , err = os . OpenFile ( lf . path , os . O_RDONLY , 0666 ) if err != nil { return errors . Wrapf ( err , " " , lf . path ) } fi , err := lf . fd . Stat ( ) if err != nil { return errors . Wrapf ( err , " " , lf . path ) } y . AssertTrue ( fi . Size ( ) <= math . MaxUint32 ) lf . size = uint32 ( fi . Size ( ) ) if err = lf . mmap ( fi . Size ( ) ) ; err != nil { _ = lf . fd . Close ( ) return y . Wrapf ( err , " " ) } return nil }

// openReadOnly assumes that we have a write lock on logFile.

openReadOnly assumes that we have a write lock on logFile .


func (vlog *valueLog) iterate(lf *logFile, offset uint32, fn logEntry) (uint32, error) {
	fi, err := lf.fd.Stat()
	if err != nil {
		return 0, err
	}
	if int64(offset) == fi.Size() {
		// We're at the end of the file already. No need to do anything.
		return offset, nil
	}
	if vlog.opt.ReadOnly {
		// We're not at the end of the file. We'd need to replay the entries, or
		// possibly truncate the file.
		return 0, ErrReplayNeeded
	}

	// We're not at the end of the file. Let's Seek to the offset and start reading.
	if _, err := lf.fd.Seek(int64(offset), io.SeekStart); err != nil {
		return 0, errFile(err, lf.path, "Unable to seek")
	}

	reader := bufio.NewReader(lf.fd)
	read := &safeRead{
		k:            make([]byte, 10),
		v:            make([]byte, 10),
		recordOffset: offset,
	}

	var lastCommit uint64
	var validEndOffset uint32
	for {
		e, err := read.Entry(reader)
		if err == io.EOF {
			break
		} else if err == io.ErrUnexpectedEOF || err == errTruncate {
			break
		} else if err != nil {
			return 0, err
		} else if e == nil {
			continue
		}

		var vp valuePointer
		vp.Len = uint32(headerBufSize + len(e.Key) + len(e.Value) + crc32.Size)
		read.recordOffset += vp.Len

		vp.Offset = e.offset
		vp.Fid = lf.fid

		if e.meta&bitTxn > 0 {
			txnTs := y.ParseTs(e.Key)
			if lastCommit == 0 {
				lastCommit = txnTs
			}
			if lastCommit != txnTs {
				break
			}

		} else if e.meta&bitFinTxn > 0 {
			txnTs, err := strconv.ParseUint(string(e.Value), 10, 64)
			if err != nil || lastCommit != txnTs {
				break
			}
			// Got the end of txn. Now we can store them.
			lastCommit = 0
			validEndOffset = read.recordOffset

		} else {
			if lastCommit != 0 {
				// This is most likely an entry which was moved as part of GC.
				// We shouldn't get this entry in the middle of a transaction.
				break
			}
			validEndOffset = read.recordOffset
		}

		if err := fn(*e, vp); err != nil {
			if err == errStop {
				break
			}
			return 0, errFile(err, lf.path, "Iteration function")
		}
	}
	return validEndOffset, nil
}

// iterate iterates over log file. It doesn't not allocate new memory for every kv pair.
// Therefore, the kv pair is only valid for the duration of fn call.

iterate iterates over log file . It doesn t not allocate new memory for every kv pair . Therefore the kv pair is only valid for the duration of fn call .


func (vlog *valueLog) sortedFids() []uint32 {
	toBeDeleted := make(map[uint32]struct{})
	for _, fid := range vlog.filesToBeDeleted {
		toBeDeleted[fid] = struct{}{}
	}
	ret := make([]uint32, 0, len(vlog.filesMap))
	for fid := range vlog.filesMap {
		if _, ok := toBeDeleted[fid]; !ok {
			ret = append(ret, fid)
		}
	}
	sort.Slice(ret, func(i, j int) bool {
		return ret[i] < ret[j]
	})
	return ret
}

func ( vlog * valueLog ) sortedFids ( ) [ ] uint32 { toBeDeleted := make ( map [ uint32 ] struct { } ) for _ , fid := range vlog . filesToBeDeleted { toBeDeleted [ fid ] = struct { } { } } ret := make ( [ ] uint32 , 0 , len ( vlog . filesMap ) ) for fid := range vlog . filesMap { if _ , ok := toBeDeleted [ fid ] ; ! ok { ret = append ( ret , fid ) } } sort . Slice ( ret , func ( i , j int ) bool { return ret [ i ] < ret [ j ] } ) return ret }

// sortedFids returns the file id's not pending deletion, sorted.  Assumes we have shared access to
// filesMap.

sortedFids returns the file id s not pending deletion sorted . Assumes we have shared access to filesMap .


func (vlog *valueLog) write(reqs []*request) error {
	vlog.filesLock.RLock()
	maxFid := atomic.LoadUint32(&vlog.maxFid)
	curlf := vlog.filesMap[maxFid]
	vlog.filesLock.RUnlock()

	var buf bytes.Buffer
	toDisk := func() error {
		if buf.Len() == 0 {
			return nil
		}
		vlog.elog.Printf("Flushing %d blocks of total size: %d", len(reqs), buf.Len())
		n, err := curlf.fd.Write(buf.Bytes())
		if err != nil {
			return errors.Wrapf(err, "Unable to write to value log file: %q", curlf.path)
		}
		buf.Reset()
		y.NumWrites.Add(1)
		y.NumBytesWritten.Add(int64(n))
		vlog.elog.Printf("Done")
		atomic.AddUint32(&vlog.writableLogOffset, uint32(n))

		if vlog.woffset() > uint32(vlog.opt.ValueLogFileSize) ||
			vlog.numEntriesWritten > vlog.opt.ValueLogMaxEntries {
			var err error
			if err = curlf.doneWriting(vlog.woffset()); err != nil {
				return err
			}

			newid := atomic.AddUint32(&vlog.maxFid, 1)
			y.AssertTruef(newid > 0, "newid has overflown uint32: %v", newid)
			newlf, err := vlog.createVlogFile(newid)
			if err != nil {
				return err
			}
			curlf = newlf
		}
		return nil
	}

	for i := range reqs {
		b := reqs[i]
		b.Ptrs = b.Ptrs[:0]
		for j := range b.Entries {
			e := b.Entries[j]
			var p valuePointer

			p.Fid = curlf.fid
			// Use the offset including buffer length so far.
			p.Offset = vlog.woffset() + uint32(buf.Len())
			plen, err := encodeEntry(e, &buf) // Now encode the entry into buffer.
			if err != nil {
				return err
			}
			p.Len = uint32(plen)
			b.Ptrs = append(b.Ptrs, p)
		}
		vlog.numEntriesWritten += uint32(len(b.Entries))
		// We write to disk here so that all entries that are part of the same transaction are
		// written to the same vlog file.
		writeNow :=
			vlog.woffset()+uint32(buf.Len()) > uint32(vlog.opt.ValueLogFileSize) ||
				vlog.numEntriesWritten > uint32(vlog.opt.ValueLogMaxEntries)
		if writeNow {
			if err := toDisk(); err != nil {
				return err
			}
		}
	}
	return toDisk()
}

// write is thread-unsafe by design and should not be called concurrently.

write is thread - unsafe by design and should not be called concurrently .


func (vlog *valueLog) populateDiscardStats() error {
	discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, math.MaxUint64)
	vs, err := vlog.db.get(discardStatsKey)
	if err != nil {
		return err
	}

	// check if value is Empty
	if vs.Value == nil || len(vs.Value) == 0 {
		vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)}
		return nil
	}

	var statsMap map[uint32]int64
	if err := json.Unmarshal(vs.Value, &statsMap); err != nil {
		return err
	}
	vlog.opt.Debugf("Value Log Discard stats: %v", statsMap)
	vlog.lfDiscardStats = &lfDiscardStats{m: statsMap}
	return nil
}

func
(
vlog
*
valueLog
)
populateDiscardStats
(
)
error
{
discardStatsKey
:=
y
.
KeyWithTs
(
lfDiscardStatsKey
,
math
.
MaxUint64
)
vs
,
err
:=
vlog
.
db
.
get
(
discardStatsKey
)
if
err
!=
{
return
err
}
// check if value is Empty
if
vs
.
Value
==
||
len
(
vs
.
Value
)
==
0
{
vlog
.
lfDiscardStats
=
&
lfDiscardStats
{
m
:
make
(
map
[
uint32
]
int64
)
}
return
}
var
statsMap
map
[
uint32
]
int64
if
err
:=
json
.
Unmarshal
(
vs
.
Value
,
&
statsMap
)
;
err
!=
{
return
err
}
vlog
.
opt
.
Debugf
(
"
"
,
statsMap
)
vlog
.
lfDiscardStats
=
&
lfDiscardStats
{
m
:
statsMap
}
return
}

// populateDiscardStats populates vlog.lfDiscardStats
// This function will be called while initializing valueLog

populateDiscardStats populates vlog . lfDiscardStats This function will be called while initializing valueLog


func (db *DB) Backup(w io.Writer, since uint64) (uint64, error) {
	stream := db.NewStream()
	stream.LogPrefix = "DB.Backup"
	return stream.Backup(w, since)
}

func ( db * DB ) Backup ( w io . Writer , since uint64 ) ( uint64 , error ) { stream := db . NewStream ( ) stream . LogPrefix = " " return stream . Backup ( w , since ) }

// Backup is a wrapper function over Stream.Backup to generate full and incremental backups of the
// DB. For more control over how many goroutines are used to generate the backup, or if you wish to
// backup only a certain range of keys, use Stream.Backup directly.

Backup is a wrapper function over Stream . Backup to generate full and incremental backups of the DB . For more control over how many goroutines are used to generate the backup or if you wish to backup only a certain range of keys use Stream . Backup directly .


func (st *Stream) ToList(key []byte, itr *Iterator) (*pb.KVList, error) {
	list := &pb.KVList{}
	for ; itr.Valid(); itr.Next() {
		item := itr.Item()
		if item.IsDeletedOrExpired() {
			break
		}
		if !bytes.Equal(key, item.Key()) {
			// Break out on the first encounter with another key.
			break
		}

		valCopy, err := item.ValueCopy(nil)
		if err != nil {
			return nil, err
		}
		kv := &pb.KV{
			Key:       item.KeyCopy(nil),
			Value:     valCopy,
			UserMeta:  []byte{item.UserMeta()},
			Version:   item.Version(),
			ExpiresAt: item.ExpiresAt(),
		}
		list.Kv = append(list.Kv, kv)
		if st.db.opt.NumVersionsToKeep == 1 {
			break
		}

		if item.DiscardEarlierVersions() {
			break
		}
	}
	return list, nil
}

// ToList is a default implementation of KeyToList. It picks up all valid versions of the key,
// skipping over deleted or expired keys.

ToList is a default implementation of KeyToList . It picks up all valid versions of the key skipping over deleted or expired keys .


func (st *Stream) produceRanges(ctx context.Context) {
	splits := st.db.KeySplits(st.Prefix)
	start := y.SafeCopy(nil, st.Prefix)
	for _, key := range splits {
		st.rangeCh <- keyRange{left: start, right: y.SafeCopy(nil, []byte(key))}
		start = y.SafeCopy(nil, []byte(key))
	}
	// Edge case: prefix is empty and no splits exist. In that case, we should have at least one
	// keyRange output.
	st.rangeCh <- keyRange{left: start}
	close(st.rangeCh)
}

// keyRange is [start, end), including start, excluding end. Do ensure that the start,
// end byte slices are owned by keyRange struct.

keyRange is [ start end ) including start excluding end . Do ensure that the start end byte slices are owned by keyRange struct .


func (st *Stream) produceKVs(ctx context.Context) error {
	var size int
	var txn *Txn
	if st.readTs > 0 {
		txn = st.db.NewTransactionAt(st.readTs, false)
	} else {
		txn = st.db.NewTransaction(false)
	}
	defer txn.Discard()

	iterate := func(kr keyRange) error {
		iterOpts := DefaultIteratorOptions
		iterOpts.AllVersions = true
		iterOpts.Prefix = st.Prefix
		iterOpts.PrefetchValues = false
		itr := txn.NewIterator(iterOpts)
		defer itr.Close()

		outList := new(pb.KVList)
		var prevKey []byte
		for itr.Seek(kr.left); itr.Valid(); {
			// it.Valid would only return true for keys with the provided Prefix in iterOpts.
			item := itr.Item()
			if bytes.Equal(item.Key(), prevKey) {
				itr.Next()
				continue
			}
			prevKey = append(prevKey[:0], item.Key()...)

			// Check if we reached the end of the key range.
			if len(kr.right) > 0 && bytes.Compare(item.Key(), kr.right) >= 0 {
				break
			}
			// Check if we should pick this key.
			if st.ChooseKey != nil && !st.ChooseKey(item) {
				continue
			}

			// Now convert to key value.
			list, err := st.KeyToList(item.KeyCopy(nil), itr)
			if err != nil {
				return err
			}
			if list == nil || len(list.Kv) == 0 {
				continue
			}
			outList.Kv = append(outList.Kv, list.Kv...)
			size += list.Size()
			if size >= pageSize {
				st.kvChan <- outList
				outList = new(pb.KVList)
				size = 0
			}
		}
		if len(outList.Kv) > 0 {
			st.kvChan <- outList
		}
		return nil
	}

	for {
		select {
		case kr, ok := <-st.rangeCh:
			if !ok {
				// Done with the keys.
				return nil
			}
			if err := iterate(kr); err != nil {
				return err
			}
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

// produceKVs picks up ranges from rangeCh, generates KV lists and sends them to kvChan.

produceKVs picks up ranges from rangeCh generates KV lists and sends them to kvChan .


func (st *Stream) Orchestrate(ctx context.Context) error {
	st.rangeCh = make(chan keyRange, 3) // Contains keys for posting lists.

	// kvChan should only have a small capacity to ensure that we don't buffer up too much data if
	// sending is slow. Page size is set to 4MB, which is used to lazily cap the size of each
	// KVList. To get around 64MB buffer, we can set the channel size to 16.
	st.kvChan = make(chan *pb.KVList, 16)

	if st.KeyToList == nil {
		st.KeyToList = st.ToList
	}

	// Picks up ranges from Badger, and sends them to rangeCh.
	go st.produceRanges(ctx)

	errCh := make(chan error, 1) // Stores error by consumeKeys.
	var wg sync.WaitGroup
	for i := 0; i < st.NumGo; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			// Picks up ranges from rangeCh, generates KV lists, and sends them to kvChan.
			if err := st.produceKVs(ctx); err != nil {
				select {
				case errCh <- err:
				default:
				}
			}
		}()
	}

	// Pick up key-values from kvChan and send to stream.
	kvErr := make(chan error, 1)
	go func() {
		// Picks up KV lists from kvChan, and sends them to Output.
		kvErr <- st.streamKVs(ctx)
	}()
	wg.Wait()        // Wait for produceKVs to be over.
	close(st.kvChan) // Now we can close kvChan.

	select {
	case err := <-errCh: // Check error from produceKVs.
		return err
	default:
	}

	// Wait for key streaming to be over.
	err := <-kvErr
	return err
}

// Orchestrate runs Stream. It picks up ranges from the SSTables, then runs NumGo number of
// goroutines to iterate over these ranges and batch up KVs in lists. It concurrently runs a single
// goroutine to pick these lists, batch them up further and send to Output.Send. Orchestrate also
// spits logs out to Infof, using provided LogPrefix. Note that all calls to Output.Send
// are serial. In case any of these steps encounter an error, Orchestrate would stop execution and
// return that error. Orchestrate can be called multiple times, but in serial order.

Orchestrate runs Stream . It picks up ranges from the SSTables then runs NumGo number of goroutines to iterate over these ranges and batch up KVs in lists . It concurrently runs a single goroutine to pick these lists batch them up further and send to Output . Send . Orchestrate also spits logs out to Infof using provided LogPrefix . Note that all calls to Output . Send are serial . In case any of these steps encounter an error Orchestrate would stop execution and return that error . Orchestrate can be called multiple times but in serial order .


func (db *DB) NewStream() *Stream {
	if db.opt.managedTxns {
		panic("This API can not be called in managed mode.")
	}
	return db.newStream()
}

func ( db * DB ) NewStream ( ) * Stream { if db . opt . managedTxns { panic ( " " ) } return db . newStream ( ) }


func (db *DB) NewStreamAt(readTs uint64) *Stream {
	if !db.opt.managedTxns {
		panic("This API can only be called in managed mode.")
	}
	stream := db.newStream()
	stream.readTs = readTs
	return stream
}

func ( db * DB ) NewStreamAt ( readTs uint64 ) * Stream { if ! db . opt . managedTxns { panic ( " " ) } stream := db . newStream ( ) stream . readTs = readTs return stream }

// NewStreamAt creates a new Stream at a particular timestamp. Should only be used with managed DB.

NewStreamAt creates a new Stream at a particular timestamp . Should only be used with managed DB .


func (t *Table) DecrRef() error {
	newRef := atomic.AddInt32(&t.ref, -1)
	if newRef == 0 {
		// We can safely delete this file, because for all the current files, we always have
		// at least one reference pointing to them.

		// It's necessary to delete windows files
		if t.loadingMode == options.MemoryMap {
			y.Munmap(t.mmap)
		}
		if err := t.fd.Truncate(0); err != nil {
			// This is very important to let the FS know that the file is deleted.
			return err
		}
		filename := t.fd.Name()
		if err := t.fd.Close(); err != nil {
			return err
		}
		if err := os.Remove(filename); err != nil {
			return err
		}
	}
	return nil
}

// DecrRef decrements the refcount and possibly deletes the table


func ParseFileID(name string) (uint64, bool) {
	name = path.Base(name)
	if !strings.HasSuffix(name, fileSuffix) {
		return 0, false
	}
	//	suffix := name[len(fileSuffix):]
	name = strings.TrimSuffix(name, fileSuffix)
	id, err := strconv.Atoi(name)
	if err != nil {
		return 0, false
	}
	y.AssertTrue(id >= 0)
	return uint64(id), true
}

func
ParseFileID
(
name
string
)
(
uint64
,
bool
)
{
name
=
path
.
Base
(
name
)
if
!
strings
.
HasSuffix
(
name
,
fileSuffix
)
{
return
0
,
false
}
//	suffix := name[len(fileSuffix):]
name
=
strings
.
TrimSuffix
(
name
,
fileSuffix
)
id
,
err
:=
strconv
.
Atoi
(
name
)
if
err
!=
{
return
0
,
false
}
y
.
AssertTrue
(
id
>=
0
)
return
uint64
(
id
)
,
true
}

// ParseFileID reads the file id out of a filename.

ParseFileID reads the file id out of a filename .


func (db *DB) PrintHistogram(keyPrefix []byte) {
	if db == nil {
		fmt.Println("\nCannot build histogram: DB is nil.")
		return
	}
	histogram := db.buildHistogram(keyPrefix)
	fmt.Printf("Histogram of key sizes (in bytes)\n")
	histogram.keySizeHistogram.printHistogram()
	fmt.Printf("Histogram of value sizes (in bytes)\n")
	histogram.valueSizeHistogram.printHistogram()
}

func ( db * DB ) PrintHistogram ( keyPrefix [ ] byte ) { if db == nil { fmt . Println ( " \n " ) return } histogram := db . buildHistogram ( keyPrefix ) fmt . Printf ( " \n " ) histogram . keySizeHistogram . printHistogram ( ) fmt . Printf ( " \n " ) histogram . valueSizeHistogram . printHistogram ( ) }

// PrintHistogram builds and displays the key-value size histogram.
// When keyPrefix is set, only the keys that have prefix "keyPrefix" are
// considered for creating the histogram

PrintHistogram builds and displays the key - value size histogram . When keyPrefix is set only the keys that have prefix keyPrefix are considered for creating the histogram


func newSizeHistogram() *sizeHistogram {
	// TODO(ibrahim): find appropriate bin size.
	keyBins := createHistogramBins(1, 16)
	valueBins := createHistogramBins(1, 30)
	return &sizeHistogram{
		keySizeHistogram: histogramData{
			bins:        keyBins,
			countPerBin: make([]int64, len(keyBins)+1),
			max:         math.MinInt64,
			min:         math.MaxInt64,
			sum:         0,
		},
		valueSizeHistogram: histogramData{
			bins:        valueBins,
			countPerBin: make([]int64, len(valueBins)+1),
			max:         math.MinInt64,
			min:         math.MaxInt64,
			sum:         0,
		},
	}
}

// newSizeHistogram returns a new instance of keyValueSizeHistogram with
// properly initialized fields.

newSizeHistogram returns a new instance of keyValueSizeHistogram with properly initialized fields .