Repository: etcd-io/etcd
File: server/etcdserver/bootstrap.go (Go)

Code before:
		},
	)
}

func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot) *bootstrappedWAL {
	wal, st, ents, snap, meta := openWALFromSnapshot(cfg, snapshot)
	bwal := &bootstrappedWAL{
		lg:       cfg.Logger,
		w:        wal,
		st:       st,
		ents:     ents,
		snapshot: snap,
		meta:     meta,
		haveWAL:  true,
	}

	if cfg.ForceNewCluster {
		// discard the previously uncommitted entries
		bwal.ents = bwal.CommitedEntries()
		entries := bwal.NewConfigChangeEntries()
		// force commit config change entries
		bwal.AppendAndCommitEntries(entries)
		cfg.Logger.Info(
			"forcing restart member",
			zap.String("cluster-id", meta.clusterID.String()),
			zap.String("local-member-id", meta.nodeID.String()),
			zap.Uint64("commit-index", bwal.st.Commit),
		)
	} else {
		cfg.Logger.Info(
			"restarting local member",
			zap.String("cluster-id", meta.clusterID.String()),
			zap.String("local-member-id", meta.nodeID.String()),
			zap.Uint64("commit-index", bwal.st.Commit),
		)
	}
	return bwal
}

// openWALFromSnapshot reads the WAL at the given snap and returns the wal, its latest HardState and cluster ID, and all entries that appear
// after the position of the given snap in the WAL.
// The snap must have been previously saved to the WAL, or this call will panic.
func openWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot) (*wal.WAL, *raftpb.HardState, []raftpb.Entry, *raftpb.Snapshot, *snapshotMetadata) {
	var walsnap walpb.Snapshot
	if snapshot != nil {
		walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term
	}
	repaired := false
	for {
		w, err := wal.Open(cfg.Logger, cfg.WALDir(), walsnap)
		if err != nil {

Review comment (bug):
Are you leaving any log in place, or the index/term of the entry at which the log is truncated? One caveat to mention is, depending how synchronization between the log and the state machine works in your case, you may end up not just with a higher applied index, but also one for each the pre-image entry does not exist in the log.

Scenario:
1. Log is `[1][1][1][2][2][2]`, with indices 1..6
2. The node receives an overriding append / `MsgApp` from a new leader, which also says that all these entries are committed: log is now `[1][1][3][3]`, with commit index = 4. Sends this mutation to log storage.
3. Entries `[1][1][3][3]` are sent for state machine application.

If steps (2) and (3) are parallel, it's possible that on crash/restart you end up with:
- state machine applied `[1][1][3][3]`
- the log is `[1][1][1][2][2][2]`, i.e. it forgot the last append

In this case, the state machine's last entry at index 4 (known to be committed) does not match the log. Which is fine: it just means that the state machine completely obsoletes the log, and the log should be removed. But you might need to detect this case.

Code after:
			storage:     serverstorage.NewStorage(b.lg, wal, ss),
		},
	)
}

func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot, ci cindex.ConsistentIndexer) *bootstrappedWAL {
	wal, st, ents, snap, meta := openWALFromSnapshot(cfg, snapshot)
	bwal := &bootstrappedWAL{
		lg:       cfg.Logger,
		w:        wal,
		st:       st,
		ents:     ents,
		snapshot: snap,
		meta:     meta,
		haveWAL:  true,
	}

	if cfg.ForceNewCluster {
		consistentIndex := ci.ConsistentIndex()
		oldCommitIndex := bwal.st.Commit
		// If only `HardState.Commit` increases, HardState won't be persisted
		// to disk, even though the committed entries might have already been
		// applied. This can result in consistent_index > CommitIndex.
		//
		// When restarting etcd with `--force-new-cluster`, all uncommitted
		// entries are dropped. To avoid losing entries that were actually
		// committed, we reset Commit to max(HardState.Commit, consistent_index).
		//
		// See: https://github.com/etcd-io/raft/pull/300 for more details.
		bwal.st.Commit = max(oldCommitIndex, consistentIndex)

		// discard the previously uncommitted entries
		bwal.ents = bwal.CommitedEntries()
		entries := bwal.NewConfigChangeEntries()
		// force commit config change entries
		bwal.AppendAndCommitEntries(entries)
		cfg.Logger.Info(
			"forcing restart member",
			zap.String("cluster-id", meta.clusterID.String()),
			zap.String("local-member-id", meta.nodeID.String()),
			zap.Uint64("wal-commit-index", oldCommitIndex),
			zap.Uint64("commit-index", bwal.st.Commit),
		)
	} else {
		cfg.Logger.Info(
			"restarting local member",
			zap.String("cluster-id", meta.clusterID.String()),
			zap.String("local-member-id", meta.nodeID.String()),
			zap.Uint64("commit-index", bwal.st.Commit),
		)
	}