Repository: etcd-io/etcd
File: tests/e2e/cluster_downgrade_test.go (Go)

Code before:
	t.Logf("Cluster created")
	if len(epc.Procs) > 1 {
		t.Log("Waiting health interval to required to make membership changes")
		time.Sleep(etcdserver.HealthInterval)
	}

	t.Log("Adding member to test membership, but a learner avoid breaking quorum")
	resp, err := cc.MemberAddAsLearner(context.Background(), "fake1", []string{"http://127.0.0.1:1001"})
	require.NoError(t, err)
	if triggerSnapshot {
		t.Logf("Generating snapshot")
		generateSnapshot(t, snapshotCount, cc)
		verifySnapshot(t, epc)
	}
	t.Log("Removing learner to test membership")
	_, err = cc.MemberRemove(context.Background(), resp.Member.ID)
	require.NoError(t, err)
	beforeMembers, beforeKV := getMembersAndKeys(t, cc)

	if triggerCancellation == cancelRightBeforeEnable {
		t.Logf("Cancelling downgrade before enabling")
		e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
		return // No need to perform downgrading, end the test here
	}
	e2e.DowngradeEnable(t, epc, lastVersion)
	if triggerCancellation == cancelRightAfterEnable {
		t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)")
		e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
		return // No need to perform downgrading, end the test here
	}

	t.Logf("Starting downgrade process to %q", lastVersionStr)
	err = e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion)
	require.NoError(t, err)
	e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")

	t.Log("Downgrade complete")
	afterMembers, afterKV := getMembersAndKeys(t, cc)
	assert.Equal(t, beforeKV.Kvs, afterKV.Kvs)
	assert.Equal(t, beforeMembers.Members, afterMembers.Members)

	if len(epc.Procs) > 1 {
		t.Log("Waiting health interval to required to make membership changes")
		time.Sleep(etcdserver.HealthInterval)
	}
	t.Log("Adding learner to test membership, but avoid breaking quorum")
	resp, err = cc.MemberAddAsLearner(context.Background(), "fake2", []string{"http://127.0.0.1:1002"})
	require.NoError(t, err)
	if triggerSnapshot {
		t.Logf("Generating snapshot")
		generateSnapshot(t, snapshotCount, cc)

Review comment (suggestion):
I think the test should have some validation, like check if cluster version of all members. For cancellation before and right after enable we the cluster version should stay the same. Also would be good to confirm the state of downgrade. 

Code after:
	t.Logf("Cluster created")
	if len(epc.Procs) > 1 {
		t.Log("Waiting health interval to required to make membership changes")
		time.Sleep(etcdserver.HealthInterval)
	}

	t.Log("Adding member to test membership, but a learner avoid breaking quorum")
	resp, err := cc.MemberAddAsLearner(context.Background(), "fake1", []string{"http://127.0.0.1:1001"})
	require.NoError(t, err)
	if triggerSnapshot {
		t.Logf("Generating snapshot")
		generateSnapshot(t, snapshotCount, cc)
		verifySnapshot(t, epc)
	}
	t.Log("Removing learner to test membership")
	_, err = cc.MemberRemove(context.Background(), resp.Member.ID)
	require.NoError(t, err)
	beforeMembers, beforeKV := getMembersAndKeys(t, cc)

	if triggerCancellation == cancelRightBeforeEnable {
		t.Logf("Cancelling downgrade before enabling")
		e2e.DowngradeCancel(t, epc)
		t.Log("Downgrade cancelled, validating if cluster is in the right state")
		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))

		return // No need to perform downgrading, end the test here
	}
	e2e.DowngradeEnable(t, epc, lastVersion)
	if triggerCancellation == cancelRightAfterEnable {
		t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)")
		e2e.DowngradeCancel(t, epc)
		t.Log("Downgrade cancelled, validating if cluster is in the right state")
		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
		return // No need to perform downgrading, end the test here
	}

	t.Logf("Starting downgrade process to %q", lastVersionStr)
	err = e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion)
	require.NoError(t, err)
	e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")

	t.Log("Downgrade complete")
	afterMembers, afterKV := getMembersAndKeys(t, cc)
	assert.Equal(t, beforeKV.Kvs, afterKV.Kvs)
	assert.Equal(t, beforeMembers.Members, afterMembers.Members)

	if len(epc.Procs) > 1 {
		t.Log("Waiting health interval to required to make membership changes")
		time.Sleep(etcdserver.HealthInterval)
	}
	t.Log("Adding learner to test membership, but avoid breaking quorum")