Repository: hashicorp/nomad
File: e2e/scheduler_system/systemsched_test.go (Go)

Code before:

	deploymentsApi := job2.DeploymentsApi()

	// wait for the canary allocations to become healthy
	timeout := time.After(30 * time.Second)
CANARYWAIT:
	for {
		select {
		case <-timeout:
			must.Unreachable(t, must.Sprint("timeout reached waiting for healthy status of canary allocs"))
		default:
		}

		deployments, _, err := deploymentsApi.List(nil)
		must.NoError(t, err)
		for _, d := range deployments {
			for _, tg := range d.TaskGroups { // this job has 1 tg
				if d.JobVersion == 1 && tg.HealthyAllocs >= tg.DesiredCanaries {
					break CANARYWAIT
				}
			}
		}
	}

	// filter allocation from v1 version of the job, they should all be canaries
	// and there should be exactly 2
	canaryAllocs := []*api.AllocationListStub{}
	for _, a := range allocs {
		if a.JobVersion == 1 {
			must.True(t, a.DeploymentStatus.Canary)
			canaryAllocs = append(canaryAllocs, a)
		}
	}
	must.SliceLen(t, 2, canaryAllocs, must.Sprint("expected 2 canary allocs"))

	// promote canaries
	deployments, _, err := deploymentsApi.List(nil)
	must.NoError(t, err)
	must.SliceLen(t, 2, deployments)
	_, _, err = deploymentsApi.PromoteAll(deployments[0].ID, nil)
	must.NoError(t, err)

	// wait for the promotions to become healthy
PROMOTIONWAIT:
	for {
		select {
		case <-timeout:
			must.Unreachable(t, must.Sprint("timeout reached waiting for healthy status of promoted allocs"))
		default:
		}


Review comment (suggestion):
The allocations we have at this point are stale -- we've been waiting for the deployment to reach a particular point for a while. We should be querying the active allocs here. It looks like this is technically safe here but it would be easy to add a new assertion that would be operating on the stale allocs (like checking their health).

Code after:
	deploymentsApi := job2.DeploymentsApi()
	deploymentsList, _, err := deploymentsApi.List(nil)
	must.NoError(t, err)

	var deployment *api.Deployment
	for _, d := range deploymentsList {
		if d.JobID == job2.JobID() && d.Status == api.DeploymentStatusRunning {
			deployment = d
		}
	}
	must.NotNil(t, deployment)

	// wait for the canary allocations to become healthy
	timeout, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	job2.WaitForDeploymentFunc(timeout, deployment.ID, func(d *api.Deployment) bool {
		for _, tg := range d.TaskGroups { // we only have 1 tg in this job
			if d.JobVersion == 1 && tg.HealthyAllocs >= tg.DesiredCanaries {
				return true
			}
		}
		return false
	})

	// find allocations from v1 version of the job, they should all be canaries
	// and there should be exactly 2
	count := 0
	for _, a := range allocs {
		if a.JobVersion == 1 {
			must.True(t, a.DeploymentStatus.Canary)
			count += 1
		}
	}
	must.Eq(t, numberOfEligibleNodes/2, count, must.Sprint("expected canaries to be placed on 50% of eligible nodes"))

	// promote canaries
	deployments, _, err := deploymentsApi.List(nil)
	must.NoError(t, err)
	must.SliceLen(t, 2, deployments)
	_, _, err = deploymentsApi.PromoteAll(deployments[0].ID, nil)
	must.NoError(t, err)

	// promoting canaries on a system job should result in a new deployment
	deploymentsList, _, err = deploymentsApi.List(nil)
	must.NoError(t, err)

	for _, d := range deploymentsList {
		if d.JobID == job2.JobID() && d.Status == api.DeploymentStatusRunning {
			deployment = d
			break