Repository: hashicorp/nomad
File: scheduler/reconciler/reconcile_node.go (Go)

Code before:
	// Canary deployments deploy to the TaskGroup.UpdateStrategy.Canary
	// percentage of eligible nodes, so we create a mapping of task group name
	// to a list of nodes that canaries should be placed on.
	canaryNodes, canariesPerTG := computeCanaryNodes(required, nodeAllocs, eligibleNodes)

	result := new(NodeReconcileResult)
	deploymentComplete := true
	for nodeID, allocs := range nodeAllocs {
		diff, deploymentCompleteForNode := nr.computeForNode(job, nodeID, eligibleNodes,
			notReadyNodes, taintedNodes, canaryNodes[nodeID], canariesPerTG, required,
			allocs, terminal, serverSupportsDisconnectedClients)
		deploymentComplete = deploymentComplete && deploymentCompleteForNode
		result.Append(diff)
	}

	nr.DeploymentUpdates = append(nr.DeploymentUpdates, nr.setDeploymentStatusAndUpdates(deploymentComplete, job)...)

	return result
}

// computeCanaryNodes is a helper function that, given required task groups, a
// mapping of nodes to their allocs and a map of eligible nodes, outputs a
// map[nodeID] -> map[TG] -> bool which indicates which TGs this node is a
// canary for, and a map[TG] -> int to indicate how many total canaries are to
// be placed for a TG.
func computeCanaryNodes(required map[string]*structs.TaskGroup, nodeAllocs map[string][]*structs.Allocation,
	eligibleNodes map[string]*structs.Node) (map[string]map[string]bool, map[string]int) {

	canaryNodes := map[string]map[string]bool{}
	eligibleNodesList := slices.Collect(maps.Values(eligibleNodes))
	canariesPerTG := map[string]int{}

	for _, tg := range required {
		if tg.Update.IsEmpty() || tg.Update.Canary == 0 {
			continue
		}

		// round up to the nearest integer
		numberOfCanaryNodes := int(math.Ceil(float64(tg.Update.Canary) * float64(len(eligibleNodes)) / 100))
		canariesPerTG[tg.Name] = numberOfCanaryNodes

		// check if there are any allocations on any nodes that are/were
		// canaries.
		for nodeID, allocs := range nodeAllocs {
			for _, a := range allocs {
				if a.DeploymentStatus == nil || a.DeploymentStatus.Canary == false {
					continue
				}
				if a.TaskGroup == tg.Name {
					canaryNodes[nodeID] = map[string]bool{}
					canaryNodes[nodeID][tg.Name] = true

Review comment (bug):
The `nodeAllocs` map only includes _live_ allocations, so we need to also pass in the `terminal` map from the caller and do the exact same loop you've added here over that map. That will ensure we're placing canaries on the same node even if the previous canary has failed.

Code after:
	// Canary deployments deploy to the TaskGroup.UpdateStrategy.Canary
	// percentage of eligible nodes, so we create a mapping of task group name
	// to a list of nodes that canaries should be placed on.
	canaryNodes, canariesPerTG := nr.computeCanaryNodes(required, nodeAllocs, terminal, eligibleNodes)

	result := new(NodeReconcileResult)
	deploymentComplete := true
	for nodeID, allocs := range nodeAllocs {
		diff, deploymentCompleteForNode := nr.computeForNode(job, nodeID, eligibleNodes,
			notReadyNodes, taintedNodes, canaryNodes[nodeID], canariesPerTG, required,
			allocs, terminal, serverSupportsDisconnectedClients)
		deploymentComplete = deploymentComplete && deploymentCompleteForNode
		result.Append(diff)
	}

	nr.DeploymentUpdates = append(nr.DeploymentUpdates, nr.setDeploymentStatusAndUpdates(deploymentComplete, job)...)

	return result
}

// computeCanaryNodes is a helper function that, given required task groups,
// mappings of nodes to their live allocs and terminal allocs, and a map of
// eligible nodes, outputs a map[nodeID] -> map[TG] -> bool which indicates
// which TGs this node is a canary for, and a map[TG] -> int to indicate how
// many total canaries are to be placed for a TG.
func (nr *NodeReconciler) computeCanaryNodes(required map[string]*structs.TaskGroup,
	liveAllocs map[string][]*structs.Allocation, terminalAllocs structs.TerminalByNodeByName,
	eligibleNodes map[string]*structs.Node) (map[string]map[string]bool, map[string]int) {

	canaryNodes := map[string]map[string]bool{}
	eligibleNodesList := slices.Collect(maps.Values(eligibleNodes))
	canariesPerTG := map[string]int{}

	for _, tg := range required {
		if tg.Update.IsEmpty() || tg.Update.Canary == 0 {
			continue
		}

		// round up to the nearest integer
		numberOfCanaryNodes := int(math.Ceil(float64(tg.Update.Canary) * float64(len(eligibleNodes)) / 100))
		canariesPerTG[tg.Name] = numberOfCanaryNodes

		// check if there are any live allocations on any nodes that are/were
		// canaries.
		for nodeID, allocs := range liveAllocs {
			for _, a := range allocs {
				eligibleNodesList, numberOfCanaryNodes = nr.findOldCanaryNodes(
					eligibleNodesList, numberOfCanaryNodes, a, tg, canaryNodes, nodeID)
			}
		}