Repository: microsoft/onefuzz
File: src/ApiService/ApiService/Functions/AgentEvents.cs (C#)

Code before:
                _log.Info($"stopping free node with reset flags: {machineId:Tag:MachineId}");
                // discard result: node not used after this point
                _ = await _context.NodeOperations.Stop(node);
                return null;
            }

            if (await _context.NodeOperations.CouldShrinkScaleset(node)) {
                _log.Info($"stopping free node to resize scaleset: {machineId:Tag:MachineId}");
                // discard result: node not used after this point
                _ = await _context.NodeOperations.SetHalt(node);
                return null;
            }
        }

        if (ev.State == NodeState.Init) {
            if (node.DeleteRequested) {
                _log.Info($"stopping node (init and delete_requested): {machineId:Tag:MachineId}");
                // discard result: node not used after this point
                _ = await _context.NodeOperations.Stop(node);
                return null;
            }

            // Don’t check reimage_requested, as nodes only send 'init' state once.  If
            // they send 'init' with reimage_requested, it's because the node was reimaged
            // successfully.
            // Node has to be in state other than Init, otherwise SetState ignore the change, and therefore
            // InitializedAt field is never set
            node = node with { ReimageRequested = false, InitializedAt = DateTimeOffset.UtcNow };
            // discard result: node not used after this point
            _ = await _context.NodeOperations.SetState(node, ev.State);
            return null;
        }

        node = await _context.NodeOperations.SetState(node, ev.State);

        if (ev.State == NodeState.Free) {
            _log.Info($"node now available for work: {machineId:Tag:MachineId}");
        } else if (ev.State == NodeState.SettingUp) {
            if (ev.Data is NodeSettingUpEventData settingUpData) {
                if (!settingUpData.Tasks.Any()) {
                    return new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
                        $"setup without tasks.  machine_id: {machineId}",
                    });
                }

                foreach (var taskId in settingUpData.Tasks) {
                    var task = await _context.TaskOperations.GetByTaskId(taskId);
                    if (task is null) {
                        return new Error(
                            ErrorCode.INVALID_REQUEST,
                            Errors: new string[] { $"unable to find task: {taskId}" });

Review comment (bug):
Explanation on why we need a new state added to nodes. And this is bug number two, that would force nodes to go from Free to Done state right away, since InitializedAt is not set, and so the node is considered to be outdated right after it is created. Since we perform the check on timer and LInux is faster to provision than Windows - Linux nodes succeed more often than Windows

Code after:
                _log.Info($"stopping free node with reset flags: {machineId:Tag:MachineId}");
                // discard result: node not used after this point
                _ = await _context.NodeOperations.Stop(node);
                return null;
            }

            if (await _context.NodeOperations.CouldShrinkScaleset(node)) {
                _log.Info($"stopping free node to resize scaleset: {machineId:Tag:MachineId}");
                // discard result: node not used after this point
                _ = await _context.NodeOperations.SetHalt(node);
                return null;
            }
        }

        if (ev.State == NodeState.Init) {
            if (node.DeleteRequested) {
                _log.Info($"stopping node (init and delete_requested): {machineId:Tag:MachineId}");
                // discard result: node not used after this point
                _ = await _context.NodeOperations.Stop(node);
                return null;
            }

            // Don’t check reimage_requested, as nodes only send 'init' state once.  If
            // they send 'init' with reimage_requested, it's because the node was reimaged
            // successfully.
            node = node with { ReimageRequested = false, InitializedAt = DateTimeOffset.UtcNow };
            // discard result: node not used after this point
            _ = await _context.NodeOperations.SetState(node, ev.State);
            return null;
        }

        node = await _context.NodeOperations.SetState(node, ev.State);

        if (ev.State == NodeState.Free) {
            _log.Info($"node now available for work: {machineId:Tag:MachineId}");
        } else if (ev.State == NodeState.SettingUp) {
            if (ev.Data is NodeSettingUpEventData settingUpData) {
                if (!settingUpData.Tasks.Any()) {
                    return new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
                        $"setup without tasks.  machine_id: {machineId}",
                    });
                }

                foreach (var taskId in settingUpData.Tasks) {
                    var task = await _context.TaskOperations.GetByTaskId(taskId);
                    if (task is null) {
                        return new Error(
                            ErrorCode.INVALID_REQUEST,
                            Errors: new string[] { $"unable to find task: {taskId}" });
                    }