Repository: firecracker-microvm/firecracker
File: src/vmm/src/lib.rs (Rust)

Code before:
            let exit_evt = self
                .vcpus_exit_evt
                .try_clone()
                .map_err(HotplugVcpuError::EventFd)?;
            let vcpu =
                Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
            vcpus.push(vcpu);
        }

        let seccomp_filters = get_empty_filters();
        self.start_vcpus(
            vcpus,
            seccomp_filters
                .get("vcpu")
                .unwrap()
                // .ok_or_else(|| VmmError::SeccompFilters(()))?
                .clone(),
        )
        .map_err(HotplugVcpuError::VcpuStart)?;

        println!("Num vCPUs after hotplug: {}", self.vcpus_handles.len());

        // Update VM config to reflect new CPUs added

        // Resume VM
        self.resume_vm().unwrap();
        Ok(())
    }

    /// Retrieves the KVM dirty bitmap for each of the guest's memory regions.
    pub fn reset_dirty_bitmap(&self) {
        self.guest_memory
            .iter()
            .enumerate()
            .for_each(|(slot, region)| {
                let _ = self
                    .vm
                    .fd()
                    .get_dirty_log(u32::try_from(slot).unwrap(), u64_to_usize(region.len()));
            });
    }

    /// Retrieves the KVM dirty bitmap for each of the guest's memory regions.
    pub fn get_dirty_bitmap(&self) -> Result<DirtyBitmap, VmmError> {
        let mut bitmap: DirtyBitmap = HashMap::new();
        self.guest_memory
            .iter()
            .enumerate()
            .try_for_each(|(slot, region)| {
                let bitmap_region = self
                    .vm

Review comment (suggestion):
You don't actually need to resume the VM here. vCPU hotplug can happen on both paused and running virtual machines. If the VM is running, then the interrupt that Firecracker will inject will simply be immediately handled by the guest (by re-reading specific ACPI table entries). If its paused, then KVM will "buffer" the interrupt and insert it whenever we next call `KVM_RUN`. The vcpu hotplug handler here should be completely agnostic to whether the VM is paused or running, and should not try to modify its state.

Code after:
            return Err(HotplugVcpuError::VcpuCountTooLow);
        } else if self
            .vcpus_handles
            .len()
            .checked_add(config.add.into())
            .ok_or(HotplugVcpuError::VcpuCountTooHigh)?
            > MAX_SUPPORTED_VCPUS.into()
        {
            return Err(HotplugVcpuError::VcpuCountTooHigh);
        }

        // Create and start new vcpus
        let mut vcpus = Vec::with_capacity(config.add.into());

        #[allow(clippy::cast_possible_truncation)]
        let start_idx = self.vcpus_handles.len().try_into().unwrap();
        for cpu_idx in start_idx..(start_idx + config.add) {
            let exit_evt = self
                .vcpus_exit_evt
                .try_clone()
                .map_err(HotplugVcpuError::EventFd)?;
            let vcpu =
                Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
            vcpus.push(vcpu);
        }

        self.start_vcpus(
            vcpus,
            self.seccomp_filters
                .get("vcpu")
                .ok_or_else(|| HotplugVcpuError::MissingSeccompFilters("vcpu".to_string()))?
                .clone(),
        )
        .map_err(HotplugVcpuError::VcpuStart)?;

        #[allow(clippy::cast_lossless)]
        METRICS.hotplug.vcpus_added.add(config.add.into());

        // Update VM config to reflect new CPUs added
        #[allow(clippy::cast_possible_truncation)]
        let new_machine_config = MachineConfigUpdate {
            vcpu_count: Some(self.vcpus_handles.len() as u8),
            mem_size_mib: None,
            smt: None,
            cpu_template: None,
            track_dirty_pages: None,
            huge_pages: None,
        };

        Ok(new_machine_config)
    }