Repository: lxc/incus
Pull request #1945: Memory hotplug support for VMs
Author: login: presztak
Labels: name: Documentation
description: Documentation needs updating
name: API
description: Changes to the REST API
Created: 2025-04-15T21:54:45Z | Merged: 2025-04-17T00:24:14Z

This PR adds support for hot-plugging memory during VM runtime.
It reuses the same approach and set of functions that are used for generating QEMU CPU and memory configurations. The generated configuration is then translated into a format compatible with QMP. This approach respects both hugepages and NUMA node topology.

Closes: #986


Diff:
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 1594a64299a..c04b718dfa3 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -2791,3 +2791,7 @@ Adds a `snat` configuration option for network forwards which will cause any DNA
 So new connections from the target will appear as coming from the network forward address.
 
 This is limited to bridged networks as OVN doesn't support flexible enough SNAT for this.
+
+## `memory_hotplug`
+
+This adds memory hotplugging for VMs, allowing them to add memory at runtime without rebooting.
diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go
index c9fea906fbb..d92c256fdf4 100644
--- a/internal/server/instance/drivers/driver_qemu.go
+++ b/internal/server/instance/drivers/driver_qemu.go
@@ -3860,9 +3860,8 @@ func (d *qemu) writeQemuConfigFile(configPath string) error {
 	return os.WriteFile(configPath, []byte(sb.String()), 0o640)
 }
 
-// addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory.
-// If sb is nil then no config is written.
-func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) error {
+// getCPUOpts retrieves configuration options for virtualized CPUs and memory.
+func (d *qemu) getCPUOpts(cpuInfo *cpuTopology, memSizeBytes int64) (*qemuCPUOpts, error) {
 	// Figure out what memory object layout we're going to use.
 	// Before v6.0 or if version unknown, we use the "repeated" format, otherwise we use "indexed" format.
 	qemuMemObjectFormat := "repeated"
@@ -3877,8 +3876,6 @@ func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) err
 		qemuMemObjectFormat: qemuMemObjectFormat,
 	}
 
-	cpuPinning := false
-
 	hostNodes := []uint64{}
 	if cpuInfo.vcpus == nil {
 		// If not pinning, default to exposing cores.
@@ -3908,14 +3905,12 @@ func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) err
 			// Parse the NUMA restriction.
 			numaNodeSet, err := resources.ParseNumaNodeSet(numaNodes)
 			if err != nil {
-				return err
+				return nil, err
 			}
 
 			cpuOpts.memoryHostNodes = numaNodeSet
 		}
 	} else {
-		cpuPinning = true
-
 		// Figure out socket-id/core-id/thread-id for all vcpus.
 		vcpuSocket := map[uint64]uint64{}
 		vcpuCore := map[uint64]uint64{}
@@ -3962,22 +3957,11 @@ func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) err
 		cpuOpts.cpuNumaHostNodes = hostNodes
 	}
 
-	// Configure memory limit.
-	memSize := d.expandedConfig["limits.memory"]
-	if memSize == "" {
-		memSize = qemudefault.MemSize // Default if no memory limit specified.
-	}
-
-	memSizeBytes, err := ParseMemoryStr(memSize)
-	if err != nil {
-		return fmt.Errorf("limits.memory invalid: %w", err)
-	}
-
 	cpuOpts.hugepages = ""
 	if util.IsTrue(d.expandedConfig["limits.memory.hugepages"]) {
 		hugetlb, err := localUtil.HugepagesPath()
 		if err != nil {
-			return err
+			return nil, err
 		}
 
 		cpuOpts.hugepages = hugetlb
@@ -3988,9 +3972,33 @@ func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) err
 	nodeMemory := int64(memSizeMB / int64(len(hostNodes)))
 	cpuOpts.memory = nodeMemory
 
+	return &cpuOpts, nil
+}
+
+// addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory.
+// If sb is nil then no config is written.
+func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) error {
+	// Configure memory limit.
+	memSize := d.expandedConfig["limits.memory"]
+	if memSize == "" {
+		memSize = qemudefault.MemSize // Default if no memory limit specified.
+	}
+
+	memSizeBytes, err := ParseMemoryStr(memSize)
+	if err != nil {
+		return fmt.Errorf("limits.memory invalid: %w", err)
+	}
+
+	cpuOpts, err := d.getCPUOpts(cpuInfo, memSizeBytes)
+	if err != nil {
+		return err
+	}
+
+	cpuPinning := cpuInfo.vcpus != nil
+
 	if conf != nil {
-		*conf = append(*conf, qemuMemory(&qemuMemoryOpts{memSizeMB})...)
-		*conf = append(*conf, qemuCPU(&cpuOpts, cpuPinning)...)
+		*conf = append(*conf, qemuMemory(&qemuMemoryOpts{memSizeBytes / 1024 / 1024})...)
+		*conf = append(*conf, qemuCPU(cpuOpts, cpuPinning)...)
 	}
 
 	return nil
@@ -6162,7 +6170,7 @@ func (d *qemu) updateMemoryLimit(newLimit string) error {
 	if curSizeMB == newSizeMB {
 		return nil
 	} else if baseSizeMB < newSizeMB {
-		return fmt.Errorf("Cannot increase memory size beyond boot time size when VM is running (Boot time size %dMiB, new size %dMiB)", baseSizeMB, newSizeMB)
+		return d.hotplugMemory(monitor, newSizeBytes-curSizeBytes)
 	}
 
 	// Set effective memory size.
@@ -6198,6 +6206,79 @@ func (d *qemu) updateMemoryLimit(newLimit string) error {
 	return fmt.Errorf("Failed setting memory to %dMiB (currently %dMiB) as it was taking too long", newSizeMB, curSizeMB)
 }
 
+// hotplugMemory attaches a memory device to a running VM,
+// respecting NUMA node placement and hugepages.
+func (d *qemu) hotplugMemory(monitor *qmp.Monitor, sizeBytes int64) error {
+	// Get CPU information.
+	cpuInfo, err := d.cpuTopology(d.expandedConfig["limits.cpu"])
+	if err != nil {
+		return err
+	}
+
+	// Fetch memory configuration
+	cpuOpts, err := d.getCPUOpts(cpuInfo, sizeBytes)
+	if err != nil {
+		return err
+	}
+
+	cpuPinning := cpuInfo.vcpus != nil
+
+	// Get CPUs and memory configuration
+	conf := qemuCPU(cpuOpts, cpuPinning)
+
+	memoryObjects := map[int]cfg.Section{}
+	for _, section := range conf {
+		// Name is in the form 'object "mem0"', so the last quote needs to be removed.
+		// This allows proper parsing of the memory object index.
+		sectionName := section.Name[:len(section.Name)-1]
+		index, err := extractTrailingNumber(sectionName, "object \"mem")
+		if err != nil {
+			continue
+		}
+
+		memoryObjects[index] = section
+	}
+
+	// Find first available memory object index.
+	nextMemIndex, err := findNextMemoryIndex(monitor)
+	if err != nil {
+		return err
+	}
+
+	// Find first available pc-dimm device index.
+	nextDimmIndex, err := findNextDimmIndex(monitor)
+	if err != nil {
+		return err
+	}
+
+	for index, memory := range memoryObjects {
+		memIndex := nextMemIndex + index
+		dimmIndex := nextDimmIndex + index
+
+		memObj := memoryConfigSectionToMap(&memory)
+		memObj["id"] = fmt.Sprintf("mem%d", memIndex)
+
+		err = monitor.AddObject(memObj)
+		if err != nil {
+			return err
+		}
+
+		memDev := map[string]any{
+			"driver": "pc-dimm",
+			"id":     fmt.Sprintf("dimm%d", dimmIndex),
+			"memdev": fmt.Sprintf("mem%d", memIndex),
+			"node":   index,
+		}
+
+		err = monitor.AddDevice(memDev)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (d *qemu) removeUnixDevices() error {
 	// Check that we indeed have devices to remove.
 	if !util.PathExists(d.DevicesPath()) {
diff --git a/internal/server/instance/drivers/driver_qemu_config_test.go b/internal/server/instance/drivers/driver_qemu_config_test.go
index 35a2c5b0b6e..8f1d5eb289f 100644
--- a/internal/server/instance/drivers/driver_qemu_config_test.go
+++ b/internal/server/instance/drivers/driver_qemu_config_test.go
@@ -102,12 +102,16 @@ func TestQemuConfigTemplates(t *testing.T) {
 			qemuMemoryOpts{4096},
 			`# Memory
 			[memory]
-			size = "4096M"`,
+			size = "4096M"
+			slots = "16"
+			maxmem = "1T"`,
 		}, {
 			qemuMemoryOpts{8192},
 			`# Memory
 			[memory]
-			size = "8192M"`,
+			size = "8192M"
+			slots = "16"
+			maxmem = "1T"`,
 		}}
 		for _, tc := range testCases {
 			runTest(tc.expected, qemuMemory(&tc.opts))
diff --git a/internal/server/instance/drivers/driver_qemu_templates.go b/internal/server/instance/drivers/driver_qemu_templates.go
index ea99449d947..6a5427cab86 100644
--- a/internal/server/instance/drivers/driver_qemu_templates.go
+++ b/internal/server/instance/drivers/driver_qemu_templates.go
@@ -119,10 +119,15 @@ type qemuMemoryOpts struct {
 }
 
 func qemuMemory(opts *qemuMemoryOpts) []cfg.Section {
+	// Sets fixed values for slots and maxmem to support memory hotplug.
 	return []cfg.Section{{
 		Name:    "memory",
 		Comment: "Memory",
-		Entries: []cfg.Entry{{Key: "size", Value: fmt.Sprintf("%dM", opts.memSizeMB)}},
+		Entries: []cfg.Entry{
+			{Key: "size", Value: fmt.Sprintf("%dM", opts.memSizeMB)},
+			{Key: "slots", Value: "16"},
+			{Key: "maxmem", Value: "1T"},
+		},
 	}}
 }
 
diff --git a/internal/server/instance/drivers/qmp/commands.go b/internal/server/instance/drivers/qmp/commands.go
index f5429fe18bf..7b8d3bbf2ee 100644
--- a/internal/server/instance/drivers/qmp/commands.go
+++ b/internal/server/instance/drivers/qmp/commands.go
@@ -74,6 +74,35 @@ type CPUModel struct {
 	Flags map[string]any `json:"props"`
 }
 
+// MemoryDevice contains information about a memory device.
+type MemoryDevice struct {
+	Type string          `json:"type"`
+	Data json.RawMessage `json:"data"`
+}
+
+// PCDimmDevice contains information about a memory device of type pc-dimm.
+type PCDimmDevice struct {
+	ID           string `json:"id"`
+	Addr         uint64 `json:"addr"`
+	Slot         int    `json:"slot"`
+	Node         int    `json:"node"`
+	Memdev       string `json:"memdev"`
+	Hotpluggable bool   `json:"hotpluggable"`
+	Hotplugged   bool   `json:"hotplugged"`
+}
+
+// MemDev contains information about a memory device.
+type MemDev struct {
+	ID        string `json:"id"`
+	Size      int    `json:"size"`
+	Merge     bool   `json:"merge"`
+	Dump      bool   `json:"dump"`
+	Prealloc  bool   `json:"prealloc"`
+	Share     bool   `json:"share"`
+	Reserve   bool   `json:"reserve"`
+	HostNodes []int  `json:"host-nodes"`
+}
+
 // QueryCPUs returns a list of CPUs.
 func (m *Monitor) QueryCPUs() ([]CPU, error) {
 	// Prepare the response.
@@ -562,6 +591,71 @@ func (m *Monitor) SetMemoryBalloonSizeBytes(sizeBytes int64) error {
 	return m.Run("balloon", args, nil)
 }
 
+// GetMemdev retrieves memory devices by executing the query-memdev QMP command.
+func (m *Monitor) GetMemdev() ([]MemDev, error) {
+	// Prepare the response.
+	var resp struct {
+		Return []MemDev `json:"return"`
+	}
+
+	err := m.Run("query-memdev", nil, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.Return, nil
+}
+
+// GetMemoryDevices retrieves memory devices by executing the query-memory-devices QMP command.
+func (m *Monitor) GetMemoryDevices() ([]MemoryDevice, error) {
+	// Prepare the response.
+	var resp struct {
+		Return []MemoryDevice `json:"return"`
+	}
+
+	err := m.Run("query-memory-devices", nil, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.Return, nil
+}
+
+// GetDimmDevices returns a list of memory devices of type pc-dimm.
+func (m *Monitor) GetDimmDevices() ([]PCDimmDevice, error) {
+	devices, err := m.GetMemoryDevices()
+	if err != nil {
+		return nil, err
+	}
+
+	result := []PCDimmDevice{}
+	for _, dev := range devices {
+		if dev.Type != "dimm" {
+			continue
+		}
+
+		var dimmData PCDimmDevice
+		err := json.Unmarshal(dev.Data, &dimmData)
+		if err != nil {
+			continue
+		}
+
+		result = append(result, dimmData)
+	}
+
+	return result, nil
+}
+
+// AddObject adds a new object.
+func (m *Monitor) AddObject(args map[string]any) error {
+	err := m.Run("object-add", &args, nil)
+	if err != nil {
+		return fmt.Errorf("Failed adding object: %w", err)
+	}
+
+	return nil
+}
+
 // AddBlockDevice adds a block device.
 func (m *Monitor) AddBlockDevice(blockDev map[string]any, device map[string]any) error {
 	revert := revert.New()
diff --git a/internal/server/instance/drivers/util.go b/internal/server/instance/drivers/util.go
index bda9dcd59ba..2b0291e088f 100644
--- a/internal/server/instance/drivers/util.go
+++ b/internal/server/instance/drivers/util.go
@@ -14,6 +14,8 @@ import (
 
 	"github.com/lxc/incus/v6/internal/linux"
 	"github.com/lxc/incus/v6/internal/server/db"
+	"github.com/lxc/incus/v6/internal/server/instance/drivers/cfg"
+	"github.com/lxc/incus/v6/internal/server/instance/drivers/qmp"
 	"github.com/lxc/incus/v6/internal/server/instance/instancetype"
 	"github.com/lxc/incus/v6/internal/server/state"
 	internalUtil "github.com/lxc/incus/v6/internal/util"
@@ -131,3 +133,116 @@ func ParseMemoryStr(memory string) (valueInt int64, err error) {
 func qemuEscapeCmdline(value string) string {
 	return strings.ReplaceAll(value, ",", ",,")
 }
+
+// roundDownToBlockSize returns the largest multiple of blockSize less than or equal to the input value.
+func roundDownToBlockSize(value int64, blockSize int64) int64 {
+	if value%blockSize == 0 {
+		return value
+	}
+
+	return ((value / blockSize) - 1) * blockSize
+}
+
+// memoryConfigSectionToMap converts a memory object of type cfg.Section to type map[string]any.
+func memoryConfigSectionToMap(section *cfg.Section) map[string]any {
+	const blockSize = 128 * 1024 * 1024 // 128MiB
+	obj := map[string]any{}
+	hostNodes := []int{}
+
+	for _, entry := range section.Entries {
+		if strings.HasPrefix(entry.Key, "host-nodes") {
+			hostNode, err := strconv.Atoi(entry.Value)
+			if err != nil {
+				continue
+			}
+
+			hostNodes = append(hostNodes, hostNode)
+		} else if entry.Key == "size" {
+			// Size in the config is specified in the format: 1024M, so the last character needs to be removed before parsing.
+			memSizeMB, err := strconv.Atoi(entry.Value[:len(entry.Value)-1])
+			if err != nil {
+				continue
+			}
+
+			obj["size"] = roundDownToBlockSize(int64(memSizeMB)*1024*1024, blockSize)
+		} else if entry.Key == "merge" || entry.Key == "dump" || entry.Key == "prealloc" || entry.Key == "share" || entry.Key == "reserve" {
+			val := false
+			if entry.Value == "on" {
+				val = true
+			}
+
+			obj[entry.Key] = val
+		} else {
+			obj[entry.Key] = entry.Value
+		}
+	}
+
+	if len(hostNodes) > 0 {
+		obj["host-nodes"] = hostNodes
+	}
+
+	return obj
+}
+
+// extractTrailingNumber extracts the trailing number from a string.
+// For example, given "dimm1", it returns 1.
+func extractTrailingNumber(s string, prefix string) (int, error) {
+	if !strings.HasPrefix(s, prefix) {
+		return -1, fmt.Errorf("Prefix %s not found in %s", prefix, s)
+	}
+
+	trimmed := strings.TrimPrefix(s, prefix)
+	num, err := strconv.Atoi(trimmed)
+	if err != nil {
+		return -1, err
+	}
+
+	return num, nil
+}
+
+// findNextDimmIndex finds the next available index for a pc-dimm device
+// whose ID starts with the prefix "dimm".
+func findNextDimmIndex(monitor *qmp.Monitor) (int, error) {
+	devices, err := monitor.GetDimmDevices()
+	if err != nil {
+		return -1, err
+	}
+
+	index := -1
+	for _, dev := range devices {
+		i, err := extractTrailingNumber(dev.ID, "dimm")
+		if err != nil {
+			continue
+		}
+
+		if i > index {
+			index = i
+		}
+	}
+
+	return index + 1, nil
+}
+
+// findNextMemoryIndex finds the next available index for a memory object
+// whose ID starts with the prefix "mem".
+func findNextMemoryIndex(monitor *qmp.Monitor) (int, error) {
+	memDevs, err := monitor.GetMemdev()
+	if err != nil {
+		return -1, err
+	}
+
+	memIndex := -1
+	for _, mem := range memDevs {
+		var index int
+		index, err := extractTrailingNumber(mem.ID, "mem")
+		if err != nil {
+			continue
+		}
+
+		if index > memIndex {
+			memIndex = index
+		}
+	}
+
+	return memIndex + 1, nil
+}
diff --git a/internal/server/instance/drivers/util_test.go b/internal/server/instance/drivers/util_test.go
new file mode 100644
index 00000000000..2a5923d4dae
--- /dev/null
+++ b/internal/server/instance/drivers/util_test.go
@@ -0,0 +1,63 @@
+package drivers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/lxc/incus/v6/internal/server/instance/drivers/cfg"
+)
+
+// Test roundDownToBlockSize.
+func TestRoundDownToBlockSize(t *testing.T) {
+	const blockSize = 128 * 1024 * 1024
+
+	value := roundDownToBlockSize(1073741824, blockSize)
+	assert.Equal(t, int64(1073741824), value)
+
+	value = roundDownToBlockSize(1000000000, blockSize)
+	assert.Equal(t, int64(805306368), value)
+}
+
+// Test memoryConfigSectionToMap.
+func TestMemoryConfigSectionToMap(t *testing.T) {
+	result := memoryConfigSectionToMap(
+		&cfg.Section{
+			Name: "object \"mem0\"",
+			Entries: []cfg.Entry{
+				{Key: "size", Value: "1024M"},
+				{Key: "host-nodes.0", Value: "0"},
+				{Key: "host-nodes.1", Value: "1"},
+				{Key: "policy", Value: "bind"},
+				{Key: "share", Value: "on"},
+			},
+		},
+	)
+
+	expected := map[string]any{
+		"size":       int64(1073741824),
+		"host-nodes": []int{0, 1},
+		"policy":     "bind",
+		"share":      true,
+	}
+
+	assert.Equal(t, expected, result)
+}
+
+// Test extractTraiingNumber.
+func TestExtractTrailingNumber(t *testing.T) {
+	value, _ := extractTrailingNumber("mem0", "mem")
+	assert.Equal(t, 0, value)
+
+	value, _ = extractTrailingNumber("mem34", "mem")
+	assert.Equal(t, 34, value)
+
+	value, _ = extractTrailingNumber("dimm1", "dimm")
+	assert.Equal(t, 1, value)
+
+	expectedErr := "Prefix mem not found in dimm1"
+	_, err := extractTrailingNumber("dimm1", "mem")
+	if err.Error() != expectedErr {
+		t.Errorf("unexpected error message: got %q, want %q", err.Error(), expectedErr)
+	}
+}
diff --git a/internal/version/api.go b/internal/version/api.go
index 6ac2aee99d4..037c28dec37 100644
--- a/internal/version/api.go
+++ b/internal/version/api.go
@@ -479,6 +479,7 @@ var APIExtensions = []string{
 	"network_address_set",
 	"server_logging",
 	"network_forward_snat",
+	"memory_hotplug",
 }
 
 // APIExtensionsCount returns the number of available API extensions.


Reviews:
user: login: stgraber
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-16T15:51:48Z
user: login: presztak
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T16:35:53Z

Review comments:
diff_hunk: @@ -74,6 +74,35 @@ type CPUModel struct {
 	Flags map[string]any `json:"props"`
 }
 
+// MemoryDevice contains information about a memory device.
+type MemoryDevice struct {
+	Type string          `json:"type"`
+	Data json.RawMessage `json:"data"`
+}
+
+// PcDimmDevice contains information about a memory device of type pc-dimm.
+type PcDimmDevice struct {
path: internal/server/instance/drivers/qmp/commands.go
user: login: stgraber
body: Nit, should be capitalized as `PCIDimmDevice` I think.
created_at: 2025-04-16T15:51:48Z
author_association: MEMBER
diff_hunk: @@ -74,6 +74,35 @@ type CPUModel struct {
 	Flags map[string]any `json:"props"`
 }
 
+// MemoryDevice contains information about a memory device.
+type MemoryDevice struct {
+	Type string          `json:"type"`
+	Data json.RawMessage `json:"data"`
+}
+
+// PcDimmDevice contains information about a memory device of type pc-dimm.
+type PcDimmDevice struct {
path: internal/server/instance/drivers/qmp/commands.go
user: login: presztak
body: Done.
created_at: 2025-04-16T16:35:53Z
author_association: CONTRIBUTOR

PR comments:
user: login: stgraber
created_at: 2025-04-16T15:57:58Z
body: We also need to add an API extension.
author_association: MEMBER
user: login: stgraber
created_at: 2025-04-16T16:01:01Z
body: And it'd be nice to split the second commit in two, first half being the function split which doesn't introduce anything user visible, and then second half will be the hotplug.
author_association: MEMBER
user: login: presztak
created_at: 2025-04-16T16:36:39Z
body: All comments have been addressed.
author_association: CONTRIBUTOR