Configuring and managing virtualization Red Hat Enterprise Linux 9 Setting up your host, creating and administering virtual machines, and understanding virtualization features Red Hat Customer Content Services

Commands:
dnf install qemu-kvm libvirt virt-install virt-viewer
for drv in qemu network nodedev nwfilter secret storage interface; do systemctl start virtUSD{drv}d{,-ro,-admin}.socket; done
virt-host-validate [...] QEMU: Checking for device assignment IOMMU support : PASS QEMU: Checking if IOMMU is enabled by kernel : WARN (IOMMU appears to be disabled in kernel. Add intel_iommu=on to kernel cmdline arguments) LXC: Checking for Linux >= 2.6.26 : PASS [...] LXC: Checking for cgroup 'blkio' controller mount-point : PASS LXC: Checking if device /sys/fs/fuse/connections exists : FAIL (Load the 'fuse' module to enable /proc/ overrides)
QEMU: Checking for hardware virtualization: FAIL (Only emulated CPUs are available, performance will be significantly limited)
grep sie /proc/cpuinfo features : esan3 zarch stfle msa ldisp eimm dfp edat etf3eh highgprs te sie
dnf install qemu-kvm libvirt virt-install
for drv in qemu network nodedev nwfilter secret storage interface; do systemctl start virtUSD{drv}d{,-ro,-admin}.socket; done
virt-host-validate [...] QEMU: Checking if device /dev/kvm is accessible : PASS QEMU: Checking if device /dev/vhost-net exists : PASS QEMU: Checking if device /dev/net/tun exists : PASS QEMU: Checking for cgroup 'memory' controller support : PASS QEMU: Checking for cgroup 'memory' controller mount-point : PASS [...]
QEMU: Checking for hardware virtualization: FAIL (Only emulated CPUs are available, performance will be significantly limited)
dnf install qemu-kvm libvirt virt-install
for drv in qemu network nodedev nwfilter secret storage interface; do systemctl start virtUSD{drv}d{,-ro,-admin}.socket; done
virt-host-validate [...] QEMU: Checking if device /dev/vhost-net exists : PASS QEMU: Checking if device /dev/net/tun exists : PASS QEMU: Checking for cgroup 'memory' controller support : PASS QEMU: Checking for cgroup 'memory' controller mount-point : PASS [...] QEMU: Checking for cgroup 'blkio' controller support : PASS QEMU: Checking for cgroup 'blkio' controller mount-point : PASS QEMU: Checking if IOMMU is enabled by kernel : WARN (Unknown if this platform has IOMMU support)
touch qemuga.xml
<channel type='unix'> <source mode='bind' path='/var/lib/libvirt/qemu/f16x86_64.agent'/> <target type='virtio' name='org.qemu.guest_agent.0'/> </channel>
virsh attach-device <vm-name> qemuga.xml --live --config
virsh attach-device <vm-name> qemuga.xml --config
dnf install qemu-guest-agent
systemctl start qemu-guest-agent
setsebool virt_qemu_ga_read_nonsecurity_files on
setsebool virt_qemu_ga_manage_ssh on
virt-install --name demo-guest1 --memory 2048 --vcpus 2 --disk size=80 --os-variant win10 --cdrom /home/username/Downloads/Win10install.iso
virt-install --name demo-guest2 --memory 4096 --vcpus 4 --disk none --livecd --os-variant rhel9.0 --cdrom /home/username/Downloads/rhel9.iso
virt-install --name demo-guest3 --memory 2048 --vcpus 2 --os-variant rhel9.0 --import --disk /home/username/backup/disk.qcow2
virt-install --name demo-guest4 --memory 2048 --vcpus 2 --disk size=160 --os-variant rhel9.0 --location http://example.com/OS-install --initrd-inject /home/username/ks.cfg --extra-args="inst.ks=file:/ks.cfg console=tty0 console=ttyS0,115200n8"
%packages -kernel kernel-64k %end
virt-install --name demo-guest5 --memory 16384 --vcpus 16 --disk size=280 --os-variant rhel9.0 --location RHEL9.iso --graphics none --extra-args='console=ttyS0'
virt-install --connect qemu+ssh://root@192.0.2.1/system --name demo-guest6 --memory 16384 --vcpus 16 --disk size=280 --os-variant rhel9.0 --location RHEL9.iso --graphics none --extra-args='console=ttyS0'
virt-install --name demo-guest7 --memory 16384 --vcpus 16 --disk size=280 --os-variant rhel9.0 --location RHEL9.iso --graphics none --disk none --hostdev mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8,address.type=ccw,address.cssid=0xfe,address.ssid=0x0,address.devno=0x1111,boot-order=1 --extra-args 'rd.dasd=0.0.1111'
{PackageManagerCommand} info libvirt-daemon-config-network Installed Packages Name : libvirt-daemon-config-network [...]
virsh net-list --all Name State Autostart Persistent -------------------------------------------- default active yes yes
virsh net-autostart default Network default marked as autostarted virsh net-start default Network default started
error: failed to get network 'default' error: Network not found: no network with matching name 'default'
{PackageManagerCommand} reinstall libvirt-daemon-config-network
error: Failed to start network default error: internal error: Network is already in use by interface ens2
virsh start demo-guest1 Domain 'demo-guest1' started
virsh -c qemu+ssh://root@192.0.2.1/system start demo-guest1 root@192.0.2.1's password: Domain 'demo-guest1' started
virsh autostart demo-guest1 Domain ' demo-guest1 ' marked as autostarted
mkdir -p /etc/systemd/system/virtqemud.service.d/
touch /etc/systemd/system/virtqemud.service.d/10-network-online.conf
[Unit] After=network-online.target
virsh dominfo demo-guest1 Id: 2 Name: demo-guest1 UUID: e46bc81c-74e2-406e-bd7a-67042bae80d1 OS Type: hvm State: running CPU(s): 2 CPU time: 385.9s Max memory: 4194304 KiB Used memory: 4194304 KiB Persistent: yes Autostart: enable Managed save: no Security model: selinux Security DOI: 0 Security label: system_u:system_r:svirt_t:s0:c873,c919 (enforcing)
cat /etc/systemd/system/virtqemud.service.d/10-network-online.conf [Unit] After=network-online.target
virt-viewer guest-name
virt-viewer --direct --connect qemu+ssh://root@192.0.2.1/system guest-name root@192.0.2.1's password:
dnf install libvirt-nss
passwd: compat shadow: compat group: compat hosts: files libvirt_guest dns
ssh root@192.0.2.1 root@192.0.2.1's password: Last login: Mon Sep 24 12:05:36 2021 root~#
ssh root@testguest1 root@testguest1's password: Last login: Wed Sep 12 12:05:36 2018 root~]#
virsh list --all Id Name State ---------------------------------------------------- 2 testguest1 running - testguest2 shut off
sudo grep GRUB_TERMINAL /etc/default/grub GRUB_TERMINAL=serial
virsh dumpxml vm-name | grep console <console type='pty' tty='/dev/pts/2'> </console>
cat /proc/cmdline BOOT_IMAGE=/vmlinuz-3.10.0-948.el7.x86_64 root=/dev/mapper/rhel-root ro console=tty0 console=ttyS0 ,9600n8 rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb
grubby --update-kernel=ALL --args="console=ttyS0"
grub2-editenv - unset kernelopts
systemctl status serial-getty@ttyS0.service ○ serial-getty@ttyS0.service - Serial Getty on ttyS0 Loaded: loaded (/usr/lib/systemd/system/serial-getty@.service; enabled; preset: enabled)
virsh console guest1 --safe Connected to domain 'guest1' Escape character is ^] Subscription-name Kernel 3.10.0-948.el7.x86_64 on an x86_64 localhost login:
virsh -c qemu+ssh://root@ 192.0.2.1/system list root@192.0.2.1's password: Id Name State --------------------------------- 1 remote-guest running
virsh -c remote-host list root@192.0.2.1's password: Id Name State --------------------------------- 1 remote-guest running
vi ~/.ssh/config Host example-host-alias User root Hostname 192.0.2.1
vi /etc/libvirt/libvirt.conf uri_aliases = [ " example-qemu-host-alias =qemu+ssh:// example-host-alias /system", ]
virsh -c example-qemu-host-alias list root@192.0.2.1's password: Id Name State ---------------------------------------- 1 example-remote-guest running
These can be used in cases when no URI is supplied by the application (@uri_default also prevents probing of the hypervisor driver). # uri_default = "example-qemu-host-alias"
virsh list root@192.0.2.1's password: Id Name State --------------------------------- 1 example-remote-guest running
virsh dumpxml <vm-name> | grep graphics <graphics type='vnc' ports='-1' autoport= yes listen= 127.0.0.1 > </graphics>
virsh edit <vm-name>
<graphics type='vnc' ports='-1' autoport= yes listen= 127.0.0.1 passwd=' <password> '>
<graphics type='vnc' ports='-1' autoport= yes listen= 127.0.0.1 passwd=' <password> ' passwdValidTo='2025-02-01T15:30:00' >
virsh start <vm-name>
virt-viewer <vm-name>
virsh shutdown demo-guest1 Domain 'demo-guest1' is being shutdown
virsh -c qemu+ssh://root@192.0.2.1/system shutdown demo-guest1 root@192.0.2.1's password: Domain 'demo-guest1' is being shutdown
virsh destroy demo-guest1 Domain 'demo-guest1' destroyed
virsh list --all Id Name State ------------------------------------------ 1 demo-guest1 shut off
virsh undefine guest1 --remove-all-storage --nvram Domain 'guest1' has been undefined Volume 'vda'(/home/images/guest1.qcow2) removed.
dnf install cockpit-machines
virsh list --all Id Name State ---------------------------------- 1 testguest1 running - testguest2 shut off - testguest3 shut off - testguest4 shut off
virsh dominfo testguest1 Id: 1 Name: testguest1 UUID: a973666f-2f6e-415a-8949-75a7a98569e1 OS Type: hvm State: running CPU(s): 2 CPU time: 188.3s Max memory: 4194304 KiB Used memory: 4194304 KiB Persistent: yes Autostart: disable Managed save: no Security model: selinux Security DOI: 0 Security label: system_u:system_r:svirt_t:s0:c486,c538 (enforcing)
virsh dumpxml testguest2 <domain type='kvm' id='1'> <name>testguest2</name> <uuid>a973434f-2f6e-4esa-8949-76a7a98569e1</uuid> <metadata> [...]
virsh domblklist testguest3 Target Source --------------------------------------------------------------- vda /var/lib/libvirt/images/testguest3.qcow2 sda - sdb /home/username/Downloads/virt-p2v-1.36.10-1.el7.iso
virsh domfsinfo testguest3 Mountpoint Name Type Target ------------------------------------ / dm-0 xfs /boot vda1 xfs
virsh vcpuinfo testguest4 VCPU: 0 CPU: 3 State: running CPU time: 103.1s CPU Affinity: yyyy VCPU: 1 CPU: 0 State: running CPU time: 88.6s CPU Affinity: yyyy
virsh net-list --all Name State Autostart Persistent --------------------------------------------- default active yes yes labnet active yes yes
virsh net-info default Name: default UUID: c699f9f6-9202-4ca8-91d0-6b8cb9024116 Active: yes Persistent: yes Autostart: yes Bridge: virbr0
virsh dumpxml testguest1
<domain type='kvm'> <name>Testguest1</name> <uuid>ec6fbaa1-3eb4-49da-bf61-bb02fbec4967</uuid> <memory unit='KiB'>1048576</memory> <currentMemory unit='KiB'>1048576</currentMemory>
<vcpu placement='static'>1</vcpu>
<os> <type arch='x86_64' machine='pc-q35-rhel9.0.0'>hvm</type> <boot dev='hd'/> </os>
<features> <acpi/> <apic/> </features>
<cpu mode='host-model' check='partial'/>
<clock offset='utc'> <timer name='rtc' tickpolicy='catchup'/> <timer name='pit' tickpolicy='delay'/> <timer name='hpet' present='no'/> </clock>
<on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>destroy</on_crash>
<pm> <suspend-to-mem enabled='no'/> <suspend-to-disk enabled='no'/> </pm>
<devices> <emulator>/usr/libexec/qemu-kvm</emulator> <disk type='file' device='disk'> <driver name='qemu' type='qcow2'/> <source file='/var/lib/libvirt/images/Testguest.qcow2'/> <target dev='vda' bus='virtio'/> </disk> <disk type='file' device='cdrom'> <driver name='qemu' type='raw'/> <target dev='sdb' bus='sata'/> <readonly/> </disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'/> <controller type='sata' index='0'/> <controller type='pci' index='0' model='pcie-root'/> <controller type='pci' index='1' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='1' port='0x10'/> </controller> <controller type='pci' index='2' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='2' port='0x11'/> </controller> <controller type='pci' index='3' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='3' port='0x12'/> </controller> <controller type='pci' index='4' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='4' port='0x13'/> </controller> <controller type='pci' index='5' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='5' port='0x14'/> </controller> <controller type='pci' index='6' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='6' port='0x15'/> </controller> <controller type='pci' index='7' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='7' port='0x16'/> </controller> <controller type='virtio-serial' index='0'/>
<interface type='network'> <mac address='52:54:00:65:29:21'/> <source network='default'/> <model type='virtio'/> </interface>
<serial type='pty'> <target type='isa-serial' port='0'> <model name='isa-serial'/> </target> </serial> <console type='pty'> <target type='serial' port='0'/> </console> <channel type='unix'> <target type='virtio' name='org.qemu.guest_agent.0'/> <address type='virtio-serial' controller='0' bus='0' port='1'/> </channel>
<input type='tablet' bus='usb'> <address type='usb' bus='0' port='1'/> </input> <input type='mouse' bus='ps2'/> <input type='keyboard' bus='ps2'/>
<graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1'> <listen type='address' address='127.0.0.1'/> </graphics>
<redirdev bus='usb' type='tcp'> <source mode='connect' host='localhost' service='4000'/> <protocol type='raw'/> </redirdev> <memballoon model='virtio'> <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/> </memballoon> </devices> </domain>
virsh managedsave demo-guest1 Domain 'demo-guest1' saved by libvirt
virsh list --managed-save --all Id Name State ---------------------------------------------------- - demo-guest1 saved - demo-guest2 shut off
virsh list --with-managed-save --all Id Name State ---------------------------------------------------- - demo-guest1 shut off
virsh start demo-guest1 Domain 'demo-guest1' started
virsh -c qemu+ssh://root@192.0.2.1/system start demo-guest1 root@192.0.2.1's password: Domain 'demo-guest1' started
dnf install guestfs-tools
ls -la /var/lib/libvirt/images -rw-------. 1 root root 9665380352 Jul 23 14:50 a-really-important-vm.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 an-actual-vm-that-i-use.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 totally-not-a-fake-vm.qcow2 -rw-------. 1 root root 10739318784 Sep 20 17:57 another-vm-example.qcow2
whoami root
cp /var/lib/libvirt/images/a-really-important-vm.qcow2 /var/lib/libvirt/images/a-really-important-vm-original.qcow2
virt-sysprep -a /var/lib/libvirt/images/a-really-important-vm.qcow2 [ 0.0] Examining the guest [ 7.3] Performing "abrt-data" [ 7.3] Performing "backup-files" [ 9.6] Performing "bash-history" [ 9.6] Performing "blkid-tab" [...]
virt-diff -a /var/lib/libvirt/images/a-really-important-vm-orig.qcow2 -A /var/lib/libvirt/images/a-really-important-vm.qcow2 - - 0644 1001 /etc/group- - - 0000 797 /etc/gshadow- = - 0444 33 /etc/machine-id [...] - - 0600 409 /home/username/.bash_history - d 0700 6 /home/username/.ssh - - 0600 868 /root/.bash_history [...]
ls -la /var/lib/libvirt/images -rw-------. 1 root root 9665380352 Jul 23 14:50 a-really-important-vm.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 an-actual-vm-that-i-use.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 totally-not-a-fake-vm.qcow2 -rw-------. 1 root root 10739318784 Sep 20 17:57 another-vm-example.qcow2
rm -f /etc/udev/rules.d/70-persistent-net.rules
*ID=ExampleNetwork BOOTPROTO="dhcp" HWADDR="AA:BB:CC:DD:EE:FF" <- REMOVE NM_CONTROLLED="yes" ONBOOT="yes" TYPE="Ethernet" UUID="954bd22c-f96c-4b59-9445-b39dd86ac8ab" <- REMOVE
rm /etc/sysconfig/rhn/systemid
subscription-manager unsubscribe --all # subscription-manager unregister # subscription-manager clean
subscription-manager clean
subscription-manager register --consumerid=71rd64fx-6216-4409-bf3a-e4b7c7bd8ac9
rm -rf /etc/ssh/ssh_host_example
rm /etc/lvm/devices/system.devices
rm ~/.config/gnome-initial-setup-done
virt-clone --original example-VM-1 --auto-clone Allocating 'example-VM-1-clone.qcow2' | 50.0 GB 00:05:37 Clone 'example-VM-1-clone' created successfully.
virt-clone --original example-VM-2 --name example-VM-3 --file /var/lib/libvirt/images/ disk-1-example-VM-2 .qcow2 --file /var/lib/libvirt/images/ disk-2-example-VM-2 .qcow2 Allocating 'disk-1-example-VM-2-clone.qcow2' | 78.0 GB 00:05:37 Allocating 'disk-2-example-VM-2-clone.qcow2' | 80.0 GB 00:05:37 Clone 'example-VM-3' created successfully.
virsh migrate --offline --persistent example-VM-3 qemu+ssh://root@192.0.2.1/system root@192.0.2.1's password: scp /var/lib/libvirt/images/ <disk-1-example-VM-2-clone> .qcow2 root@192.0.2.1/ <user@remote_host.com> ://var/lib/libvirt/images/ scp /var/lib/libvirt/images/ <disk-2-example-VM-2-clone> .qcow2 root@192.0.2.1/ <user@remote_host.com> ://var/lib/libvirt/images/
virsh list --all Id Name State --------------------------------------- - example-VM-1 shut off - example-VM-1-clone shut off
virsh start example-VM-1-clone Domain 'example-VM-1-clone' started
virsh domdirtyrate-calc <example_VM> 30
virsh domstats <example_VM> --dirtyrate Domain: 'example-VM' dirtyrate.calc_status=2 dirtyrate.calc_start_time=200942 dirtyrate.calc_period=30 dirtyrate.megabytes_per_second=2
systemctl enable --now virtqemud.socket
systemctl enable --now virtproxyd-tls.socket
systemctl enable --now virtproxyd-tcp.socket
virsh migrate --offline --persistent <example_VM> qemu+ssh:// example-destination /system
virsh migrate --live --persistent <example_VM> qemu+ssh:// example-destination /system
virsh migrate --live --persistent --parallel --parallel-connections 4 <example_VM> qemu+ssh:// <example-destination> /system
virsh migrate-setmaxdowntime <example_VM> <time_interval_in_milliseconds>
virsh migrate --live --persistent --postcopy --timeout <time_interval_in_seconds> --timeout-postcopy <example_VM> qemu+ssh:// <example-destination> /system
virsh migrate --live --persistent --auto-converge <example_VM> qemu+ssh:// <example-destination> /system
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 shut off
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 running
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 shut off
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 running
virsh domdirtyrate-calc vm-name 30
virsh domstats vm-name --dirtyrate Domain: ' vm-name ' dirtyrate.calc_status=2 dirtyrate.calc_start_time=200942 dirtyrate.calc_period=30 dirtyrate.megabytes_per_second=2
dnf install mstflint
mstconfig -d <device_pci_address> query | grep -i VF_migration VF_MIGRATION_MODE MIGRATION_ENABLED(2)
mstconfig -d <device_pci_address> set VF_MIGRATION_MODE=2
dnf install openvswitch
devlink dev eswitch set pci/ <device_pci_address> mode switchdev
echo 1 > /sys/bus/pci/devices/0000\:e1\:00.0/sriov_numvfs
virsh nodedev-detach <vf_pci_address> --driver pci-stub
lshw -c network -businfo Bus info Device Class Description =========================================================================== pci@0000:e1:00.0 enp225s0np0 network MT2910 Family [ConnectX-7] pci@0000:e1:00.1 enp225s0v0 network ConnectX Family mlx5Gen Virtual Function
devlink port function set pci/0000:e1:00.0/1 migratable enable
systemctl start openvswitch
ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
ovs-vsctl set Open_vSwitch . other_config:max-idle=300000
ovs-vsctl add-br <bridge_name>
systemctl restart openvswitch
ovs-vsctl add-port <bridge_name> enp225s0np0
ovs-vsctl add-port <bridge_name> enp225s0npf0vf0
<interface type='hostdev' managed='yes'> <mac address='52:54:00:56:8c:f7'/> <source> <address type='pci' domain='0x0000' bus='0xe1' slot='0x00' function='0x1'/> </source> </interface>
virsh attach-device <vm_name> mlx_vf.xml --live --config
virsh migrate --live --domain <vm_name> --desturi qemu+ssh:// <destination_host_ip_address> /system
ifconfig eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 192.168.1.10 netmask 255.255.255.0 broadcast 192.168.1.255 inet6 fe80::a00:27ff:fe4e:66a1 prefixlen 64 scopeid 0x20<link> ether 08:00:27:4e:66:a1 txqueuelen 1000 (Ethernet) RX packets 100000 bytes 6543210 (6.5 MB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 100000 bytes 6543210 (6.5 MB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 enp4s0f0v0 : flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 192.168.3.10 netmask 255.255.255.0 broadcast 192.168.3.255 inet6 fe80::a00:27ff:fe4e:66c3 prefixlen 64 scopeid 0x20<link> ether 08:00:27:4e:66:c3 txqueuelen 1000 (Ethernet) RX packets 200000 bytes 12345678 (12.3 MB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 200000 bytes 12345678 (12.3 MB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ping -I <VF_interface_name> 8.8.8.8 PING 8.8.8.8 (8.8.8.8) from 192.168.3.10 <VF_interface_name> : 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=57 time=27.4 ms 64 bytes from 8.8.8.8: icmp_seq=2 ttl=57 time=26.9 ms --- 8.8.8.8 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1002ms rtt min/avg/max/mdev = 26.944/27.046/27.148/0.102 ms
setsebool virt_use_nfs 1
ssh root@ example-shared-storage root@example-shared-storage's password: Last login: Mon Sep 24 12:05:36 2019 root~#
mkdir /var/lib/libvirt/shared-images
scp /var/lib/libvirt/images/ example-disk-1 .qcow2 root@ example-shared-storage :/var/lib/libvirt/shared-images/ example-disk-1 .qcow2
/var/lib/libvirt/shared-images example-source-machine (rw,no_root_squash) example-destination-machine (rw,no\_root_squash)
exportfs -a
mount example-shared-storage :/var/lib/libvirt/shared-images /var/lib/libvirt/images
virsh domcapabilities | xmllint --xpath "//cpu/mode[@name='host-model']" - > domCaps-CPUs.xml
cat domCaps-CPUs.xml <cpu> <model fallback="forbid">Skylake-Client-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="clflushopt"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaves"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="ibrs"/> <feature policy="require" name="amd-stibp"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="rsba"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> <feature policy="disable" name="hle"/> <feature policy="disable" name="rtm"/> </cpu>
virsh domcapabilities | xmllint --xpath "//cpu/mode[@name='host-model']" - <mode name="host-model" supported="yes"> <model fallback="forbid">IvyBridge-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="pcid"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="arat"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaveopt"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> </mode>
cat domCaps-CPUs.xml <cpu> <model fallback="forbid">Skylake-Client-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="clflushopt"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaves"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="ibrs"/> <feature policy="require" name="amd-stibp"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="rsba"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> <feature policy="disable" name="hle"/> <feature policy="disable" name="rtm"/> </cpu> <cpu> <model fallback="forbid">IvyBridge-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="pcid"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="arat"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaveopt"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> </cpu>
virsh hypervisor-cpu-baseline domCaps-CPUs.xml <cpu mode='custom' match='exact'> <model fallback='forbid'>IvyBridge-IBRS</model> <vendor>Intel</vendor> <feature policy='require' name='ss'/> <feature policy='require' name='vmx'/> <feature policy='require' name='pdcm'/> <feature policy='require' name='pcid'/> <feature policy='require' name='hypervisor'/> <feature policy='require' name='arat'/> <feature policy='require' name='tsc_adjust'/> <feature policy='require' name='umip'/> <feature policy='require' name='md-clear'/> <feature policy='require' name='stibp'/> <feature policy='require' name='arch-capabilities'/> <feature policy='require' name='ssbd'/> <feature policy='require' name='xsaveopt'/> <feature policy='require' name='pdpe1gb'/> <feature policy='require' name='invtsc'/> <feature policy='require' name='ibpb'/> <feature policy='require' name='amd-ssbd'/> <feature policy='require' name='skip-l1dfl-vmentry'/> <feature policy='require' name='pschange-mc-no'/> </cpu>
virsh edit <vm_name>
virsh shutdown <vm_name> virsh start <vm_name>
virsh snapshot-dumpxml <vm-name> <snapshot-name>
<domainsnapshot> <name>sample-snapshot-name-1<name> <state>shutoff</state> <creationTime>1706658764</creationTime> <memory snapshot=' no '/> <disks> <disk name='vda' snapshot=' external ' type=' file '> <driver type='qcow2'/> <source file='/var/lib/libvirt/images/vm-name.sample-snapshot-name-1'/> </disk> </disks> <domain type='kvm'> [...]
<domainsnapshot> <name>sample-snapshot-name-2</name> <state>running</state> <creationTime>1653396424</creationTime> <memory snapshot= 'internal' /> <disks> <disk name='vda' snapshot=' internal '/> <disk name='sda' snapshot='no'/> </disks> <domain type='kvm'> [...]
virsh dumpxml <vm-name> | grep "disk type" <disk type=' file ' device='disk'> <disk type='file' device='cdrom'>
virsh snapshot-create-as <vm-name> <snapshot-name> <optional-description> <additional-parameters>
virsh snapshot-create-as Testguest1 Snapshot1 --disk-only Domain snapshot Snapshot1 created.
virsh snapshot-create-as Testguest2 Snapshot2 "clean system install" --disk-only --quiesce Domain snapshot Snapshot2 created.
virsh snapshot-create-as Testguest3 Snapshot3 --memspec /var/lib/libvirt/images/saved_memory.img Domain snapshot Snapshot3 created.
virsh snapshot-create-as Testguest4 Snapshot4 --live --memspec /var/lib/libvirt/images/saved_memory2.img Domain snapshot Snapshot4 created.
virsh snapshot-list <Testguest1> Name Creation Time State -------------------------------------------------------------- Snapshot1 2024-01-30 18:34:58 +0100 shutoff
virsh snapshot-dumpxml <Testguest1> <Snapshot1> | grep external <disk name='vda' snapshot=' external ' type=' file '>
virsh snapshot-dumpxml <Testguest1> <Snapshot1> | grep external <disk name='vda' snapshot=' external ' type=' file '>
virsh snapshot-revert Testguest2 clean-install Domain snapshot clean-install reverted
virsh snapshot-current Testguest2 --name clean-install
virsh snapshot-list --tree <vm-name> Clean-install-snapshot | +- Redundant-snapshot | +- Latest-snapshot
virsh snapshot-delete Testguest1 Redundant-snapshot Domain snapshot Redundant-snapshot deleted
virsh snapshot-list --tree <Testguest1> Clean-install-snapshot | +- Latest-snapshot
virt-xml --network=? --network options: [...] address.unit boot_order clearxml driver_name [...]
virt-xml testguest --add-device --disk /var/lib/libvirt/images/newdisk.qcow2,format=qcow2,size=20 Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virt-xml testguest2 --add-device --update --hostdev 002.004 Device hotplug successful. Domain 'testguest2' defined successfully.
virsh dumpxml testguest [...] <hostdev mode='subsystem' type='usb' managed='yes'> <source> <vendor id='0x4146'/> <product id='0x902e'/> <address bus='2' device='4'/> </source> <alias name='hostdev0'/> <address type='usb' bus='0' port='3'/> </hostdev> [...]
virt-xml --network=? --network options: [...] address.unit boot_order clearxml driver_name [...]
virsh dumpxml testguest1 > testguest1.xml cat testguest1.xml <domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>testguest1</name> <uuid>ede29304-fe0c-4ca4-abcd-d246481acd18</uuid> [...] </domain>
virt-xml testguest --edit --cpu host-model,clearxml=yes Domain 'testguest' defined successfully.
virsh dumpxml testguest [...] <cpu mode='host-model' check='partial'> <model fallback='allow'/> </cpu> [...]
virsh define testguest.xml
virsh dumpxml testguest1 > testguest1.xml cat testguest1.xml <domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>testguest1</name> <uuid>ede29304-fe0c-4ca4-abcd-d246481acd18</uuid> [...] </domain>
virt-xml testguest --remove-device --disk target=vdb Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virt-xml testguest2 --remove-device --update --hostdev type=usb Device hotunplug successful. Domain 'testguest2' defined successfully.
virsh define testguest.xml
virsh edit example-VM-1
virsh dumpxml example-VM-1 > example-VM-1 .xml
virsh edit <example-VM-1>
virsh dumpxml testguest1 > testguest1.xml cat testguest1.xml <domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>testguest1</name> <uuid>ede29304-fe0c-4ca4-abcd-d246481acd18</uuid> [...] </domain>
virsh define testguest1.xml
lsusb [...] Bus 001 Device 003: ID 2567:0a2b Intel Corp. Bus 001 Device 005: ID 0407:6252 Kingston River 2.0 [...]
virt-xml example-VM-1 --add-device --hostdev 001.005 Domain 'example-VM-1' defined successfully.
virsh dumpxml example-VM-1 [...] <hostdev mode='subsystem' type='usb' managed='yes'> <source> <vendor id='0x0407'/> <product id='0x6252'/> <address bus='1' device='5'/> </source> <alias name='hostdev0'/> <address type='usb' bus='0' port='3'/> </hostdev> [...]
lsusb [...] Bus 001 Device 003: ID 2567:0a2b Intel Corp. Bus 001 Device 005: ID 0407:6252 Kingston River 2.0 [...]
virt-xml example-VM-1 --remove-device --hostdev 001.005 Domain 'example-VM-1' defined successfully.
virt-xml example-VM-name --add-device --disk /home/username/Downloads/example-ISO-name.iso ,device=cdrom Domain 'example-VM-name' defined successfully.
virt-xml vmname --add-device --disk target.dev=sda,device=cdrom
virsh dumpxml example-VM-name <disk> <source file=' USD(/home/username/Downloads/example-ISO-name.iso) '/> <target dev='sda' bus='sata'/> </disk>
virt-xml example-VM-name --edit target=sda --disk /dev/cdrom/example-ISO-name-2.iso Domain 'example-VM-name' defined successfully.
virsh dumpxml example-VM-name <disk> <source file=' USD(/home/username/Downloads/example-ISO-name.iso) '/> <target dev='sda' bus='sata'/> </disk>
virt-xml example-VM-name --edit target=sda --disk path= Domain 'example-VM-name' defined successfully.
virsh dumpxml example-VM-name <disk type='file' device='cdrom'> <driver name='qemu' type='raw'/> <target dev='sda' bus='sata'/> </disk>
virt-xml example-VM-name --remove-device --disk target=sda Domain 'example-VM-name' defined successfully.
lspci -v [...] 02:00.0 Ethernet controller: Intel Corporation 82576 Gigabit Network Connection (rev 01) Subsystem: Intel Corporation Gigabit ET Dual Port Server Adapter Flags: bus master, fast devsel, latency 0, IRQ 16, NUMA node 0 Memory at fcba0000 (32-bit, non-prefetchable) [size=128K] [...] Capabilities: [150] Alternative Routing-ID Interpretation (ARI) Capabilities: [160] Single Root I/O Virtualization (SR-IOV) Kernel driver in use: igb Kernel modules: igb [...]
ip link set eth1 up ip link show eth1 8: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT qlen 1000 link/ether a0:36:9f:8f:3f:b8 brd ff:ff:ff:ff:ff:ff vf 0 MAC 00:00:00:00:00:00, spoof checking on, link-state auto vf 1 MAC 00:00:00:00:00:00, spoof checking on, link-state auto vf 2 MAC 00:00:00:00:00:00, spoof checking on, link-state auto vf 3 MAC 00:00:00:00:00:00, spoof checking on, link-state auto
grubby --args="intel_iommu=on iommu=pt" --update-kernel=ALL
grubby --args="iommu=pt" --update-kernel=ALL
cat /sys/class/net/eth1/device/sriov_totalvfs 7
echo VF-number > /sys/class/net/ network-interface /device/sriov_numvfs
echo 2 > /sys/class/net/eth1/device/sriov_numvfs
lspci | grep Ethernet 82:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) 82:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) 82:10.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) 82:10.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="ixgbe", ATTR{device/sriov_numvfs}="2"
virsh attach-interface testguest1 hostdev 0000:82:10.0 --managed --live --config
dnf install driverctl
lsmod | grep vfio
lscss -d 0.0.002c Device Subchan. DevType CU Type Use PIM PAM POM CHPIDs ---------------------------------------------------------------------- 0.0.002c 0.0.29a8 3390/0c 3990/e9 yes f0 f0 ff 02111221 00000000
cio_ignore -r 0.0.002c
cio_ignore=all,!condev, ! 0.0.002c
driverctl -b css set-override 0.0.29a8 vfio_ccw
cat nodedev.xml <device> <parent>css_0_0_29a8</parent> <capability type="mdev"> <type id="vfio_ccw-io"/> </capability> </device> virsh nodedev-define nodedev.xml Node device 'mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8' defined from 'nodedev.xml' virsh nodedev-start mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 Device mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 started
virsh nodedev-dumpxml mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 <device> <name>mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8</name> <parent>css_0_0_29a8</parent> <capability type='mdev'> <type id='vfio_ccw-io'/> <uuid>30820a6f-b1a5-4503-91ca-0c10ba12345a</uuid> <iommuGroup number='0'/> <attr name='assign_adapter' value='0x02'/> <attr name='assign_domain' value='0x002b'/> </capability> </device>
<hostdev mode='subsystem' type='mdev' model='vfio-ccw'> <source> <address uuid="30820a6f-b1a5-4503-91ca-0c10ba12345a"/> </source> </hostdev>
virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8
virsh nodedev-info mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 Name: mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 Parent: css_0_0_0121 Active: yes Persistent: yes Autostart: yes
virsh dumpxml vm-name <domain> [...] <hostdev mode='subsystem' type='mdev' managed='no' model='vfio-ccw'> <source> <address uuid='10620d2f-ed4d-437b-8aff-beda461541f9'/> </source> <alias name='hostdev0'/> <address type='ccw' cssid='0xfe' ssid='0x0' devno='0x0009 '/> </hostdev> [...] </domain>
lscss | grep 0.0.0009 0.0.0009 0.0.0007 3390/0c 3990/e9 f0 f0 ff 12212231 00000000
chccwdev -e 0.0009 Setting device 0.0.0009 online Done
lsmod | grep vfio
lspci -nkD 0000:00:00.0 0000: 1014:04ed Kernel driver in use: ism Kernel modules: ism 0001:00:00.0 0000: 1014:04ed Kernel driver in use: ism Kernel modules: ism 0002:00:00.0 0200: 15b3:1016 Subsystem: 15b3:0062 Kernel driver in use: mlx5_core Kernel modules: mlx5_core 0003:00:00.0 0200: 15b3:1016 Subsystem: 15b3:0062 Kernel driver in use: mlx5_core Kernel modules: mlx5_core
virsh edit vm-name
<hostdev mode="subsystem" type="pci" managed="yes"> <driver name="vfio"/> <source> <address domain=" 0x0003 " bus=" 0x00 " slot=" 0x00 " function=" 0x0 "/> </source> <address type="pci"/> </hostdev>
<hostdev mode="subsystem" type="pci" managed="yes"> <driver name="vfio"/> <source> <address domain=" 0x0003 " bus=" 0x00 " slot=" 0x00 " function=" 0x0 "/> </source> <address type="pci"> <zpci uid="0x0008" fid="0x001807"/> </address> </hostdev>
virsh shutdown vm-name
lspci -nkD | grep 0003:00:00.0 0003:00:00.0 8086:9a09 (rev 01)
virsh vol-info --pool guest_images firstimage Name: firstimage Type: block Capacity: 20.00 GB Allocation: 20.00 GB
virsh pool-list --all --details Name State Autostart Persistent Capacity Allocation Available default running yes yes 48.97 GiB 23.93 GiB 25.03 GiB Downloads running yes yes 175.62 GiB 62.02 GiB 113.60 GiB RHEL-Storage-Pool running yes yes 214.62 GiB 93.02 GiB 168.60 GiB
virsh pool-capabilities | grep "'dir' supported='yes'"
virsh pool-define-as guest_images_dir dir --target "/guest_images" Pool guest_images_dir defined
virsh pool-build guest_images_dir Pool guest_images_dir built ls -la /guest_images total 8 drwx------. 2 root root 4096 May 31 19:38 . dr-xr-xr-x. 25 root root 4096 May 31 19:38 ..
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_dir inactive no
virsh pool-start guest_images_dir Pool guest_images_dir started
virsh pool-autostart guest_images_dir Pool guest_images_dir marked as autostarted
virsh pool-info guest_images_dir Name: guest_images_dir UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'disk' supported='yes'"
GRUB_DISABLE_OS_PROBER=true
GRUB_OS_PROBER_SKIP_LIST="5ef6313a-257c-4d43@/dev/sdb1"
virsh pool-define-as guest_images_disk disk --source-format=gpt --source-dev=/dev/sdb --target /dev Pool guest_images_disk defined
virsh pool-build guest_images_disk Pool guest_images_disk built
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_disk inactive no
virsh pool-start guest_images_disk Pool guest_images_disk started
virsh pool-autostart guest_images_disk Pool guest_images_disk marked as autostarted
virsh pool-info guest_images_disk Name: guest_images_disk UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'fs' supported='yes'"
GRUB_DISABLE_OS_PROBER=true
GRUB_OS_PROBER_SKIP_LIST="5ef6313a-257c-4d43@/dev/sdb1"
virsh pool-define-as guest_images_fs fs --source-dev /dev/sdc1 --target /guest_images Pool guest_images_fs defined
virsh pool-build guest_images_fs Pool guest_images_fs built ls -la /guest_images total 8 drwx------. 2 root root 4096 May 31 19:38 . dr-xr-xr-x. 25 root root 4096 May 31 19:38 ..
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_fs inactive no
virsh pool-start guest_images_fs Pool guest_images_fs started
virsh pool-autostart guest_images_fs Pool guest_images_fs marked as autostarted
virsh pool-info guest_images_fs Name: guest_images_fs UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
mount | grep /guest_images /dev/sdc1 on /guest_images type ext4 (rw) ls -la /guest_images total 24 drwxr-xr-x. 3 root root 4096 May 31 19:47 . dr-xr-xr-x. 25 root root 4096 May 31 19:38 .. drwx------. 2 root root 16384 May 31 14:18 lost+found
virsh pool-capabilities | grep "'iscsi' supported='yes'"
virsh pool-define-as --name guest_images_iscsi --type iscsi --source-host server1.example.com --source-dev iqn.2010-05.com.example.server1:iscsirhel7guest --target /dev/disk/by-path Pool guest_images_iscsi defined
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_iscsi inactive no
virsh pool-start guest_images_iscsi Pool guest_images_iscsi started
virsh pool-autostart guest_images_iscsi Pool guest_images_iscsi marked as autostarted
virsh pool-info guest_images_iscsi Name: guest_images_iscsi UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'logical' supported='yes'"
virsh pool-define-as guest_images_lvm logical --source-name lvm_vg --target /dev/lvm_vg Pool guest_images_lvm defined
virsh pool-list --all Name State Autostart ------------------------------------------- default active yes guest_images_lvm inactive no
virsh pool-start guest_images_lvm Pool guest_images_lvm started
virsh pool-autostart guest_images_lvm Pool guest_images_lvm marked as autostarted
virsh pool-info guest_images_lvm Name: guest_images_lvm UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "<value>nfs</value>"
virsh pool-define-as --name guest_images_netfs --type netfs --source-host='111.222.111.222' --source-path='/home/net_mount' --source-format='nfs' --target='/var/lib/libvirt/images/nfspool'
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_netfs inactive no
virsh pool-start guest_images_netfs Pool guest_images_netfs started
virsh pool-autostart guest_images_netfs Pool guest_images_netfs marked as autostarted
virsh pool-info guest_images_netfs Name: guest_images_netfs UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'scsi' supported='yes'"
virsh pool-define-as guest_images_vhba scsi --adapter-parent scsi_host3 --adapter-wwnn 5001a4a93526d0a1 --adapter-wwpn 5001a4ace3ee047d --target /dev/disk/ Pool guest_images_vhba defined
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_vhba inactive no
virsh pool-start guest_images_vhba Pool guest_images_vhba started
virsh pool-autostart guest_images_vhba Pool guest_images_vhba marked as autostarted
virsh pool-info guest_images_vhba Name: guest_images_vhba UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-list --all Name State Autostart ------------------------------------------- default active yes Downloads active yes RHEL-Storage-Pool active yes
virsh pool-destroy Downloads Pool Downloads destroyed
virsh pool-delete Downloads Pool Downloads deleted
virsh pool-undefine Downloads Pool Downloads has been undefined
virsh pool-list --all Name State Autostart ------------------------------------------- default active yes rhel-Storage-Pool active yes
virsh pool-define ~/guest_images.xml Pool defined from guest_images_dir
<pool type='dir'> <name>dirpool</name> <target> <path>/guest_images</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_disk
<pool type='disk'> <name>phy_disk</name> <source> <device path='/dev/sdb'/> <format type='gpt'/> </source> <target> <path>/dev</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_fs
<pool type='fs'> <name>guest_images_fs</name> <source> <device path='/dev/sdc1'/> <format type='auto'/> </source> <target> <path>/guest_images</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_iscsi
<pool type='iscsi'> <name>iSCSI_pool</name> <source> <host name='server1.example.com'/> <device path='iqn.2010-05.com.example.server1:iscsirhel7guest'/> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_logical
<source> <device path='/dev/sda1'/> <device path='/dev/sdb3'/> <device path='/dev/sdc2'/> </source>
<pool type='logical'> <name>guest_images_lvm</name> <source> <device path='/dev/sdc'/> <name>libvirt_lvm</name> <format type='lvm2'/> </source> <target> <path>/dev/libvirt_lvm</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_netfs
<pool type='netfs'> <name>nfspool</name> <source> <host name='file_server'/> <format type='nfs'/> <dir path='/home/net_mount'/> </source> <target> <path>/var/lib/libvirt/images/nfspool</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_vhba
<pool type='scsi'> <name>vhbapool_host3</name> <source> <adapter type='fc_host' wwnn='5001a4a93526d0a1' wwpn='5001a4ace3ee047d'/> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
<pool type='scsi'> <name>vhbapool_host3</name> <source> <adapter type='fc_host' parent='scsi_host3' wwnn='5001a4a93526d0a1' wwpn='5001a4ace3ee047d'/> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
virsh vol-list --pool RHEL-Storage-Pool --details Name Path Type Capacity Allocation --------------------------------------------------------------------------------------------- .bash_history /home/VirtualMachines/.bash_history file 18.70 KiB 20.00 KiB .bash_logout /home/VirtualMachines/.bash_logout file 18.00 B 4.00 KiB .bash_profile /home/VirtualMachines/.bash_profile file 193.00 B 4.00 KiB .bashrc /home/VirtualMachines/.bashrc file 1.29 KiB 4.00 KiB .git-prompt.sh /home/VirtualMachines/.git-prompt.sh file 15.84 KiB 16.00 KiB .gitconfig /home/VirtualMachines/.gitconfig file 167.00 B 4.00 KiB RHEL_Volume.qcow2 /home/VirtualMachines/RHEL8_Volume.qcow2 file 60.00 GiB 13.93 GiB
virsh vol-info --pool RHEL-Storage-Pool --vol RHEL_Volume.qcow2 Name: RHEL_Volume.qcow2 Type: file Capacity: 60.00 GiB Allocation: 13.93 GiB
virsh pool-list --details Name State Autostart Persistent Capacity Allocation Available -------------------------------------------------------------------------------------------- default running yes yes 48.97 GiB 36.34 GiB 12.63 GiB Downloads running yes yes 175.92 GiB 121.20 GiB 54.72 GiB VM-disks running yes yes 175.92 GiB 121.20 GiB 54.72 GiB
virsh vol-create-as --pool guest-images-fs --name vm-disk1 --capacity 20 --format qcow2
<disk type='volume' device='disk'> <driver name='qemu' type='qcow2'/> <source pool='guest-images-fs' volume='vm-disk1'/> <target dev='hdk' bus='ide'/> </disk>
<disk type='block' device='disk'> <driver name='qemu' type='raw'/> <source dev='/dev/mapper/mpatha' /> <target dev='sda' bus='scsi'/> </disk>
<disk type='network' device='disk'> <driver name='qemu' type='raw'/> <source protocol='rbd' name='pool/image'> <host name='mon1.example.org' port='6321'/> </source> <target dev='vdc' bus='virtio'/> </disk>
virsh attach-device --config testguest1 ~/vm-disk1.xml
virsh vol-list --pool RHEL-SP Name Path --------------------------------------------------------------- .bash_history /home/VirtualMachines/.bash_history .bash_logout /home/VirtualMachines/.bash_logout .bash_profile /home/VirtualMachines/.bash_profile .bashrc /home/VirtualMachines/.bashrc .git-prompt.sh /home/VirtualMachines/.git-prompt.sh .gitconfig /home/VirtualMachines/.gitconfig vm-disk1 /home/VirtualMachines/vm-disk1
virsh vol-wipe --pool RHEL-SP vm-disk1 Vol vm-disk1 wiped
virsh vol-delete --pool RHEL-SP vm-disk1 Vol vm-disk1 deleted
virsh vol-list --pool RHEL-SP Name Path --------------------------------------------------------------- .bash_history /home/VirtualMachines/.bash_history .bash_logout /home/VirtualMachines/.bash_logout .bash_profile /home/VirtualMachines/.bash_profile .bashrc /home/VirtualMachines/.bashrc .git-prompt.sh /home/VirtualMachines/.git-prompt.sh .gitconfig /home/VirtualMachines/.gitconfig
qemu-img create -f <format> <image-name> <size>
qemu-img create -f qcow2 test-image 30G Formatting 'test-img', fmt=qcow2 cluster_size=65536 extended_l2=off compression_type=zlib size=32212254720 lazy_refcounts=off refcount_bits=16
qemu-img info <test-img> image: test-img file format: qcow2 virtual size: 30 GiB (32212254720 bytes) disk size: 196 KiB cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib lazy refcounts: false refcount bits: 16 corrupt: false extended l2: false
qemu-img check <test-name.qcow2> No errors were found on the image. 327434/327680 = 99.92% allocated, 0.00% fragmented, 0.00% compressed clusters Image end offset: 21478375424
167 errors were found on the image. Data may be corrupted, or further writes to the image may corrupt it. 453368 leaked clusters were found on the image. This means waste of disk space, but no harm to data. 259 internal errors have occurred during the check. Image end offset: 21478375424
qemu-img check -r all <test-name.qcow2> [...] 122 errors were found on the image. Data may be corrupted, or further writes to the image may corrupt it. 250 internal errors have occurred during the check. Image end offset: 27071414272
virsh domblklist <vm-name> Target Source ---------------------------------------------------------- vda /home/username/disk-images/ example-image.qcow2
cp <example-image.qcow2> <example-image-backup.qcow2>
qemu-img resize <example-image.qcow2> +10G
qemu-img info <converted-image.qcow2> image: converted-image.qcow2 file format: qcow2 virtual size: 30 GiB (32212254720 bytes) disk size: 196 KiB cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib lazy refcounts: false refcount bits: 16 corrupt: false extended l2: false
qemu-img convert -f raw <original-image.img> -O qcow2 <converted-image.qcow2>
qemu-img info <converted-image.qcow2> image: converted-image.qcow2 file format: qcow2 virtual size: 30 GiB (32212254720 bytes) disk size: 196 KiB cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib lazy refcounts: false refcount bits: 16 corrupt: false extended l2: false
<secret ephemeral='no' private='yes'> <description>Passphrase for the iSCSI example.com server</description> <usage type='iscsi'> <target>iscsirhel7secret</target> </usage> </secret>
virsh secret-define secret.xml
virsh secret-list UUID Usage -------------------------------------------------------------- 2d7891af-20be-4e5e-af83-190e8a922360 iscsi iscsirhel7secret
virsh secret-set-value --interactive 2d7891af-20be-4e5e-af83-190e8a922360 Enter new value for secret: Secret value set
<pool type='iscsi'> <name>iscsirhel7pool</name> <source> <host name='192.0.2.1'/> <device path='iqn.2010-05.com.example.server1:iscsirhel7guest'/> <auth type='chap' username='_example-user_'> <secret usage='iscsirhel7secret'/> </auth> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
<auth username='redhat'> <secret type='iscsi' usage='iscsirhel7secret'/> </auth>
virsh pool-destroy iscsirhel7pool virsh pool-start iscsirhel7pool
virsh nodedev-list --cap vports scsi_host3 scsi_host4
virsh nodedev-dumpxml scsi_host3
<device> <name>scsi_host3</name> <path>/sys/devices/pci0000:00/0000:00:04.0/0000:10:00.0/host3</path> <parent>pci_0000_10_00_0</parent> <capability type='scsi_host'> <host>3</host> <unique_id>0</unique_id> <capability type='fc_host'> <wwnn>20000000c9848140</wwnn> <wwpn>10000000c9848140</wwpn> <fabric_wwn>2002000573de9a81</fabric_wwn> </capability> <capability type='vport_ops'> <max_vports>127</max_vports> <vports>0</vports> </capability> </capability> </device>
<device> <parent>scsi_host3</parent> <capability type='scsi_host'> <capability type='fc_host'> </capability> </capability> </device>
<device> <name>vhba</name> <parent wwnn='20000000c9848140' wwpn='10000000c9848140'/> <capability type='scsi_host'> <capability type='fc_host'> </capability> </capability> </device>
virsh nodedev-create vhba_host3 Node device scsi_host5 created from vhba_host3.xml
virsh nodedev-dumpxml scsi_host5 <device> <name>scsi_host5</name> <path>/sys/devices/pci0000:00/0000:00:04.0/0000:10:00.0/host3/vport-3:0-0/host5</path> <parent>scsi_host3</parent> <capability type='scsi_host'> <host>5</host> <unique_id>2</unique_id> <capability type='fc_host'> <wwnn>5001a4a93526d0a1</wwnn> <wwpn>5001a4ace3ee047d</wwpn> <fabric_wwn>2002000573de9a81</fabric_wwn> </capability> </capability> </device>
grubby --args="intel_iommu=on iommu_pt" --update-kernel DEFAULT
grubby --args="iommu=pt" --update-kernel DEFAULT
lspci -Dnn | grep VGA 0000:02:00.0 VGA compatible controller [0300]: NVIDIA Corporation GK106GL [Quadro K4000] [ 10de:11fa ] (rev a1)
grubby --args="pci-stub.ids=10de:11fa" --update-kernel DEFAULT
virsh nodedev-dumpxml pci_0000_02_00_0
<device> <name>pci_0000_02_00_0</name> <path>/sys/devices/pci0000:00/0000:00:03.0/0000:02:00.0</path> <parent>pci_0000_00_03_0</parent> <driver> <name>pci-stub</name> </driver> <capability type='pci'> <domain>0</domain> <bus>2</bus> <slot>0</slot> <function>0</function> <product id='0x11fa'>GK106GL [Quadro K4000]</product> <vendor id='0x10de'>NVIDIA Corporation</vendor> <iommuGroup number='13'> <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/> <address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/> </iommuGroup> <pci-express> <link validity='cap' port='0' speed='8' width='16'/> <link validity='sta' speed='2.5' width='16'/> </pci-express> </capability> </device>
driverctl set-override 0000:02:00.1 vfio-pci
<hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/> </source> </hostdev>
virsh attach-device System1 --file /home/GPU-Assign.xml --persistent Device attached successfully.
lshw -C display *-display description: 3D controller product: GP104GL [Tesla P4] vendor: NVIDIA Corporation physical id: 0 bus info: pci@0000:01:00.0 version: a1 width: 64 bits clock: 33MHz capabilities: pm msi pciexpress cap_list configuration: driver=vfio-pci latency=0 resources: irq:16 memory:f6000000-f6ffffff memory:e0000000-efffffff memory:f0000000-f1ffffff
blacklist nouveau options nouveau modeset=0
dracut --force reboot
lsmod | grep nvidia_vgpu_vfio nvidia_vgpu_vfio 45011 0 nvidia 14333621 10 nvidia_vgpu_vfio mdev 20414 2 vfio_mdev,nvidia_vgpu_vfio vfio 32695 3 vfio_mdev,nvidia_vgpu_vfio,vfio_iommu_type1 systemctl status nvidia-vgpu-mgr.service nvidia-vgpu-mgr.service - NVIDIA vGPU Manager Daemon Loaded: loaded (/usr/lib/systemd/system/nvidia-vgpu-mgr.service; enabled; vendor preset: disabled) Active: active (running) since Fri 2018-03-16 10:17:36 CET; 5h 8min ago Main PID: 1553 (nvidia-vgpu-mgr) [...]
uuidgen 30820a6f-b1a5-4503-91ca-0c10ba58692a
<device> <parent>pci_0000_01_00_0</parent> <capability type="mdev"> <type id="nvidia-63"/> <uuid>30820a6f-b1a5-4503-91ca-0c10ba58692a</uuid> </capability> </device>
virsh nodedev-define vgpu-test.xml Node device mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 created from vgpu-test.xml
virsh nodedev-list --cap mdev --inactive mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-start mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Device mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 started
virsh nodedev-list --cap mdev mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Device mdev_d196754e_d8ed_4f43_bf22_684ed698b08b_0000_9b_00_0 marked as autostarted
<hostdev mode='subsystem' type='mdev' managed='no' model='vfio-pci' display='on'> <source> <address uuid='30820a6f-b1a5-4503-91ca-0c10ba58692a'/> </source> </hostdev>
virsh nodedev-info mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Name: virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Parent: pci_0000_01_00_0 Active: yes Persistent: yes Autostart: yes
lspci -d 10de: -k 07:00.0 VGA compatible controller: NVIDIA Corporation GV100GL [Tesla V100 SXM2 32GB] (rev a1) Subsystem: NVIDIA Corporation Device 12ce Kernel driver in use: nvidia Kernel modules: nouveau, nvidia_drm, nvidia
virsh nodedev-list --cap mdev mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-destroy mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Destroyed node device 'mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0'
virsh nodedev-info mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Name: virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Parent: pci_0000_01_00_0 Active: no Persistent: yes Autostart: yes
<hostdev mode='subsystem' type='mdev' managed='no' model='vfio-pci'> <source> <address uuid='30820a6f-b1a5-4503-91ca-0c10ba58692a'/> </source> </hostdev>
virsh nodedev-undefine mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Undefined node device 'mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0'
virsh nodedev-list --cap mdev --inactive mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-list --cap mdev
virsh nodedev-list --cap mdev_types pci_0000_5b_00_0 pci_0000_9b_00_0
virsh nodedev-dumpxml pci_0000_9b_00_0 <device> <name>pci_0000_9b_00_0</name> <path>/sys/devices/pci0000:9a/0000:9a:00.0/0000:9b:00.0</path> <parent>pci_0000_9a_00_0</parent> <driver> <name>nvidia</name> </driver> <capability type='pci'> <class>0x030000</class> <domain>0</domain> <bus>155</bus> <slot>0</slot> <function>0</function> <product id='0x1e30'>TU102GL [Quadro RTX 6000/8000]</product> <vendor id='0x10de'>NVIDIA Corporation</vendor> <capability type='mdev_types'> <type id='nvidia-346'> <name>GRID RTX6000-12C</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>2</availableInstances> </type> <type id='nvidia-439'> <name>GRID RTX6000-3A</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>8</availableInstances> </type> [...] <type id='nvidia-440'> <name>GRID RTX6000-4A</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>6</availableInstances> </type> <type id='nvidia-261'> <name>GRID RTX6000-8Q</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>3</availableInstances> </type> </capability> <iommuGroup number='216'> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x3'/> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x1'/> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x2'/> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x0'/> </iommuGroup> <numa node='2'/> <pci-express> <link validity='cap' port='0' speed='8' width='16'/> <link validity='sta' speed='2.5' width='8'/> </pci-express> </capability> </device>
ip addr show virbr0 3: virbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN link/ether 1b:c4:94:cf:fd:17 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global virbr0
ip addr [...] enp0s25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 54:ee:75:49:dc:46 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global dynamic noprefixroute enp0s25
virt-xml testguest --edit --network bridge=bridge0 Domain 'testguest' defined successfully.
virsh start testguest
ip link show master bridge0 2: enp0s25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master bridge0 state UP mode DEFAULT group default qlen 1000 link/ether 54:ee:75:49:dc:46 brd ff:ff:ff:ff:ff:ff 10: vnet0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master bridge0 state UNKNOWN mode DEFAULT group default qlen 1000 link/ether fe:54:00:89:15:40 brd ff:ff:ff:ff:ff:ff
ip addr [...] enp0s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 52:54:00:09:15:46 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global dynamic noprefixroute enp0s0
ssh root@192.0.2.1 root@192.0.2.1's password: Last login: Mon Sep 24 12:05:36 2019 root~#*
ip addr [...] enp0s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 52:54:00:09:15:46 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global dynamic noprefixroute enp0s0
ssh root@192.0.2.1 root@192.0.2.1's password: Last login: Mon Sep 24 12:05:36 2019 root~#*
chmod -R a+r /var/lib/tftpboot
chown -R nobody: /var/lib/tftpboot
chcon -R --reference /usr/sbin/dnsmasq /var/lib/tftpboot chcon -R --reference /usr/libexec/libvirt_leaseshelper /var/lib/tftpboot
virsh net-destroy default
virsh net-edit default
<ip address='192.0.2.1' netmask='255.255.255.0'> <tftp root='/var/lib/tftpboot'/> <dhcp> <range start='192.0.2.2' end='192.0.2.254' /> <bootp file=' example-pxelinux '/> </dhcp> </ip>
virsh net-start default
virsh net-list Name State Autostart Persistent --------------------------------------------------- default active no no
virt-install --pxe --network network=default --memory 2048 --vcpus 2 --disk size=10
<interface type='network' > <mac address='52:54:00:66:79:14'/> <source network='default'/> <target dev='vnet0'/> <alias name='net0'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> <boot order='1'/> </interface>
virt-install --pxe --network bridge=breth0 --memory 2048 --vcpus 2 --disk size=10
<interface type='bridge' > <mac address='52:54:00:5a:ad:cb'/> <source bridge='breth0'/> <target dev='vnet0'/> <alias name='net0'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> <boot order='1'/> </interface>
nmcli connection add type bond con-name bond0 ifname bond0 bond.options "mode=active-backup"
nmcli connection add type ethernet slave-type bond con-name bond0-port1 ifname enp7s0 master bond0 nmcli connection add type ethernet slave-type bond con-name bond0-port2 ifname enp8s0 master bond0
nmcli connection modify bond0 ipv4.addresses 192.0.2.1/24 ipv4.gateway 192.0.2.254 ipv4.dns 192.0.2.253 ipv4.dns-search example.com ipv4.method manual
nmcli connection modify bond0 ipv6.addresses 2001:db8:1::1/64 ipv6.gateway 2001:db8:1::fffe ipv6.dns 2001:db8:1::fffd ipv6.dns-search example.com ipv6.method manual
nmcli connection modify bond0-port1 bond-port.<parameter> <value>
nmcli connection modify bond0 connection.autoconnect-ports 1
nmcli connection up bond0
cat /proc/net/bonding/bond0
nmcli connection add type bridge con-name br0 ifname br0 ipv4.method disabled ipv6.method disabled
nmcli connection modify bond0 master br0
nmcli connection modify br0 connection.autoconnect-ports 1
nmcli connection up br0
ip link show master br0 6: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue master br0 state UP mode DEFAULT group default qlen 1000 link/ether 52:54:00:38:a9:4d brd ff:ff:ff:ff:ff:ff
bridge link show 6: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 master br0 state forwarding priority 32 cost 100
<network> <name>bond0-bridge</name> <forward mode="bridge" /> <bridge name="br0" /> </network>
virsh net-define ~/bond0-bridge.xml
rm ~/bond0-bridge.xml
virsh net-start bond0-bridge
virsh net-autostart bond0-bridge
virsh net-list Name State Autostart Persistent ---------------------------------------------------- bond0-bridge active yes yes
virt-install ... --network network:bond0-bridge
virt-xml < example_vm > --edit --network network=bond0-bridge
virsh shutdown < example_vm > virsh start < example_vm >
virsh domiflist < example_vm > Interface Type Source Model MAC ------------------------------------------------------------------- vnet1 bridge bond0-bridge virtio 52:54:00:c5:98:1c
ip link show master br0 18: bond0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br0 state UNKNOWN mode DEFAULT group default qlen 1000 link/ether 2a:53:bd:d5:b3:0a brd ff:ff:ff:ff:ff:ff 19: vnet1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br0 state UNKNOWN mode DEFAULT group default qlen 1000 link/ether 52:54:00:c5:98:1c brd ff:ff:ff:ff:ff:ff
dnf install passt
virsh edit <testguest1>
<devices> [...] <interface type='user'> <backend type='passt'/> </interface> </devices>
<devices> [...] <interface type='user'> <backend type='passt'/> <mac address="52:54:00:98:d8:b7"/> <source dev='eth0'/> <ip family='ipv4' address='192.0.2.1' prefix='24'/> <ip family='ipv6' address='::ffff:c000:201'/> <portForward proto='tcp'> <range start='2022' to='22'/> </portForward> <portForward proto='udp' address='1.2.3.4'> <range start='5000' end='5020' to='6000'/> <range start='5010' end='5015' exclude='yes'/> </portForward> <portForward proto='tcp' address='2001:db8:ac10:fd01::1:10' dev='eth0'> <range start='8080'/> <range start='4433' to='3444'/> </portForward> </interface> </devices>
virsh reboot <vm-name> virsh start <vm-name>
tuned-adm list Available profiles: - balanced - General non-specialized TuneD profile - desktop - Optimize for the desktop use-case [...] - virtual-guest - Optimize for running inside a virtual guest - virtual-host - Optimize for running KVM guests Current active profile: balanced
tuned-adm profile selected-profile
tuned-adm profile virtual-host
tuned-adm profile virtual-guest
tuned-adm active Current active profile: virtual-host
tuned-adm verify Verification succeeded, current system settings match the preset profile. See tuned log file ('/var/log/tuned/tuned.log') for details.
systemctl is-active libvirtd.service active
systemctl stop libvirtd.service systemctl stop libvirtd{,-ro,-admin,-tcp,-tls}.socket
systemctl disable libvirtd.service systemctl disable libvirtd{,-ro,-admin,-tcp,-tls}.socket
for drv in qemu interface network nodedev nwfilter secret storage; do systemctl unmask virtUSD{drv}d.service; systemctl unmask virtUSD{drv}d{,-ro,-admin}.socket; systemctl enable virtUSD{drv}d.service; systemctl enable virtUSD{drv}d{,-ro,-admin}.socket; done
for drv in qemu network nodedev nwfilter secret storage; do systemctl start virtUSD{drv}d{,-ro,-admin}.socket; done
grep listen_tls /etc/libvirt/libvirtd.conf listen_tls = 0
systemctl unmask virtproxyd.service systemctl unmask virtproxyd{,-ro,-admin}.socket systemctl enable virtproxyd.service systemctl enable virtproxyd{,-ro,-admin}.socket systemctl start virtproxyd{,-ro,-admin}.socket
systemctl unmask virtproxyd.service systemctl unmask virtproxyd{,-ro,-admin,-tls}.socket systemctl enable virtproxyd.service systemctl enable virtproxyd{,-ro,-admin,-tls}.socket systemctl start virtproxyd{,-ro,-admin,-tls}.socket
virsh uri qemu:///system
systemctl is-active virtqemud.service active
virsh dumpxml testguest | grep memballoon <memballoon model='virtio'> </memballoon>
virsh dominfo testguest Max memory: 2097152 KiB Used memory: 2097152 KiB
virsh dumpxml testguest | grep memballoon <memballoon model='virtio'> </memballoon>
virsh dominfo testguest Max memory: 2097152 KiB Used memory: 2097152 KiB
virt-xml testguest --edit --memory memory=4096,currentMemory=4096 Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virsh setmem testguest --current 2048
virsh dominfo testguest Max memory: 4194304 KiB Used memory: 2097152 KiB
virsh domstats --balloon testguest Domain: 'testguest' balloon.current=365624 balloon.maximum=4194304 balloon.swap_in=0 balloon.swap_out=0 balloon.major_fault=306 balloon.minor_fault=156117 balloon.unused=3834448 balloon.available=4035008 balloon.usable=3746340 balloon.last-update=1587971682 balloon.disk_caches=75444 balloon.hugetlb_pgalloc=0 balloon.hugetlb_pgfail=0 balloon.rss=1005456
virsh edit testguest
<memoryBacking> <hugepages> <page size='1' unit='GiB'/> </hugepages> </memoryBacking>
cat /proc/meminfo | grep Huge HugePages_Total: 4 HugePages_Free: 2 HugePages_Rsvd: 1 Hugepagesize: 1024000 kB
grubby --update-kernel=ALL --remove-args=memhp_default_state --args=memhp_default_state=online_movable
grubby --update-kernel=ALL --remove-args=memhp_default_state --args=memhp_default_state=online_kernel
grubby --update-kernel=ALL --remove-args=memhp_default_state --args=memhp_default_state=online
grubby --update-kernel=ALL --remove-args="memory_hotplug.online_policy" --args=memory_hotplug.online_policy=auto-movable
grubby --update-kernel=ALL --remove-args="memory_hotplug.auto_movable_ratio" --args=memory_hotplug.auto_movable_ratio= <percentage> grubby --update-kernel=ALL --remove-args="memory_hotplug.memory_auto_movable_numa_aware" --args=memory_hotplug.auto_movable_numa_aware= <y/n>
cat /sys/devices/system/memory/auto_online_blocks online_movable
cat /sys/devices/system/memory/auto_online_blocks online_kernel
cat /sys/devices/system/memory/auto_online_blocks online
cat /sys/module/memory_hotplug/parameters/online_policy auto-movable
cat /sys/module/memory_hotplug/parameters/auto_movable_ratio 301
cat /sys/module/memory_hotplug/parameters/auto_movable_numa_aware y
virsh edit testguest1 <domain type='kvm'> <name>testguest1</name> <maxMemory unit='GiB'>128</maxMemory> </domain>
vim virtio-mem-device.xml
<memory model='virtio-mem'> <target> <size unit='GiB'>48</size> <node>0</node> <block unit='MiB'>2</block> <requested unit='GiB'>16</requested> <current unit='GiB'>16</current> </target> <alias name='ua-virtiomem0'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> </memory> <memory model='virtio-mem'> <target> <size unit='GiB'>48</size> <node>1</node> <block unit='MiB'>2</block> <requested unit='GiB'>0</requested> <current unit='GiB'>0</current> </target> <alias name='ua-virtiomem1'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> </memory>
virsh attach-device testguest1 virtio-mem-device.xml --live --config
virsh update-memory-device testguest1 --alias ua-virtiomem0 --requested-size 4GiB
virsh detach-device testguest1 virtio-mem-device.xml
virsh update-memory-device testguest1 --alias ua-virtiomem0 --requested-size 0
virsh detach-device testguest1 virtio-mem-device.xml --config
free -h total used free shared buff/cache available Mem: 31Gi 5.5Gi 14Gi 1.3Gi 11Gi 23Gi Swap: 8.0Gi 0B 8.0Gi
numactl -H available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 node 0 size: 29564 MB node 0 free: 13351 MB node distances: node 0 0: 10
virsh dumpxml testguest1 <domain type='kvm'> <name>testguest1</name> <currentMemory unit='GiB'>31</currentMemory> <memory model='virtio-mem'> <target> <size unit='GiB'>48</size> <node>0</node> <block unit='MiB'>2</block> <requested unit='GiB'>16</requested> <current unit='GiB'>16</current> </target> <alias name='ua-virtiomem0'/> <address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/> </domain>
<domain> [...] <blkiotune> <weight>800</weight> <device> <path>/dev/sda</path> <weight>1000</weight> </device> <device> <path>/dev/sdb</path> <weight>500</weight> </device> </blkiotune> [...] </domain>
virsh blkiotune VM-name --device-weights device , I/O-weight
virsh blkiotune testguest1 --device-weights /dev/sda, 500
virsh blkiotune testguest1 Block I/O tuning parameters for domain testguest1: weight : 800 device_weight : [ {"sda": 500}, ]
virsh domblklist rollin-coal Target Source ------------------------------------------------ vda /var/lib/libvirt/images/rollin-coal.qcow2 sda - sdb /home/horridly-demanding-processes.iso
lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT zram0 252:0 0 4G 0 disk [SWAP] nvme0n1 259:0 0 238.5G 0 disk ├─nvme0n1p1 259:1 0 600M 0 part /boot/efi ├─nvme0n1p2 259:2 0 1G 0 part /boot └─nvme0n1p3 259:3 0 236.9G 0 part └─luks-a1123911-6f37-463c-b4eb-fxzy1ac12fea 253:0 0 236.9G 0 crypt /home
virsh blkiotune VM-name --parameter device , limit
virsh blkiotune rollin-coal --device-read-iops-sec /dev/nvme0n1p3,1000 --device-write-iops-sec /dev/nvme0n1p3,1000 --device-write-bytes-sec /dev/nvme0n1p3,52428800 --device-read-bytes-sec /dev/nvme0n1p3,52428800
virsh edit <example_vm>
<disk type='block' device='disk'> <driver name='qemu' type='raw' queues='N' /> <source dev='/dev/sda'/> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> </disk>
<controller type='scsi' index='0' model='virtio-scsi'> <driver queues='N' /> </controller>
virsh edit <testguest1> <domain type='kvm'> <name>testguest1</name> <vcpu placement='static'>8</vcpu> <iothreads>1</iothreads> </domain>
virsh edit <testguest1> <domain type='kvm'> <name>testguest1</name> <devices> <disk type='file' device='disk'> <driver name='qemu' type='raw' cache='none' io='native' iothread='1' /> <source file='/var/lib/libvirt/images/test-disk.raw'/> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/> </disk> </devices> </domain>
virsh edit <testguest1> <domain type='kvm'> <name>testguest1</name> <devices> <controller type='scsi' index='0' model='virtio-scsi'> <driver iothread='1' /> <address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/> </controller> </devices> </domain>
virsh edit <vm_name>
<domain type='kvm'> <name>testguest1</name> <devices> <disk type='file' device='disk'> <driver name='qemu' type='raw' cache='none' io='native' iothread='1'/> <source file='/var/lib/libvirt/images/test-disk.raw'/> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/> </disk> </devices> </domain>
virt-xml testguest1 --edit --cpu host-model
virsh vcpucount testguest maximum config 4 maximum live 2 current config 2 current live 1
virsh setvcpus testguest 8 --maximum --config
virsh setvcpus testguest 4 --live
virsh setvcpus testguest 1 --config
virsh vcpucount testguest maximum config 8 maximum live 4 current config 1 current live 4
virsh nodeinfo CPU model: x86_64 CPU(s): 48 CPU frequency: 1200 MHz CPU socket(s): 1 Core(s) per socket: 12 Thread(s) per core: 2 NUMA cell(s): 2 Memory size: 67012964 KiB
dnf install numactl
virt-xml testguest5 --edit --vcpus placement=auto virt-xml testguest5 --edit --numatune mode=preferred
echo 1 > /proc/sys/kernel/numa_balancing
systemctl start numad
numactl --hardware available: 2 nodes (0-1) node 0 size: 18156 MB node 0 free: 9053 MB node 1 size: 18180 MB node 1 free: 6853 MB node distances: node 0 1 0: 10 20 1: 20 10
virsh edit <testguest6> <domain type='kvm'> <name>testguest6</name> <vcpu placement='static'>16</vcpu> <cpu ...> <numa> <cell id='0' cpus='0-7' memory='16' unit='GiB'/> <cell id='1' cpus='8-15' memory='16' unit='GiB'/> </numa> </domain>
lscpu -p=node,cpu Node,CPU 0,0 0,1 0,2 0,3 0,4 0,5 0,6 0,7 1,0 1,1 1,2 1,3 1,4 1,5 1,6 1,7
lscpu -p=node,cpu Node,CPU 0,0 0,1 0,2 0,3
virsh vcpupin testguest6 0 1 virsh vcpupin testguest6 1 3 virsh vcpupin testguest6 2 5 virsh vcpupin testguest6 3 7
virsh vcpupin testguest6 VCPU CPU Affinity ---------------------- 0 1 1 3 2 5 3 7
virsh emulatorpin testguest6 2,4 virsh emulatorpin testguest6 emulator: CPU Affinity ---------------------------------- *: 2,4
virsh schedinfo <vm_name> Scheduler : posix cpu_shares : 0 vcpu_period : 0 vcpu_quota : 0 emulator_period: 0 emulator_quota : 0 global_period : 0 global_quota : 0 iothread_period: 0 iothread_quota : 0
virsh schedinfo <vm_name> --set vcpu_period=100000
virsh schedinfo <vm_name> --set vcpu_quota=50000
virsh schedinfo <vm_name> Scheduler : posix cpu_shares : 2048 vcpu_period : 100000 vcpu_quota : 50000
virsh schedinfo <vm_name> Scheduler : posix cpu_shares : 1024 vcpu_period : 0 vcpu_quota : 0 emulator_period: 0 emulator_quota : 0 global_period : 0 global_quota : 0 iothread_period: 0 iothread_quota : 0
virsh schedinfo <vm_name> --set cpu_shares=2048 Scheduler : posix cpu_shares : 2048 vcpu_period : 0 vcpu_quota : 0 emulator_period: 0 emulator_quota : 0 global_period : 0 global_quota : 0 iothread_period: 0 iothread_quota : 0
{PackageManagerCommand} install ksmtuned
systemctl start ksm systemctl start ksmtuned
systemctl enable ksm Created symlink /etc/systemd/system/multi-user.target.wants/ksm.service -> /usr/lib/systemd/system/ksm.service systemctl enable ksmtuned Created symlink /etc/systemd/system/multi-user.target.wants/ksmtuned.service -> /usr/lib/systemd/system/ksmtuned.service
systemctl stop ksm systemctl stop ksmtuned
systemctl disable ksm Removed /etc/systemd/system/multi-user.target.wants/ksm.service. systemctl disable ksmtuned Removed /etc/systemd/system/multi-user.target.wants/ksmtuned.service.
echo 2 > /sys/kernel/mm/ksm/run
lsmod | grep vhost vhost_net 32768 1 vhost 53248 1 vhost_net tap 24576 1 vhost_net tun 57344 6 vhost_net
modprobe vhost_net
<interface type='network'> <source network='default'/> <model type='virtio'/> <driver name='vhost' queues='N'/> </interface>
ethtool -C tap0 rx-frames 64
dnf install perf
perf kvm stat report Analyze events for all VMs, all VCPUs: VM-EXIT Samples Samples% Time% Min Time Max Time Avg time EXTERNAL_INTERRUPT 365634 31.59% 18.04% 0.42us 58780.59us 204.08us ( +- 0.99% ) MSR_WRITE 293428 25.35% 0.13% 0.59us 17873.02us 1.80us ( +- 4.63% ) PREEMPTION_TIMER 276162 23.86% 0.23% 0.51us 21396.03us 3.38us ( +- 5.19% ) PAUSE_INSTRUCTION 189375 16.36% 11.75% 0.72us 29655.25us 256.77us ( +- 0.70% ) HLT 20440 1.77% 69.83% 0.62us 79319.41us 14134.56us ( +- 0.79% ) VMCALL 12426 1.07% 0.03% 1.02us 5416.25us 8.77us ( +- 7.36% ) EXCEPTION_NMI 27 0.00% 0.00% 0.69us 1.34us 0.98us ( +- 3.50% ) EPT_MISCONFIG 5 0.00% 0.00% 5.15us 10.85us 7.88us ( +- 11.67% ) Total Samples:1157497, Total events handled time:413728274.66us.
numastat -c qemu-kvm Per-node process memory usage (in MBs) PID Node 0 Node 1 Node 2 Node 3 Node 4 Node 5 Node 6 Node 7 Total --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- 51722 (qemu-kvm) 68 16 357 6936 2 3 147 598 8128 51747 (qemu-kvm) 245 11 5 18 5172 2532 1 92 8076 53736 (qemu-kvm) 62 432 1661 506 4851 136 22 445 8116 53773 (qemu-kvm) 1393 3 1 2 12 0 0 6702 8114 --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- Total 1769 463 2024 7462 10037 2672 169 7837 32434
numastat -c qemu-kvm Per-node process memory usage (in MBs) PID Node 0 Node 1 Node 2 Node 3 Node 4 Node 5 Node 6 Node 7 Total --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- 51747 (qemu-kvm) 0 0 7 0 8072 0 1 0 8080 53736 (qemu-kvm) 0 0 7 0 0 0 8113 0 8120 53773 (qemu-kvm) 0 0 7 0 0 0 1 8110 8118 59065 (qemu-kvm) 0 0 8050 0 0 0 0 0 8051 --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- Total 0 0 8072 0 8072 0 8114 8110 32368
getenforce Enforcing
dnf install edk2-ovmf
virt-install --name rhel8sb --memory 4096 --vcpus 4 --os-variant rhel9.0 --boot uefi,nvram_template=/usr/share/OVMF/OVMF_VARS.secboot.fd --disk boot_order=2,size=10 --disk boot_order=1,device=cdrom,bus=scsi,path=/images/RHEL-9.0-installation.iso
mokutil --sb-state SecureBoot enabled
ls /usr/share/polkit-1/actions | grep libvirt ls /usr/share/polkit-1/rules.d | grep libvirt
find /etc/libvirt/ -name virt*d.conf -exec sed -i 's/#access_drivers = \[ "polkit" \]/access_drivers = \[ "polkit" \]/g' {} +
systemctl try-restart virtqemud
virsh -c qemu:///system list --all Id Name State -------------------------------
getsebool -a | grep virt [...] virt_sandbox_use_netlink --> off virt_sandbox_use_sys_admin --> off virt_transition_userdomain --> off virt_use_comm --> off virt_use_execmem --> off virt_use_fusefs --> off [...]
grep facilities /proc/cpuinfo | grep 158
ls /sys/firmware | grep uv
virsh domcapabilities | grep unpack <feature policy='require' name='unpack'/>
virsh dumpxml <vm_name> | grep "<cpu mode='host-model'/>"
dnf install genprotimg
dnf install guestfs-tools
grubby --update-kernel=ALL --args="prot_virt=1"
[...] </memballoon> </devices> <launchSecurity type="s390-pv"/> </domain>
touch ~/secure-parameters
ls /boot/loader/entries -l [...] -rw-r--r--. 1 root root 281 Oct 9 15:51 3ab27a195c2849429927b00679db15c1-4.18.0-240.el8.s390x.conf
cat /boot/loader/entries/3ab27a195c2849429927b00679db15c1-4.18.0-240.el8.s390x.conf | grep options options root=/dev/mapper/rhel-root rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap
echo "root=/dev/mapper/rhel-root rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap swiotlb=262144" > ~/secure-parameters
genprotimg -i /boot/vmlinuz-4.18.0-240.el8.s390x -r /boot/initramfs-4.18.0-240.el8.s390x.img -p ~/secure-parameters -k HKD-8651-00020089A8.crt -o /boot/secure-image
cat /boot/loader/entries/3ab27a195c2849429927b00679db15c1-4.18.0-240.el8.s390x.conf title Red Hat Enterprise Linux 8.3 version 4.18.0-240.el8.s390x linux /boot/secure-image [...]
zipl -V
shred /boot/vmlinuz-4.18.0-240.el8.s390x shred /boot/initramfs-4.18.0-240.el8.s390x.img shred secure-parameters
#!/usr/bin/bash echo "USD(cat /proc/cmdline) swiotlb=262144" > parmfile cat > ./HKD.crt << EOF -----BEGIN CERTIFICATE----- 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 xLPRGYwhmXzKDg== -----END CERTIFICATE----- EOF version=USD(uname -r) kernel=/boot/vmlinuz-USDversion initrd=/boot/initramfs-USDversion.img genprotimg -k ./HKD.crt -p ./parmfile -i USDkernel -r USDinitrd -o /boot/secure-linux --no-verify cat >> /etc/zipl.conf<< EOF [secure] target=/boot image=/boot/secure-linux EOF zipl -V shutdown -h now
virt-customize -a <vm_image_path> --selinux-relabel --firstboot <script_path>
virsh dumpxml vm-name [...] <cpu mode='host-model'/> <devices> <disk type='file' device='disk'> <driver name='qemu' type='qcow2' cache='none' io='native'> <source file='/var/lib/libvirt/images/secure-guest.qcow2'/> <target dev='vda' bus='virtio'/> </disk> <interface type='network'> <source network='default'/> <model type='virtio'/> </interface> <console type='pty'/> <memballoon model='none'/> </devices> <launchSecurity type="s390-pv"/> </domain>
lszcrypt -V CARD.DOMAIN TYPE MODE STATUS REQUESTS PENDING HWTYPE QDEPTH FUNCTIONS DRIVER -------------------------------------------------------------------------------------------- 05 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4card 05.0004 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue 05.00ab CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue
lsmod | grep vfio_ap vfio_ap 24576 0 [...]
modprobe vfio_ap
lszdev --list-types ap Cryptographic Adjunct Processor (AP) device
echo "obase=10; ibase=16; 04" | bc 4 echo "obase=10; ibase=16; AB" | bc 171
chzdev -t ap apmask=-5 aqmask=-4,-171
lszcrypt -V CARD.DOMAIN TYPE MODE STATUS REQUESTS PENDING HWTYPE QDEPTH FUNCTIONS DRIVER -------------------------------------------------------------------------------------------- 05 CEX5C CCA-Coproc - 1 0 11 08 S--D--N-- cex4card 05.0004 CEX5C CCA-Coproc - 1 0 11 08 S--D--N-- vfio_ap 05.00ab CEX5C CCA-Coproc - 1 0 11 08 S--D--N-- vfio_ap
vim vfio_ap.xml <device> <parent>ap_matrix</parent> <capability type="mdev"> <type id="vfio_ap-passthrough"/> <attr name='assign_adapter' value='0x05'/> <attr name='assign_domain' value='0x0004'/> <attr name='assign_domain' value='0x00ab'/> <attr name='assign_control_domain' value='0x00ab'/> </capability> </device>
virsh nodedev-define vfio_ap.xml Node device 'mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix' defined from 'vfio_ap.xml'
virsh nodedev-start mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix Device mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix started
cat /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/ 669d9b23-fe1b-4ecb-be08-a2fabca99b71 /matrix 05.0004 05.00ab
virsh nodedev-dumpxml mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix <device> <name>mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix</name> <parent>ap_matrix</parent> <capability type='mdev'> <type id='vfio_ap-passthrough'/> <uuid>8f9c4a73-1411-48d2-895d-34db9ac18f85</uuid> <iommuGroup number='0'/> <attr name='assign_adapter' value='0x05'/> <attr name='assign_domain' value='0x0004'/> <attr name='assign_domain' value='0x00ab'/> <attr name='assign_control_domain' value='0x00ab'/> </capability> </device>
vim crypto-dev.xml
<hostdev mode='subsystem' type='mdev' managed='no' model='vfio-ap'> <source> <address uuid='8f9c4a73-1411-48d2-895d-34db9ac18f85'/> </source> </hostdev>
virsh attach-device testguest1 crypto-dev.xml --live --config
lszcrypt -V CARD.DOMAIN TYPE MODE STATUS REQUESTS PENDING HWTYPE QDEPTH FUNCTIONS DRIVER -------------------------------------------------------------------------------------------- 05 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4card 05.0004 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue 05.00ab CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue
lszcrypt -d C DOMAIN 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ------------------------------------------------------ 00 . . . . U . . . . . . . . . . . 10 . . . . . . . . . . . . . . . . 20 . . . . . . . . . . . . . . . . 30 . . . . . . . . . . . . . . . . 40 . . . . . . . . . . . . . . . . 50 . . . . . . . . . . . . . . . . 60 . . . . . . . . . . . . . . . . 70 . . . . . . . . . . . . . . . . 80 . . . . . . . . . . . . . . . . 90 . . . . . . . . . . . . . . . . a0 . . . . . . . . . . . B . . . . b0 . . . . . . . . . . . . . . . . c0 . . . . . . . . . . . . . . . . d0 . . . . . . . . . . . . . . . . e0 . . . . . . . . . . . . . . . . f0 . . . . . . . . . . . . . . . . ------------------------------------------------------ C: Control domain U: Usage domain B: Both (Control + Usage domain)
{PackageManagerCommand} install edk2-ovmf
{PackageManagerCommand} install swtpm libtpms
<devices> [...] <tpm model='tpm-crb'> <backend type='emulator' version='2.0'/> </tpm> [...] </devices>
Your device meets the requirements for standard hardware security.
virsh edit Example-L1
<cpu mode='custom' match='exact' check='partial'> <model fallback='allow'>Skylake-Client-IBRS</model> <topology sockets='1' dies='1' cores='4' threads='1'/> <feature policy='require' name='vmx'/> </cpu>
Your device meets the requirements for enhanced hardware security.
dnf install nfs-utils -y
mkdir shared-files
virsh domifaddr testguest1 Name MAC address Protocol Address ---------------------------------------------------------------- vnet0 52:53:00:84:57:90 ipv4 192.0.2.2/24 virsh domifaddr testguest2 Name MAC address Protocol Address ---------------------------------------------------------------- vnet1 52:53:00:65:29:21 ipv4 192.0.2.3/24
/home/<username>/Downloads/<shared_directory>/ <VM1-IP(options)> <VM2-IP(options)>
/usr/local/shared-files/ 192.0.2.2(rw,sync) 192.0.2.3(rw,sync)
exportfs -a
systemctl start nfs-server
ip addr 5: virbr0: [BROADCAST,MULTICAST,UP,LOWER_UP] mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 52:54:00:32:ff:a5 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global virbr0 valid_lft forever preferred_lft forever
mount 192.0.2.1:/usr/local/shared-files /mnt/host-share
Install-WindowsFeature NFS-Client
Enable-WindowsOptionalFeature -FeatureName ServicesForNFS-ClientOnly, ClientForNFS-Infrastructure -Online -NoRestart
C:\Windows\system32\mount.exe -o anon \\192.0.2.1\usr\local\shared-files Z:
ls <mount_point> shared-file1 shared-file2 shared-file3
mkdir /root/shared-files
virsh edit vm-name
<filesystem type='mount' accessmode='passthrough'> <driver type='virtiofs'/> <binary path='/usr/libexec/virtiofsd' xattr='on'/> <source dir='/root/shared-files'/> <target dir='host-file-share'/> </filesystem>
<domain> [...] <memoryBacking> <access mode='shared'/> </memoryBacking> [...] </domain>
virsh start vm-name
mount -t virtiofs host-file-share /mnt
sc start VirtioFsSvc
sc stop VirtioFsSvc sc config VirtioFsSvc start=demand
"C:\Program Files (x86)\WinFsp\bin\fsreg.bat" virtiofs " <path to the binary> \virtiofs.exe" "-t %1 -m %2"
"C:\Program Files (x86)\WinFsp\bin\launchctl-x64.exe" start virtiofs viofsY mount_tag0 Y:
"C:\Program Files (x86)\WinFsp\bin\launchctl-x64.exe" stop virtiofs viofsY
mkdir /home/centurion
--disk path= /usr/share/virtio-win/virtio-win.iso ,device=cdrom
--os-variant win10
osinfo-query os
--boot uefi
virsh edit windows-vm
<os firmware='efi' > <type arch='x86_64' machine='pc-q35-6.2'>hvm</type> <boot dev='hd'/> </os>
<devices> <tpm model='tpm-crb'> <backend type='emulator' version='2.0'/> </tpm> </devices>
subscription-manager refresh All local data refreshed
dnf install -y virtio-win
dnf upgrade -y virtio-win
ls /usr/share/virtio-win/ drivers/ guest-agent/ virtio-win-1.9.9.iso virtio-win.iso
virt-xml WindowsVM --add-device --disk virtio-win.iso,device=cdrom Domain 'WindowsVM' defined successfully.
C:\WINDOWS\system32\netsh dump > backup.txt
C:\WINDOWS\system32\msiexec.exe /i X :\virtio-win-gt-x86.msi /passive /norestart
C:\WINDOWS\system32\netsh -f backup.txt
virsh edit windows-vm
<features> [...] <hyperv> <relaxed state='on'/> <vapic state='on'/> <spinlocks state='on' retries='8191'/> <vendor_id state='on' value='KVM Hv'/> <vpindex state='on'/> <runtime state='on' /> <synic state='on'/> <stimer state='on'> <direct state='on'/> </stimer> <frequencies state='on'/> <reset state='on'/> <tlbflush state='on'/> <reenlightenment state='on'/> <ipi state='on'/> <evmcs state='on'/> </hyperv> [...] </features>
<clock offset='localtime'> <timer name='hypervclock' present='yes'/> </clock>
<hyperv> <relaxed state='on'/> <vapic state='on'/> <spinlocks state='on' retries='8191'/> <vendor_id state='on' value='KVM Hv'/> <vpindex state='on'/> <runtime state='on' /> <synic state='on'/> <frequencies state='on'/> <reset state='on'/> <tlbflush state='on'/> <reenlightenment state='on'/> <stimer state='on'> <direct state='on'/> </stimer> <ipi state='on'/> <evmcs state='on'/> </hyperv> <clock offset='localtime'> <timer name='hypervclock' present='yes'/> </clock>
bcdedit /set useplatformclock No
{PackageManagerCommand} install edk2-ovmf
{PackageManagerCommand} install swtpm libtpms
<devices> [...] <tpm model='tpm-crb'> <backend type='emulator' version='2.0'/> </tpm> [...] </devices>
Your device meets the requirements for standard hardware security.
virsh edit Example-L1
<cpu mode='custom' match='exact' check='partial'> <model fallback='allow'>Skylake-Client-IBRS</model> <topology sockets='1' dies='1' cores='4' threads='1'/> <feature policy='require' name='vmx'/> </cpu>
Your device meets the requirements for enhanced hardware security.
sudo dnf install libguestfs-tools libguestfs-winsupport
sudo dnf install guestfs-tools guestfs-winsupport
modprobe kvm hpage=1 nested=1 modprobe: ERROR: could not insert 'kvm': Invalid argument dmesg |tail -1 [90226.508366] kvm-s390: A KVM host that supports nesting cannot back its KVM guests with huge pages
cat /sys/module/kvm_intel/parameters/nested
modprobe -r kvm_intel
modprobe kvm_intel nested=1
options kvm_intel nested=1
virsh edit Intel-L1
<cpu mode='host-passthrough' />
<cpu mode ='custom' match ='exact' check='partial'> <model fallback='allow'> Haswell-noTSX </model> <feature policy='require' name='vmx'/> </cpu>
cat /sys/module/kvm_amd/parameters/nested
modprobe -r kvm_amd
modprobe kvm_amd nested=1
options kvm_amd nested=1
virsh edit AMD-L1
<cpu mode='host-passthrough' />
<cpu mode="custom" match="exact" check="none"> <model fallback="allow"> EPYC-IBPB </model> <feature policy="require" name="svm"/> </cpu>
cat /sys/module/kvm/parameters/nested
modprobe -r kvm
modprobe kvm nested=1
options kvm nested=1
log_filters="3:remote 4:event 3:util.json 3:rpc" log_outputs="1:file:/var/log/libvirt/libvirt.log"
systemctl restart virtqemud.service
virt-admin -c virtqemud:///system daemon-log-filters >> virt-filters-backup
virt-admin -c virtqemud:///system daemon-log-filters "3:remote 4:event 3:util.json 3:rpc"
virt-admin -c virtqemud:///system daemon-log-outputs "1:file:/var/log/libvirt/libvirt.log"
virt-admin -c virtqemud:///system daemon-log-filters Logging filters:
virsh dump lander1 /core/file/gargantua.file --memory-only Domain 'lander1' dumped to /core/file/gargantua.file
crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M
pgrep libvirt 22014 22025
gstack 22014 Thread 3 (Thread 0x7f33edaf7700 (LWP 22017)): #0 0x00007f33f81aef21 in poll () from /lib64/libc.so.6 #1 0x00007f33f89059b6 in g_main_context_iterate.isra () from /lib64/libglib-2.0.so.0 #2 0x00007f33f8905d72 in g_main_loop_run () from /lib64/libglib-2.0.so.0
virsh dumpxml VM-name | grep machine=
/usr/libexec/qemu-kvm -M help
<domain type='qemu'>
<domain type='kvm'>
<disk type="block" device="lun">
<hostdev mode='subsystem' type='scsi'>
<disk type='file' device='disk'> <driver name='qemu' type='qcow2'/> <source file='/path/to/qcow2'/> <target dev='vda' bus='virtio'/> <address type='ccw' cssid='0xfe' ssid='0x0' devno='0x0000'/> <boot order='1' loadparm='2'/> </disk>
<devices> <watchdog model='diag288' action='poweroff'/> </devices>
pxelinux default linux label linux kernel kernel.img initrd initrd.img append ip=dhcp inst.repo=example.com/redhat/BaseOS/s390x/os/
%packages -kernel kernel-64k %end
grep -m 1 Features /proc/cpuinfo | grep -w sve Features: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm fcma dcpop sve
virsh edit vm-name
<cpu mode='host-passthrough' check='none'> <feature policy='require' name='sve'/> <feature policy='require' name='sve128'/> <feature policy='require' name='sve256'/> <feature policy='disable' name='sve384'/> <feature policy='require' name='sve512'/> </cpu>