Configuring and managing virtualization Red Hat Enterprise Linux 8 Setting up your host, creating and administering virtual machines, and understanding virtualization features in Red Hat Enterprise Linux 8 Red Hat Customer Content Services

Commands:
yum module install virt
yum install virt-install virt-viewer
systemctl start libvirtd
virt-host-validate [...] QEMU: Checking for device assignment IOMMU support : PASS QEMU: Checking if IOMMU is enabled by kernel : WARN (IOMMU appears to be disabled in kernel. Add intel_iommu=on to kernel cmdline arguments) LXC: Checking for Linux >= 2.6.26 : PASS [...] LXC: Checking for cgroup 'blkio' controller mount-point : PASS LXC: Checking if device /sys/fs/fuse/connections exists : FAIL (Load the 'fuse' module to enable /proc/ overrides)
QEMU: Checking for hardware virtualization: FAIL (Only emulated CPUs are available, performance will be significantly limited)
virt-install --graphics vnc --name demo-guest1 --memory 2048 --vcpus 2 --disk size=80 --os-variant win10 --cdrom /home/username/Downloads/Win10install.iso
virt-install --graphics vnc --name demo-guest2 --memory 4096 --vcpus 4 --disk none --livecd --os-variant rhel8.0 --cdrom /home/username/Downloads/rhel8.iso
virt-install --graphics vnc --name demo-guest3 --memory 2048 --vcpus 2 --os-variant rhel8.0 --import --disk /home/username/backup/disk.qcow2
virt-install --graphics vnc --name demo-guest4 --memory 2048 --vcpus 2 --disk size=160 --os-variant rhel8.0 --location http://example.com/OS-install --initrd-inject /home/username/ks.cfg --extra-args="inst.ks=file:/ks.cfg console=tty0 console=ttyS0,115200n8"
virt-install --name demo-guest5 --memory 16384 --vcpus 16 --disk size=280 --os-variant rhel8.0 --location RHEL8.iso --graphics none --extra-args='console=ttyS0'
virt-install --connect qemu+ssh://root@192.0.2.1/system --name demo-guest6 --memory 16384 --vcpus 16 --disk size=280 --os-variant rhel8.0 --location RHEL8.iso --graphics none --extra-args='console=ttyS0'
{PackageManagerCommand} info libvirt-daemon-config-network Installed Packages Name : libvirt-daemon-config-network [...]
virsh net-list --all Name State Autostart Persistent -------------------------------------------- default active yes yes
virsh net-autostart default Network default marked as autostarted virsh net-start default Network default started
error: failed to get network 'default' error: Network not found: no network with matching name 'default'
{PackageManagerCommand} reinstall libvirt-daemon-config-network
error: Failed to start network default error: internal error: Network is already in use by interface ens2
virsh start demo-guest1 Domain 'demo-guest1' started
virsh -c qemu+ssh://root@192.0.2.1/system start demo-guest1 root@192.0.2.1's password: Domain 'demo-guest1' started
virsh autostart demo-guest1 Domain ' demo-guest1 ' marked as autostarted
mkdir -p /etc/systemd/system/libvirtd.service.d/
touch /etc/systemd/system/libvirtd.service.d/10-network-online.conf
[Unit] After=network-online.target
virsh dominfo demo-guest1 Id: 2 Name: demo-guest1 UUID: e46bc81c-74e2-406e-bd7a-67042bae80d1 OS Type: hvm State: running CPU(s): 2 CPU time: 385.9s Max memory: 4194304 KiB Used memory: 4194304 KiB Persistent: yes Autostart: enable Managed save: no Security model: selinux Security DOI: 0 Security label: system_u:system_r:svirt_t:s0:c873,c919 (enforcing)
cat /etc/systemd/system/libvirtd.service.d/10-network-online.conf [Unit] After=network-online.target
virt-viewer guest-name
virt-viewer --direct --connect qemu+ssh://root@192.0.2.1/system guest-name root@192.0.2.1's password:
yum install libvirt-nss
passwd: compat shadow: compat group: compat hosts: files libvirt_guest dns
ssh root@192.0.2.1 root@192.0.2.1's password: Last login: Mon Sep 24 12:05:36 2021 root~#
ssh root@testguest1 root@testguest1's password: Last login: Wed Sep 12 12:05:36 2018 root~]#
virsh list --all Id Name State ---------------------------------------------------- 2 testguest1 running - testguest2 shut off
sudo grep GRUB_TERMINAL /etc/default/grub GRUB_TERMINAL=serial
virsh dumpxml vm-name | grep console <console type='pty' tty='/dev/pts/2'> </console>
cat /proc/cmdline BOOT_IMAGE=/vmlinuz-3.10.0-948.el7.x86_64 root=/dev/mapper/rhel-root ro console=tty0 console=ttyS0 ,9600n8 rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb
grubby --update-kernel=ALL --args="console=ttyS0"
grub2-editenv - unset kernelopts
systemctl status serial-getty@ttyS0.service ○ serial-getty@ttyS0.service - Serial Getty on ttyS0 Loaded: loaded (/usr/lib/systemd/system/serial-getty@.service; enabled; preset: enabled)
virsh console guest1 --safe Connected to domain 'guest1' Escape character is ^] Subscription-name Kernel 3.10.0-948.el7.x86_64 on an x86_64 localhost login:
virsh -c qemu+ssh://root@ 192.0.2.1/system list root@192.0.2.1's password: Id Name State --------------------------------- 1 remote-guest running
virsh -c remote-host list root@192.0.2.1's password: Id Name State --------------------------------- 1 remote-guest running
vi ~/.ssh/config Host example-host-alias User root Hostname 192.0.2.1
vi /etc/libvirt/libvirt.conf uri_aliases = [ " example-qemu-host-alias =qemu+ssh:// example-host-alias /system", ]
virsh -c example-qemu-host-alias list root@192.0.2.1's password: Id Name State ---------------------------------------- 1 example-remote-guest running
These can be used in cases when no URI is supplied by the application (@uri_default also prevents probing of the hypervisor driver). # uri_default = "example-qemu-host-alias"
virsh list root@192.0.2.1's password: Id Name State --------------------------------- 1 example-remote-guest running
virsh dumpxml <vm-name> | grep graphics <graphics type='vnc' ports='-1' autoport= yes listen= 127.0.0.1 > </graphics>
virsh edit <vm-name>
<graphics type='vnc' ports='-1' autoport= yes listen= 127.0.0.1 passwd=' <password> '>
<graphics type='vnc' ports='-1' autoport= yes listen= 127.0.0.1 passwd=' <password> ' passwdValidTo='2025-02-01T15:30:00' >
virsh start <vm-name>
virt-viewer <vm-name>
virsh shutdown demo-guest1 Domain 'demo-guest1' is being shutdown
virsh -c qemu+ssh://root@192.0.2.1/system shutdown demo-guest1 root@192.0.2.1's password: Domain 'demo-guest1' is being shutdown
virsh destroy demo-guest1 Domain 'demo-guest1' destroyed
virsh list --all Id Name State ------------------------------------------ 1 demo-guest1 shut off
virsh undefine guest1 --remove-all-storage --nvram Domain 'guest1' has been undefined Volume 'vda'(/home/images/guest1.qcow2) removed.
grep ^platform /proc/cpuinfo/ platform : PowerNV
modprobe kvm_hv
lsmod | grep kvm
yum module install virt
yum install virt-install
systemctl start libvirtd
virt-host-validate [...] QEMU: Checking if device /dev/vhost-net exists : PASS QEMU: Checking if device /dev/net/tun exists : PASS QEMU: Checking for cgroup 'memory' controller support : PASS QEMU: Checking for cgroup 'memory' controller mount-point : PASS [...] QEMU: Checking for cgroup 'blkio' controller support : PASS QEMU: Checking for cgroup 'blkio' controller mount-point : PASS QEMU: Checking if IOMMU is enabled by kernel : PASS
QEMU: Checking for hardware virtualization: FAIL (Only emulated CPUs are available, performance will be significantly limited)
qemu-kvm: Failed to allocate KVM HPT of order 33 (try smaller maxmem?): Cannot allocate memory
grep sie /proc/cpuinfo features : esan3 zarch stfle msa ldisp eimm dfp edat etf3eh highgprs te sie
modprobe kvm
lsmod | grep kvm
yum module install virt:rhel/common
for drv in qemu network nodedev nwfilter secret storage interface; do systemctl start virtUSD{drv}d{,-ro,-admin}.socket; done
virt-host-validate [...] QEMU: Checking if device /dev/kvm is accessible : PASS QEMU: Checking if device /dev/vhost-net exists : PASS QEMU: Checking if device /dev/net/tun exists : PASS QEMU: Checking for cgroup 'memory' controller support : PASS QEMU: Checking for cgroup 'memory' controller mount-point : PASS [...]
QEMU: Checking for hardware virtualization: FAIL (Only emulated CPUs are available, performance will be significantly limited)
hostnamectl | grep "Operating System" Operating System: Red Hat Enterprise Linux 8.5 (Ootpa) yum module list --installed [...] Advanced Virtualization for RHEL 8 IBM Z Systems (RPMs) Name Stream Profiles Summary virt av [e] common [i] Virtualization module
yum disable virt:av
yum module reset virt -y
yum update
yum module info virt Name : virt Stream : rhel [d][e][a] Version : 8050020211203195115 [...]
<disk type='file' device='disk'> <driver name='qemu' type='qcow2'/> <source file='/path/to/qcow2'/> <target dev='vda' bus='virtio'/> <address type='ccw' cssid='0xfe' ssid='0x0' devno='0x0000'/> <boot order='1' loadparm='2'/> </disk>
<devices> <watchdog model='diag288' action='poweroff'/> </devices>
pxelinux default linux label linux kernel kernel.img initrd initrd.img append ip=dhcp inst.repo=example.com/redhat/BaseOS/s390x/os/
<cpu mode='host-model' check='partial'> <model fallback='allow'/> </cpu>
<cpu mode='custom' match='exact' check='partial'> <model fallback='allow'>zEC12</model> <feature policy='force' name='ppa15'/> <feature policy='force' name='bpb'/> </cpu>
touch qemuga.xml
<channel type='unix'> <source mode='bind' path='/var/lib/libvirt/qemu/f16x86_64.agent'/> <target type='virtio' name='org.qemu.guest_agent.0'/> </channel>
virsh attach-device <vm-name> qemuga.xml --live --config
virsh attach-device <vm-name> qemuga.xml --config
yum install qemu-guest-agent
systemctl start qemu-guest-agent
setsebool virt_qemu_ga_read_nonsecurity_files on
setsebool virt_qemu_ga_manage_ssh on
yum install cockpit-machines
virsh list --all Id Name State ---------------------------------- 1 testguest1 running - testguest2 shut off - testguest3 shut off - testguest4 shut off
virsh dominfo testguest1 Id: 1 Name: testguest1 UUID: a973666f-2f6e-415a-8949-75a7a98569e1 OS Type: hvm State: running CPU(s): 2 CPU time: 188.3s Max memory: 4194304 KiB Used memory: 4194304 KiB Persistent: yes Autostart: disable Managed save: no Security model: selinux Security DOI: 0 Security label: system_u:system_r:svirt_t:s0:c486,c538 (enforcing)
virsh dumpxml testguest2 <domain type='kvm' id='1'> <name>testguest2</name> <uuid>a973434f-2f6e-4esa-8949-76a7a98569e1</uuid> <metadata> [...]
virsh domblklist testguest3 Target Source --------------------------------------------------------------- vda /var/lib/libvirt/images/testguest3.qcow2 sda - sdb /home/username/Downloads/virt-p2v-1.36.10-1.el7.iso
virsh domfsinfo testguest3 Mountpoint Name Type Target ------------------------------------ / dm-0 xfs /boot vda1 xfs
virsh vcpuinfo testguest4 VCPU: 0 CPU: 3 State: running CPU time: 103.1s CPU Affinity: yyyy VCPU: 1 CPU: 0 State: running CPU time: 88.6s CPU Affinity: yyyy
virsh net-list --all Name State Autostart Persistent --------------------------------------------- default active yes yes labnet active yes yes
virsh net-info default Name: default UUID: c699f9f6-9202-4ca8-91d0-6b8cb9024116 Active: yes Persistent: yes Autostart: yes Bridge: virbr0
virsh dumpxml testguest1
<domain type='kvm'> <name>Testguest1</name> <uuid>ec6fbaa1-3eb4-49da-bf61-bb02fbec4967</uuid> <memory unit='KiB'>1048576</memory> <currentMemory unit='KiB'>1048576</currentMemory>
<vcpu placement='static'>1</vcpu>
<os> <type arch='x86_64' machine='pc-q35-4.1'>hvm</type> <boot dev='hd'/> </os>
<features> <acpi/> <apic/> </features>
<cpu mode='host-model' check='partial'/>
<clock offset='utc'> <timer name='rtc' tickpolicy='catchup'/> <timer name='pit' tickpolicy='delay'/> <timer name='hpet' present='no'/> </clock>
<on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>destroy</on_crash>
<pm> <suspend-to-mem enabled='no'/> <suspend-to-disk enabled='no'/> </pm>
<devices> <emulator>/usr/bin/qemu-kvm</emulator> <disk type='file' device='disk'> <driver name='qemu' type='qcow2'/> <source file='/var/lib/libvirt/images/Testguest.qcow2'/> <target dev='hda' bus='ide'/> </disk> <disk type='file' device='cdrom'> <driver name='qemu' type='raw'/> <target dev='hdb' bus='ide'/> <readonly/> </disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'/> <controller type='sata' index='0'/> <controller type='pci' index='0' model='pcie-root'/> <controller type='pci' index='1' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='1' port='0x10'/> </controller> <controller type='pci' index='2' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='2' port='0x11'/> </controller> <controller type='pci' index='3' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='3' port='0x12'/> </controller> <controller type='pci' index='4' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='4' port='0x13'/> </controller> <controller type='pci' index='5' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='5' port='0x14'/> </controller> <controller type='pci' index='6' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='6' port='0x15'/> </controller> <controller type='pci' index='7' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='7' port='0x16'/> </controller> <controller type='virtio-serial' index='0'/>
<interface type='network'> <mac address='52:54:00:65:29:21'/> <source network='default'/> <model type='rtl8139'/> </interface>
<serial type='pty'> <target type='isa-serial' port='0'> <model name='isa-serial'/> </target> </serial> <console type='pty'> <target type='serial' port='0'/> </console> <channel type='unix'> <target type='virtio' name='org.qemu.guest_agent.0'/> <address type='virtio-serial' controller='0' bus='0' port='1'/> </channel> <channel type='spicevmc'> <target type='virtio' name='com.redhat.spice.0'/> <address type='virtio-serial' controller='0' bus='0' port='2'/> </channel>
<input type='tablet' bus='usb'> <address type='usb' bus='0' port='1'/> </input> <input type='mouse' bus='ps2'/> <input type='keyboard' bus='ps2'/>
<graphics type='spice' autoport='yes' listen='127.0.0.1'> <listen type='address' address='127.0.0.1'/> <image compression='off'/> </graphics> <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1'> <listen type='address' address='127.0.0.1'/> </graphics>
<sound model='ich6'> <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> </sound> <video> <model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> </video>
<redirdev bus='usb' type='spicevmc'> <address type='usb' bus='0' port='1'/> </redirdev> <redirdev bus='usb' type='spicevmc'> <address type='usb' bus='0' port='2'/> </redirdev> <memballoon model='virtio'> <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/> </memballoon> </devices> </domain>
virsh managedsave demo-guest1 Domain 'demo-guest1' saved by libvirt
virsh list --managed-save --all Id Name State ---------------------------------------------------- - demo-guest1 saved - demo-guest2 shut off
virsh list --with-managed-save --all Id Name State ---------------------------------------------------- - demo-guest1 shut off
virsh start demo-guest1 Domain 'demo-guest1' started
virsh -c qemu+ssh://root@192.0.2.1/system start demo-guest1 root@192.0.2.1's password: Domain 'demo-guest1' started
yum install libguestfs-tools-c
ls -la /var/lib/libvirt/images -rw-------. 1 root root 9665380352 Jul 23 14:50 a-really-important-vm.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 an-actual-vm-that-i-use.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 totally-not-a-fake-vm.qcow2 -rw-------. 1 root root 10739318784 Sep 20 17:57 another-vm-example.qcow2
whoami root
cp /var/lib/libvirt/images/a-really-important-vm.qcow2 /var/lib/libvirt/images/a-really-important-vm-original.qcow2
virt-sysprep -a /var/lib/libvirt/images/a-really-important-vm.qcow2 [ 0.0] Examining the guest [ 7.3] Performing "abrt-data" [ 7.3] Performing "backup-files" [ 9.6] Performing "bash-history" [ 9.6] Performing "blkid-tab" [...]
virt-diff -a /var/lib/libvirt/images/a-really-important-vm-orig.qcow2 -A /var/lib/libvirt/images/a-really-important-vm.qcow2 - - 0644 1001 /etc/group- - - 0000 797 /etc/gshadow- = - 0444 33 /etc/machine-id [...] - - 0600 409 /home/username/.bash_history - d 0700 6 /home/username/.ssh - - 0600 868 /root/.bash_history [...]
ls -la /var/lib/libvirt/images -rw-------. 1 root root 9665380352 Jul 23 14:50 a-really-important-vm.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 an-actual-vm-that-i-use.qcow2 -rw-------. 1 root root 8591507456 Jul 26 2017 totally-not-a-fake-vm.qcow2 -rw-------. 1 root root 10739318784 Sep 20 17:57 another-vm-example.qcow2
rm -f /etc/udev/rules.d/70-persistent-net.rules
DEVICE=eth[x] BOOTPROTO=none ONBOOT=yes #NETWORK=192.0.2.0 <- REMOVE #NETMASK=255.255.255.0 <- REMOVE #IPADDR=192.0.2.1 <- REMOVE #HWADDR=xx:xx:xx:xx:xx <- REMOVE #USERCTL=no <- REMOVE # Remove any other *unique or non-desired settings, such as UUID.*
DEVICE=eth[x] BOOTPROTO=dhcp ONBOOT=yes
rm /etc/sysconfig/rhn/systemid
subscription-manager unsubscribe --all # subscription-manager unregister # subscription-manager clean
subscription-manager clean
subscription-manager register --consumerid=71rd64fx-6216-4409-bf3a-e4b7c7bd8ac9
rm -rf /etc/ssh/ssh_host_example
rm /etc/lvm/devices/system.devices
rm ~/.config/gnome-initial-setup-done
virt-clone --original example-VM-1 --auto-clone Allocating 'example-VM-1-clone.qcow2' | 50.0 GB 00:05:37 Clone 'example-VM-1-clone' created successfully.
virt-clone --original example-VM-2 --name example-VM-3 --file /var/lib/libvirt/images/ disk-1-example-VM-2 .qcow2 --file /var/lib/libvirt/images/ disk-2-example-VM-2 .qcow2 Allocating 'disk-1-example-VM-2-clone.qcow2' | 78.0 GB 00:05:37 Allocating 'disk-2-example-VM-2-clone.qcow2' | 80.0 GB 00:05:37 Clone 'example-VM-3' created successfully.
virsh migrate --offline --persistent example-VM-3 qemu+ssh://root@192.0.2.1/system root@192.0.2.1's password: scp /var/lib/libvirt/images/ <disk-1-example-VM-2-clone> .qcow2 root@192.0.2.1/ <user@remote_host.com> ://var/lib/libvirt/images/ scp /var/lib/libvirt/images/ <disk-2-example-VM-2-clone> .qcow2 root@192.0.2.1/ <user@remote_host.com> ://var/lib/libvirt/images/
virsh list --all Id Name State --------------------------------------- - example-VM-1 shut off - example-VM-1-clone shut off
virsh start example-VM-1-clone Domain 'example-VM-1-clone' started
virsh domdirtyrate-calc <example_VM> 30
virsh domstats <example_VM> --dirtyrate Domain: 'example-VM' dirtyrate.calc_status=2 dirtyrate.calc_start_time=200942 dirtyrate.calc_period=30 dirtyrate.megabytes_per_second=2
systemctl enable --now libvirtd.service
virsh migrate --offline --persistent <example_VM> qemu+ssh:// example-destination /system
virsh migrate --live --persistent <example_VM> qemu+ssh:// example-destination /system
virsh migrate --live --persistent --parallel --parallel-connections 4 <example_VM> qemu+ssh:// <example-destination> /system
virsh migrate-setmaxdowntime <example_VM> <time_interval_in_milliseconds>
virsh migrate --live --persistent --postcopy --timeout <time_interval_in_seconds> --timeout-postcopy <example_VM> qemu+ssh:// <example-destination> /system
virsh migrate --live --persistent --auto-converge <example_VM> qemu+ssh:// <example-destination> /system
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 shut off
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 running
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 shut off
virsh list --all Id Name State ---------------------------------- 10 example-VM-1 running
virsh domdirtyrate-calc vm-name 30
virsh domstats vm-name --dirtyrate Domain: ' vm-name ' dirtyrate.calc_status=2 dirtyrate.calc_start_time=200942 dirtyrate.calc_period=30 dirtyrate.megabytes_per_second=2
setsebool virt_use_nfs 1
ssh root@ example-shared-storage root@example-shared-storage's password: Last login: Mon Sep 24 12:05:36 2019 root~#
mkdir /var/lib/libvirt/shared-images
scp /var/lib/libvirt/images/ example-disk-1 .qcow2 root@ example-shared-storage :/var/lib/libvirt/shared-images/ example-disk-1 .qcow2
/var/lib/libvirt/shared-images example-source-machine (rw,no_root_squash) example-destination-machine (rw,no\_root_squash)
exportfs -a
mount example-shared-storage :/var/lib/libvirt/shared-images /var/lib/libvirt/images
virsh domcapabilities | xmllint --xpath "//cpu/mode[@name='host-model']" - > domCaps-CPUs.xml
cat domCaps-CPUs.xml <cpu> <model fallback="forbid">Skylake-Client-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="clflushopt"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaves"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="ibrs"/> <feature policy="require" name="amd-stibp"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="rsba"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> <feature policy="disable" name="hle"/> <feature policy="disable" name="rtm"/> </cpu>
virsh domcapabilities | xmllint --xpath "//cpu/mode[@name='host-model']" - <mode name="host-model" supported="yes"> <model fallback="forbid">IvyBridge-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="pcid"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="arat"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaveopt"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> </mode>
cat domCaps-CPUs.xml <cpu> <model fallback="forbid">Skylake-Client-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="clflushopt"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaves"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="ibrs"/> <feature policy="require" name="amd-stibp"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="rsba"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> <feature policy="disable" name="hle"/> <feature policy="disable" name="rtm"/> </cpu> <cpu> <model fallback="forbid">IvyBridge-IBRS</model> <vendor>Intel</vendor> <feature policy="require" name="ss"/> <feature policy="require" name="vmx"/> <feature policy="require" name="pdcm"/> <feature policy="require" name="pcid"/> <feature policy="require" name="hypervisor"/> <feature policy="require" name="arat"/> <feature policy="require" name="tsc_adjust"/> <feature policy="require" name="umip"/> <feature policy="require" name="md-clear"/> <feature policy="require" name="stibp"/> <feature policy="require" name="arch-capabilities"/> <feature policy="require" name="ssbd"/> <feature policy="require" name="xsaveopt"/> <feature policy="require" name="pdpe1gb"/> <feature policy="require" name="invtsc"/> <feature policy="require" name="ibpb"/> <feature policy="require" name="amd-ssbd"/> <feature policy="require" name="skip-l1dfl-vmentry"/> <feature policy="require" name="pschange-mc-no"/> </cpu>
virsh hypervisor-cpu-baseline domCaps-CPUs.xml <cpu mode='custom' match='exact'> <model fallback='forbid'>IvyBridge-IBRS</model> <vendor>Intel</vendor> <feature policy='require' name='ss'/> <feature policy='require' name='vmx'/> <feature policy='require' name='pdcm'/> <feature policy='require' name='pcid'/> <feature policy='require' name='hypervisor'/> <feature policy='require' name='arat'/> <feature policy='require' name='tsc_adjust'/> <feature policy='require' name='umip'/> <feature policy='require' name='md-clear'/> <feature policy='require' name='stibp'/> <feature policy='require' name='arch-capabilities'/> <feature policy='require' name='ssbd'/> <feature policy='require' name='xsaveopt'/> <feature policy='require' name='pdpe1gb'/> <feature policy='require' name='invtsc'/> <feature policy='require' name='ibpb'/> <feature policy='require' name='amd-ssbd'/> <feature policy='require' name='skip-l1dfl-vmentry'/> <feature policy='require' name='pschange-mc-no'/> </cpu>
virsh edit <vm_name>
virsh shutdown <vm_name> virsh start <vm_name>
virt-xml --network=? --network options: [...] address.unit boot_order clearxml driver_name [...]
virt-xml testguest --add-device --disk /var/lib/libvirt/images/newdisk.qcow2,format=qcow2,size=20 Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virt-xml testguest2 --add-device --update --hostdev 002.004 Device hotplug successful. Domain 'testguest2' defined successfully.
virsh dumpxml testguest [...] <hostdev mode='subsystem' type='usb' managed='yes'> <source> <vendor id='0x4146'/> <product id='0x902e'/> <address bus='2' device='4'/> </source> <alias name='hostdev0'/> <address type='usb' bus='0' port='3'/> </hostdev> [...]
virt-xml --network=? --network options: [...] address.unit boot_order clearxml driver_name [...]
virsh dumpxml testguest1 > testguest1.xml cat testguest1.xml <domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>testguest1</name> <uuid>ede29304-fe0c-4ca4-abcd-d246481acd18</uuid> [...] </domain>
virt-xml testguest --edit --cpu host-model,clearxml=yes Domain 'testguest' defined successfully.
virsh dumpxml testguest [...] <cpu mode='host-model' check='partial'> <model fallback='allow'/> </cpu> [...]
virsh define testguest.xml
virsh dumpxml testguest1 > testguest1.xml cat testguest1.xml <domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>testguest1</name> <uuid>ede29304-fe0c-4ca4-abcd-d246481acd18</uuid> [...] </domain>
virt-xml testguest --remove-device --disk target=vdb Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virt-xml testguest2 --remove-device --update --hostdev type=usb Device hotunplug successful. Domain 'testguest2' defined successfully.
virsh define testguest.xml
virsh edit example-VM-1
virsh dumpxml example-VM-1 > example-VM-1 .xml
virsh edit <example-VM-1>
virsh dumpxml testguest1 > testguest1.xml cat testguest1.xml <domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>testguest1</name> <uuid>ede29304-fe0c-4ca4-abcd-d246481acd18</uuid> [...] </domain>
virsh define testguest1.xml
lsusb [...] Bus 001 Device 003: ID 2567:0a2b Intel Corp. Bus 001 Device 005: ID 0407:6252 Kingston River 2.0 [...]
virt-xml example-VM-1 --add-device --hostdev 001.005 Domain 'example-VM-1' defined successfully.
virsh dumpxml example-VM-1 [...] <hostdev mode='subsystem' type='usb' managed='yes'> <source> <vendor id='0x0407'/> <product id='0x6252'/> <address bus='1' device='5'/> </source> <alias name='hostdev0'/> <address type='usb' bus='0' port='3'/> </hostdev> [...]
lsusb [...] Bus 001 Device 003: ID 2567:0a2b Intel Corp. Bus 001 Device 005: ID 0407:6252 Kingston River 2.0 [...]
virt-xml example-VM-1 --remove-device --hostdev 001.005 Domain 'example-VM-1' defined successfully.
virt-xml testguest --add-device --smartcard mode=passthrough,type=spicevmc Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virsh dumpxml testguest
<smartcard mode='passthrough' type='spicevmc'/>
virt-xml example-VM-name --add-device --disk /home/username/Downloads/example-ISO-name.iso ,device=cdrom Domain 'example-VM-name' defined successfully.
virt-xml vmname --add-device --disk target.dev=sda,device=cdrom
virsh dumpxml example-VM-name <disk> <source file=' USD(/home/username/Downloads/example-ISO-name.iso) '/> <target dev='sda' bus='sata'/> </disk>
virt-xml example-VM-name --edit target=sda --disk /dev/cdrom/example-ISO-name-2.iso Domain 'example-VM-name' defined successfully.
virsh dumpxml example-VM-name <disk> <source file=' USD(/home/username/Downloads/example-ISO-name.iso) '/> <target dev='sda' bus='sata'/> </disk>
virt-xml example-VM-name --edit target=sda --disk path= Domain 'example-VM-name' defined successfully.
virsh dumpxml example-VM-name <disk type='file' device='cdrom'> <driver name='qemu' type='raw'/> <target dev='sda' bus='sata'/> </disk>
virt-xml example-VM-name --remove-device --disk target=sda Domain 'example-VM-name' defined successfully.
lspci -v [...] 02:00.0 Ethernet controller: Intel Corporation 82576 Gigabit Network Connection (rev 01) Subsystem: Intel Corporation Gigabit ET Dual Port Server Adapter Flags: bus master, fast devsel, latency 0, IRQ 16, NUMA node 0 Memory at fcba0000 (32-bit, non-prefetchable) [size=128K] [...] Capabilities: [150] Alternative Routing-ID Interpretation (ARI) Capabilities: [160] Single Root I/O Virtualization (SR-IOV) Kernel driver in use: igb Kernel modules: igb [...]
ip link set eth1 up ip link show eth1 8: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT qlen 1000 link/ether a0:36:9f:8f:3f:b8 brd ff:ff:ff:ff:ff:ff vf 0 MAC 00:00:00:00:00:00, spoof checking on, link-state auto vf 1 MAC 00:00:00:00:00:00, spoof checking on, link-state auto vf 2 MAC 00:00:00:00:00:00, spoof checking on, link-state auto vf 3 MAC 00:00:00:00:00:00, spoof checking on, link-state auto
grubby --args="intel_iommu=on iommu=pt" --update-kernel=ALL
grubby --args="iommu=pt" --update-kernel=ALL
cat /sys/class/net/eth1/device/sriov_totalvfs 7
echo VF-number > /sys/class/net/ network-interface /device/sriov_numvfs
echo 2 > /sys/class/net/eth1/device/sriov_numvfs
lspci | grep Ethernet 82:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) 82:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) 82:10.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) 82:10.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="ixgbe", ATTR{device/sriov_numvfs}="2"
virsh attach-interface testguest1 hostdev 0000:82:10.0 --managed --live --config
yum install driverctl
lsmod | grep vfio
lscss -d 0.0.002c Device Subchan. DevType CU Type Use PIM PAM POM CHPIDs ---------------------------------------------------------------------- 0.0.002c 0.0.29a8 3390/0c 3990/e9 yes f0 f0 ff 02111221 00000000
cio_ignore -r 0.0.002c
cio_ignore=all,!condev, ! 0.0.002c
driverctl -b css set-override 0.0.29a8 vfio_ccw
cat nodedev.xml <device> <parent>css_0_0_29a8</parent> <capability type="mdev"> <type id="vfio_ccw-io"/> </capability> </device> virsh nodedev-define nodedev.xml Node device 'mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8' defined from 'nodedev.xml' virsh nodedev-start mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 Device mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 started
virsh nodedev-dumpxml mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 <device> <name>mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8</name> <parent>css_0_0_29a8</parent> <capability type='mdev'> <type id='vfio_ccw-io'/> <uuid>30820a6f-b1a5-4503-91ca-0c10ba12345a</uuid> <iommuGroup number='0'/> <attr name='assign_adapter' value='0x02'/> <attr name='assign_domain' value='0x002b'/> </capability> </device>
<hostdev mode='subsystem' type='mdev' model='vfio-ccw'> <source> <address uuid="30820a6f-b1a5-4503-91ca-0c10ba12345a"/> </source> </hostdev>
virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8
virsh nodedev-info mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 Name: mdev_30820a6f_b1a5_4503_91ca_0c10ba12345a_0_0_29a8 Parent: css_0_0_0121 Active: yes Persistent: yes Autostart: yes
virsh dumpxml vm-name <domain> [...] <hostdev mode='subsystem' type='mdev' managed='no' model='vfio-ccw'> <source> <address uuid='10620d2f-ed4d-437b-8aff-beda461541f9'/> </source> <alias name='hostdev0'/> <address type='ccw' cssid='0xfe' ssid='0x0' devno='0x0009 '/> </hostdev> [...] </domain>
lscss | grep 0.0.0009 0.0.0009 0.0.0007 3390/0c 3990/e9 f0 f0 ff 12212231 00000000
chccwdev -e 0.0009 Setting device 0.0.0009 online Done
lsmod | grep vfio
lspci -nkD 0000:00:00.0 0000: 1014:04ed Kernel driver in use: ism Kernel modules: ism 0001:00:00.0 0000: 1014:04ed Kernel driver in use: ism Kernel modules: ism 0002:00:00.0 0200: 15b3:1016 Subsystem: 15b3:0062 Kernel driver in use: mlx5_core Kernel modules: mlx5_core 0003:00:00.0 0200: 15b3:1016 Subsystem: 15b3:0062 Kernel driver in use: mlx5_core Kernel modules: mlx5_core
virsh edit vm-name
<hostdev mode="subsystem" type="pci" managed="yes"> <driver name="vfio"/> <source> <address domain=" 0x0003 " bus=" 0x00 " slot=" 0x00 " function=" 0x0 "/> </source> <address type="pci"/> </hostdev>
<hostdev mode="subsystem" type="pci" managed="yes"> <driver name="vfio"/> <source> <address domain=" 0x0003 " bus=" 0x00 " slot=" 0x00 " function=" 0x0 "/> </source> <address type="pci"> <zpci uid="0x0008" fid="0x001807"/> </address> </hostdev>
virsh shutdown vm-name
lspci -nkD | grep 0003:00:00.0 0003:00:00.0 8086:9a09 (rev 01)
virsh vol-info --pool guest_images firstimage Name: firstimage Type: block Capacity: 20.00 GB Allocation: 20.00 GB
virsh pool-list --all --details Name State Autostart Persistent Capacity Allocation Available default running yes yes 48.97 GiB 23.93 GiB 25.03 GiB Downloads running yes yes 175.62 GiB 62.02 GiB 113.60 GiB RHEL-Storage-Pool running yes yes 214.62 GiB 93.02 GiB 168.60 GiB
virsh pool-capabilities | grep "'dir' supported='yes'"
virsh pool-define-as guest_images_dir dir --target "/guest_images" Pool guest_images_dir defined
virsh pool-build guest_images_dir Pool guest_images_dir built ls -la /guest_images total 8 drwx------. 2 root root 4096 May 31 19:38 . dr-xr-xr-x. 25 root root 4096 May 31 19:38 ..
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_dir inactive no
virsh pool-start guest_images_dir Pool guest_images_dir started
virsh pool-autostart guest_images_dir Pool guest_images_dir marked as autostarted
virsh pool-info guest_images_dir Name: guest_images_dir UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'disk' supported='yes'"
GRUB_DISABLE_OS_PROBER=true
GRUB_OS_PROBER_SKIP_LIST="5ef6313a-257c-4d43@/dev/sdb1"
virsh pool-define-as guest_images_disk disk --source-format=gpt --source-dev=/dev/sdb --target /dev Pool guest_images_disk defined
virsh pool-build guest_images_disk Pool guest_images_disk built
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_disk inactive no
virsh pool-start guest_images_disk Pool guest_images_disk started
virsh pool-autostart guest_images_disk Pool guest_images_disk marked as autostarted
virsh pool-info guest_images_disk Name: guest_images_disk UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'fs' supported='yes'"
GRUB_DISABLE_OS_PROBER=true
GRUB_OS_PROBER_SKIP_LIST="5ef6313a-257c-4d43@/dev/sdb1"
virsh pool-define-as guest_images_fs fs --source-dev /dev/sdc1 --target /guest_images Pool guest_images_fs defined
virsh pool-build guest_images_fs Pool guest_images_fs built ls -la /guest_images total 8 drwx------. 2 root root 4096 May 31 19:38 . dr-xr-xr-x. 25 root root 4096 May 31 19:38 ..
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_fs inactive no
virsh pool-start guest_images_fs Pool guest_images_fs started
virsh pool-autostart guest_images_fs Pool guest_images_fs marked as autostarted
virsh pool-info guest_images_fs Name: guest_images_fs UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
mount | grep /guest_images /dev/sdc1 on /guest_images type ext4 (rw) ls -la /guest_images total 24 drwxr-xr-x. 3 root root 4096 May 31 19:47 . dr-xr-xr-x. 25 root root 4096 May 31 19:38 .. drwx------. 2 root root 16384 May 31 14:18 lost+found
gluster volume status Status of volume: gluster-vol1 Gluster process Port Online Pid ------------------------------------------------------------ Brick 222.111.222.111:/gluster-vol1 49155 Y 18634 Task Status of Volume gluster-vol1 ------------------------------------------------------------ There are no active volume tasks
setsebool virt_use_fusefs on getsebool virt_use_fusefs virt_use_fusefs --> on
virsh pool-capabilities | grep "'gluster' supported='yes'"
virsh pool-define-as --name guest_images_glusterfs --type gluster --source-host 111.222.111.222 --source-name gluster-vol1 --source-path / Pool guest_images_glusterfs defined
virsh pool-list --all Name State Autostart -------------------------------------------- default active yes guest_images_glusterfs inactive no
virsh pool-start guest_images_glusterfs Pool guest_images_glusterfs started
virsh pool-autostart guest_images_glusterfs Pool guest_images_glusterfs marked as autostarted
virsh pool-info guest_images_glusterfs Name: guest_images_glusterfs UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'iscsi' supported='yes'"
virsh pool-define-as --name guest_images_iscsi --type iscsi --source-host server1.example.com --source-dev iqn.2010-05.com.example.server1:iscsirhel7guest --target /dev/disk/by-path Pool guest_images_iscsi defined
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_iscsi inactive no
virsh pool-start guest_images_iscsi Pool guest_images_iscsi started
virsh pool-autostart guest_images_iscsi Pool guest_images_iscsi marked as autostarted
virsh pool-info guest_images_iscsi Name: guest_images_iscsi UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'logical' supported='yes'"
virsh pool-define-as guest_images_lvm logical --source-name lvm_vg --target /dev/lvm_vg Pool guest_images_lvm defined
virsh pool-list --all Name State Autostart ------------------------------------------- default active yes guest_images_lvm inactive no
virsh pool-start guest_images_lvm Pool guest_images_lvm started
virsh pool-autostart guest_images_lvm Pool guest_images_lvm marked as autostarted
virsh pool-info guest_images_lvm Name: guest_images_lvm UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "<value>nfs</value>"
virsh pool-define-as --name guest_images_netfs --type netfs --source-host='111.222.111.222' --source-path='/home/net_mount' --source-format='nfs' --target='/var/lib/libvirt/images/nfspool'
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_netfs inactive no
virsh pool-start guest_images_netfs Pool guest_images_netfs started
virsh pool-autostart guest_images_netfs Pool guest_images_netfs marked as autostarted
virsh pool-info guest_images_netfs Name: guest_images_netfs UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-capabilities | grep "'scsi' supported='yes'"
virsh pool-define-as guest_images_vhba scsi --adapter-parent scsi_host3 --adapter-wwnn 5001a4a93526d0a1 --adapter-wwpn 5001a4ace3ee047d --target /dev/disk/ Pool guest_images_vhba defined
virsh pool-list --all Name State Autostart ----------------------------------------- default active yes guest_images_vhba inactive no
virsh pool-start guest_images_vhba Pool guest_images_vhba started
virsh pool-autostart guest_images_vhba Pool guest_images_vhba marked as autostarted
virsh pool-info guest_images_vhba Name: guest_images_vhba UUID: c7466869-e82a-a66c-2187-dc9d6f0877d0 State: running Persistent: yes Autostart: yes Capacity: 458.39 GB Allocation: 197.91 MB Available: 458.20 GB
virsh pool-list --all Name State Autostart ------------------------------------------- default active yes Downloads active yes RHEL-Storage-Pool active yes
virsh pool-destroy Downloads Pool Downloads destroyed
virsh pool-delete Downloads Pool Downloads deleted
virsh pool-undefine Downloads Pool Downloads has been undefined
virsh pool-list --all Name State Autostart ------------------------------------------- default active yes rhel-Storage-Pool active yes
virsh pool-define ~/guest_images.xml Pool defined from guest_images_dir
<pool type='dir'> <name>dirpool</name> <target> <path>/guest_images</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_disk
<pool type='disk'> <name>phy_disk</name> <source> <device path='/dev/sdb'/> <format type='gpt'/> </source> <target> <path>/dev</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_fs
<pool type='fs'> <name>guest_images_fs</name> <source> <device path='/dev/sdc1'/> <format type='auto'/> </source> <target> <path>/guest_images</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_glusterfs
<pool type='gluster'> <name>Gluster_pool</name> <source> <host name='111.222.111.222'/> <dir path='/'/> <name>gluster-vol1</name> </source> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_iscsi
<pool type='iscsi'> <name>iSCSI_pool</name> <source> <host name='server1.example.com'/> <device path='iqn.2010-05.com.example.server1:iscsirhel7guest'/> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_logical
<source> <device path='/dev/sda1'/> <device path='/dev/sdb3'/> <device path='/dev/sdc2'/> </source>
<pool type='logical'> <name>guest_images_lvm</name> <source> <device path='/dev/sdc'/> <name>libvirt_lvm</name> <format type='lvm2'/> </source> <target> <path>/dev/libvirt_lvm</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_netfs
<pool type='netfs'> <name>nfspool</name> <source> <host name='file_server'/> <format type='nfs'/> <dir path='/home/net_mount'/> </source> <target> <path>/var/lib/libvirt/images/nfspool</path> </target> </pool>
virsh pool-define ~/guest_images.xml Pool defined from guest_images_vhba
<pool type='scsi'> <name>vhbapool_host3</name> <source> <adapter type='fc_host' wwnn='5001a4a93526d0a1' wwpn='5001a4ace3ee047d'/> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
<pool type='scsi'> <name>vhbapool_host3</name> <source> <adapter type='fc_host' parent='scsi_host3' wwnn='5001a4a93526d0a1' wwpn='5001a4ace3ee047d'/> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
virsh vol-list --pool RHEL-Storage-Pool --details Name Path Type Capacity Allocation --------------------------------------------------------------------------------------------- .bash_history /home/VirtualMachines/.bash_history file 18.70 KiB 20.00 KiB .bash_logout /home/VirtualMachines/.bash_logout file 18.00 B 4.00 KiB .bash_profile /home/VirtualMachines/.bash_profile file 193.00 B 4.00 KiB .bashrc /home/VirtualMachines/.bashrc file 1.29 KiB 4.00 KiB .git-prompt.sh /home/VirtualMachines/.git-prompt.sh file 15.84 KiB 16.00 KiB .gitconfig /home/VirtualMachines/.gitconfig file 167.00 B 4.00 KiB RHEL_Volume.qcow2 /home/VirtualMachines/RHEL8_Volume.qcow2 file 60.00 GiB 13.93 GiB
virsh vol-info --pool RHEL-Storage-Pool --vol RHEL_Volume.qcow2 Name: RHEL_Volume.qcow2 Type: file Capacity: 60.00 GiB Allocation: 13.93 GiB
virsh pool-list --details Name State Autostart Persistent Capacity Allocation Available -------------------------------------------------------------------------------------------- default running yes yes 48.97 GiB 36.34 GiB 12.63 GiB Downloads running yes yes 175.92 GiB 121.20 GiB 54.72 GiB VM-disks running yes yes 175.92 GiB 121.20 GiB 54.72 GiB
virsh vol-create-as --pool guest-images-fs --name vm-disk1 --capacity 20 --format qcow2
<disk type='volume' device='disk'> <driver name='qemu' type='qcow2'/> <source pool='guest-images-fs' volume='vm-disk1'/> <target dev='hdk' bus='ide'/> </disk>
<disk type='network' device='disk'> <driver name='qemu' type='raw'/> <source protocol='gluster' name='Volume1/Image'> <host name='example.org' port='6000'/> </source> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> </disk>
<disk type='block' device='disk'> <driver name='qemu' type='raw'/> <source dev='/dev/mapper/mpatha' /> <target dev='sda' bus='scsi'/> </disk>
<disk type='network' device='disk'> <driver name='qemu' type='raw'/> <source protocol='rbd' name='pool/image'> <host name='mon1.example.org' port='6321'/> </source> <target dev='vdc' bus='virtio'/> </disk>
virsh attach-device --config testguest1 ~/vm-disk1.xml
virsh vol-list --pool RHEL-SP Name Path --------------------------------------------------------------- .bash_history /home/VirtualMachines/.bash_history .bash_logout /home/VirtualMachines/.bash_logout .bash_profile /home/VirtualMachines/.bash_profile .bashrc /home/VirtualMachines/.bashrc .git-prompt.sh /home/VirtualMachines/.git-prompt.sh .gitconfig /home/VirtualMachines/.gitconfig vm-disk1 /home/VirtualMachines/vm-disk1
virsh vol-wipe --pool RHEL-SP vm-disk1 Vol vm-disk1 wiped
virsh vol-delete --pool RHEL-SP vm-disk1 Vol vm-disk1 deleted
virsh vol-list --pool RHEL-SP Name Path --------------------------------------------------------------- .bash_history /home/VirtualMachines/.bash_history .bash_logout /home/VirtualMachines/.bash_logout .bash_profile /home/VirtualMachines/.bash_profile .bashrc /home/VirtualMachines/.bashrc .git-prompt.sh /home/VirtualMachines/.git-prompt.sh .gitconfig /home/VirtualMachines/.gitconfig
qemu-img create -f <format> <image-name> <size>
qemu-img create -f qcow2 test-image 30G Formatting 'test-img', fmt=qcow2 cluster_size=65536 extended_l2=off compression_type=zlib size=32212254720 lazy_refcounts=off refcount_bits=16
qemu-img info <test-img> image: test-img file format: qcow2 virtual size: 30 GiB (32212254720 bytes) disk size: 196 KiB cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib lazy refcounts: false refcount bits: 16 corrupt: false extended l2: false
qemu-img check <test-name.qcow2> No errors were found on the image. 327434/327680 = 99.92% allocated, 0.00% fragmented, 0.00% compressed clusters Image end offset: 21478375424
167 errors were found on the image. Data may be corrupted, or further writes to the image may corrupt it. 453368 leaked clusters were found on the image. This means waste of disk space, but no harm to data. 259 internal errors have occurred during the check. Image end offset: 21478375424
qemu-img check -r all <test-name.qcow2> [...] 122 errors were found on the image. Data may be corrupted, or further writes to the image may corrupt it. 250 internal errors have occurred during the check. Image end offset: 27071414272
virsh domblklist <vm-name> Target Source ---------------------------------------------------------- vda /home/username/disk-images/ example-image.qcow2
cp <example-image.qcow2> <example-image-backup.qcow2>
qemu-img resize <example-image.qcow2> +10G
qemu-img info <converted-image.qcow2> image: converted-image.qcow2 file format: qcow2 virtual size: 30 GiB (32212254720 bytes) disk size: 196 KiB cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib lazy refcounts: false refcount bits: 16 corrupt: false extended l2: false
qemu-img convert -f raw <original-image.img> -O qcow2 <converted-image.qcow2>
qemu-img info <converted-image.qcow2> image: converted-image.qcow2 file format: qcow2 virtual size: 30 GiB (32212254720 bytes) disk size: 196 KiB cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib lazy refcounts: false refcount bits: 16 corrupt: false extended l2: false
<secret ephemeral='no' private='yes'> <description>Passphrase for the iSCSI example.com server</description> <usage type='iscsi'> <target>iscsirhel7secret</target> </usage> </secret>
virsh secret-define secret.xml
virsh secret-list UUID Usage -------------------------------------------------------------- 2d7891af-20be-4e5e-af83-190e8a922360 iscsi iscsirhel7secret
virsh secret-set-value --interactive 2d7891af-20be-4e5e-af83-190e8a922360 Enter new value for secret: Secret value set
<pool type='iscsi'> <name>iscsirhel7pool</name> <source> <host name='192.0.2.1'/> <device path='iqn.2010-05.com.example.server1:iscsirhel7guest'/> <auth type='chap' username='_example-user_'> <secret usage='iscsirhel7secret'/> </auth> </source> <target> <path>/dev/disk/by-path</path> </target> </pool>
<auth username='redhat'> <secret type='iscsi' usage='iscsirhel7secret'/> </auth>
virsh pool-destroy iscsirhel7pool virsh pool-start iscsirhel7pool
virsh nodedev-list --cap vports scsi_host3 scsi_host4
virsh nodedev-dumpxml scsi_host3
<device> <name>scsi_host3</name> <path>/sys/devices/pci0000:00/0000:00:04.0/0000:10:00.0/host3</path> <parent>pci_0000_10_00_0</parent> <capability type='scsi_host'> <host>3</host> <unique_id>0</unique_id> <capability type='fc_host'> <wwnn>20000000c9848140</wwnn> <wwpn>10000000c9848140</wwpn> <fabric_wwn>2002000573de9a81</fabric_wwn> </capability> <capability type='vport_ops'> <max_vports>127</max_vports> <vports>0</vports> </capability> </capability> </device>
<device> <parent>scsi_host3</parent> <capability type='scsi_host'> <capability type='fc_host'> </capability> </capability> </device>
<device> <name>vhba</name> <parent wwnn='20000000c9848140' wwpn='10000000c9848140'/> <capability type='scsi_host'> <capability type='fc_host'> </capability> </capability> </device>
virsh nodedev-create vhba_host3 Node device scsi_host5 created from vhba_host3.xml
virsh nodedev-dumpxml scsi_host5 <device> <name>scsi_host5</name> <path>/sys/devices/pci0000:00/0000:00:04.0/0000:10:00.0/host3/vport-3:0-0/host5</path> <parent>scsi_host3</parent> <capability type='scsi_host'> <host>5</host> <unique_id>2</unique_id> <capability type='fc_host'> <wwnn>5001a4a93526d0a1</wwnn> <wwpn>5001a4ace3ee047d</wwpn> <fabric_wwn>2002000573de9a81</fabric_wwn> </capability> </capability> </device>
grubby --args="intel_iommu=on iommu_pt" --update-kernel DEFAULT
grubby --args="iommu=pt" --update-kernel DEFAULT
lspci -Dnn | grep VGA 0000:02:00.0 VGA compatible controller [0300]: NVIDIA Corporation GK106GL [Quadro K4000] [ 10de:11fa ] (rev a1)
grubby --args="pci-stub.ids=10de:11fa" --update-kernel DEFAULT
virsh nodedev-dumpxml pci_0000_02_00_0
<device> <name>pci_0000_02_00_0</name> <path>/sys/devices/pci0000:00/0000:00:03.0/0000:02:00.0</path> <parent>pci_0000_00_03_0</parent> <driver> <name>pci-stub</name> </driver> <capability type='pci'> <domain>0</domain> <bus>2</bus> <slot>0</slot> <function>0</function> <product id='0x11fa'>GK106GL [Quadro K4000]</product> <vendor id='0x10de'>NVIDIA Corporation</vendor> <iommuGroup number='13'> <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/> <address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/> </iommuGroup> <pci-express> <link validity='cap' port='0' speed='8' width='16'/> <link validity='sta' speed='2.5' width='16'/> </pci-express> </capability> </device>
driverctl set-override 0000:02:00.1 vfio-pci
<hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/> </source> </hostdev>
virsh attach-device System1 --file /home/GPU-Assign.xml --persistent Device attached successfully.
lshw -C display *-display description: 3D controller product: GP104GL [Tesla P4] vendor: NVIDIA Corporation physical id: 0 bus info: pci@0000:01:00.0 version: a1 width: 64 bits clock: 33MHz capabilities: pm msi pciexpress cap_list configuration: driver=vfio-pci latency=0 resources: irq:16 memory:f6000000-f6ffffff memory:e0000000-efffffff memory:f0000000-f1ffffff
blacklist nouveau options nouveau modeset=0
dracut --force reboot
lsmod | grep nvidia_vgpu_vfio nvidia_vgpu_vfio 45011 0 nvidia 14333621 10 nvidia_vgpu_vfio mdev 20414 2 vfio_mdev,nvidia_vgpu_vfio vfio 32695 3 vfio_mdev,nvidia_vgpu_vfio,vfio_iommu_type1 systemctl status nvidia-vgpu-mgr.service nvidia-vgpu-mgr.service - NVIDIA vGPU Manager Daemon Loaded: loaded (/usr/lib/systemd/system/nvidia-vgpu-mgr.service; enabled; vendor preset: disabled) Active: active (running) since Fri 2018-03-16 10:17:36 CET; 5h 8min ago Main PID: 1553 (nvidia-vgpu-mgr) [...]
uuidgen 30820a6f-b1a5-4503-91ca-0c10ba58692a
<device> <parent>pci_0000_01_00_0</parent> <capability type="mdev"> <type id="nvidia-63"/> <uuid>30820a6f-b1a5-4503-91ca-0c10ba58692a</uuid> </capability> </device>
virsh nodedev-define vgpu-test.xml Node device mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 created from vgpu-test.xml
virsh nodedev-list --cap mdev --inactive mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-start mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Device mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 started
virsh nodedev-list --cap mdev mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Device mdev_d196754e_d8ed_4f43_bf22_684ed698b08b_0000_9b_00_0 marked as autostarted
<hostdev mode='subsystem' type='mdev' managed='no' model='vfio-pci' display='on'> <source> <address uuid='30820a6f-b1a5-4503-91ca-0c10ba58692a'/> </source> </hostdev>
virsh nodedev-info mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Name: virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Parent: pci_0000_01_00_0 Active: yes Persistent: yes Autostart: yes
lspci -d 10de: -k 07:00.0 VGA compatible controller: NVIDIA Corporation GV100GL [Tesla V100 SXM2 32GB] (rev a1) Subsystem: NVIDIA Corporation Device 12ce Kernel driver in use: nvidia Kernel modules: nouveau, nvidia_drm, nvidia
virsh nodedev-list --cap mdev mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-destroy mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Destroyed node device 'mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0'
virsh nodedev-info mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Name: virsh nodedev-autostart mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Parent: pci_0000_01_00_0 Active: no Persistent: yes Autostart: yes
<hostdev mode='subsystem' type='mdev' managed='no' model='vfio-pci'> <source> <address uuid='30820a6f-b1a5-4503-91ca-0c10ba58692a'/> </source> </hostdev>
virsh nodedev-undefine mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0 Undefined node device 'mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0'
virsh nodedev-list --cap mdev --inactive mdev_30820a6f_b1a5_4503_91ca_0c10ba58692a_0000_01_00_0
virsh nodedev-list --cap mdev
virsh nodedev-list --cap mdev_types pci_0000_5b_00_0 pci_0000_9b_00_0
virsh nodedev-dumpxml pci_0000_9b_00_0 <device> <name>pci_0000_9b_00_0</name> <path>/sys/devices/pci0000:9a/0000:9a:00.0/0000:9b:00.0</path> <parent>pci_0000_9a_00_0</parent> <driver> <name>nvidia</name> </driver> <capability type='pci'> <class>0x030000</class> <domain>0</domain> <bus>155</bus> <slot>0</slot> <function>0</function> <product id='0x1e30'>TU102GL [Quadro RTX 6000/8000]</product> <vendor id='0x10de'>NVIDIA Corporation</vendor> <capability type='mdev_types'> <type id='nvidia-346'> <name>GRID RTX6000-12C</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>2</availableInstances> </type> <type id='nvidia-439'> <name>GRID RTX6000-3A</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>8</availableInstances> </type> [...] <type id='nvidia-440'> <name>GRID RTX6000-4A</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>6</availableInstances> </type> <type id='nvidia-261'> <name>GRID RTX6000-8Q</name> <deviceAPI>vfio-pci</deviceAPI> <availableInstances>3</availableInstances> </type> </capability> <iommuGroup number='216'> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x3'/> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x1'/> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x2'/> <address domain='0x0000' bus='0x9b' slot='0x00' function='0x0'/> </iommuGroup> <numa node='2'/> <pci-express> <link validity='cap' port='0' speed='8' width='16'/> <link validity='sta' speed='2.5' width='8'/> </pci-express> </capability> </device>
ip addr show virbr0 3: virbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN link/ether 1b:c4:94:cf:fd:17 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global virbr0
ip addr [...] enp0s25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 54:ee:75:49:dc:46 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global dynamic noprefixroute enp0s25
virt-xml testguest --edit --network bridge=bridge0 Domain 'testguest' defined successfully.
virsh start testguest
ip link show master bridge0 2: enp0s25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master bridge0 state UP mode DEFAULT group default qlen 1000 link/ether 54:ee:75:49:dc:46 brd ff:ff:ff:ff:ff:ff 10: vnet0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master bridge0 state UNKNOWN mode DEFAULT group default qlen 1000 link/ether fe:54:00:89:15:40 brd ff:ff:ff:ff:ff:ff
ip addr [...] enp0s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 52:54:00:09:15:46 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global dynamic noprefixroute enp0s0
ssh root@192.0.2.1 root@192.0.2.1's password: Last login: Mon Sep 24 12:05:36 2019 root~#*
ip addr [...] enp0s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 52:54:00:09:15:46 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global dynamic noprefixroute enp0s0
ssh root@192.0.2.1 root@192.0.2.1's password: Last login: Mon Sep 24 12:05:36 2019 root~#*
chmod -R a+r /var/lib/tftpboot
chown -R nobody: /var/lib/tftpboot
chcon -R --reference /usr/sbin/dnsmasq /var/lib/tftpboot chcon -R --reference /usr/libexec/libvirt_leaseshelper /var/lib/tftpboot
virsh net-destroy default
virsh net-edit default
<ip address='192.0.2.1' netmask='255.255.255.0'> <tftp root='/var/lib/tftpboot'/> <dhcp> <range start='192.0.2.2' end='192.0.2.254' /> <bootp file=' example-pxelinux '/> </dhcp> </ip>
virsh net-start default
virsh net-list Name State Autostart Persistent --------------------------------------------------- default active no no
virt-install --pxe --network network=default --memory 2048 --vcpus 2 --disk size=10
<interface type='network' > <mac address='52:54:00:66:79:14'/> <source network='default'/> <target dev='vnet0'/> <alias name='net0'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> <boot order='1'/> </interface>
virt-install --pxe --network bridge=breth0 --memory 2048 --vcpus 2 --disk size=10
<interface type='bridge' > <mac address='52:54:00:5a:ad:cb'/> <source bridge='breth0'/> <target dev='vnet0'/> <alias name='net0'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> <boot order='1'/> </interface>
yum install nfs-utils -y
mkdir shared-files
virsh domifaddr testguest1 Name MAC address Protocol Address ---------------------------------------------------------------- vnet0 52:53:00:84:57:90 ipv4 192.0.2.2/24 virsh domifaddr testguest2 Name MAC address Protocol Address ---------------------------------------------------------------- vnet1 52:53:00:65:29:21 ipv4 192.0.2.3/24
/home/<username>/Downloads/<shared_directory>/ <VM1-IP(options)> <VM2-IP(options)>
/usr/local/shared-files/ 192.0.2.2(rw,sync) 192.0.2.3(rw,sync)
exportfs -a
systemctl start nfs-server
ip addr 5: virbr0: [BROADCAST,MULTICAST,UP,LOWER_UP] mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 52:54:00:32:ff:a5 brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/24 brd 192.0.2.255 scope global virbr0 valid_lft forever preferred_lft forever
mount 192.0.2.1:/usr/local/shared-files /mnt/host-share
Install-WindowsFeature NFS-Client
Enable-WindowsOptionalFeature -FeatureName ServicesForNFS-ClientOnly, ClientForNFS-Infrastructure -Online -NoRestart
C:\Windows\system32\mount.exe -o anon \\192.0.2.1\usr\local\shared-files Z:
ls <mount_point> shared-file1 shared-file2 shared-file3
getenforce Enforcing
yum install edk2-ovmf
virt-install --name rhel8sb --memory 4096 --vcpus 4 --os-variant rhel8.0 --boot uefi,nvram_template=/usr/share/OVMF/OVMF_VARS.secboot.fd --disk boot_order=2,size=10 --disk boot_order=1,device=cdrom,bus=scsi,path=/images/RHEL-8.0-installation.iso
mokutil --sb-state SecureBoot enabled
ls /usr/share/polkit-1/actions | grep libvirt ls /usr/share/polkit-1/rules.d | grep libvirt
sed -i 's/#access_drivers = \[ "polkit" \]/access_drivers = \[ "polkit" \]/' /etc/libvirt/libvirtd.conf
systemctl restart libvirtd
virsh -c qemu:///system list --all Id Name State -------------------------------
getsebool -a | grep virt [...] virt_sandbox_use_netlink --> off virt_sandbox_use_sys_admin --> off virt_transition_userdomain --> off virt_use_comm --> off virt_use_execmem --> off virt_use_fusefs --> off [...]
grep facilities /proc/cpuinfo | grep 158
ls /sys/firmware | grep uv
virsh domcapabilities | grep unpack <feature policy='require' name='unpack'/>
virsh dumpxml <vm_name> | grep "<cpu mode='host-model'/>"
yum install guestfs-tools
grubby --update-kernel=ALL --args="prot_virt=1"
[...] </memballoon> </devices> <launchSecurity type="s390-pv"/> </domain>
touch ~/secure-parameters
ls /boot/loader/entries -l [...] -rw-r--r--. 1 root root 281 Oct 9 15:51 3ab27a195c2849429927b00679db15c1-4.18.0-240.el8.s390x.conf
cat /boot/loader/entries/3ab27a195c2849429927b00679db15c1-4.18.0-240.el8.s390x.conf | grep options options root=/dev/mapper/rhel-root crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap
echo "root=/dev/mapper/rhel-root crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap swiotlb=262144" > ~/secure-parameters
genprotimg -i /boot/vmlinuz-4.18.0-240.el8.s390x -r /boot/initramfs-4.18.0-240.el8.s390x.img -p ~/secure-parameters -k HKD-8651-00020089A8.crt -o /boot/secure-image
cat /boot/loader/entries/3ab27a195c2849429927b00679db15c1-4.18.0-240.el8.s390x.conf title Red Hat Enterprise Linux 8.3 version 4.18.0-240.el8.s390x linux /boot/secure-image [...]
zipl -V
shred /boot/vmlinuz-4.18.0-240.el8.s390x shred /boot/initramfs-4.18.0-240.el8.s390x.img shred secure-parameters
#!/usr/bin/bash echo "USD(cat /proc/cmdline) swiotlb=262144" > parmfile cat > ./HKD.crt << EOF -----BEGIN CERTIFICATE----- 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 1234569901234569901234569901234569901234569901234569901234569900 xLPRGYwhmXzKDg== -----END CERTIFICATE----- EOF version=USD(uname -r) kernel=/boot/vmlinuz-USDversion initrd=/boot/initramfs-USDversion.img genprotimg -k ./HKD.crt -p ./parmfile -i USDkernel -r USDinitrd -o /boot/secure-linux --no-verify cat >> /etc/zipl.conf<< EOF [secure] target=/boot image=/boot/secure-linux EOF zipl -V shutdown -h now
virt-customize -a <vm_image_path> --selinux-relabel --firstboot <script_path>
virsh dumpxml vm-name [...] <cpu mode='host-model'/> <devices> <disk type='file' device='disk'> <driver name='qemu' type='qcow2' cache='none' io='native'> <source file='/var/lib/libvirt/images/secure-guest.qcow2'/> <target dev='vda' bus='virtio'/> </disk> <interface type='network'> <source network='default'/> <model type='virtio'/> </interface> <console type='pty'/> <memballoon model='none'/> </devices> <launchSecurity type="s390-pv"/> </domain>
lszcrypt -V CARD.DOMAIN TYPE MODE STATUS REQUESTS PENDING HWTYPE QDEPTH FUNCTIONS DRIVER -------------------------------------------------------------------------------------------- 05 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4card 05.0004 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue 05.00ab CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue
lsmod | grep vfio_ap vfio_ap 24576 0 [...]
modprobe vfio_ap
lszdev --list-types ap Cryptographic Adjunct Processor (AP) device
echo "obase=10; ibase=16; 04" | bc 4 echo "obase=10; ibase=16; AB" | bc 171
chzdev -t ap apmask=-5 aqmask=-4,-171
lszcrypt -V CARD.DOMAIN TYPE MODE STATUS REQUESTS PENDING HWTYPE QDEPTH FUNCTIONS DRIVER -------------------------------------------------------------------------------------------- 05 CEX5C CCA-Coproc - 1 0 11 08 S--D--N-- cex4card 05.0004 CEX5C CCA-Coproc - 1 0 11 08 S--D--N-- vfio_ap 05.00ab CEX5C CCA-Coproc - 1 0 11 08 S--D--N-- vfio_ap
vim vfio_ap.xml <device> <parent>ap_matrix</parent> <capability type="mdev"> <type id="vfio_ap-passthrough"/> <attr name='assign_adapter' value='0x05'/> <attr name='assign_domain' value='0x0004'/> <attr name='assign_domain' value='0x00ab'/> <attr name='assign_control_domain' value='0x00ab'/> </capability> </device>
virsh nodedev-define vfio_ap.xml Node device 'mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix' defined from 'vfio_ap.xml'
virsh nodedev-start mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix Device mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix started
cat /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/ 669d9b23-fe1b-4ecb-be08-a2fabca99b71 /matrix 05.0004 05.00ab
virsh nodedev-dumpxml mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix <device> <name>mdev_8f9c4a73_1411_48d2_895d_34db9ac18f85_matrix</name> <parent>ap_matrix</parent> <capability type='mdev'> <type id='vfio_ap-passthrough'/> <uuid>8f9c4a73-1411-48d2-895d-34db9ac18f85</uuid> <iommuGroup number='0'/> <attr name='assign_adapter' value='0x05'/> <attr name='assign_domain' value='0x0004'/> <attr name='assign_domain' value='0x00ab'/> <attr name='assign_control_domain' value='0x00ab'/> </capability> </device>
vim crypto-dev.xml
<hostdev mode='subsystem' type='mdev' managed='no' model='vfio-ap'> <source> <address uuid='8f9c4a73-1411-48d2-895d-34db9ac18f85'/> </source> </hostdev>
virsh attach-device testguest1 crypto-dev.xml --live --config
lszcrypt -V CARD.DOMAIN TYPE MODE STATUS REQUESTS PENDING HWTYPE QDEPTH FUNCTIONS DRIVER -------------------------------------------------------------------------------------------- 05 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4card 05.0004 CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue 05.00ab CEX5C CCA-Coproc online 1 0 11 08 S--D--N-- cex4queue
lszcrypt -d C DOMAIN 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ------------------------------------------------------ 00 . . . . U . . . . . . . . . . . 10 . . . . . . . . . . . . . . . . 20 . . . . . . . . . . . . . . . . 30 . . . . . . . . . . . . . . . . 40 . . . . . . . . . . . . . . . . 50 . . . . . . . . . . . . . . . . 60 . . . . . . . . . . . . . . . . 70 . . . . . . . . . . . . . . . . 80 . . . . . . . . . . . . . . . . 90 . . . . . . . . . . . . . . . . a0 . . . . . . . . . . . B . . . . b0 . . . . . . . . . . . . . . . . c0 . . . . . . . . . . . . . . . . d0 . . . . . . . . . . . . . . . . e0 . . . . . . . . . . . . . . . . f0 . . . . . . . . . . . . . . . . ------------------------------------------------------ C: Control domain U: Usage domain B: Both (Control + Usage domain)
{PackageManagerCommand} install edk2-ovmf
{PackageManagerCommand} install swtpm libtpms
<devices> [...] <tpm model='tpm-crb'> <backend type='emulator' version='2.0'/> </tpm> [...] </devices>
Your device meets the requirements for standard hardware security.
tuned-adm list Available profiles: - balanced - General non-specialized TuneD profile - desktop - Optimize for the desktop use-case [...] - virtual-guest - Optimize for running inside a virtual guest - virtual-host - Optimize for running KVM guests Current active profile: balanced
tuned-adm profile selected-profile
tuned-adm profile virtual-host
tuned-adm profile virtual-guest
tuned-adm active Current active profile: virtual-host
tuned-adm verify Verification succeeded, current system settings match the preset profile. See tuned log file ('/var/log/tuned/tuned.log') for details.
virsh dumpxml testguest | grep memballoon <memballoon model='virtio'> </memballoon>
virsh dominfo testguest Max memory: 2097152 KiB Used memory: 2097152 KiB
virsh dumpxml testguest | grep memballoon <memballoon model='virtio'> </memballoon>
virsh dominfo testguest Max memory: 2097152 KiB Used memory: 2097152 KiB
virt-xml testguest --edit --memory memory=4096,currentMemory=4096 Domain 'testguest' defined successfully. Changes will take effect after the domain is fully powered off.
virsh setmem testguest --current 2048
virsh dominfo testguest Max memory: 4194304 KiB Used memory: 2097152 KiB
virsh domstats --balloon testguest Domain: 'testguest' balloon.current=365624 balloon.maximum=4194304 balloon.swap_in=0 balloon.swap_out=0 balloon.major_fault=306 balloon.minor_fault=156117 balloon.unused=3834448 balloon.available=4035008 balloon.usable=3746340 balloon.last-update=1587971682 balloon.disk_caches=75444 balloon.hugetlb_pgalloc=0 balloon.hugetlb_pgfail=0 balloon.rss=1005456
virsh edit testguest
<memoryBacking> <hugepages> <page size='1' unit='GiB'/> </hugepages> </memoryBacking>
cat /proc/meminfo | grep Huge HugePages_Total: 4 HugePages_Free: 2 HugePages_Rsvd: 1 Hugepagesize: 1024000 kB
<domain> [...] <blkiotune> <weight>800</weight> <device> <path>/dev/sda</path> <weight>1000</weight> </device> <device> <path>/dev/sdb</path> <weight>500</weight> </device> </blkiotune> [...] </domain>
virsh blkiotune VM-name --device-weights device , I/O-weight
virsh blkiotune testguest1 --device-weights /dev/sda, 500
virsh blkiotune testguest1 Block I/O tuning parameters for domain testguest1: weight : 800 device_weight : [ {"sda": 500}, ]
virsh domblklist rollin-coal Target Source ------------------------------------------------ vda /var/lib/libvirt/images/rollin-coal.qcow2 sda - sdb /home/horridly-demanding-processes.iso
lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT zram0 252:0 0 4G 0 disk [SWAP] nvme0n1 259:0 0 238.5G 0 disk ├─nvme0n1p1 259:1 0 600M 0 part /boot/efi ├─nvme0n1p2 259:2 0 1G 0 part /boot └─nvme0n1p3 259:3 0 236.9G 0 part └─luks-a1123911-6f37-463c-b4eb-fxzy1ac12fea 253:0 0 236.9G 0 crypt /home
virsh blkiotune VM-name --parameter device , limit
virsh blkiotune rollin-coal --device-read-iops-sec /dev/nvme0n1p3,1000 --device-write-iops-sec /dev/nvme0n1p3,1000 --device-write-bytes-sec /dev/nvme0n1p3,52428800 --device-read-bytes-sec /dev/nvme0n1p3,52428800
virsh edit <example_vm>
<disk type='block' device='disk'> <driver name='qemu' type='raw' queues='N' /> <source dev='/dev/sda'/> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> </disk>
<controller type='scsi' index='0' model='virtio-scsi'> <driver queues='N' /> </controller>
virsh edit <testguest1> <domain type='kvm'> <name>testguest1</name> <vcpu placement='static'>8</vcpu> <iothreads>1</iothreads> </domain>
virsh edit <testguest1> <domain type='kvm'> <name>testguest1</name> <devices> <disk type='file' device='disk'> <driver name='qemu' type='raw' cache='none' io='native' iothread='1' /> <source file='/var/lib/libvirt/images/test-disk.raw'/> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/> </disk> </devices> </domain>
virsh edit <testguest1> <domain type='kvm'> <name>testguest1</name> <devices> <controller type='scsi' index='0' model='virtio-scsi'> <driver iothread='1' /> <address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/> </controller> </devices> </domain>
virsh edit <vm_name>
<domain type='kvm'> <name>testguest1</name> <devices> <disk type='file' device='disk'> <driver name='qemu' type='raw' cache='none' io='native' iothread='1'/> <source file='/var/lib/libvirt/images/test-disk.raw'/> <target dev='vda' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/> </disk> </devices> </domain>
virt-xml testguest1 --edit --cpu host-model
virsh vcpucount testguest maximum config 4 maximum live 2 current config 2 current live 1
virsh setvcpus testguest 8 --maximum --config
virsh setvcpus testguest 4 --live
virsh setvcpus testguest 1 --config
virsh vcpucount testguest maximum config 8 maximum live 4 current config 1 current live 4
virsh nodeinfo CPU model: x86_64 CPU(s): 48 CPU frequency: 1200 MHz CPU socket(s): 1 Core(s) per socket: 12 Thread(s) per core: 2 NUMA cell(s): 2 Memory size: 67012964 KiB
yum install numactl
virt-xml testguest5 --edit --vcpus placement=auto virt-xml testguest5 --edit --numatune mode=preferred
echo 1 > /proc/sys/kernel/numa_balancing
systemctl start numad
numactl --hardware available: 2 nodes (0-1) node 0 size: 18156 MB node 0 free: 9053 MB node 1 size: 18180 MB node 1 free: 6853 MB node distances: node 0 1 0: 10 20 1: 20 10
virsh edit <testguest6> <domain type='kvm'> <name>testguest6</name> <vcpu placement='static'>16</vcpu> <cpu ...> <numa> <cell id='0' cpus='0-7' memory='16' unit='GiB'/> <cell id='1' cpus='8-15' memory='16' unit='GiB'/> </numa> </domain>
lscpu -p=node,cpu Node,CPU 0,0 0,1 0,2 0,3 0,4 0,5 0,6 0,7 1,0 1,1 1,2 1,3 1,4 1,5 1,6 1,7
lscpu -p=node,cpu Node,CPU 0,0 0,1 0,2 0,3
virsh vcpupin testguest6 0 1 virsh vcpupin testguest6 1 3 virsh vcpupin testguest6 2 5 virsh vcpupin testguest6 3 7
virsh vcpupin testguest6 VCPU CPU Affinity ---------------------- 0 1 1 3 2 5 3 7
virsh emulatorpin testguest6 2,4 virsh emulatorpin testguest6 emulator: CPU Affinity ---------------------------------- *: 2,4
virsh schedinfo <vm_name> Scheduler : posix cpu_shares : 0 vcpu_period : 0 vcpu_quota : 0 emulator_period: 0 emulator_quota : 0 global_period : 0 global_quota : 0 iothread_period: 0 iothread_quota : 0
virsh schedinfo <vm_name> --set vcpu_period=100000
virsh schedinfo <vm_name> --set vcpu_quota=50000
virsh schedinfo <vm_name> Scheduler : posix cpu_shares : 2048 vcpu_period : 100000 vcpu_quota : 50000
virsh schedinfo <vm_name> Scheduler : posix cpu_shares : 1024 vcpu_period : 0 vcpu_quota : 0 emulator_period: 0 emulator_quota : 0 global_period : 0 global_quota : 0 iothread_period: 0 iothread_quota : 0
virsh schedinfo <vm_name> --set cpu_shares=2048 Scheduler : posix cpu_shares : 2048 vcpu_period : 0 vcpu_quota : 0 emulator_period: 0 emulator_quota : 0 global_period : 0 global_quota : 0 iothread_period: 0 iothread_quota : 0
systemctl stop ksm systemctl stop ksmtuned
systemctl disable ksm Removed /etc/systemd/system/multi-user.target.wants/ksm.service. systemctl disable ksmtuned Removed /etc/systemd/system/multi-user.target.wants/ksmtuned.service.
echo 2 > /sys/kernel/mm/ksm/run
lsmod | grep vhost vhost_net 32768 1 vhost 53248 1 vhost_net tap 24576 1 vhost_net tun 57344 6 vhost_net
modprobe vhost_net
<interface type='network'> <source network='default'/> <model type='virtio'/> <driver name='vhost' queues='N'/> </interface>
ethtool -C tap0 rx-frames 64
yum install perf
perf kvm stat report Analyze events for all VMs, all VCPUs: VM-EXIT Samples Samples% Time% Min Time Max Time Avg time EXTERNAL_INTERRUPT 365634 31.59% 18.04% 0.42us 58780.59us 204.08us ( +- 0.99% ) MSR_WRITE 293428 25.35% 0.13% 0.59us 17873.02us 1.80us ( +- 4.63% ) PREEMPTION_TIMER 276162 23.86% 0.23% 0.51us 21396.03us 3.38us ( +- 5.19% ) PAUSE_INSTRUCTION 189375 16.36% 11.75% 0.72us 29655.25us 256.77us ( +- 0.70% ) HLT 20440 1.77% 69.83% 0.62us 79319.41us 14134.56us ( +- 0.79% ) VMCALL 12426 1.07% 0.03% 1.02us 5416.25us 8.77us ( +- 7.36% ) EXCEPTION_NMI 27 0.00% 0.00% 0.69us 1.34us 0.98us ( +- 3.50% ) EPT_MISCONFIG 5 0.00% 0.00% 5.15us 10.85us 7.88us ( +- 11.67% ) Total Samples:1157497, Total events handled time:413728274.66us.
numastat -c qemu-kvm Per-node process memory usage (in MBs) PID Node 0 Node 1 Node 2 Node 3 Node 4 Node 5 Node 6 Node 7 Total --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- 51722 (qemu-kvm) 68 16 357 6936 2 3 147 598 8128 51747 (qemu-kvm) 245 11 5 18 5172 2532 1 92 8076 53736 (qemu-kvm) 62 432 1661 506 4851 136 22 445 8116 53773 (qemu-kvm) 1393 3 1 2 12 0 0 6702 8114 --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- Total 1769 463 2024 7462 10037 2672 169 7837 32434
numastat -c qemu-kvm Per-node process memory usage (in MBs) PID Node 0 Node 1 Node 2 Node 3 Node 4 Node 5 Node 6 Node 7 Total --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- 51747 (qemu-kvm) 0 0 7 0 8072 0 1 0 8080 53736 (qemu-kvm) 0 0 7 0 0 0 8113 0 8120 53773 (qemu-kvm) 0 0 7 0 0 0 1 8110 8118 59065 (qemu-kvm) 0 0 8050 0 0 0 0 0 8051 --------------- ------ ------ ------ ------ ------ ------ ------ ------ ----- Total 0 0 8072 0 8072 0 8114 8110 32368
--disk path= /usr/share/virtio-win/virtio-win.iso ,device=cdrom
--os-variant win10
osinfo-query os
--boot uefi --tpm model=tpm-crb,backend.type=emulator,backend.version=2.0
virsh edit windows-vm
<os firmware='efi' > <type arch='x86_64' machine='pc-q35-6.2'>hvm</type> <boot dev='hd'/> </os>
<devices> <tpm model='tpm-crb'> <backend type='emulator' version='2.0'/> </tpm> </devices>
subscription-manager refresh All local data refreshed
yum install -y virtio-win
yum upgrade -y virtio-win
ls /usr/share/virtio-win/ drivers/ guest-agent/ virtio-win-1.9.9.iso virtio-win.iso
virt-xml WindowsVM --add-device --disk virtio-win.iso,device=cdrom Domain 'WindowsVM' defined successfully.
C:\WINDOWS\system32\netsh dump > backup.txt
C:\WINDOWS\system32\msiexec.exe /i X :\virtio-win-gt-x86.msi /passive /norestart
C:\WINDOWS\system32\netsh -f backup.txt
virsh edit windows-vm
<features> [...] <hyperv> <relaxed state='on'/> <vapic state='on'/> <spinlocks state='on' retries='8191'/> <vpindex state='on'/> <runtime state='on' /> <synic state='on'/> <stimer state='on'> <direct state='on'/> </stimer> <frequencies state='on'/> </hyperv> [...] </features>
<clock offset='localtime'> <timer name='hypervclock' present='yes'/> </clock>
<hyperv> <relaxed state='on'/> <vapic state='on'/> <spinlocks state='on' retries='8191'/> <vpindex state='on'/> <runtime state='on' /> <synic state='on'/> <stimer state='on'> <direct state='on'/> </stimer> <frequencies state='on'/> </hyperv> <clock offset='localtime'> <timer name='hypervclock' present='yes'/> </clock>
bcdedit /set useplatformclock No
{PackageManagerCommand} install edk2-ovmf
{PackageManagerCommand} install swtpm libtpms
<devices> [...] <tpm model='tpm-crb'> <backend type='emulator' version='2.0'/> </tpm> [...] </devices>
Your device meets the requirements for standard hardware security.
*modprobe kvm hpage=1 nested=1* modprobe: ERROR: could not insert 'kvm': Invalid argument *dmesg |tail -1* [90226.508366] kvm-s390: A KVM host that supports nesting cannot back its KVM guests with huge pages
cat /sys/module/kvm_intel/parameters/nested
modprobe -r kvm_intel
modprobe kvm_intel nested=1
options kvm_intel nested=1
virsh edit Intel-L1
<cpu mode='host-passthrough' />
<cpu mode ='custom' match ='exact' check='partial'> <model fallback='allow'> Haswell-noTSX </model> <feature policy='require' name='vmx'/> </cpu>
cat /sys/module/kvm_amd/parameters/nested
modprobe -r kvm_amd
modprobe kvm_amd nested=1
options kvm_amd nested=1
virsh edit AMD-L1
<cpu mode='host-passthrough' />
<cpu mode="custom" match="exact" check="none"> <model fallback="allow"> EPYC-IBPB </model> <feature policy="require" name="svm"/> </cpu>
cat /sys/module/kvm/parameters/nested
modprobe -r kvm
modprobe kvm nested=1
options kvm nested=1
cat /sys/module/kvm_hv/parameters/nested
modprobe -r kvm_hv
modprobe kvm_hv nested=1
options kvm_hv nested=1
<nested-hv state='on'/>
log_filters="3:remote 4:event 3:util.json 3:rpc" log_outputs="1:file:/var/log/libvirt/libvirt.log"
systemctl restart libvirtd.service
virt-admin daemon-log-filters >> virt-filters-backup
virt-admin daemon-log-filters "3:remote 4:event 3:util.json 3:rpc"
virt-admin daemon-log-outputs "1:file:/var/log/libvirt/libvirt.log"
virt-admin daemon-log-filters Logging filters:
virsh dump lander1 /core/file/gargantua.file --memory-only Domain 'lander1' dumped to /core/file/gargantua.file
crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M
pgrep libvirt 22014 22025
gstack 22014 Thread 3 (Thread 0x7f33edaf7700 (LWP 22017)): #0 0x00007f33f81aef21 in poll () from /lib64/libc.so.6 #1 0x00007f33f89059b6 in g_main_context_iterate.isra () from /lib64/libglib-2.0.so.0 #2 0x00007f33f8905d72 in g_main_loop_run () from /lib64/libglib-2.0.so.0
virsh dumpxml VM-name | grep machine=
/usr/libexec/qemu-kvm -M help
<domain type='qemu'>
<domain type='kvm'>
<disk type="block" device="lun">
<hostdev mode='subsystem' type='scsi'>