Managing, monitoring, and updating the kernel Red Hat Enterprise Linux 9 A guide to managing the Linux kernel on Red Hat Enterprise Linux 9 Red Hat Customer Content Services

Commands:
dnf repoquery <package_name>
dnf repoquery -l <package_name>
dnf install kernel- {version}
dnf update kernel
grubby --set-default USDkernel_path
grubby --info ALL | grep id grubby --set-default /boot/vmlinuz-<version>.<architecture>
grub2-reboot <index|title|id>
grubby --info=ALL | grep title
title="Red Hat Enterprise Linux (5.14.0-1.el9.x86_64) 9.0 (Plow)" title="Red Hat Enterprise Linux (0-rescue-0d772916a9724907a5d1350bcd39ac92) 9.0 (Plow)"
lsmod Module Size Used by fuse 126976 3 uinput 20480 1 xt_CHECKSUM 16384 1 ipt_MASQUERADE 16384 1 xt_conntrack 16384 1 ipt_REJECT 16384 1 nft_counter 16384 16 nf_nat_tftp 16384 0 nf_conntrack_tftp 16384 1 nf_nat_tftp tun 49152 1 bridge 192512 0 stp 16384 1 bridge llc 16384 2 bridge,stp nf_tables_set 32768 5 nft_fib_inet 16384 1 ...
modinfo < KERNEL_MODULE_NAME >
modinfo virtio_net filename: /lib/modules/5.14.0-1.el9.x86_64/kernel/drivers/net/virtio_net.ko.xz license: GPL description: Virtio network driver rhelversion: 9.0 srcversion: 8809CDDBE7202A1B00B9F1C alias: virtio:d00000001v* depends: net_failover retpoline: Y intree: Y name: virtio_net vermagic: 5.14.0-1.el9.x86_64 SMP mod_unload modversions ... parm: napi_weight:int parm: csum:bool parm: gso:bool parm: napi_tx:bool
modprobe < MODULE_NAME >
lsmod | grep < MODULE_NAME >
lsmod | grep serio_raw serio_raw 16384 0
lsmod
modprobe -r < MODULE_NAME >
lsmod | grep < MODULE_NAME >
lsmod | grep serio_raw
echo < MODULE_NAME > > /etc/modules-load.d/< MODULE_NAME >.conf
lsmod | grep < MODULE_NAME >
lsmod Module Size Used by tls 131072 0 uinput 20480 1 snd_seq_dummy 16384 0 snd_hrtimer 16384 1 ...
ls /lib/modules/4.18.0-477.20.1.el8_8.x86_64/kernel/crypto/ ansi_cprng.ko.xz chacha20poly1305.ko.xz md4.ko.xz serpent_generic.ko.xz anubis.ko.xz cmac.ko.xz...
touch /etc/modprobe.d/denylist.conf
Prevents <KERNEL-MODULE-1> from being loaded blacklist <MODULE-NAME-1> install <MODULE-NAME-1> /bin/false Prevents <KERNEL-MODULE-2> from being loaded blacklist <MODULE-NAME-2> install <MODULE-NAME-2> /bin/false ...
cp /boot/initramfs-USD(uname -r).img /boot/initramfs-USD(uname -r).bak.USD(date +%m-%d-%H%M%S).img
cp /boot/initramfs- <VERSION> .img /boot/initramfs- <VERSION> .img.bak.USD(date +%m-%d-%H%M%S)
dracut -f -v
dracut -f -v /boot/initramfs- <TARGET-VERSION> .img <CORRESPONDING-TARGET-KERNEL-VERSION>
reboot
dnf install kernel-devel-USD(uname -r) gcc elfutils-libelf-devel
#include <linux/module.h> #include <linux/kernel.h> int init_module(void) { printk("Hello World\n This is a test\n"); return 0; } void cleanup_module(void) { printk("Good Bye World"); } MODULE_LICENSE("GPL");
obj-m := test.o
make -C /lib/modules/USD(uname -r)/build M=/root/testmodule modules make: Entering directory '/usr/src/kernels/5.14.0-70.17.1.el9_0.x86_64' CC [M] /root/testmodule/test.o MODPOST /root/testmodule/Module.symvers CC [M] /root/testmodule/test.mod.o LD [M] /root/testmodule/test.ko BTF [M] /root/testmodule/test.ko Skipping BTF generation for /root/testmodule/test.ko due to unavailability of vmlinux make: Leaving directory '/usr/src/kernels/5.14.0-70.17.1.el9_0.x86_64'
ls -l /root/testmodule/ total 152 -rw-r- r--. 1 root root 16 Jul 26 08:19 Makefile -rw-r- r--. 1 root root 25 Jul 26 08:20 modules.order -rw-r- r--. 1 root root 0 Jul 26 08:20 Module.symvers -rw-r- r--. 1 root root 224 Jul 26 08:18 test.c -rw-r- r--. 1 root root 62176 Jul 26 08:20 test.ko -rw-r- r--. 1 root root 25 Jul 26 08:20 test.mod -rw-r- r--. 1 root root 849 Jul 26 08:20 test.mod.c -rw-r- r--. 1 root root 50936 Jul 26 08:20 test.mod.o -rw-r- r--. 1 root root 12912 Jul 26 08:20 test.o
cp /root/testmodule/test.ko /lib/modules/USD(uname -r)/
depmod -a
modprobe -v test insmod /lib/modules/ 5.14.0-1.el9.x86_64 /test.ko
lsmod | grep test test 16384 0
dmesg [ 74422.545004 ] Hello World This is a test
d8712ab6d4f14683c5625e87b52b6b6e-5.14.0-1.el9.x86_64.conf
title Red Hat Enterprise Linux (5.14.0-1.el9.x86_64) 9.0 (Plow) version 5.14.0-1.el9.x86_64 linux /vmlinuz-5.14.0-1.el9.x86_64 initrd /initramfs-5.14.0-1.el9.x86_64.img options root=/dev/mapper/rhel_kvm--02--guest08-root ro crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=/dev/mapper/rhel_kvm--02--guest08-swap rd.lvm.lv=rhel_kvm-02-guest08/root rd.lvm.lv=rhel_kvm-02-guest08/swap console=ttyS0,115200 grub_users USDgrub_users grub_arg --unrestricted grub_class kernel
grubby --update-kernel=ALL --args="< NEW_PARAMETER >"
zipl
grubby --update-kernel=ALL --remove-args="< PARAMETER_TO_REMOVE >"
zipl
grubby --update-kernel=/boot/vmlinuz-USD(uname -r) --args="< NEW_PARAMETER >"
zipl
grubby --update-kernel=/boot/vmlinuz-USD(uname -r) --remove-args="< PARAMETER_TO_REMOVE >"
zipl
linux (USDroot)/vmlinuz-5.14.0-63.el9.x86_64 root=/dev/mapper/rhel-root ro crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet emergency
GRUB_TERMINAL="serial" GRUB_SERIAL_COMMAND="serial --speed=9600 --unit=0 --word=8 --parity=no --stop=1"
grub2-mkconfig -o /boot/grub2/grub.cfg
grub2-mkconfig -o /boot/grub2/grub.cfg
GRUB_CMDLINE_LINUX="crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap"
grubby --update-kernel < PATH_TO_KERNEL > --args "< NEW_ARGUMENTS >"
grubby --update-kernel /boot/vmlinuz-5.14.0-362.8.1.el9_3.x86_64 --args "noapic"
grub2-mkconfig -o /boot/grub2/grub.cfg --update-bls-cmdline Generating grub configuration file ... Adding boot menu entry for UEFI Firmware Settings ... done
BOOT_IMAGE=(hd0,gpt2)/vmlinuz-4.18.0-425.3.1.el8.x86_64 root=/dev/mapper/RHELCSB-Root ro vconsole.keymap=us crashkernel=auto rd.lvm.lv=RHELCSB/Root rd.luks.uuid=luks-d8a28c4c-96aa-4319-be26-96896272151d rhgb quiet noapic rd.luks.key=d8a28c4c-96aa-4319-be26-96896272151d=/keyfile:UUID=c47d962e-4be8-41d6-8216-8cf7a0d3b911 ipv6.disable=1
sysctl -a
sysctl <TUNABLE_CLASS>.<PARAMETER>=<TARGET_VALUE>
sysctl -a
sysctl -w <TUNABLE_CLASS>.<PARAMETER>=<TARGET_VALUE> >> /etc/sysctl.conf
vim /etc/sysctl.d/< some_file.conf >
< TUNABLE_CLASS >.< PARAMETER >=< TARGET_VALUE > < TUNABLE_CLASS >.< PARAMETER >=< TARGET_VALUE >
sysctl -p /etc/sysctl.d/< some_file.conf >
ls -l /proc/sys/< TUNABLE_CLASS >/
echo < TARGET_VALUE > > /proc/sys/< TUNABLE_CLASS >/< PARAMETER >
cat /proc/sys/< TUNABLE_CLASS >/< PARAMETER >
--- - name: Configuring kernel settings hosts: managed-node-01.example.com tasks: - name: Configure hugepages, packet size for loopback device, and limits on simultaneously open files. ansible.builtin.include_role: name: rhel-system-roles.kernel_settings vars: kernel_settings_sysctl: - name: fs.file-max value: 400000 - name: kernel.threads-max value: 65536 kernel_settings_sysfs: - name: /sys/class/net/lo/mtu value: 65000 kernel_settings_transparent_hugepages: madvise kernel_settings_reboot_ok: true
ansible-playbook --syntax-check ~/playbook.yml
ansible-playbook ~/playbook.yml
ansible managed-node-01.example.com -m command -a 'sysctl fs.file-max kernel.threads-max net.ipv6.conf.lo.mtu' ansible managed-node-01.example.com -m command -a 'cat /sys/kernel/mm/transparent_hugepage/enabled'
--- - name: Configuration and management of GRUB boot loader hosts: managed-node-01.example.com tasks: - name: Update existing boot loader entries ansible.builtin.include_role: name: rhel-system-roles.bootloader vars: bootloader_settings: - kernel: path: /boot/vmlinuz-5.14.0-362.24.1.el9_3.aarch64 options: - name: quiet state: present bootloader_reboot_ok: true
ansible-playbook --syntax-check ~/playbook.yml
ansible-playbook ~/playbook.yml
ansible managed-node-01.example.com -m ansible.builtin.command -a 'grubby --info=ALL' managed-node-01.example.com | CHANGED | rc=0 >> index=1 kernel="/boot/vmlinuz-5.14.0-362.24.1.el9_3.aarch64" args="ro crashkernel=1G-4G:256M,4G-64G:320M,64G-:576M rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap USDtuned_params quiet " root="/dev/mapper/rhel-root" initrd="/boot/initramfs-5.14.0-362.24.1.el9_3.aarch64.img USDtuned_initrd" title="Red Hat Enterprise Linux (5.14.0-362.24.1.el9_3.aarch64) 9.4 (Plow)" id="2c9ec787230141a9b087f774955795ab-5.14.0-362.24.1.el9_3.aarch64"
ansible-vault create vault.yml New Vault password: <vault_password> Confirm New Vault password: <vault_password>
pwd: <password>
--- - name: Configuration and management of GRUB boot loader hosts: managed-node-01.example.com vars_files: - vault.yml tasks: - name: Set the bootloader password ansible.builtin.include_role: name: rhel-system-roles.bootloader vars: bootloader_password: "{{ pwd }}" bootloader_reboot_ok: true
ansible-playbook --syntax-check --ask-vault-pass ~/playbook.yml
ansible-playbook --ask-vault-pass ~/playbook.yml
--- - name: Configuration and management of the GRUB boot loader hosts: managed-node-01.example.com tasks: - name: Update the boot loader timeout ansible.builtin.include_role: name: rhel-system-roles.bootloader vars: bootloader_timeout: 10
ansible-playbook --syntax-check ~/playbook.yml
ansible-playbook ~/playbook.yml
ansible managed-node-01.example.com -m ansible.builtin.reboot managed-node-01.example.com | CHANGED => { "changed": true, "elapsed": 21, "rebooted": true }
ansible managed-node-01.example.com -m ansible.builtin.command -a "grep 'timeout' /boot/grub2/grub.cfg" managed-node-01.example.com | CHANGED | rc=0 >> if [ xUSDfeature_timeout_style = xy ] ; then set timeout_style=menu set timeout=10 Fallback normal timeout code in case the timeout_style feature is set timeout=10 if [ xUSDfeature_timeout_style = xy ] ; then set timeout_style=menu set timeout=10 set orig_timeout_style=USD{timeout_style} set orig_timeout=USD{timeout} # timeout_style=menu + timeout=0 avoids the countdown code keypress check set timeout_style=menu set timeout=10 set timeout_style=hidden set timeout=10 if [ xUSDfeature_timeout_style = xy ]; then if [ "USD{menu_show_once_timeout}" ]; then set timeout_style=menu set timeout=10 unset menu_show_once_timeout save_env menu_show_once_timeout
--- - name: Configuration and management of GRUB boot loader hosts: managed-node-01.example.com tasks: - name: Gather information about the boot loader configuration ansible.builtin.include_role: name: rhel-system-roles.bootloader vars: bootloader_gather_facts: true - name: Display the collected boot loader configuration information debug: var: bootloader_facts
ansible-playbook --syntax-check ~/playbook.yml
ansible-playbook ~/playbook.yml
"bootloader_facts": [ { "args": "ro crashkernel=1G-4G:256M,4G-64G:320M,64G-:576M rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap USDtuned_params quiet", "default": true, "id": "2c9ec787230141a9b087f774955795ab-5.14.0-362.24.1.el9_3.aarch64", "index": "1", "initrd": "/boot/initramfs-5.14.0-362.24.1.el9_3.aarch64.img USDtuned_initrd", "kernel": "/boot/vmlinuz-5.14.0-362.24.1.el9_3.aarch64", "root": "/dev/mapper/rhel-root", "title": "Red Hat Enterprise Linux (5.14.0-362.24.1.el9_3.aarch64) 9.4 (Plow)" } ]
uname -r 5.14.0-1.el9.x86_64
dnf search USD(uname -r)
dnf install "kpatch-patch = USD(uname -r)"
kpatch list Loaded patch modules: kpatch_5_14_0_1_0_1 [enabled] Installed patch modules: kpatch_5_14_0_1_0_1 (5.14.0-1.el9.x86_64) ...
rpm -qa | grep kpatch kpatch-dnf-0.4-3.el9.noarch kpatch-0.9.7-2.el9.noarch kpatch-patch-5_14_0-284_25_1-0-0.el9_2.x86_64
dnf list installed | grep kernel Updating Subscription Management repositories. Installed Packages kernel-core.x86_64 5.14.0-1.el9 @beaker-BaseOS kernel-core.x86_64 5.14.0-2.el9 @@commandline uname -r 5.14.0-2.el9.x86_64
dnf install kpatch-dnf
dnf kpatch auto Updating Subscription Management repositories. Last metadata expiration check: 1:38:21 ago on Fri 17 Sep 2021 07:29:53 AM EDT. Dependencies resolved. ================================================== Package Architecture ================================================== Installing: kpatch-patch-5_14_0-1 x86_64 kpatch-patch-5_14_0-2 x86_64 Transaction Summary =================================================== Install 2 Packages ...
kpatch list Loaded patch modules: kpatch_5_14_0_2_0_1 [enabled] Installed patch modules: kpatch_5_14_0_1_0_1 (5.14.0-1.el9.x86_64) kpatch_5_14_0_2_0_1 (5.14.0-2.el9.x86_64)
rpm -qa | grep kpatch kpatch-dnf-0.4-3.el9.noarch kpatch-0.9.7-2.el9.noarch kpatch-patch-5_14_0-284_25_1-0-0.el9_2.x86_64
dnf list installed | grep kernel Updating Subscription Management repositories. Installed Packages kernel-core.x86_64 5.14.0-1.el9 @beaker-BaseOS kernel-core.x86_64 5.14.0-2.el9 @@commandline uname -r 5.14.0-2.el9.x86_64
dnf kpatch manual Updating Subscription Management repositories.
yum kpatch status Updating Subscription Management repositories. Last metadata expiration check: 0:30:41 ago on Tue Jun 14 15:59:26 2022. Kpatch update setting: manual
dnf update "kpatch-patch = USD(uname -r)"
dnf update "kpatch-patch "
dnf list installed | grep kpatch-patch kpatch-patch-5_14_0-1.x86_64 0-1.el9 @@commandline ...
dnf remove kpatch-patch-5_14_0-1.x86_64
dnf list installed | grep kpatch-patch
kpatch list Loaded patch modules:
kpatch list Loaded patch modules: kpatch_5_14_0_1_0_1 [enabled] Installed patch modules: kpatch_5_14_0_1_0_1 (5.14.0-1.el9.x86_64) ...
kpatch uninstall kpatch_5_14_0_1_0_1 uninstalling kpatch_5_14_0_1_0_1 (5.14.0-1.el9.x86_64)
kpatch list Loaded patch modules: kpatch_5_14_0_1_0_1 [enabled] Installed patch modules: < NO_RESULT >
kpatch list Loaded patch modules: ...
systemctl is-enabled kpatch.service enabled
systemctl disable kpatch.service Removed /etc/systemd/system/multi-user.target.wants/kpatch.service.
kpatch list Loaded patch modules: kpatch_5_14_0_1_0_1 [enabled] Installed patch modules: kpatch_5_14_0_1_0_1 (5.14.0-1.el9.x86_64)
systemctl status kpatch.service ● kpatch.service - "Apply kpatch kernel patches" Loaded: loaded (/usr/lib/systemd/system/kpatch.service; disabled; vendor preset: disabled) Active: inactive (dead)
kpatch list Loaded patch modules: Installed patch modules: kpatch_5_14_0_1_0_1 (5.14.0-1.el9.x86_64)
sysctl kernel.printk kernel.printk = 7 4 1 7
grub2-install /dev/sda
reboot
yum reinstall grub2-efi shim
reboot
bootlist -m normal -o sda1
grub2-install partition
reboot
rm /etc/grub.d/ * rm /etc/sysconfig/grub
yum reinstall grub2-tools
yum reinstall grub2-efi shim grub2-tools
grub2-mkconfig -o /boot/grub2/grub.cfg
grub2-mkconfig -o /boot/grub2/grub.cfg
rpm -q kexec-tools
kexec-tools-2.0.22-13.el9.x86_64
package kexec-tools is not installed
dnf install kexec-tools
makedumpfile --mem-usage /proc/kcore TYPE PAGES EXCLUDABLE DESCRIPTION ------------------------------------------------------------- ZERO 501635 yes Pages filled with zero CACHE 51657 yes Cache pages CACHE_PRIVATE 5442 yes Cache pages + private USER 16301 yes User process pages FREE 77738211 yes Free pages KERN_DATA 1333192 no Dumpable kernel data
crashkernel=1G-4G:192M,4G-64G:256M,64G:512M
kdumpctl reset-crashkernel --kernel=ALL
crashkernel=192M
crashkernel=1G-4G:192M,2G-64G:256M
crashkernel=192M@16M
grubby --update-kernel ALL --args "crashkernel= <custom-value> "
reboot
echo c > /proc/sysrq-trigger
path /var/crash
kdump_post <path_to_kdump_post.sh>
*grep -v ^# /etc/kdump.conf | grep -v ^USD* ext4 /dev/mapper/vg00-varcrashvol path /var/crash core_collector makedumpfile -c --message-level 1 -d 31
path /usr/local/cores
ext4 UUID=03138356-5e61-4ab3-b58e-27507ac41937
raw /dev/sdb1
nfs penguin.example.com:/export/cores
sudo systemctl restart kdump.service
ssh john@penguin.example.com sshkey /root/.ssh/mykey
core_collector makedumpfile -l --message-level 1 -d 31
core_collector makedumpfile -l --message-level 1 -d 31
core_collector makedumpfile -c -d 31 --message-level 1
core_collector makedumpfile -p -d 31 --message-level 1
failure_action poweroff
KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug quiet log_buf_len swiotlb"
KDUMP_COMMANDLINE_APPEND="cgroup_disable=memory"
kdumpctl restart
kdumpctl status kdump:Kdump is operational
echo c > /proc/sysrq-trigger
systemctl enable kdump.service
systemctl start kdump.service
systemctl stop kdump.service
systemctl disable kdump.service
lsmod Module Size Used by fuse 126976 3 xt_CHECKSUM 16384 1 ipt_MASQUERADE 16384 1 uinput 20480 1 xt_conntrack 16384 1
KDUMP_COMMANDLINE_APPEND="rd.driver.blacklist= hv_vmbus,hv_storvsc,hv_utils,hv_netvsc,hid-hyperv "
KDUMP_COMMANDLINE_APPEND="modprobe.blacklist= emcp modprobe.blacklist= bnx2fc modprobe.blacklist= libfcoe modprobe.blacklist= fcoe "
systemctl restart kdump
*kdumpctl estimate* Encrypted kdump target requires extra memory, assuming using the keyslot with minimum memory requirement Reserved crashkernel: 256M Recommended crashkernel: 652M Kernel image size: 47M Kernel modules size: 8M Initramfs size: 20M Runtime reservation: 64M LUKS required size: 512M Large modules: <none> WARNING: Current crashkernel size is lower than recommended size 652M.
sudo grubby --update-kernel ALL --args crashkernel=512M
grubby --update-kernel=ALL --args="crashkernel=xxM"
systemctl enable --now kdump.service
systemctl status kdump.service ○ kdump.service - Crash recovery kernel arming Loaded: loaded (/usr/lib/systemd/system/kdump.service; enabled; vendor preset: disabled) Active: active (live)
ls -a /boot/vmlinuz- * /boot/vmlinuz-0-rescue-2930657cd0dc43c2b75db480e5e5b4a9 /boot/vmlinuz-4.18.0-330.el8.x86_64 /boot/vmlinuz-4.18.0-330.rt7.111.el8.x86_64
grubby --update-kernel= vmlinuz-4.18.0-330.el8.x86_64 --args="crashkernel= xxM "
systemctl enable --now kdump.service
systemctl status kdump.service ○ kdump.service - Crash recovery kernel arming Loaded: loaded (/usr/lib/systemd/system/kdump.service; enabled; vendor preset: disabled) Active: active (live)
systemctl stop kdump.service
systemctl disable kdump.service
uname -m
kdumpctl restart
kdumpctl restart
error: ../../grub-core/kern/mm.c:376:out of memory. Press any key to continue...
kdumpctl reset-crashkernel --fadump=on --kernel=ALL
grubby --update-kernel ALL --args="fadump=on crashkernel=xxM"
reboot
kernel.panic=0 kernel.unknown_nmi_panic=1
failure_action shell
subscription-manager repos --enable baseos repository
subscription-manager repos --enable appstream repository
subscription-manager repos --enable rhel-9-for-x86_64-baseos-debug-rpms
dnf install crash
dnf install kernel-debuginfo
crash /usr/lib/debug/lib/modules/5.14.0-1.el9.x86_64/vmlinux /var/crash/127.0.0.1-2021-09-13-14:05:33/vmcore
WARNING: kernel relocated [202MB]: patching 90160 gdb minimal_symbol values KERNEL: /usr/lib/debug/lib/modules/5.14.0-1.el9.x86_64/vmlinux DUMPFILE: /var/crash/127.0.0.1-2021-09-13-14:05:33/vmcore [PARTIAL DUMP] CPUS: 2 DATE: Mon Sep 13 14:05:16 2021 UPTIME: 01:03:57 LOAD AVERAGE: 0.00, 0.00, 0.00 TASKS: 586 NODENAME: localhost.localdomain RELEASE: 5.14.0-1.el9.x86_64 VERSION: #1 SMP Wed Aug 29 11:51:55 UTC 2018 MACHINE: x86_64 (2904 Mhz) MEMORY: 2.9 GB PANIC: "sysrq: SysRq : Trigger a crash" PID: 10635 COMMAND: "bash" TASK: ffff8d6c84271800 [THREAD_INFO: ffff8d6c84271800] CPU: 1 STATE: TASK_RUNNING (SYSRQ) crash>
crash> exit ~]#
crash> log ... several lines omitted EIP: 0060:[<c068124f>] EFLAGS: 00010096 CPU: 2 EIP is at sysrq_handle_crash+0xf/0x20 EAX: 00000063 EBX: 00000063 ECX: c09e1c8c EDX: 00000000 ESI: c0a09ca0 EDI: 00000286 EBP: 00000000 ESP: ef4dbf24 DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068 Process bash (pid: 5591, ti=ef4da000 task=f196d560 task.ti=ef4da000) Stack: c068146b c0960891 c0968653 00000003 00000000 00000002 efade5c0 c06814d0 <0> fffffffb c068150f b7776000 f2600c40 c0569ec4 ef4dbf9c 00000002 b7776000 <0> efade5c0 00000002 b7776000 c0569e60 c051de50 ef4dbf9c f196d560 ef4dbfb4 Call Trace: [<c068146b>] ? __handle_sysrq+0xfb/0x160 [<c06814d0>] ? write_sysrq_trigger+0x0/0x50 [<c068150f>] ? write_sysrq_trigger+0x3f/0x50 [<c0569ec4>] ? proc_reg_write+0x64/0xa0 [<c0569e60>] ? proc_reg_write+0x0/0xa0 [<c051de50>] ? vfs_write+0xa0/0x190 [<c051e8d1>] ? sys_write+0x41/0x70 [<c0409adc>] ? syscall_call+0x7/0xb Code: a0 c0 01 0f b6 41 03 19 d2 f7 d2 83 e2 03 83 e0 cf c1 e2 04 09 d0 88 41 03 f3 c3 90 c7 05 c8 1b 9e c0 01 00 00 00 0f ae f8 89 f6 <c6> 05 00 00 00 00 01 c3 89 f6 8d bc 27 00 00 00 00 8d 50 d0 83 EIP: [<c068124f>] sysrq_handle_crash+0xf/0x20 SS:ESP 0068:ef4dbf24 CR2: 0000000000000000
crash> bt PID: 5591 TASK: f196d560 CPU: 2 COMMAND: "bash" #0 [ef4dbdcc] crash_kexec at c0494922 #1 [ef4dbe20] oops_end at c080e402 #2 [ef4dbe34] no_context at c043089d #3 [ef4dbe58] bad_area at c0430b26 #4 [ef4dbe6c] do_page_fault at c080fb9b #5 [ef4dbee4] error_code (via page_fault) at c080d809 EAX: 00000063 EBX: 00000063 ECX: c09e1c8c EDX: 00000000 EBP: 00000000 DS: 007b ESI: c0a09ca0 ES: 007b EDI: 00000286 GS: 00e0 CS: 0060 EIP: c068124f ERR: ffffffff EFLAGS: 00010096 #6 [ef4dbf18] sysrq_handle_crash at c068124f #7 [ef4dbf24] __handle_sysrq at c0681469 #8 [ef4dbf48] write_sysrq_trigger at c068150a #9 [ef4dbf54] proc_reg_write at c0569ec2 #10 [ef4dbf74] vfs_write at c051de4e #11 [ef4dbf94] sys_write at c051e8cc #12 [ef4dbfb0] system_call at c0409ad5 EAX: ffffffda EBX: 00000001 ECX: b7776000 EDX: 00000002 DS: 007b ESI: 00000002 ES: 007b EDI: b7776000 SS: 007b ESP: bfcb2088 EBP: bfcb20b4 GS: 0033 CS: 0073 EIP: 00edc416 ERR: 00000004 EFLAGS: 00000246
crash> ps PID PPID CPU TASK ST %MEM VSZ RSS COMM > 0 0 0 c09dc560 RU 0.0 0 0 [swapper] > 0 0 1 f7072030 RU 0.0 0 0 [swapper] 0 0 2 f70a3a90 RU 0.0 0 0 [swapper] > 0 0 3 f70ac560 RU 0.0 0 0 [swapper] 1 0 1 f705ba90 IN 0.0 2828 1424 init ... several lines omitted 5566 1 1 f2592560 IN 0.0 12876 784 auditd 5567 1 2 ef427560 IN 0.0 12876 784 auditd 5587 5132 0 f196d030 IN 0.0 11064 3184 sshd > 5591 5587 2 f196d560 RU 0.0 5084 1648 bash
crash> vm PID: 5591 TASK: f196d560 CPU: 2 COMMAND: "bash" MM PGD RSS TOTAL_VM f19b5900 ef9c6000 1648k 5084k VMA START END FLAGS FILE f1bb0310 242000 260000 8000875 /lib/ld-2.12.so f26af0b8 260000 261000 8100871 /lib/ld-2.12.so efbc275c 261000 262000 8100873 /lib/ld-2.12.so efbc2a18 268000 3ed000 8000075 /lib/libc-2.12.so efbc23d8 3ed000 3ee000 8000070 /lib/libc-2.12.so efbc2888 3ee000 3f0000 8100071 /lib/libc-2.12.so efbc2cd4 3f0000 3f1000 8100073 /lib/libc-2.12.so efbc243c 3f1000 3f4000 100073 efbc28ec 3f6000 3f9000 8000075 /lib/libdl-2.12.so efbc2568 3f9000 3fa000 8100071 /lib/libdl-2.12.so efbc2f2c 3fa000 3fb000 8100073 /lib/libdl-2.12.so f26af888 7e6000 7fc000 8000075 /lib/libtinfo.so.5.7 f26aff2c 7fc000 7ff000 8100073 /lib/libtinfo.so.5.7 efbc211c d83000 d8f000 8000075 /lib/libnss_files-2.12.so efbc2504 d8f000 d90000 8100071 /lib/libnss_files-2.12.so efbc2950 d90000 d91000 8100073 /lib/libnss_files-2.12.so f26afe00 edc000 edd000 4040075 f1bb0a18 8047000 8118000 8001875 /bin/bash f1bb01e4 8118000 811d000 8101873 /bin/bash f1bb0c70 811d000 8122000 100073 f26afae0 9fd9000 9ffa000 100073 ... several lines omitted
crash> files PID: 5591 TASK: f196d560 CPU: 2 COMMAND: "bash" ROOT: / CWD: /root FD FILE DENTRY INODE TYPE PATH 0 f734f640 eedc2c6c eecd6048 CHR /pts/0 1 efade5c0 eee14090 f00431d4 REG /proc/sysrq-trigger 2 f734f640 eedc2c6c eecd6048 CHR /pts/0 10 f734f640 eedc2c6c eecd6048 CHR /pts/0 255 f734f640 eedc2c6c eecd6048 CHR /pts/0
systemctl is-enabled kdump.service && systemctl is-active kdump.service enabled active
dracut -f --add earlykdump
grubby --update-kernel=/boot/vmlinuz-USD(uname -r) --args="rd.earlykdump"
reboot
cat /proc/cmdline BOOT_IMAGE=(hd0,msdos1)/vmlinuz-5.14.0-1.el9.x86_64 root=/dev/mapper/rhel-root ro crashkernel=auto resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet rd.earlykdump journalctl -x | grep early-kdump Sep 13 15:46:11 redhat dracut-cmdline[304]: early-kdump is enabled. Sep 13 15:46:12 redhat dracut-cmdline[304]: kexec: loaded early-kdump kernel
dnf install pesign openssl kernel-devel mokutil keyutils
efikeygen --dbdir /etc/pki/pesign --self-sign --module --common-name 'CN= Organization signing key ' --nickname ' Custom Secure Boot key '
efikeygen --dbdir /etc/pki/pesign --self-sign --kernel --common-name 'CN= Organization signing key ' --nickname ' Custom Secure Boot key '
efikeygen --dbdir /etc/pki/pesign --self-sign --kernel --common-name 'CN= Organization signing key ' --nickname ' Custom Secure Boot key ' --token 'NSS FIPS 140-2 Certificate DB'
keyctl list %:.builtin_trusted_keys 6 keys in keyring: ...asymmetric: Red Hat Enterprise Linux Driver Update Program (key 3): bf57f3e87 ...asymmetric: Red Hat Secure Boot (CA key 1): 4016841644ce3a810408050766e8f8a29 ...asymmetric: Microsoft Corporation UEFI CA 2011: 13adbf4309bd82709c8cd54f316ed ...asymmetric: Microsoft Windows Production PCA 2011: a92902398e16c49778cd90f99e ...asymmetric: Red Hat Enterprise Linux kernel signing key: 4249689eefc77e95880b ...asymmetric: Red Hat Enterprise Linux kpatch signing key: 4d38fd864ebe18c5f0b7 keyctl list %:.platform 4 keys in keyring: ...asymmetric: VMware, Inc.: 4ad8da0472073 ...asymmetric: Red Hat Secure Boot CA 5: cc6fafe72 ...asymmetric: Microsoft Windows Production PCA 2011: a929f298e1 ...asymmetric: Microsoft Corporation UEFI CA 2011: 13adbf4e0bd82 keyctl list %:.blacklist 4 keys in keyring: ...blacklist: bin:f5ff83a ...blacklist: bin:0dfdbec ...blacklist: bin:38f1d22 ...blacklist: bin:51f831f
dmesg | egrep 'integrity.*cert' [1.512966] integrity: Loading X.509 certificate: UEFI:db [1.513027] integrity: Loaded X.509 cert 'Microsoft Windows Production PCA 2011: a929023 [1.513028] integrity: Loading X.509 certificate: UEFI:db [1.513057] integrity: Loaded X.509 cert 'Microsoft Corporation UEFI CA 2011: 13adbf4309 [1.513298] integrity: Loading X.509 certificate: UEFI:MokListRT (MOKvar table) [1.513549] integrity: Loaded X.509 cert 'Red Hat Secure Boot CA 5: cc6fa5e72868ba494e93
certutil -d /etc/pki/pesign -n ' Custom Secure Boot key ' -Lr > sb_cert.cer
mokutil --import sb_cert.cer
pesign --certificate ' Custom Secure Boot key ' --in vmlinuz- version --sign --out vmlinuz- version .signed
pesign --show-signature --in vmlinuz- version .signed
mv vmlinuz- version .signed vmlinuz- version
zcat vmlinuz- version > vmlinux- version
pesign --certificate ' Custom Secure Boot key ' --in vmlinux- version --sign --out vmlinux- version .signed
pesign --show-signature --in vmlinux- version .signed
gzip --to-stdout vmlinux- version .signed > vmlinuz- version
rm vmlinux- version *
pesign --in /boot/efi/EFI/redhat/grubx64.efi --out /boot/efi/EFI/redhat/grubx64.efi.signed --certificate ' Custom Secure Boot key ' --sign
pesign --in /boot/efi/EFI/redhat/grubx64.efi.signed --show-signature
mv /boot/efi/EFI/redhat/grubx64.efi.signed /boot/efi/EFI/redhat/grubx64.efi
pesign --in /boot/efi/EFI/redhat/grubaa64.efi --out /boot/efi/EFI/redhat/grubaa64.efi.signed --certificate ' Custom Secure Boot key ' --sign
pesign --in /boot/efi/EFI/redhat/grubaa64.efi.signed --show-signature
mv /boot/efi/EFI/redhat/grubaa64.efi.signed /boot/efi/EFI/redhat/grubaa64.efi
certutil -d /etc/pki/pesign -n ' Custom Secure Boot key ' -Lr > sb_cert.cer
pk12util -o sb_cert.p12 -n ' Custom Secure Boot key ' -d /etc/pki/pesign
openssl pkcs12 -in sb_cert.p12 -out sb_cert.priv -nocerts -noenc
/usr/src/kernels/USD(uname -r)/scripts/sign-file sha256 sb_cert.priv sb_cert.cer my_module .ko
modinfo my_module .ko | grep signer signer: Your Name Key
insmod my_module .ko
modprobe -r my_module .ko
dnf -y install kernel-modules-extra
keyctl list %:.platform
cp my_module .ko /lib/modules/USD(uname -r)/extra/
depmod -a
modprobe -v my_module
echo " my_module " > /etc/modules-load.d/ my_module .conf
lsmod | grep my_module
fwupdmgr get-devices
fwupdmgr enable-remote lvfs
fwupdmgr refresh
fwupdmgr update
fwupdmgr get-devices
fwupdmgr get-devices
ls /usr/share/dbxtool/
DBXUpdate- date - architecture .cab
fwupdmgr install /usr/share/dbxtool/DBXUpdate- date - architecture .cab
fwupdmgr get-devices
cat /sys/class/tpm/tpm0/tpm_version_major 2
TPM_DEVICE=/dev/tpm0 tsscreateprimary -hi o -st Handle 80000000 TPM_DEVICE=/dev/tpm0 tssevictcontrol -hi o -ho 80000000 -hp 81000001
tpm2_createprimary --key-algorithm=rsa2048 --key-context=key.ctxt name-alg: value: sha256 raw: 0xb ... sym-keybits: 128 rsa: xxxxxx... tpm2_evictcontrol -c key.ctxt 0x81000001 persistentHandle: 0x81000001 action: persisted
keyctl add trusted kmk "new 32 keyhandle=0x81000001" @u 642500861
keyctl show Session Keyring -3 --alswrv 500 500 keyring: ses 97833714 --alswrv 500 -1 \ keyring: uid.1000 642500861 --alswrv 500 500 \ trusted: kmk
keyctl pipe 642500861 > kmk.blob
keyctl add trusted kmk "load `cat kmk.blob`" @u 268728824
keyctl add encrypted encr-key "new trusted:kmk 32" @u 159771175
keyctl add user kmk-user "USD(dd if=/dev/urandom bs=1 count=32 2>/dev/null)" @u 427069434
keyctl add encrypted encr-key "new user:kmk-user 32" @u 1012412758
keyctl list @u 2 keys in keyring: 427069434: --alswrv 1000 1000 user: kmk-user 1012412758: --alswrv 1000 1000 encrypted: encr-key
mount securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
grep < options > pattern < files >
dmesg | grep -i -e EVM -e IMA -w [ 0.943873] ima: No TPM chip found, activating TPM-bypass! [ 0.944566] ima: Allocated hash algorithm: sha256 [ 0.944579] ima: No architecture policies found [ 0.944601] evm: Initialising EVM extended attributes: [ 0.944602] evm: security.selinux [ 0.944604] evm: security.SMACK64 (disabled) [ 0.944605] evm: security.SMACK64EXEC (disabled) [ 0.944607] evm: security.SMACK64TRANSMUTE (disabled) [ 0.944608] evm: security.SMACK64MMAP (disabled) [ 0.944609] evm: security.apparmor (disabled) [ 0.944611] evm: security.ima [ 0.944612] evm: security.capability [ 0.944613] evm: HMAC attrs: 0x1 [ 1.314520] systemd[1]: systemd 252-18.el9 running in system mode (+PAM +AUDIT +SELINUX -APPARMOR +IMA +SMACK +SECCOMP +GCRYPT +GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS -FIDO2 +IDN2 -IDN -IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY +P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -BPF_FRAMEWORK +XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified) [ 1.717675] device-mapper: core: CONFIG_IMA_DISABLE_HTABLE is disabled. Duplicate IMA measurements will not be recorded in the IMA log. [ 4.799436] systemd[1]: systemd 252-18.el9 running in system mode (+PAM +AUDIT +SELINUX -APPARMOR +IMA +SMACK +SECCOMP +GCRYPT +GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS -FIDO2 +IDN2 -IDN -IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY +P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -BPF_FRAMEWORK +XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified)
grubby --update-kernel=/boot/vmlinuz-USD(uname -r) --args="ima_policy=appraise_tcb ima_appraise=fix evm=fix"
cat /proc/cmdline BOOT_IMAGE=(hd0,msdos1)/vmlinuz-5.14.0-1.el9.x86_64 root=/dev/mapper/rhel-root ro crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=/dev/mapper/rhel-swap rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet ima_policy=appraise_tcb ima_appraise=fix evm=fix
keyctl add user kmk "USD(dd if=/dev/urandom bs=1 count=32 2> /dev/null)" @u 748544121
keyctl add encrypted evm-key "new user:kmk 64" @u 641780271
mkdir -p /etc/keys/
keyctl pipe USD(keyctl search @u user kmk) > /etc/keys/kmk
keyctl pipe USD(keyctl search @u encrypted evm-key) > /etc/keys/evm-key
keyctl show Session Keyring 974575405 --alswrv 0 0 keyring: ses 299489774 --alswrv 0 65534 \ keyring: uid.0 748544121 --alswrv 0 0 \ user: kmk 641780271 --alswrv 0 0 \_ encrypted: evm-key ls -l /etc/keys/ total 8 -rw-r--r--. 1 root root 246 Jun 24 12:44 evm-key -rw-r--r--. 1 root root 32 Jun 24 12:43 kmk
keyctl add user kmk "USD(cat /etc/keys/kmk)" @u 451342217
keyctl add encrypted evm-key "load USD(cat /etc/keys/evm-key)" @u 924537557
echo 1 > /sys/kernel/security/evm
find / -fstype xfs -type f -uid 0 -exec head -n 1 '{}' >/dev/null \;
dmesg | tail -1 [... ] evm: key initialized
echo < Test_text > > test_file
getfattr -m . -d test_file file: test_file security.evm=0sAnDIy4VPA0HArpPO/EqiutnNyBql security.ima=0sAQOEDeuUnWzwwKYk+n66h/vby3eD
dnf install rpm-plugin-ima -y
dnf reinstall '*' -y
getfattr -m security.ima -d /usr/bin/bash
'security.ima=0sAwIE0zIESQBnMGUCMFhf0iBeM7NjjhCCHVt4/ORx1eCegjrWSHzFbJMCsAhR9bYU2hNGjiWUYT2IIqWaaAIxALFGUkqGP5vDLuxQXibO9g7HFcfyZzRBY4rbKPsXcAIZRtDHVS5dQBZqM3hyS5v1MA=='
evmctl ima_verify -k /usr/share/doc/kernel-keys/USD(uname -r)/ima.cer /usr/bin/bash
'key 1: d3320449 /usr/share/doc/kernel-keys/5.14.0-359.el9.x86-64/ima.cer /usr/bin/bash:' verification is OK
mkdir -p /etc/keys/ima cp /usr/share/doc/kernel-keys/USD(uname -r)/ima.cer /etc/ima/keys
keyctl padd asymmetric RedHat-IMA %:.ima < /etc/ima/keys/ima.cer
PROC_SUPER_MAGIC = 0x9fa0 dont_appraise fsmagic=0x9fa0 SYSFS_MAGIC = 0x62656572 dont_appraise fsmagic=0x62656572 DEBUGFS_MAGIC = 0x64626720 dont_appraise fsmagic=0x64626720 TMPFS_MAGIC = 0x01021994 dont_appraise fsmagic=0x1021994 RAMFS_MAGIC dont_appraise fsmagic=0x858458f6 DEVPTS_SUPER_MAGIC=0x1cd1 dont_appraise fsmagic=0x1cd1 BINFMTFS_MAGIC=0x42494e4d dont_appraise fsmagic=0x42494e4d SECURITYFS_MAGIC=0x73636673 dont_appraise fsmagic=0x73636673 SELINUX_MAGIC=0xf97cff8c dont_appraise fsmagic=0xf97cff8c SMACK_MAGIC=0x43415d53 dont_appraise fsmagic=0x43415d53 NSFS_MAGIC=0x6e736673 dont_appraise fsmagic=0x6e736673 EFIVARFS_MAGIC dont_appraise fsmagic=0xde5e81e4 CGROUP_SUPER_MAGIC=0x27e0eb dont_appraise fsmagic=0x27e0eb CGROUP2_SUPER_MAGIC=0x63677270 dont_appraise fsmagic=0x63677270 appraise func=BPRM_CHECK appraise func=FILE_MMAP mask=MAY_EXEC
echo /etc/sysconfig/ima-policy > /sys/kernel/security/ima/policy echo USD? 0
echo 'add_dracutmodules+=" integrity "' > /etc/dracut.conf.d/98-integrity.conf dracut -f
openssl req -new -x509 -utf8 -sha256 -days 3650 -batch -config ima_ca.conf -outform DER -out custom_ima_ca.der -keyout custom_ima_ca.priv
cat ima_ca.conf [ req ] default_bits = 2048 distinguished_name = req_distinguished_name prompt = no string_mask = utf8only x509_extensions = ca [ req_distinguished_name ] O = YOUR_ORG CN = YOUR_COMMON_NAME IMA CA emailAddress = YOUR_EMAIL [ ca ] basicConstraints=critical,CA:TRUE subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer keyUsage=critical,keyCertSign,cRLSign
openssl req -new -utf8 -sha256 -days 365 -batch -config ima.conf -out custom_ima.csr -keyout custom_ima.priv
cat ima.conf [ req ] default_bits = 2048 distinguished_name = req_distinguished_name prompt = no string_mask = utf8only x509_extensions = code_signing [ req_distinguished_name ] O = YOUR_ORG CN = YOUR_COMMON_NAME IMA signing key emailAddress = YOUR_EMAIL [ code_signing ] basicConstraints=critical,CA:FALSE keyUsage=digitalSignature subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer
openssl x509 -req -in custom_ima.csr -days 365 -extfile ima.conf -extensions code_signing -CA custom_ima_ca.der -CAkey custom_ima_ca.priv -CAcreateserial -outform DER -out ima.der
evmctl ima_sign /etc/sysconfig/ima-policy -k < PATH_TO_YOUR_CUSTOM_IMA_KEY > Place your public certificate under /etc/keys/ima/ and add it to the .ima keyring
keyctl padd asymmetric CUSTOM_IMA1 %:.ima < /etc/ima/keys/my_ima.cer
echo /etc/sysconfig/ima-policy > /sys/kernel/security/ima/policy echo USD? 0
systemctl show --property < unit file option > < service name >
systemctl set-property < service name > < unit file option >=< value >
systemctl show --property < unit file option > < service name >
<name> .service
<name> .scope
<parent-name> .slice
Control group /: -.slice ├─user.slice │ ├─user-42.slice │ │ ├─session-c1.scope │ │ │ ├─ 967 gdm-session-worker [pam/gdm-launch-environment] │ │ │ ├─1035 /usr/libexec/gdm-x-session gnome-session --autostart /usr/share/gdm/greeter/autostart │ │ │ ├─1054 /usr/libexec/Xorg vt1 -displayfd 3 -auth /run/user/42/gdm/Xauthority -background none -noreset -keeptty -verbose 3 │ │ │ ├─1212 /usr/libexec/gnome-session-binary --autostart /usr/share/gdm/greeter/autostart │ │ │ ├─1369 /usr/bin/gnome-shell │ │ │ ├─1732 ibus-daemon --xim --panel disable │ │ │ ├─1752 /usr/libexec/ibus-dconf │ │ │ ├─1762 /usr/libexec/ibus-x11 --kill-daemon │ │ │ ├─1912 /usr/libexec/gsd-xsettings │ │ │ ├─1917 /usr/libexec/gsd-a11y-settings │ │ │ ├─1920 /usr/libexec/gsd-clipboard ... ├─init.scope │ └─1 /usr/lib/systemd/systemd --switched-root --system --deserialize 18 └─system.slice ├─rngd.service │ └─800 /sbin/rngd -f ├─systemd-udevd.service │ └─659 /usr/lib/systemd/systemd-udevd ├─chronyd.service │ └─823 /usr/sbin/chronyd ├─auditd.service │ ├─761 /sbin/auditd │ └─763 /usr/sbin/sedispatch ├─accounts-daemon.service │ └─876 /usr/libexec/accounts-daemon ├─example.service │ ├─ 929 /bin/bash /home/jdoe/example.sh │ └─4902 sleep 1 ...
systemctl UNIT LOAD ACTIVE SUB DESCRIPTION ... init.scope loaded active running System and Service Manager session-2.scope loaded active running Session 2 of user jdoe abrt-ccpp.service loaded active exited Install ABRT coredump hook abrt-oops.service loaded active running ABRT kernel log watcher abrt-vmcore.service loaded active exited Harvest vmcores for ABRT abrt-xorg.service loaded active running ABRT Xorg log watcher ... -.slice loaded active active Root Slice machine.slice loaded active active Virtual Machine and Container Slice system-getty.slice loaded active active system-getty.slice system-lvm2\x2dpvscan.slice loaded active active system-lvm2\x2dpvscan.slice system-sshd\x2dkeygen.slice loaded active active system-sshd\x2dkeygen.slice system-systemd\x2dhibernate\x2dresume.slice loaded active active system-systemd\x2dhibernate\x2dresume> system-user\x2druntime\x2ddir.slice loaded active active system-user\x2druntime\x2ddir.slice system.slice loaded active active System Slice user-1000.slice loaded active active User Slice of UID 1000 user-42.slice loaded active active User Slice of UID 42 user.slice loaded active active User and Session Slice ...
systemctl --all
systemctl --type service,masked
systemd-cgls Control group /: -.slice ├─user.slice │ ├─user-42.slice │ │ ├─session-c1.scope │ │ │ ├─ 965 gdm-session-worker [pam/gdm-launch-environment] │ │ │ ├─1040 /usr/libexec/gdm-x-session gnome-session --autostart /usr/share/gdm/greeter/autostart ... ├─init.scope │ └─1 /usr/lib/systemd/systemd --switched-root --system --deserialize 18 └─system.slice ... ├─example.service │ ├─6882 /bin/bash /home/jdoe/example.sh │ └─6902 sleep 1 ├─systemd-journald.service └─629 /usr/lib/systemd/systemd-journald ...
systemd-cgls memory Controller memory; Control group /: ├─1 /usr/lib/systemd/systemd --switched-root --system --deserialize 18 ├─user.slice │ ├─user-42.slice │ │ ├─session-c1.scope │ │ │ ├─ 965 gdm-session-worker [pam/gdm-launch-environment] ... └─system.slice | ... ├─chronyd.service │ └─844 /usr/sbin/chronyd ├─example.service │ ├─8914 /bin/bash /home/jdoe/example.sh │ └─8916 sleep 1 ...
systemctl status example.service ● example.service - My example service Loaded: loaded (/usr/lib/systemd/system/example.service; enabled; vendor preset: disabled) Active: active (running) since Tue 2019-04-16 12:12:39 CEST; 3s ago Main PID: 17737 (bash) Tasks: 2 (limit: 11522) Memory: 496.0K (limit: 1.5M) CGroup: /system.slice/example.service ├─17737 /bin/bash /home/jdoe/example.sh └─17743 sleep 1 Apr 16 12:12:39 redhat systemd[1]: Started My example service. Apr 16 12:12:39 redhat bash[17737]: The current time is Tue Apr 16 12:12:39 CEST 2019 Apr 16 12:12:40 redhat bash[17737]: The current time is Tue Apr 16 12:12:40 CEST 2019
cat /proc/2467/cgroup 0::/system.slice/example.service
cat /sys/fs/cgroup/system.slice/example.service/cgroup.controllers memory pids ls /sys/fs/cgroup/system.slice/example.service/ cgroup.controllers cgroup.events ... cpu.pressure cpu.stat io.pressure memory.current memory.events ... pids.current pids.events pids.max
systemd-cgtop Control Group Tasks %CPU Memory Input/s Output/s / 607 29.8 1.5G - - /system.slice 125 - 428.7M - - /system.slice/ModemManager.service 3 - 8.6M - - /system.slice/NetworkManager.service 3 - 12.8M - - /system.slice/accounts-daemon.service 3 - 1.8M - - /system.slice/boot.mount - - 48.0K - - /system.slice/chronyd.service 1 - 2.0M - - /system.slice/cockpit.socket - - 1.3M - - /system.slice/colord.service 3 - 3.5M - - /system.slice/crond.service 1 - 1.8M - - /system.slice/cups.service 1 - 3.1M - - /system.slice/dev-hugepages.mount - - 244.0K - - /system.slice/dev-mapper-rhel\x2dswap.swap - - 912.0K - - /system.slice/dev-mqueue.mount - - 48.0K - - /system.slice/example.service 2 - 2.0M - - /system.slice/firewalld.service 2 - 28.8M - -
... [Service] MemoryMax=1500K ...
systemctl daemon-reload
systemctl restart example.service
cat /sys/fs/cgroup/system.slice/example.service/memory.max 1536000
systemctl show --property <CPU affinity configuration option> <service name>
systemctl set-property <service name> CPUAffinity= <value>
systemctl restart <service name>
systemctl daemon-reload
systemctl show --property <NUMA policy configuration option> <service name>
systemctl set-property <service name> NUMAPolicy= <value>
systemctl restart <service name>
systemd daemon-reload
systemd-run --unit= <name> --slice= <name> .slice <command>
Running as unit <name> .service
systemd-run --unit= <name> --slice= <name> .slice --remain-after-exit <command>
systemctl stop < name >.service
systemctl kill < name >.service --kill-who= PID,... --signal=< signal >
mkdir /sys/fs/cgroup/Example/
ll /sys/fs/cgroup/Example/ -r- r- r--. 1 root root 0 Jun 1 10:33 cgroup.controllers -r- r- r--. 1 root root 0 Jun 1 10:33 cgroup.events -rw-r- r--. 1 root root 0 Jun 1 10:33 cgroup.freeze -rw-r- r--. 1 root root 0 Jun 1 10:33 cgroup.procs ... -rw-r- r--. 1 root root 0 Jun 1 10:33 cgroup.subtree_control -r- r- r--. 1 root root 0 Jun 1 10:33 memory.events.local -rw-r- r--. 1 root root 0 Jun 1 10:33 memory.high -rw-r- r--. 1 root root 0 Jun 1 10:33 memory.low ... -r- r- r--. 1 root root 0 Jun 1 10:33 pids.current -r- r- r--. 1 root root 0 Jun 1 10:33 pids.events -rw-r- r--. 1 root root 0 Jun 1 10:33 pids.max
cat /sys/fs/cgroup/cgroup.controllers cpuset cpu io memory hugetlb pids rdma
echo "+cpu" >> /sys/fs/cgroup/cgroup.subtree_control echo "+cpuset" >> /sys/fs/cgroup/cgroup.subtree_control
echo "+cpu +cpuset" >> /sys/fs/cgroup/Example/cgroup.subtree_control
mkdir /sys/fs/cgroup/Example/tasks/
ll /sys/fs/cgroup/Example/tasks -r- r- r--. 1 root root 0 Jun 1 11:45 cgroup.controllers -r- r- r--. 1 root root 0 Jun 1 11:45 cgroup.events -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.freeze -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.max.depth -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.max.descendants -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.procs -r- r- r--. 1 root root 0 Jun 1 11:45 cgroup.stat -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.subtree_control -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.threads -rw-r- r--. 1 root root 0 Jun 1 11:45 cgroup.type -rw-r- r--. 1 root root 0 Jun 1 11:45 cpu.max -rw-r- r--. 1 root root 0 Jun 1 11:45 cpu.pressure -rw-r- r--. 1 root root 0 Jun 1 11:45 cpuset.cpus -r- r- r--. 1 root root 0 Jun 1 11:45 cpuset.cpus.effective -rw-r- r--. 1 root root 0 Jun 1 11:45 cpuset.cpus.partition -rw-r- r--. 1 root root 0 Jun 1 11:45 cpuset.mems -r- r- r--. 1 root root 0 Jun 1 11:45 cpuset.mems.effective -r- r- r--. 1 root root 0 Jun 1 11:45 cpu.stat -rw-r- r--. 1 root root 0 Jun 1 11:45 cpu.weight -rw-r- r--. 1 root root 0 Jun 1 11:45 cpu.weight.nice -rw-r- r--. 1 root root 0 Jun 1 11:45 io.pressure -rw-r- r--. 1 root root 0 Jun 1 11:45 memory.pressure
cat /sys/fs/cgroup/Example/tasks/cgroup.controllers cpuset cpu
... ├── Example │ ├── g1 │ ├── g2 │ └── g3 ...
echo "150" > /sys/fs/cgroup/Example/g1/cpu.weight echo "100" > /sys/fs/cgroup/Example/g2/cpu.weight echo "50" > /sys/fs/cgroup/Example/g3/cpu.weight
echo "33373" > /sys/fs/cgroup/Example/g1/cgroup.procs echo "33374" > /sys/fs/cgroup/Example/g2/cgroup.procs echo "33377" > /sys/fs/cgroup/Example/g3/cgroup.procs
cat /proc/33373/cgroup /proc/33374/cgroup /proc/33377/cgroup 0::/Example/g1 0::/Example/g2 0::/Example/g3
top top - 05:17:18 up 1 day, 18:25, 1 user, load average: 3.03, 3.03, 3.00 Tasks: 95 total, 4 running, 91 sleeping, 0 stopped, 0 zombie %Cpu(s): 18.1 us, 81.6 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.3 hi, 0.0 si, 0.0 st MiB Mem : 3737.0 total, 3233.7 free, 132.8 used, 370.5 buff/cache MiB Swap: 4060.0 total, 4060.0 free, 0.0 used. 3373.1 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 33373 root 20 0 18720 1748 1460 R 49.5 0.0 415:05.87 sha1sum 33374 root 20 0 18720 1756 1464 R 32.9 0.0 412:58.33 sha1sum 33377 root 20 0 18720 1860 1568 R 16.3 0.0 411:03.12 sha1sum 760 root 20 0 416620 28540 15296 S 0.3 0.7 0:10.23 tuned 1 root 20 0 186328 14108 9484 S 0.0 0.4 0:02.00 systemd 2 root 20 0 0 0 0 S 0.0 0.0 0:00.01 kthread
grubby --update-kernel=/boot/vmlinuz-USD(uname -r) --args="systemd.unified_cgroup_hierarchy=0 systemd.legacy_systemd_cgroup_controller"
grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=0 systemd.legacy_systemd_cgroup_controller"
mount -l | grep cgroup tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,seclabel,size=4096k,nr_inodes=1024,mode=755,inode64) cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd) cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,perf_event) cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpu,cpuacct) cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,pids) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuset) cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,net_cls,net_prio) cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb) cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,memory) cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,blkio) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,devices) cgroup on /sys/fs/cgroup/misc type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,misc) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,freezer) cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,rdma)
ll /sys/fs/cgroup/ dr-xr-xr-x. 10 root root 0 Mar 16 09:34 blkio lrwxrwxrwx. 1 root root 11 Mar 16 09:34 cpu -> cpu,cpuacct lrwxrwxrwx. 1 root root 11 Mar 16 09:34 cpuacct -> cpu,cpuacct dr-xr-xr-x. 10 root root 0 Mar 16 09:34 cpu,cpuacct dr-xr-xr-x. 2 root root 0 Mar 16 09:34 cpuset dr-xr-xr-x. 10 root root 0 Mar 16 09:34 devices dr-xr-xr-x. 2 root root 0 Mar 16 09:34 freezer dr-xr-xr-x. 2 root root 0 Mar 16 09:34 hugetlb dr-xr-xr-x. 10 root root 0 Mar 16 09:34 memory dr-xr-xr-x. 2 root root 0 Mar 16 09:34 misc lrwxrwxrwx. 1 root root 16 Mar 16 09:34 net_cls -> net_cls,net_prio dr-xr-xr-x. 2 root root 0 Mar 16 09:34 net_cls,net_prio lrwxrwxrwx. 1 root root 16 Mar 16 09:34 net_prio -> net_cls,net_prio dr-xr-xr-x. 2 root root 0 Mar 16 09:34 perf_event dr-xr-xr-x. 10 root root 0 Mar 16 09:34 pids dr-xr-xr-x. 2 root root 0 Mar 16 09:34 rdma dr-xr-xr-x. 11 root root 0 Mar 16 09:34 systemd
grubby --update-kernel=/boot/vmlinuz-USD(uname -r) --args="systemd.unified_cgroup_hierarchy=0 systemd.legacy_systemd_cgroup_controller"
top top - 11:34:09 up 11 min, 1 user, load average: 0.51, 0.27, 0.22 Tasks: 267 total, 3 running, 264 sleeping, 0 stopped, 0 zombie %Cpu(s): 49.0 us, 3.3 sy, 0.0 ni, 47.5 id, 0.0 wa, 0.2 hi, 0.0 si, 0.0 st MiB Mem : 1826.8 total, 303.4 free, 1046.8 used, 476.5 buff/cache MiB Swap: 1536.0 total, 1396.0 free, 140.0 used. 616.4 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 6955 root 20 0 228440 1752 1472 R 99.3 0.1 0:32.71 sha1sum 5760 jdoe 20 0 3603868 205188 64196 S 3.7 11.0 0:17.19 gnome-shell 6448 jdoe 20 0 743648 30640 19488 S 0.7 1.6 0:02.73 gnome-terminal- 1 root 20 0 245300 6568 4116 S 0.3 0.4 0:01.87 systemd 505 root 20 0 0 0 0 I 0.3 0.0 0:00.75 kworker/u4:4-events_unbound
mkdir /sys/fs/cgroup/cpu/Example/
ll /sys/fs/cgroup/cpu/Example/ -rw-r- r--. 1 root root 0 Mar 11 11:42 cgroup.clone_children -rw-r- r--. 1 root root 0 Mar 11 11:42 cgroup.procs -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.stat -rw-r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage_all -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage_percpu -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage_percpu_sys -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage_percpu_user -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage_sys -r- r- r--. 1 root root 0 Mar 11 11:42 cpuacct.usage_user -rw-r- r--. 1 root root 0 Mar 11 11:42 cpu.cfs_period_us -rw-r- r--. 1 root root 0 Mar 11 11:42 cpu.cfs_quota_us -rw-r- r--. 1 root root 0 Mar 11 11:42 cpu.rt_period_us -rw-r- r--. 1 root root 0 Mar 11 11:42 cpu.rt_runtime_us -rw-r- r--. 1 root root 0 Mar 11 11:42 cpu.shares -r- r- r--. 1 root root 0 Mar 11 11:42 cpu.stat -rw-r- r--. 1 root root 0 Mar 11 11:42 notify_on_release -rw-r- r--. 1 root root 0 Mar 11 11:42 tasks
echo "1000000" > /sys/fs/cgroup/cpu/Example/cpu.cfs_period_us echo "200000" > /sys/fs/cgroup/cpu/Example/cpu.cfs_quota_us
cat /sys/fs/cgroup/cpu/Example/cpu.cfs_period_us /sys/fs/cgroup/cpu/Example/cpu.cfs_quota_us 1000000 200000
echo "6955" > /sys/fs/cgroup/cpu/Example/cgroup.procs
cat /proc/6955/cgroup 12:cpuset:/ 11:hugetlb:/ 10:net_cls,net_prio:/ 9:memory:/user.slice/user-1000.slice/user@1000.service 8:devices:/user.slice 7:blkio:/ 6:freezer:/ 5:rdma:/ 4:pids:/user.slice/user-1000.slice/user@1000.service 3:perf_event:/ 2:cpu,cpuacct:/Example 1:name=systemd:/user.slice/user-1000.slice/user@1000.service/gnome-terminal-server.service
top top - 12:28:42 up 1:06, 1 user, load average: 1.02, 1.02, 1.00 Tasks: 266 total, 6 running, 260 sleeping, 0 stopped, 0 zombie %Cpu(s): 11.0 us, 1.2 sy, 0.0 ni, 87.5 id, 0.0 wa, 0.2 hi, 0.0 si, 0.2 st MiB Mem : 1826.8 total, 287.1 free, 1054.4 used, 485.3 buff/cache MiB Swap: 1536.0 total, 1396.7 free, 139.2 used. 608.3 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 6955 root 20 0 228440 1752 1472 R 20.6 0.1 47:11.43 sha1sum 5760 jdoe 20 0 3604956 208832 65316 R 2.3 11.2 0:43.50 gnome-shell 6448 jdoe 20 0 743836 31736 19488 S 0.7 1.7 0:08.25 gnome-terminal- 505 root 20 0 0 0 0 I 0.3 0.0 0:03.39 kworker/u4:4-events_unbound 4217 root 20 0 74192 1612 1320 S 0.3 0.1 0:01.19 spice-vdagentd
dnf install bcc-tools
ls -l /usr/share/bcc/tools/ -rwxr-xr-x. 1 root root 4198 Dec 14 17:53 dcsnoop -rwxr-xr-x. 1 root root 3931 Dec 14 17:53 dcstat -rwxr-xr-x. 1 root root 20040 Dec 14 17:53 deadlock_detector -rw-r--r--. 1 root root 7105 Dec 14 17:53 deadlock_detector.c drwxr-xr-x. 3 root root 8192 Mar 11 10:28 doc -rwxr-xr-x. 1 root root 7588 Dec 14 17:53 execsnoop -rwxr-xr-x. 1 root root 6373 Dec 14 17:53 ext4dist -rwxr-xr-x. 1 root root 10401 Dec 14 17:53 ext4slower
/usr/share/bcc/tools/execsnoop
ls /usr/share/bcc/tools/doc/
PCOMM PID PPID RET ARGS ls 8382 8287 0 /usr/bin/ls --color=auto /usr/share/bcc/tools/doc/
/usr/share/bcc/tools/opensnoop -n uname
uname
PID COMM FD ERR PATH 8596 uname 3 0 /etc/ld.so.cache 8596 uname 3 0 /lib64/libc.so.6 8596 uname 3 0 /usr/lib/locale/locale-archive
/usr/share/bcc/tools/biotop 30
dd if=/dev/vda of=/dev/zero
PID COMM D MAJ MIN DISK I/O Kbytes AVGms 9568 dd R 252 0 vda 16294 14440636.0 3.69 48 kswapd0 W 252 0 vda 1763 120696.0 1.65 7571 gnome-shell R 252 0 vda 834 83612.0 0.33 1891 gnome-shell R 252 0 vda 1379 19792.0 0.15 7515 Xorg R 252 0 vda 280 9940.0 0.28 7579 llvmpipe-1 R 252 0 vda 228 6928.0 0.19 9515 gnome-control-c R 252 0 vda 62 6444.0 0.43 8112 gnome-terminal- R 252 0 vda 67 2572.0 1.54 7807 gnome-software R 252 0 vda 31 2336.0 0.73 9578 awk R 252 0 vda 17 2228.0 0.66 7578 llvmpipe-0 R 252 0 vda 156 2204.0 0.07 9581 pgrep R 252 0 vda 58 1748.0 0.42 7531 InputThread R 252 0 vda 30 1200.0 0.48 7504 gdbus R 252 0 vda 3 1164.0 0.30 1983 llvmpipe-1 R 252 0 vda 39 724.0 0.08 1982 llvmpipe-0 R 252 0 vda 36 652.0 0.06
/usr/share/bcc/tools/xfsslower 1
vim text
TIME COMM PID T BYTES OFF_KB LAT(ms) FILENAME 13:07:14 b'bash' 4754 R 256 0 7.11 b'vim' 13:07:14 b'vim' 4754 R 832 0 4.03 b'libgpm.so.2.1.0' 13:07:14 b'vim' 4754 R 32 20 1.04 b'libgpm.so.2.1.0' 13:07:14 b'vim' 4754 R 1982 0 2.30 b'vimrc' 13:07:14 b'vim' 4754 R 1393 0 2.52 b'getscriptPlugin.vim' 13:07:45 b'vim' 4754 S 0 0 6.71 b'text' 13:07:45 b'pool' 2588 R 16 0 5.58 b'text'