4.2. Disk The following sections showcase scripts that monitor disk and I/O activity. 4.2.1. Summarizing Disk Read/Write Traffic This section describes how to identify which processes are performing the heaviest disk reads/writes to the system. Example 4.9. disktop.stp #!/usr/bin/stap # #

Commands:
#!/usr/bin/stap # Copyright (C) 2007 Oracle Corp. # Get the status of reading/writing disk every 5 seconds, output top ten entries # This is free software,GNU General Public License (GPL); either version 2, or (at your option) any later version. # Usage: ./disktop.stp # global io_stat,device global read_bytes,write_bytes probe vfs.read.return { if (USDreturn>0) { if (devname!="N/A") {/*skip read from cache*/ io_stat[pid(),execname(),uid(),ppid(),"R"] += USDreturn device[pid(),execname(),uid(),ppid(),"R"] = devname read_bytes += USDreturn } } } probe vfs.write.return { if (USDreturn>0) { if (devname!="N/A") { /*skip update cache*/ io_stat[pid(),execname(),uid(),ppid(),"W"] += USDreturn device[pid(),execname(),uid(),ppid(),"W"] = devname write_bytes += USDreturn } } } probe timer.ms(5000) { /* skip non-read/write disk */ if (read_bytes+write_bytes) { printf("\n%-25s, %-8s%4dKb/sec, %-7s%6dKb, %-7s%6dKb\n\n", ctime(gettimeofday_s()), "Average:", ((read_bytes+write_bytes)/1024)/5, "Read:",read_bytes/1024, "Write:",write_bytes/1024) /* print header */ printf("%8s %8s %8s %25s %8s %4s %12s\n", "UID","PID","PPID","CMD","DEVICE","T","BYTES") } /* print top ten I/O */ foreach ([process,cmd,userid,parent,action] in io_stat- limit 10) printf("%8d %8d %8d %25s %8s %4s %12d\n", userid,process,parent,cmd, device[process,cmd,userid,parent,action], action,io_stat[process,cmd,userid,parent,action]) /* clear data */ delete io_stat delete device read_bytes = 0 write_bytes = 0 } probe end{ delete io_stat delete device delete read_bytes delete write_bytes }
[...] Mon Sep 29 03:38:28 2008 , Average: 19Kb/sec, Read: 7Kb, Write: 89Kb UID PID PPID CMD DEVICE T BYTES 0 26319 26294 firefox sda5 W 90229 0 2758 2757 pam_timestamp_c sda5 R 8064 0 2885 1 cupsd sda5 W 1678 Mon Sep 29 03:38:38 2008 , Average: 1Kb/sec, Read: 7Kb, Write: 1Kb UID PID PPID CMD DEVICE T BYTES 0 2758 2757 pam_timestamp_c sda5 R 8064 0 2885 1 cupsd sda5 W 1678
global start global entry_io global fd_io global time_io function timestamp:long() { return gettimeofday_us() - start } function proc:string() { return sprintf("%d (%s)", pid(), execname()) } probe begin { start = gettimeofday_us() } global filenames global filehandles global fileread global filewrite probe syscall.open { filenames[pid()] = user_string(USDfilename) } probe syscall.open.return { if (USDreturn != -1) { filehandles[pid(), USDreturn] = filenames[pid()] fileread[pid(), USDreturn] = 0 filewrite[pid(), USDreturn] = 0 } else { printf("%d %s access %s fail\n", timestamp(), proc(), filenames[pid()]) } delete filenames[pid()] } probe syscall.read { if (USDcount > 0) { fileread[pid(), USDfd] += USDcount } t = gettimeofday_us(); p = pid() entry_io[p] = t fd_io[p] = USDfd } probe syscall.read.return { t = gettimeofday_us(); p = pid() fd = fd_io[p] time_io[p,fd] <<< t - entry_io[p] } probe syscall.write { if (USDcount > 0) { filewrite[pid(), USDfd] += USDcount } t = gettimeofday_us(); p = pid() entry_io[p] = t fd_io[p] = USDfd } probe syscall.write.return { t = gettimeofday_us(); p = pid() fd = fd_io[p] time_io[p,fd] <<< t - entry_io[p] } probe syscall.close { if (filehandles[pid(), USDfd] != "") { printf("%d %s access %s read: %d write: %d\n", timestamp(), proc(), filehandles[pid(), USDfd], fileread[pid(), USDfd], filewrite[pid(), USDfd]) if (@count(time_io[pid(), USDfd])) printf("%d %s iotime %s time: %d\n", timestamp(), proc(), filehandles[pid(), USDfd], @sum(time_io[pid(), USDfd])) } delete fileread[pid(), USDfd] delete filewrite[pid(), USDfd] delete filehandles[pid(), USDfd] delete fd_io[pid()] delete entry_io[pid()] delete time_io[pid(),USDfd] }
[...] 825946 3364 (NetworkManager) access /sys/class/net/eth0/carrier read: 8190 write: 0 825955 3364 (NetworkManager) iotime /sys/class/net/eth0/carrier time: 9 [...] 117061 2460 (pcscd) access /dev/bus/usb/003/001 read: 43 write: 0 117065 2460 (pcscd) iotime /dev/bus/usb/003/001 time: 7 [...] 3973737 2886 (sendmail) access /proc/loadavg read: 4096 write: 0 3973744 2886 (sendmail) iotime /proc/loadavg time: 11 [...]
#! /usr/bin/env stap traceio.stp Copyright (C) 2007 Red Hat, Inc., Eugene Teo <eteo@redhat.com> Copyright (C) 2009 Kai Meyer <kai@unixlords.com> Fixed a bug that allows this to run longer And added the humanreadable function # This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation. # global reads, writes, total_io probe vfs.read.return { reads[pid(),execname()] += USDreturn total_io[pid(),execname()] += USDreturn } probe vfs.write.return { writes[pid(),execname()] += USDreturn total_io[pid(),execname()] += USDreturn } function humanreadable(bytes) { if (bytes > 1024*1024*1024) { return sprintf("%d GiB", bytes/1024/1024/1024) } else if (bytes > 1024*1024) { return sprintf("%d MiB", bytes/1024/1024) } else if (bytes > 1024) { return sprintf("%d KiB", bytes/1024) } else { return sprintf("%d B", bytes) } } probe timer.s(1) { foreach([p,e] in total_io- limit 10) printf("%8d %15s r: %12s w: %12s\n", p, e, humanreadable(reads[p,e]), humanreadable(writes[p,e])) printf("\n") # Note we don't zero out reads, writes and total_io, # so the values are cumulative since the script started. }
[...] Xorg r: 583401 KiB w: 0 KiB floaters r: 96 KiB w: 7130 KiB multiload-apple r: 538 KiB w: 537 KiB sshd r: 71 KiB w: 72 KiB pam_timestamp_c r: 138 KiB w: 0 KiB staprun r: 51 KiB w: 51 KiB snmpd r: 46 KiB w: 0 KiB pcscd r: 28 KiB w: 0 KiB irqbalance r: 27 KiB w: 4 KiB cupsd r: 4 KiB w: 18 KiB Xorg r: 588140 KiB w: 0 KiB floaters r: 97 KiB w: 7143 KiB multiload-apple r: 543 KiB w: 542 KiB sshd r: 72 KiB w: 72 KiB pam_timestamp_c r: 138 KiB w: 0 KiB staprun r: 51 KiB w: 51 KiB snmpd r: 46 KiB w: 0 KiB pcscd r: 28 KiB w: 0 KiB irqbalance r: 27 KiB w: 4 KiB cupsd r: 4 KiB w: 18 KiB
#! /usr/bin/env stap global device_of_interest probe begin { /* The following is not the most efficient way to do this. One could directly put the result of usrdev2kerndev() into device_of_interest. However, want to test out the other device functions */ dev = usrdev2kerndev(USD1) device_of_interest = MKDEV(MAJOR(dev), MINOR(dev)) } probe vfs.write, vfs.read { if (dev == device_of_interest) printf ("%s(%d) %s 0x%x\n", execname(), pid(), probefunc(), dev) }
[...] synergyc(3722) vfs_read 0x800005 synergyc(3722) vfs_read 0x800005 cupsd(2889) vfs_write 0x800005 cupsd(2889) vfs_write 0x800005 cupsd(2889) vfs_write 0x800005 [...]
#! /usr/bin/env stap probe vfs.write, vfs.read { # dev and ino are defined by vfs.write and vfs.read if (dev == MKDEV(USD1,USD2) # major/minor device && ino == USD3) printf ("%s(%d) %s 0x%x/%u\n", execname(), pid(), probefunc(), dev, ino) }
805 1078319
cat(16437) vfs_read 0x800005/1078319 cat(16437) vfs_read 0x800005/1078319
global ATTR_MODE = 1 probe kernel.function("inode_setattr") { dev_nr = USDinode->i_sb->s_dev inode_nr = USDinode->i_ino if (dev_nr == (USD1 << 20 | USD2) # major/minor device && inode_nr == USD3 && USDattr->ia_valid & ATTR_MODE) printf ("%s(%d) %s 0x%x/%u %o %d\n", execname(), pid(), probefunc(), dev_nr, inode_nr, USDattr->ia_mode, uid()) }
chmod(17448) inode_setattr 0x800005/6011835 100777 500 chmod(17449) inode_setattr 0x800005/6011835 100666 500