StackExchange/scollector — pull request #94 by kylebrandt (2014-11-03 17:04:29+00:00)
Title: Processes linux.go

This is to replace a python script we are currently using:
```
#!/usr/bin/python
"""
Monitor Process that match the filters in etc/processes.conf
Kyle Brandt November 22, 2013
"""
import os
import sys
import time

#from collectors.lib import utils
#from collectors.etc import processes

#This will be loaded from the conf file
# This acts like a group by, metric will be tagged with the cmd and filters tags to make them unique
def main():
COLLECTION_INTERVAL = 15 # seconds
#http://stackoverflow.com/questions/4189123/python-how-to-get-number-of-mili-seconds-per-jiffy
if os.sysconf(os.sysconf_names['SC_CLK_TCK']) != 100:
print("Warning, we are assuming the clock tick is 100 which doesn't seem to be the case")

#TODO Add aliases for these so we can just have a single string instead of two tags
watched_conf = [["redis-server", "core.conf", "redis-core"],
["redis-server", "coreml.conf", "redis-coreml"],
["redis-server", "gossip.conf", "redis-gossip"],
["redis-server", "careers.conf", "redis-careers"],
["ruby", "puppet", "puppet-agent"],
["httpd", "", "apache"],
["/usr/sbin/named", "", "bind"],
["/usr/sbin/exim", "", "exim"],
["haproxy-t1", "", "haproxy-t1"],
["haproxy-t2", "", "haproxy-t2"],
["haproxy-t3", "", "haproxy-t3"],
["haproxy-t4", "", "haproxy-t4"],
["haproxy-t20", "", "haproxy-t20"],
["/opt/realog/dash", "", "realog"],
["/opt/scollector/scollector", "", "scollector"],
["/usr/libexec/mysqld", "", "mysql"],
["unicorn", "worker", "unicorn-worker"],
["postgres", "", "postgres-db"],
["nginx", "", "nginx"],
["java", "opentsdb", "opentsdb"],
["/usr/sbin/keepalived", "", "keepalived"],
["/usr/local/bin/node", "", "node"],
["ruby", "patcher-update.rb", "patcher"],
["/opt/bosun/bosun", "", "bosun"]]

class IDPool(object):
def __init__(self):
self.free = set()
self.next = 0
def get(self):
if len(self.free) > 0:
min_id = min(n for n in self.free)
self.free.remove(min_id)
return(min_id)
self.next += 1
return(self.next)
def put(self, x):
self.free.add(x)

class WatchedProcess(object):
def __init__(self, cmd, argmatch, name):
self.cmd = cmd
self.argmatch = argmatch
self.name = name
self.pids = {}
self.idpool = IDPool()
self.pids = {}

def add_pid(self, pid):
if pid not in self.pids:
self.pids[pid] = self.idpool.get()

watched = []
for watch in watched_conf:
watched.append(WatchedProcess(watch[0], watch[1], watch[2]))

while True:
# Gather all the data from the proc filesystem
raw_pids = [ pid for pid in os.listdir('/proc') if pid.isdigit() ]
# Fetch the cmd name assuming the command still exists
for pid in raw_pids:
try:
cmdline = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read().split('\0')
cmd = cmdline[0]
arguments = cmdline[1:]
#Support regex matching instead of just substring, maybe join all the args and regex against that
for watch in watched:
if watch.cmd in cmd:
for arg in arguments:
if watch.argmatch in arg:
watch.add_pid(pid)
#Don't want to add multiple times if there are multiple argument matches
break
except:
pass
ts = int(time.time())

for watch in watched:
for pid in watch.pids.keys():
try:
stats = open(os.path.join('/proc', pid, 'stat'), 'rb').read().split(' ')
iof = open(os.path.join('/proc', pid, 'io'), 'rb').read()
#mstats = open(os.path.join('/proc', pid, 'statm'), 'rb').read().split(' ')
except:
id = watch.pids.pop(pid)
watch.idpool.put(id)
#break since the process is gone (keyerror otherwise)
break
#make io in the same format as stat
io = [ x.split(' ')[1] for x in iof.split('\n') if len(x) > 1]
tags = "name=%s id=%s" % (watch.name, watch.pids[pid])
print("linux.proc.cpu %d %s type=%s %s" % (ts, stats[13], "user", tags))
print("linux.proc.cpu %d %s type=%s %s" % (ts, stats[14], "system", tags))
print("linux.proc.mem.fault %d %s type=%s %s" % (ts, stats[9], "minflt", tags))
print("linux.proc.mem.fault %d %s type=%s %s" % (ts, stats[11], "majflt", tags))
print("linux.proc.mem %d %s type=%s %s" % (ts, stats[22], "vsize", tags))
print("linux.proc.mem %d %s type=%s %s" % (ts, stats[23], "rss", tags))
print("linux.proc.char_io %d %s type=%s %s" % (ts, io[0], "read", tags))
print("linux.proc.char_io %d %s type=%s %s" % (ts, io[1], "write", tags))
print("linux.proc.syscall %d %s type=%s %s" % (ts, io[2], "read", tags))
print("linux.proc.syscall %d %s type=%s %s" % (ts, io[3], "write", tags))
print("linux.proc.io_bytes %d %s type=%s %s" % (ts, io[4], "read", tags))
print("linux.proc.io_bytes %d %s type=%s %s" % (ts, io[5], "write", tags))
#Ecluding below for now, I think the memory information from stat is fine
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[0], "size", tags))
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[1], "resident", tags))
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[2], "share", tags))
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[3], "text", tags))
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[4], "lib", tags))
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[5], "data", tags))
#print("process.stat.mem %d %s type=%s %s" % (ts, mstats[6], "dirty", tags))

sys.stdout.flush()
time.sleep(COLLECTION_INTERVAL)

if __name__ == "__main__":
main()
```