Repository: DataDog/dd-agent

Issue:
Process check by pid_file results in error when process is down
When monitoring a process with the pid_file option and the process is terminated, it results in an agent error, instead of giving notice of the process being down. Result is error in the agent and process staying ok in the monitoring. Error trace:

2016-11-26 10:03:23 CET | ERROR | dd.collector | checks.process(__init__.py:784) | Check 'process' instance #0 failed
Traceback (most recent call last):
  File "/root/.datadog-agent/agent/checks/__init__.py", line 767, in run
    self.check(copy.deepcopy(instance))
  File "/root/.datadog-agent/agent/checks.d/process.py", line 347, in check
    pids = set([psutil.Process(int(pid_line)).pid])
  File "/root/.datadog-agent/venv/local/lib/python2.7/site-packages/psutil/__init__.py", line 364, in __init__
    self._init(pid)
  File "/root/.datadog-agent/venv/local/lib/python2.7/site-packages/psutil/__init__.py", line 403, in _init
    raise NoSuchProcess(pid, None, msg)
NoSuchProcess: psutil.NoSuchProcess no process found with pid 590
Process check by pid_file results in error when process is down
When monitoring a process with the pid_file option and the process is terminated, it results in an agent error, instead of giving notice of the process being down. Result is error in the agent and process staying ok in the monitoring. Error trace:

2016-11-26 10:03:23 CET | ERROR | dd.collector | checks.process(__init__.py:784) | Check 'process' instance #0 failed
Traceback (most recent call last):
  File "/root/.datadog-agent/agent/checks/__init__.py", line 767, in run
    self.check(copy.deepcopy(instance))
  File "/root/.datadog-agent/agent/checks.d/process.py", line 347, in check
    pids = set([psutil.Process(int(pid_line)).pid])
  File "/root/.datadog-agent/venv/local/lib/python2.7/site-packages/psutil/__init__.py", line 364, in __init__
    self._init(pid)
  File "/root/.datadog-agent/venv/local/lib/python2.7/site-packages/psutil/__init__.py", line 403, in _init
    raise NoSuchProcess(pid, None, msg)
NoSuchProcess: psutil.NoSuchProcess no process found with pid 590


Discussion:

Resolving patch:
diff --git a/checks.d/process.py b/checks.d/process.py
--- a/checks.d/process.py
+++ b/checks.d/process.py
@@ -307,7 +307,7 @@ def check(self, instance):
         search_string = instance.get('search_string', None)
         ignore_ad = _is_affirmative(instance.get('ignore_denied_access', True))
         pid = instance.get('pid')
-        pid_file = instance.get('pid_file', None)
+        pid_file = instance.get('pid_file')
 
         if self._conflicting_procfs:
             self.warning('The `procfs_path` defined in `process.yaml` is different from the one defined in '
@@ -340,11 +340,11 @@ def check(self, instance):
         elif pid is not None:
             # we use Process(pid) as a means to search, if pid not found
             # psutil.NoSuchProcess is raised.
-            pids = set([psutil.Process(pid).pid])
+            pids = self._get_pid_set(pid)
         elif pid_file is not None:
             with open(pid_file, 'r') as file_pid:
                 pid_line = file_pid.readline().strip()
-                pids = set([psutil.Process(int(pid_line)).pid])
+                pids = self._get_pid_set(int(pid_line))
         else:
             raise ValueError('The "search_string" or "pid" options are required for process identification')
 
@@ -376,13 +376,19 @@ def check(self, instance):
 
         self._process_service_check(name, len(pids), instance.get('thresholds', None))
 
+    def _get_pid_set(self, pid):
+        try:
+            return {psutil.Process(pid).pid}
+        except psutil.NoSuchProcess:
+            return set()
+
     def _process_service_check(self, name, nb_procs, bounds):
-        '''
+        """
         Report a service check, for each process in search_string.
         Report as OK if the process is in the warning thresholds
                    CRITICAL             out of the critical thresholds
                    WARNING              out of the warning thresholds
-        '''
+        """
         tag = ["process:%s" % name]
         status = AgentCheck.OK
         message_str = "PROCS %s: %s processes found for %s"


Test patch: