Repository: DataDog/dd-agent

Issue:
[process] Correctly handle psutil.NoProcessFound when getting process state
In situations where a process disappears after we discover its PID, we should be skipping over the subsequent interrogation of the process [here](https://github.com/DataDog/dd-agent/blob/cc5a67bb47110ef145e58561ab4471b946354742/checks.d/process.py#L173-L181). Currently the check is throwing errors like:

```
  File "/opt/datadog-agent/agent/checks.d/process.py", line 181, in get_process_state
    meminfo = self.psutil_wrapper(p, 'memory_info', ['rss', 'vms'])
UnboundLocalError: local variable 'p' referenced before assignment
```



Discussion:
We're seeing this too, thanks for attending to it.

+1 noticed this as well


Resolving patch:
diff --git a/checks.d/disk.py b/checks.d/disk.py
--- a/checks.d/disk.py
+++ b/checks.d/disk.py
@@ -52,6 +52,9 @@ def _load_conf(self, instance):
         self._all_partitions = _is_affirmative(
             instance.get('all_partitions', False))
 
+        # Force exclusion of CDROM (iso9660) from disk check
+        self._excluded_filesystems.append('iso9660')
+
         # FIXME: 6.x, drop use_mount option in datadog.conf
         self._load_legacy_option(instance, 'use_mount', False,
                                  operation=_is_affirmative)
@@ -96,10 +99,21 @@ def collect_metrics_psutil(self):
             tags = [part.fstype] if self._tag_by_filesystem else []
             device_name = part.mountpoint if self._use_mount else part.device
 
+            # Note: psutil (0.3.0 to at least 3.1.1) calculates in_use as (used / total)
+            #       The problem here is that total includes reserved space the user
+            #       doesn't have access to. This causes psutil to calculate a misleadng
+            #       percentage for in_use; a lower percentage than df shows.
+
+            # Calculate in_use w/o reserved space; consistent w/ df's Use% metric.
+            pmets = self._collect_part_metrics(part, disk_usage)
+            used = 'system.disk.used'
+            free = 'system.disk.free'
+            pmets['system.disk.in_use'] = pmets[used] / (pmets[used] + pmets[free])
+
             # legacy check names c: vs psutil name C:\\
             if Platform.is_win32():
                 device_name = device_name.strip('\\').lower()
-            for metric_name, metric_value in self._collect_part_metrics(part, disk_usage).iteritems():
+            for metric_name, metric_value in pmets.iteritems():
                 self.gauge(metric_name, metric_value,
                            tags=tags, device_name=device_name)
         # And finally, latency metrics, a legacy gift from the old Windows Check
@@ -176,14 +190,18 @@ def collect_metrics_manually(self):
 
     def _collect_metrics_manually(self, device):
         result = {}
+
+        used = float(device[3])
+        free = float(device[4])
+
         # device is
         # ["/dev/sda1", "ext4", 524288,  171642,  352646, "33%", "/"]
         result[self.METRIC_DISK.format('total')] = float(device[2])
-        result[self.METRIC_DISK.format('used')] = float(device[3])
-        result[self.METRIC_DISK.format('free')] = float(device[4])
-        if len(device[5]) > 1 and device[5][-1] == '%':
-            result[self.METRIC_DISK.format('in_use')] = \
-                float(device[5][:-1]) / 100.0
+        result[self.METRIC_DISK.format('used')] = used
+        result[self.METRIC_DISK.format('free')] = free
+
+        # Rather than grabbing in_use, let's calculate it to be more precise
+        result[self.METRIC_DISK.format('in_use')] = used / (used + free)
 
         result.update(self._collect_inodes_metrics(device[-1]))
         return result
diff --git a/checks.d/process.py b/checks.d/process.py
--- a/checks.d/process.py
+++ b/checks.d/process.py
@@ -175,8 +175,9 @@ def get_process_state(self, name, pids, cpu_check_interval):
             # Skip processes dead in the meantime
             except psutil.NoSuchProcess:
                 self.warning('Process %s disappeared while scanning' % pid)
-                # reset the PID cache now, something chaned
+                # reset the PID cache now, something changed
                 self.last_pid_cache_ts[name] = 0
+                continue
 
             meminfo = self.psutil_wrapper(p, 'memory_info', ['rss', 'vms'])
             st['rss'].append(meminfo.get('rss'))
@@ -259,10 +260,10 @@ def check(self, instance):
 
     def _process_service_check(self, name, nb_procs, bounds):
         '''
-        Repport a service check, for each processes in search_string.
-        Repport as OK if the process is in the warning thresolds
-                   CRITICAL             out of the critical thresolds
-                   WARNING              out of the warning thresolds
+        Report a service check, for each process in search_string.
+        Report as OK if the process is in the warning thresholds
+                   CRITICAL             out of the critical thresholds
+                   WARNING              out of the warning thresholds
         '''
         tag = ["process:%s" % name]
         status = AgentCheck.OK
diff --git a/config.py b/config.py
--- a/config.py
+++ b/config.py
@@ -28,7 +28,7 @@
 import yaml
 
 # CONSTANTS
-AGENT_VERSION = "5.4.5"
+AGENT_VERSION = "5.4.6"
 DATADOG_CONF = "datadog.conf"
 DEFAULT_CHECK_FREQUENCY = 15   # seconds
 LOGGING_MAX_BYTES = 5 * 1024 * 1024


Test patch:
diff --git a/tests/checks/mock/test_disk.py b/tests/checks/mock/test_disk.py
--- a/tests/checks/mock/test_disk.py
+++ b/tests/checks/mock/test_disk.py
@@ -163,7 +163,7 @@ def test_default_options(self):
         self.check._load_conf({})
 
         self.assertFalse(self.check._use_mount)
-        self.assertEqual(self.check._excluded_filesystems, [])
+        self.assertEqual(self.check._excluded_filesystems, ['iso9660'])
         self.assertEqual(self.check._excluded_disks, [])
         self.assertFalse(self.check._tag_by_filesystem)
         self.assertFalse(self.check._all_partitions)