Repository: giampaolo/psutil
Pull request #2108 (2022-05-18 19:13:11)

Linux smaps rollup
## Summary

* OS: Linux
* Bug fix: no
* Type: performance
* Fixes: #2107
* 
## Description

`Process.memory_full_info()` (reporting proecss USS/PSS/Swap memory) now reads ``/proc/pid/smaps_rollup`` instead of ``/proc/pids/smaps`` which makes it 5 times faster.

Without patch:
```
~/svn/psutil {linux-smaps-rollup}$ python3 -m timeit -s "import psutil; p = psutil.Process()" "p.memory_full_info()"
500 loops, best of 5: 518 usec per loop
```

With patch (5 times faster):
```
~/svn/psutil {linux-smaps-rollup}$ python3 -m timeit -s "import psutil; p = psutil.Process()" "p.memory_full_info()"
2000 loops, best of 5: 111 usec per loop
```

----

`make test-memleaks` suite, who heavily rely on `Process.memory_full_info()`, also received a nice speedup:

Before patch:

```
$ make test-memleaks
----------------------------------------------------------------------
Ran 99 tests in 1.646s

OK (skipped=9)
SUCCESS
```

After patch:

```
$ make test-memleaks
----------------------------------------------------------------------
Ran 99 tests in 1.195s

OK (skipped=9)
SUCCESS
```


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 8f2aaa32..49242a6e 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -12,6 +12,9 @@ XXXX-XX-XX
   ``/proc`` pseudo files line by line. This should help having more consistent
   results.
 - 2057_, [OpenBSD]: add support for `cpu_freq()`_.
+- 2107_ [Linux]: `Process.memory_full_info()`_ (reporting process USS/PSS/Swap
+  memory) now reads ``/proc/pid/smaps_rollup`` instead of ``/proc/pids/smaps``,
+  which makes it 5 times faster.
 
 **Bug fixes**
 
diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index afcb78f5..e7cef439 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -77,7 +77,8 @@ __extra__all__ = [
 
 
 POWER_SUPPLY_PATH = "/sys/class/power_supply"
-HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid())
+HAS_PROC_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid())
+HAS_PROC_SMAPS_ROLLUP = os.path.exists('/proc/%s/smaps_rollup' % os.getpid())
 HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get")
 HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get")
 
@@ -1875,18 +1876,42 @@ class Process(object):
                 [int(x) * PAGESIZE for x in f.readline().split()[:7]]
         return pmem(rss, vms, shared, text, lib, data, dirty)
 
-    # /proc/pid/smaps does not exist on kernels < 2.6.14 or if
-    # CONFIG_MMU kernel configuration option is not enabled.
-    if HAS_SMAPS:
+    if HAS_PROC_SMAPS_ROLLUP or HAS_PROC_SMAPS:
 
         @wrap_exceptions
-        def memory_full_info(
+        def _parse_smaps_rollup(self):
+            # /proc/pid/smaps_rollup was added to Linux in 2017. Faster
+            # than /proc/pid/smaps. It reports higher PSS than */smaps
+            # (from 1k up to 200k higher; tested against all processes).
+            uss = pss = swap = 0
+            try:
+                with open_binary("{}/{}/smaps_rollup".format(
+                        self._procfs_path, self.pid)) as f:
+                    for line in f:
+                        if line.startswith(b"Private_"):
+                            # Private_Clean, Private_Dirty, Private_Hugetlb
+                            uss += int(line.split()[1]) * 1024
+                        elif line.startswith(b"Pss:"):
+                            pss = int(line.split()[1]) * 1024
+                        elif line.startswith(b"Swap:"):
+                            swap = int(line.split()[1]) * 1024
+            except ProcessLookupError:  # happens on readline()
+                if not pid_exists(self.pid):
+                    raise NoSuchProcess(self.pid, self._name)
+                else:
+                    raise ZombieProcess(self.pid, self._name, self._ppid)
+            return (uss, pss, swap)
+
+        @wrap_exceptions
+        def _parse_smaps(
                 self,
                 # Gets Private_Clean, Private_Dirty, Private_Hugetlb.
                 _private_re=re.compile(br"\nPrivate.*:\s+(\d+)"),
                 _pss_re=re.compile(br"\nPss\:\s+(\d+)"),
                 _swap_re=re.compile(br"\nSwap\:\s+(\d+)")):
-            basic_mem = self.memory_info()
+            # /proc/pid/smaps does not exist on kernels < 2.6.14 or if
+            # CONFIG_MMU kernel configuration option is not enabled.
+
             # Note: using 3 regexes is faster than reading the file
             # line by line.
             # XXX: on Python 3 the 2 regexes are 30% slower than on
@@ -1905,12 +1930,20 @@ class Process(object):
             uss = sum(map(int, _private_re.findall(smaps_data))) * 1024
             pss = sum(map(int, _pss_re.findall(smaps_data))) * 1024
             swap = sum(map(int, _swap_re.findall(smaps_data))) * 1024
+            return (uss, pss, swap)
+
+        def memory_full_info(self):
+            if HAS_PROC_SMAPS_ROLLUP:  # faster
+                uss, pss, swap = self._parse_smaps_rollup()
+            else:
+                uss, pss, swap = self._parse_smaps()
+            basic_mem = self.memory_info()
             return pfullmem(*basic_mem + (uss, pss, swap))
 
     else:
         memory_full_info = memory_info
 
-    if HAS_SMAPS:
+    if HAS_PROC_SMAPS:
 
         @wrap_exceptions
         def memory_maps(self):


Test patch:
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index 8dd4caae..51e8be51 100755
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -1775,28 +1775,19 @@ class TestSensorsFans(PsutilTestCase):
 class TestProcess(PsutilTestCase):
 
     @retry_on_failure()
-    def test_memory_full_info(self):
-        testfn = self.get_testfn()
-        src = textwrap.dedent("""
-            import time
-            with open("%s", "w") as f:
-                time.sleep(10)
-            """ % testfn)
-        sproc = self.pyrun(src)
-        call_until(lambda: os.listdir('.'), "'%s' not in ret" % testfn)
-        p = psutil.Process(sproc.pid)
-        time.sleep(.1)
-        mem = p.memory_full_info()
-        maps = p.memory_maps(grouped=False)
+    def test_parse_smaps_vs_memory_maps(self):
+        sproc = self.spawn_testproc()
+        uss, pss, swap = psutil._pslinux.Process(sproc.pid)._parse_smaps()
+        maps = psutil.Process(sproc.pid).memory_maps(grouped=False)
         self.assertAlmostEqual(
-            mem.uss, sum([x.private_dirty + x.private_clean for x in maps]),
+            uss, sum([x.private_dirty + x.private_clean for x in maps]),
             delta=4096)
         self.assertAlmostEqual(
-            mem.pss, sum([x.pss for x in maps]), delta=4096)
+            pss, sum([x.pss for x in maps]), delta=4096)
         self.assertAlmostEqual(
-            mem.swap, sum([x.swap for x in maps]), delta=4096)
+            swap, sum([x.swap for x in maps]), delta=4096)
 
-    def test_memory_full_info_mocked(self):
+    def test_parse_smaps_mocked(self):
         # See: https://github.com/giampaolo/psutil/issues/1222
         with mock_open_content(
             "/proc/%s/smaps" % os.getpid(),
@@ -1823,12 +1814,12 @@ class TestProcess(PsutilTestCase):
                 Locked:                19 kB
                 VmFlags: rd ex
                 """).encode()) as m:
-            p = psutil.Process()
-            mem = p.memory_full_info()
+            p = psutil._pslinux.Process(os.getpid())
+            uss, pss, swap = p._parse_smaps()
             assert m.called
-            self.assertEqual(mem.uss, (6 + 7 + 14) * 1024)
-            self.assertEqual(mem.pss, 3 * 1024)
-            self.assertEqual(mem.swap, 15 * 1024)
+            self.assertEqual(uss, (6 + 7 + 14) * 1024)
+            self.assertEqual(pss, 3 * 1024)
+            self.assertEqual(swap, 15 * 1024)
 
     # On PYPY file descriptors are not closed fast enough.
     @unittest.skipIf(PYPY, "unreliable on PYPY")