Repository: elastic/apm-agent-python

Issue:
Reporting and showing cgroup-based metrics
See https://github.com/elastic/apm/issues/291 for implementation details


Discussion:

Resolving patch:
diff --git a/elasticapm/metrics/sets/cpu_linux.py b/elasticapm/metrics/sets/cpu_linux.py
--- a/elasticapm/metrics/sets/cpu_linux.py
+++ b/elasticapm/metrics/sets/cpu_linux.py
@@ -28,6 +28,7 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import logging
 import os
 import re
 import resource
@@ -38,19 +39,42 @@
 SYS_STATS = "/proc/stat"
 MEM_STATS = "/proc/meminfo"
 PROC_STATS = "/proc/self/stat"
+CGROUP1_MEMORY_LIMIT = "memory.limit_in_bytes"
+CGROUP1_MEMORY_USAGE = "memory.usage_in_bytes"
+CGROUP1_MEMORY_STAT = "memory.stat"
+CGROUP2_MEMORY_LIMIT = "memory.max"
+CGROUP2_MEMORY_USAGE = "memory.current"
+CGROUP2_MEMORY_STAT = "memory.stat"
+UNLIMITED = 0x7FFFFFFFFFFFF000
+PROC_SELF_CGROUP = "/proc/self/cgroup"
+PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"
+SYS_FS_CGROUP = "/sys/fs/cgroup"
 
 CPU_FIELDS = ("user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice")
 MEM_FIELDS = ("MemTotal", "MemAvailable", "MemFree", "Buffers", "Cached")
 
 whitespace_re = re.compile(r"\s+")
 
+MEMORY_CGROUP = re.compile(r"^\d+:memory:.*")
+CGROUP_V1_MOUNT_POINT = re.compile(r"^\d+? \d+? .+? .+? (.*?) .*cgroup.*memory.*")
+CGROUP_V2_MOUNT_POINT = re.compile(r"^\d+? \d+? .+? .+? (.*?) .*cgroup2.*cgroup.*")
 
 if not os.path.exists(SYS_STATS):
     raise ImportError("This metric set is only available on Linux")
 
+logger = logging.getLogger("elasticapm.metrics.cpu_linux")
+
 
 class CPUMetricSet(MetricsSet):
-    def __init__(self, registry, sys_stats_file=SYS_STATS, process_stats_file=PROC_STATS, memory_stats_file=MEM_STATS):
+    def __init__(
+        self,
+        registry,
+        sys_stats_file=SYS_STATS,
+        process_stats_file=PROC_STATS,
+        memory_stats_file=MEM_STATS,
+        proc_self_cgroup=PROC_SELF_CGROUP,
+        mount_info=PROC_SELF_MOUNTINFO,
+    ):
         self.page_size = resource.getpagesize()
         self.previous = {}
         self._read_data_lock = threading.Lock()
@@ -59,10 +83,91 @@ def __init__(self, registry, sys_stats_file=SYS_STATS, process_stats_file=PROC_S
         self.memory_stats_file = memory_stats_file
         self._sys_clock_ticks = os.sysconf("SC_CLK_TCK")
         with self._read_data_lock:
+            try:
+                self.cgroup_files = self.get_cgroup_file_paths(proc_self_cgroup, mount_info)
+            except Exception:
+                logger.debug("Reading/Parsing of cgroup memory files failed, skipping cgroup metrics", exc_info=True)
             self.previous.update(self.read_process_stats())
             self.previous.update(self.read_system_stats())
         super(CPUMetricSet, self).__init__(registry)
 
+    def get_cgroup_file_paths(self, proc_self_cgroup, mount_info):
+        """
+        Try and find the paths for CGROUP memory limit files, first trying to find the root path
+        in /proc/self/mountinfo, then falling back to the default location /sys/fs/cgroup
+        :param proc_self_cgroup: path to "self" cgroup file, usually /proc/self/cgroup
+        :param mount_info: path to "mountinfo" file, usually proc/self/mountinfo
+        :return: a 3-tuple of memory info files, or None
+        """
+        line_cgroup = None
+        try:
+            with open(proc_self_cgroup, "r") as proc_self_cgroup_file:
+                for line in proc_self_cgroup_file:
+                    if line_cgroup is None and line.startswith("0:"):
+                        line_cgroup = line
+                    if MEMORY_CGROUP.match(line):
+                        line_cgroup = line
+                        break
+        except IOError:
+            logger.debug("Cannot read %s, skipping cgroup metrics", proc_self_cgroup, exc_info=True)
+            return
+        if line_cgroup is None:
+            return
+        try:
+            with open(mount_info, "r") as mount_info_file:
+                for line in mount_info_file:
+                    # cgroup v2
+                    matcher = CGROUP_V2_MOUNT_POINT.match(line)
+                    if matcher is not None:
+                        files = self._get_cgroup_v2_file_paths(line_cgroup, matcher.group(1))
+                        if files:
+                            return files
+                    # cgroup v1
+                    matcher = CGROUP_V1_MOUNT_POINT.match(line)
+                    if matcher is not None:
+                        files = self._get_cgroup_v1_file_paths(matcher.group(1))
+                        if files:
+                            return files
+        except IOError:
+            logger.debug("Cannot read %s, skipping cgroup metrics", mount_info, exc_info=True)
+            return
+        # discovery of cgroup path failed, try with default path
+        files = self._get_cgroup_v2_file_paths(line_cgroup, SYS_FS_CGROUP)
+        if files:
+            return files
+        files = self._get_cgroup_v1_file_paths(os.path.join(SYS_FS_CGROUP, "memory"))
+        if files:
+            return files
+        logger.debug("Location of cgroup files failed, skipping cgroup metrics")
+
+    def _get_cgroup_v2_file_paths(self, line_cgroup, mount_discovered):
+        line_split = line_cgroup.strip().split(":")
+        slice_path = line_split[-1][1:]
+        try:
+            with open(os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_LIMIT), "r") as memfile:
+                line_mem = memfile.readline().strip()
+                if line_mem != "max":
+                    return (
+                        os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_LIMIT),
+                        os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_USAGE),
+                        os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_STAT),
+                    )
+        except IOError:
+            pass
+
+    def _get_cgroup_v1_file_paths(self, mount_discovered):
+        try:
+            with open(os.path.join(mount_discovered, CGROUP1_MEMORY_LIMIT), "r") as memfile:
+                mem_max = int(memfile.readline().strip())
+                if mem_max < UNLIMITED:
+                    return (
+                        os.path.join(mount_discovered, CGROUP1_MEMORY_LIMIT),
+                        os.path.join(mount_discovered, CGROUP1_MEMORY_USAGE),
+                        os.path.join(mount_discovered, CGROUP1_MEMORY_STAT),
+                    )
+        except IOError:
+            pass
+
     def before_collect(self):
         new = self.read_process_stats()
         new.update(self.read_system_stats())
@@ -83,6 +188,13 @@ def before_collect(self):
             self.gauge("system.memory.actual.free").val = mem_free
             self.gauge("system.memory.total").val = new["MemTotal"]
 
+            if "cgroup_mem_total" in new:
+                self.gauge("system.process.cgroup.memory.mem.limit.bytes").val = new["cgroup_mem_total"]
+            if "cgroup_mem_used" in new:
+                self.gauge("system.process.cgroup.memory.mem.usage.bytes").val = new["cgroup_mem_used"]
+            if "cgroup_mem_inactive" in new:
+                self.gauge("system.process.cgroup.memory.stats.inactive_file.bytes").val = new["cgroup_mem_inactive"]
+
             try:
                 cpu_process_percent = delta["proc_total_time"] / delta["cpu_total"]
             except ZeroDivisionError:
@@ -115,6 +227,20 @@ def read_system_stats(self):
                     )
                     stats["cpu_usage"] = stats["cpu_total"] - (f["idle"] + f["iowait"])
                     break
+        if self.cgroup_files:
+            with open(self.cgroup_files[0], "r") as memfile:
+                stats["cgroup_mem_total"] = int(memfile.readline())
+            with open(self.cgroup_files[1], "r") as memfile:
+                usage = int(memfile.readline())
+                stats["cgroup_mem_used"] = usage
+            with open(self.cgroup_files[2], "r") as memfile:
+                sum = 0
+                for line in memfile:
+                    (metric_name, value) = line.split(" ")
+                    if metric_name == "inactive_file":
+                        sum = sum + int(value)
+                stats["cgroup_mem_used"] = stats["cgroup_mem_used"] - sum
+                stats["cgroup_mem_inactive"] = sum
         with open(self.memory_stats_file, "r") as memfile:
             for line in memfile:
                 metric_name = line.split(":")[0]


Test patch:
diff --git a/tests/metrics/cpu_linux_tests.py b/tests/metrics/cpu_linux_tests.py
--- a/tests/metrics/cpu_linux_tests.py
+++ b/tests/metrics/cpu_linux_tests.py
@@ -86,17 +86,45 @@
 Inactive:        5468500 kB
 """
 
+TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES = "9223372036854771712"
+TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES_LIMITED = "7964778496"
+TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES = "964778496"
+
+TEMPLATE_CGROUP_MEM_STAT = """pgmajfault 0
+inactive_anon 0
+active_anon 778702848
+inactive_file 10407936
+active_file 0
+unevictable 0
+hierarchical_memory_limit 1073741824
+hierarchical_memsw_limit 2147483648
+"""
+
 
 @pytest.mark.parametrize("proc_stat_template", [TEMPLATE_PROC_STAT_DEBIAN, TEMPLATE_PROC_STAT_RHEL])
 def test_cpu_mem_from_proc(elasticapm_client, proc_stat_template, tmpdir):
     proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
     proc_stat = os.path.join(tmpdir.strpath, "stat")
     proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
+    cgroup_memory_limit = os.path.join(tmpdir.strpath, "memory", "memory.limit_in_bytes")
+    cgroup_memory_usage = os.path.join(tmpdir.strpath, "memory", "memory.usage_in_bytes")
+    proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    os.mkdir(os.path.join(tmpdir.strpath, "memory"))
+    proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
 
     for path, content in (
         (proc_stat, proc_stat_template.format(user=0, idle=0)),
         (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
         (proc_meminfo, TEMPLATE_PROC_MEMINFO),
+        (cgroup_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES),
+        (cgroup_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
+        (proc_self_cgroup, "9:memory:/slice"),
+        (
+            proc_self_mount,
+            "39 30 0:35 / "
+            + tmpdir.strpath
+            + "/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n",
+        ),
     ):
         with open(path, mode="w") as f:
             f.write(content)
@@ -105,6 +133,8 @@ def test_cpu_mem_from_proc(elasticapm_client, proc_stat_template, tmpdir):
         sys_stats_file=proc_stat,
         process_stats_file=proc_stat_self,
         memory_stats_file=proc_meminfo,
+        proc_self_cgroup=proc_self_cgroup,
+        mount_info=proc_self_mount,
     )
 
     for path, content in (
@@ -180,3 +210,193 @@ def test_cpu_usage_when_cpu_total_is_zero(elasticapm_client, tmpdir):
     cpu_total_expected = 0
     assert data["samples"]["system.cpu.total.norm.pct"]["value"] == cpu_total_expected
     assert data["samples"]["system.process.cpu.total.norm.pct"]["value"] == cpu_total_expected
+
+
+def test_mem_from_cgroup1(elasticapm_client, tmpdir):
+    proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
+    proc_stat = os.path.join(tmpdir.strpath, "stat")
+    proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
+    cgroup_memory_limit = os.path.join(tmpdir.strpath, "memory", "memory.limit_in_bytes")
+    cgroup_memory_usage = os.path.join(tmpdir.strpath, "memory", "memory.usage_in_bytes")
+    cgroup_memory_stat = os.path.join(tmpdir.strpath, "memory", "memory.stat")
+    proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    os.mkdir(os.path.join(tmpdir.strpath, "memory"))
+    proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
+
+    for path, content in (
+        (proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
+        (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
+        (proc_meminfo, TEMPLATE_PROC_MEMINFO),
+        (cgroup_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES_LIMITED),
+        (cgroup_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
+        (cgroup_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
+        (proc_self_cgroup, "9:memory:/slice"),
+        (
+            proc_self_mount,
+            "39 30 0:35 / "
+            + tmpdir.strpath
+            + "/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n",
+        ),
+    ):
+        with open(path, mode="w") as f:
+            f.write(content)
+    metricset = CPUMetricSet(
+        MetricsRegistry(elasticapm_client),
+        sys_stats_file=proc_stat,
+        process_stats_file=proc_stat_self,
+        memory_stats_file=proc_meminfo,
+        proc_self_cgroup=proc_self_cgroup,
+        mount_info=proc_self_mount,
+    )
+
+    data = next(metricset.collect())
+
+    assert data["samples"]["system.memory.total"]["value"] == 16552841216
+    assert data["samples"]["system.memory.actual.free"]["value"] == 6670774272
+
+    assert data["samples"]["system.process.memory.rss.bytes"]["value"] == 47738880
+    assert data["samples"]["system.process.memory.size"]["value"] == 3686981632
+
+    assert data["samples"]["system.process.cgroup.memory.mem.limit.bytes"]["value"] == 7964778496
+    assert data["samples"]["system.process.cgroup.memory.mem.usage.bytes"]["value"] == 954370560
+    assert data["samples"]["system.process.cgroup.memory.stats.inactive_file.bytes"]["value"] == 10407936
+
+
+def test_mem_from_cgroup2(elasticapm_client, tmpdir):
+    proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
+    proc_stat = os.path.join(tmpdir.strpath, "stat")
+    proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
+    cgroup2_memory_limit = os.path.join(tmpdir.strpath, "slice", "memory.max")
+    cgroup2_memory_usage = os.path.join(tmpdir.strpath, "slice", "memory.current")
+    cgroup2_memory_stat = os.path.join(tmpdir.strpath, "slice", "memory.stat")
+    cgroup2_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    os.mkdir(os.path.join(tmpdir.strpath, "slice"))
+    proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
+
+    for path, content in (
+        (proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
+        (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
+        (proc_meminfo, TEMPLATE_PROC_MEMINFO),
+        (cgroup2_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES_LIMITED),
+        (cgroup2_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
+        (cgroup2_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
+        (cgroup2_self_cgroup, "9:memory:/slice"),
+        (
+            proc_self_mount,
+            "30 23 0:26 / "
+            + tmpdir.strpath
+            + " rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n",
+        ),
+    ):
+        with open(path, mode="w") as f:
+            f.write(content)
+    metricset = CPUMetricSet(
+        MetricsRegistry(elasticapm_client),
+        sys_stats_file=proc_stat,
+        process_stats_file=proc_stat_self,
+        memory_stats_file=proc_meminfo,
+        proc_self_cgroup=proc_self_cgroup,
+        mount_info=proc_self_mount,
+    )
+
+    data = next(metricset.collect())
+
+    assert data["samples"]["system.memory.total"]["value"] == 16552841216
+    assert data["samples"]["system.memory.actual.free"]["value"] == 6670774272
+
+    assert data["samples"]["system.process.memory.rss.bytes"]["value"] == 47738880
+    assert data["samples"]["system.process.memory.size"]["value"] == 3686981632
+
+    assert data["samples"]["system.process.cgroup.memory.mem.limit.bytes"]["value"] == 7964778496
+    assert data["samples"]["system.process.cgroup.memory.mem.usage.bytes"]["value"] == 954370560
+    assert data["samples"]["system.process.cgroup.memory.stats.inactive_file.bytes"]["value"] == 10407936
+
+
+def test_mem_from_cgroup1_max_handling(elasticapm_client, tmpdir):
+    proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
+    proc_stat = os.path.join(tmpdir.strpath, "stat")
+    proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
+    cgroup_memory_limit = os.path.join(tmpdir.strpath, "memory", "memory.limit_in_bytes")
+    cgroup_memory_usage = os.path.join(tmpdir.strpath, "memory", "memory.usage_in_bytes")
+    cgroup_memory_stat = os.path.join(tmpdir.strpath, "memory", "memory.stat")
+    proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    os.mkdir(os.path.join(tmpdir.strpath, "memory"))
+    proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
+
+    for path, content in (
+        (proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
+        (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
+        (proc_meminfo, TEMPLATE_PROC_MEMINFO),
+        (cgroup_memory_limit, str(0x7FFFFFFFFFFFF000)),
+        (cgroup_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
+        (cgroup_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
+        (proc_self_cgroup, "9:memory:/slice"),
+        (
+            proc_self_mount,
+            "39 30 0:35 / "
+            + tmpdir.strpath
+            + "/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n",
+        ),
+    ):
+        with open(path, mode="w") as f:
+            f.write(content)
+    metricset = CPUMetricSet(
+        MetricsRegistry(elasticapm_client),
+        sys_stats_file=proc_stat,
+        process_stats_file=proc_stat_self,
+        memory_stats_file=proc_meminfo,
+        proc_self_cgroup=proc_self_cgroup,
+        mount_info=proc_self_mount,
+    )
+
+    data = next(metricset.collect())
+
+    assert "system.process.cgroup.memory.mem.limit.bytes" not in data["samples"]
+    assert "system.process.cgroup.memory.mem.usage.bytes" not in data["samples"]
+    assert "system.process.cgroup.memory.stats.inactive_file.bytes" not in data["samples"]
+
+
+def test_mem_from_cgroup2_max_handling(elasticapm_client, tmpdir):
+    proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
+    proc_stat = os.path.join(tmpdir.strpath, "stat")
+    proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
+    cgroup2_memory_limit = os.path.join(tmpdir.strpath, "slice", "memory.max")
+    cgroup2_memory_usage = os.path.join(tmpdir.strpath, "slice", "memory.current")
+    cgroup2_memory_stat = os.path.join(tmpdir.strpath, "slice", "memory.stat")
+    cgroup2_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    os.mkdir(os.path.join(tmpdir.strpath, "slice"))
+    proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
+
+    for path, content in (
+        (proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
+        (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
+        (proc_meminfo, TEMPLATE_PROC_MEMINFO),
+        (cgroup2_memory_limit, "max"),
+        (cgroup2_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
+        (cgroup2_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
+        (cgroup2_self_cgroup, "9:memory:/slice"),
+        (
+            proc_self_mount,
+            "30 23 0:26 / "
+            + tmpdir.strpath
+            + " rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n",
+        ),
+    ):
+        with open(path, mode="w") as f:
+            f.write(content)
+    metricset = CPUMetricSet(
+        MetricsRegistry(elasticapm_client),
+        sys_stats_file=proc_stat,
+        process_stats_file=proc_stat_self,
+        memory_stats_file=proc_meminfo,
+        proc_self_cgroup=proc_self_cgroup,
+        mount_info=proc_self_mount,
+    )
+
+    data = next(metricset.collect())
+
+    assert "system.process.cgroup.memory.mem.limit.bytes" not in data["samples"]
+    assert "system.process.cgroup.memory.mem.usage.bytes" not in data["samples"]
+    assert "system.process.cgroup.memory.stats.inactive_file.bytes" not in data["samples"]