Repository: elastic/apm-agent-python

Issue:
FileNotFoundError in Google App Engine
**Describe the bug**: 

Whenever running on a Linux environment where the file `/sys/fs/cgroup/memory/memory.usage_in_bytes` is not available (e.g. Google App Engine), a FileNotFoundError is raised when loading elasticapm installed to a Django web application.

**To Reproduce**

1. Create a Django application.
2. Add elasticapm.contrib.django as an INSTALLED_APP configured with a SERVICE_NAME, SECRET_TOKEN, and SERVER_URL.
3. Run the application in Google App Engine.
4. Load a web page.

**Environment (please complete the following information)**
- OS: Linux
- Python version: Python 3.8
- Framework and version: Django 3.1
- APM Server version: 7
- Agent version: 5.10.0


**Additional context**

Here is a copy of the error:
```python
FileNotFoundError: [Errno 2] No such file or directory: '/sys/fs/cgroup/memory/memory.usage_in_bytes'
at read_system_stats (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/metrics/sets/cpu_linux.py:233)
at __init__ (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/metrics/sets/cpu_linux.py:91)
at register (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/metrics/base_metrics.py:70)
at __init__ (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/base.py:191)
at __init__ (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/contrib/django/client.py:88)
at get_client (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/contrib/django/client.py:72)
at ready (/layers/google.python.pip/pip/lib/python3.8/site-packages/elasticapm/contrib/django/apps.py:65)
at populate (/layers/google.python.pip/pip/lib/python3.8/site-packages/django/apps/registry.py:122)
at setup (/layers/google.python.pip/pip/lib/python3.8/site-packages/django/__init__.py:24)
at get_wsgi_application (/layers/google.python.pip/pip/lib/python3.8/site-packages/django/core/wsgi.py:12)
at <module> (/srv/****omitted***/wsgi.py:16)
at <module> (/srv/main.py:1)
at _call_with_frames_removed (<frozen importlib._bootstrap>:219)
at exec_module (<frozen importlib._bootstrap_external>:783)
at _load_unlocked (<frozen importlib._bootstrap>:671)
at _find_and_load_unlocked (<frozen importlib._bootstrap>:975)
at _find_and_load (<frozen importlib._bootstrap>:991)
at _gcd_import (<frozen importlib._bootstrap>:1014)
at import_module (/opt/python3.8/lib/python3.8/importlib/__init__.py:127)
at import_app (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/util.py:358)
at load_wsgiapp (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/app/wsgiapp.py:39)
at load (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/app/wsgiapp.py:49)
at wsgi (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/app/base.py:67)
at load_wsgi (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/workers/base.py:144)
at init_process (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/workers/base.py:119)
at init_process (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/workers/gthread.py:92)
at spawn_worker (/layers/google.python.pip/pip/lib/python3.8/site-packages/gunicorn/arbiter.py:583)
```


Discussion:

Resolving patch:
diff --git a/elasticapm/metrics/sets/cpu_linux.py b/elasticapm/metrics/sets/cpu_linux.py
--- a/elasticapm/metrics/sets/cpu_linux.py
+++ b/elasticapm/metrics/sets/cpu_linux.py
@@ -33,7 +33,6 @@
 import re
 import resource
 import threading
-from collections import namedtuple
 
 from elasticapm.metrics.base_metrics import MetricsSet
 
@@ -65,7 +64,12 @@
 
 logger = logging.getLogger("elasticapm.metrics.cpu_linux")
 
-CGroupFiles = namedtuple("CGroupFiles", ("limit", "usage", "stat"))
+
+class CGroupFiles(object):
+    def __init__(self, limit, usage, stat):
+        self.limit = limit if os.access(limit, os.R_OK) else None
+        self.usage = usage if os.access(usage, os.R_OK) else None
+        self.stat = stat if os.access(stat, os.R_OK) else None
 
 
 class CPUMetricSet(MetricsSet):
@@ -229,11 +233,13 @@ def read_system_stats(self):
                     stats["cpu_usage"] = stats["cpu_total"] - (f["idle"] + f["iowait"])
                     break
         if self.cgroup_files:
-            with open(self.cgroup_files.limit, "r") as memfile:
-                stats["cgroup_mem_total"] = int(memfile.readline())
-            with open(self.cgroup_files.usage, "r") as memfile:
-                usage = int(memfile.readline())
-                stats["cgroup_mem_used"] = usage
+            if self.cgroup_files.limit:
+                with open(self.cgroup_files.limit, "r") as memfile:
+                    stats["cgroup_mem_total"] = int(memfile.readline())
+            if self.cgroup_files.usage:
+                with open(self.cgroup_files.usage, "r") as memfile:
+                    usage = int(memfile.readline())
+                    stats["cgroup_mem_used"] = usage
         with open(self.memory_stats_file, "r") as memfile:
             for line in memfile:
                 metric_name = line.split(":")[0]


Test patch:
diff --git a/tests/metrics/cpu_linux_tests.py b/tests/metrics/cpu_linux_tests.py
--- a/tests/metrics/cpu_linux_tests.py
+++ b/tests/metrics/cpu_linux_tests.py
@@ -396,3 +396,54 @@ def test_mem_from_cgroup2_max_handling(elasticapm_client, tmpdir):
 
     assert "system.process.cgroup.memory.mem.limit.bytes" not in data["samples"]
     assert "system.process.cgroup.memory.mem.usage.bytes" not in data["samples"]
+
+
+def test_mem_from_cgroup_files_dont_exist(elasticapm_client, tmpdir):
+    # it appears that on Google App engine, there is a possibility of
+    # memory.limit_in_bytes existing while memory.usage_in_bytes does not.
+    # See https://github.com/elastic/apm-agent-python/issues/985
+    proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
+    proc_stat = os.path.join(tmpdir.strpath, "stat")
+    proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
+    cgroup_memory_limit = os.path.join(tmpdir.strpath, "memory", "memory.limit_in_bytes")
+    # intentionally commented out
+    # cgroup_memory_usage = os.path.join(tmpdir.strpath, "memory", "memory.usage_in_bytes")
+    cgroup_memory_stat = os.path.join(tmpdir.strpath, "memory", "memory.stat")
+    proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
+    os.mkdir(os.path.join(tmpdir.strpath, "memory"))
+    proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
+
+    for path, content in (
+        (proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
+        (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
+        (proc_meminfo, TEMPLATE_PROC_MEMINFO),
+        (cgroup_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES_LIMITED),
+        # intentionally commented out
+        # (cgroup_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
+        (cgroup_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
+        (proc_self_cgroup, "9:memory:/slice"),
+        (
+            proc_self_mount,
+            "39 30 0:35 / "
+            + tmpdir.strpath
+            + "/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n",
+        ),
+    ):
+        with open(path, mode="w") as f:
+            f.write(content)
+    metricset = CPUMetricSet(
+        MetricsRegistry(elasticapm_client),
+        sys_stats_file=proc_stat,
+        process_stats_file=proc_stat_self,
+        memory_stats_file=proc_meminfo,
+        proc_self_cgroup=proc_self_cgroup,
+        mount_info=proc_self_mount,
+    )
+
+    assert metricset.cgroup_files.limit is not None
+    assert metricset.cgroup_files.usage is None
+
+    data = next(metricset.collect())
+
+    assert "system.process.cgroup.memory.mem.limit.bytes" in data["samples"]
+    assert "system.process.cgroup.memory.mem.usage.bytes" not in data["samples"]