Repository: pulp/pulpcore

Issue:
As a user or developer, I can enable the memory recording of my tasks
This feature will be enabled by a setting named `RECORD_TASK_METRICS` which will default to `False`. When enabled, the directory `/var/lib/pulp/task_metrics/` will have files added to it.


The memory usage of the each task should be written periodically to the file `/var/lib/pulp/task_metrics/80/54bdac-1cfe-4c1b-91c2-1139fc316aeb.datum` where the UUID is split at the first two chars and the filename is the remaining portion of the UUID with `.datum`. on the end of it.

The memory should be recorded periodically.


Discussion:
Use `/var/tmp/pulp/` instead of  `/var/lib/pulp/`.  The RPM plugin already uses it for dumping diagnostics, and they get cleaned up automatically by the OS after a period of time (I believe the default is 1 month).

https://github.com/pulp/pulp_rpm/blob/main/pulp_rpm/app/depsolving.py#L849-L851

I would also recommend creating a full directory for the task and calling the files inside things like `memory_tracking.datum` - that would let us put more than one kind of diagnostic there.
Great idea re the `/var/tmp/pulp/` dir.

Resolving patch:
diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py
--- a/pulpcore/app/settings.py
+++ b/pulpcore/app/settings.py
@@ -296,6 +296,8 @@
 
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
+TASK_DIAGNOSTICS = False
+
 # HERE STARTS DYNACONF EXTENSION LOAD (Keep at the very bottom of settings.py)
 # Read more at https://dynaconf.readthedocs.io/en/latest/guides/django.html
 import dynaconf  # noqa
diff --git a/pulpcore/constants.py b/pulpcore/constants.py
--- a/pulpcore/constants.py
+++ b/pulpcore/constants.py
@@ -1,6 +1,10 @@
+from pathlib import Path
 from types import SimpleNamespace
 
 
+VAR_TMP_PULP = Path("/var/tmp/pulp")
+
+
 #: All valid task states.
 TASK_STATES = SimpleNamespace(
     WAITING="waiting",
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
--- a/pulpcore/tasking/pulpcore_worker.py
+++ b/pulpcore/tasking/pulpcore_worker.py
@@ -5,10 +5,13 @@
 import logging
 import os
 import random
+import resource
 import select
 import signal
 import socket
 import sys
+import threading
+import time
 import traceback
 from contextlib import suppress
 from datetime import timedelta
@@ -37,6 +40,7 @@
 from pulpcore.constants import (  # noqa: E402: module level not at top of file
     TASK_STATES,
     TASK_INCOMPLETE_STATES,
+    VAR_TMP_PULP,
 )
 
 from pulpcore.exceptions import AdvisoryLockError  # noqa: E402: module level not at top of file
@@ -377,6 +381,23 @@ def run_forever(self):
             self.shutdown()
 
 
+def write_memory_usage(task_pk):
+    taskdata_dir = VAR_TMP_PULP / str(task_pk)
+    taskdata_dir.mkdir(parents=True, exist_ok=True)
+    memory_file_dir = taskdata_dir / "memory.datum"
+    _logger.info("Writing task memory data to {}".format(memory_file_dir))
+
+    with open(memory_file_dir, "w") as file:
+        file.write("# Seconds\tMemory in MB\n")
+        seconds = 0
+        while True:
+            current_mb_in_use = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
+            file.write(f"{seconds}\t{current_mb_in_use:.2f}\n")
+            file.flush()
+            time.sleep(5)
+            seconds += 5
+
+
 def child_signal_handler(sig, frame):
     # Reset signal handlers to default
     # If you kill the process a second time it's not graceful anymore.
@@ -394,6 +415,11 @@ def _perform_task(task_pk, task_working_dir_rel_path):
     signal.signal(signal.SIGINT, child_signal_handler)
     signal.signal(signal.SIGTERM, child_signal_handler)
     signal.signal(signal.SIGUSR1, child_signal_handler)
+    if settings.TASK_DIAGNOSTICS:
+        # It would be better to have this recording happen in the parent process instead of here
+        # https://github.com/pulp/pulpcore/issues/2337
+        normal_thread = threading.Thread(target=write_memory_usage, args=(task_pk,), daemon=True)
+        normal_thread.start()
     # All processes need to create their own postgres connection
     connection.connection = None
     task = Task.objects.get(pk=task_pk)


Test patch: