Repository: oppia/oppia

Issue:
Error Terminating Processes with managed_firebase_auth_emulator
In [this Github Actions CI run](https://github.com/oppia/oppia/pull/11564/checks?check_run_id=1647670671), there's an error from the `managed_process` call in `managed_firebase_auth_emulator`:

```
Traceback (most recent call last):
  File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/home/runner/work/oppia/oppia/scripts/run_e2e_tests.py", line 598, in <module>
    main()
  File "/home/runner/work/oppia/oppia/scripts/run_e2e_tests.py", line 576, in main
    output, return_code = run_tests(parsed_args)
  File "/home/runner/work/oppia/oppia/scripts/run_e2e_tests.py", line 564, in run_tests
    return output_lines, p.returncode
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 479, in __exit__
    _reraise_with_existing_context(exc_details)
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 353, in _reraise_with_existing_context
    exec("raise exc_type, exc_value, exc_tb")
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 468, in __exit__
    if cb(*exc_details):
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 396, in _exit_wrapper
    return cm_exit(cm, *exc_details)
  File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/contextlib.py", line 24, in __exit__
    self.gen.next()
  File "scripts/common.py", line 875, in managed_firebase_auth_emulator
    yield stack.enter_context(managed_process(emulator_args, shell=True))
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 479, in __exit__
    _reraise_with_existing_context(exc_details)
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 353, in _reraise_with_existing_context
    exec("raise exc_type, exc_value, exc_tb")
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 468, in __exit__
    if cb(*exc_details):
  File "/home/runner/work/oppia/oppia/third_party/python_libs/contextlib2.py", line 396, in _exit_wrapper
    return cm_exit(cm, *exc_details)
  File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/contextlib.py", line 24, in __exit__
    self.gen.next()
  File "scripts/common.py", line 786, in managed_process
    proc.terminate()
  File "/home/runner/work/oppia/oppia/../oppia_tools/psutil-5.7.3/psutil/__init__.py", line 269, in wrapper
    raise NoSuchProcess(self.pid, self._name)
psutil.NoSuchProcess: psutil.NoSuchProcess process no longer exists (pid=13215)
```

This may have been introduced by #11555.


Discussion:
@brianrodri this problem is causing CI tests to fail. It seems to be caused by `managed_process` trying to shut down a process that was already killed. The `managed_process` call in question is in the `managed_firebase_auth_emulator` function you introduced by #11555, so I thought you might have some insights about what's going on. Could you take a look?
@U8NWXD just to check, is this serious enough to warrant reverting the problematic PR in the meantime? (Do we know for sure what the problematic PR is?)

Resolving patch:
diff --git a/scripts/common.py b/scripts/common.py
--- a/scripts/common.py
+++ b/scripts/common.py
@@ -799,15 +799,33 @@ def managed_process(command_args, shell=False, timeout_secs=60, **kwargs):
     try:
         yield popen_proc
     finally:
-        if popen_proc.is_running():
-            procs = popen_proc.children(recursive=True) + [popen_proc]
-            for proc in procs:
+        procs_to_kill = (
+            popen_proc.children(recursive=True) if popen_proc.is_running() else
+            [])
+        # Children must be terminated before the parent, otherwise they risk
+        # becoming zombies.
+        procs_to_kill.append(popen_proc)
+
+        get_debug_info = lambda proc: (
+            'Process(name=%r, pid=%d)' % (proc.name(), proc.pid)
+            if proc.is_running() else 'Process(pid=%d)' % (proc.pid,))
+
+        procs_still_alive = []
+        for proc in procs_to_kill:
+            if proc.is_running():
+                procs_still_alive.append(proc)
                 proc.terminate()
-
-            _, still_running = psutil.wait_procs(procs, timeout=timeout_secs)
-            for proc in still_running:
-                proc.kill()
-                logging.warn('Process killed (pid=%d)' % proc.pid)
+                logging.info('Terminating %s...' % get_debug_info(proc))
+            else:
+                logging.info('%s has ended.' % get_debug_info(proc))
+
+        procs_gone, procs_still_alive = (
+            psutil.wait_procs(procs_still_alive, timeout=timeout_secs))
+        for proc in procs_gone:
+            logging.info('%s has ended.' % get_debug_info(proc))
+        for proc in procs_still_alive:
+            proc.kill()
+            logging.warn('Forced to kill %s!' % get_debug_info(proc))
 
 
 @contextlib.contextmanager


Test patch:
diff --git a/core/tests/test_utils.py b/core/tests/test_utils.py
--- a/core/tests/test_utils.py
+++ b/core/tests/test_utils.py
@@ -28,6 +28,7 @@
 import json
 import logging
 import os
+import re
 import unittest
 
 from constants import constants
@@ -840,6 +841,45 @@ def assertRaisesRegexp(  # pylint: disable=keyword-arg-before-vararg
             expected_exception, expected_regexp,
             callable_obj=callable_obj, *args, **kwargs)
 
+    def assert_matches_regexps(self, items, regexps):
+        """Asserts that each item is a full match of the corresponding regexp.
+
+        Every regexp in the list must be a full match, substring matches are
+        still treated as errors.
+
+        If there are any missing or extra items that do not correspond to a
+        regexp element, then that is also an error.
+
+        Args:
+            items: list(str). The string elements being matched.
+            regexps: list(str|RegexObject). The patterns that each item is
+                expected to match.
+
+        Raises:
+            AssertionError. At least one item does not match its corresponding
+                pattern, or the number of items does not match the number of
+                regexp patterns.
+        """
+        differences = [
+            '~ [i=%d]:\t%r does not match: %r' % (i, item, regexp)
+            for i, (regexp, item) in enumerate(python_utils.ZIP(regexps, items))
+            if re.match(regexp, item) is None
+        ]
+        if len(items) < len(regexps):
+            extra_regexps = regexps[len(items):]
+            differences.extend(
+                '- [i=%d]:\tmissing item expected to match: %r' % (i, regexp)
+                for i, regexp in enumerate(extra_regexps, start=len(items)))
+        if len(regexps) < len(items):
+            extra_items = items[len(regexps):]
+            differences.extend(
+                '+ [i=%d]:\textra item %r' % (i, item)
+                for i, item in enumerate(extra_items, start=len(regexps)))
+
+        if differences:
+            error_message = 'Lists differ:\n\t%s' % '\n\t'.join(differences)
+            raise AssertionError(error_message)
+
 
 class AppEngineTestBase(TestBase):
     """Base class for tests requiring App Engine services."""
diff --git a/scripts/common_test.py b/scripts/common_test.py
--- a/scripts/common_test.py
+++ b/scripts/common_test.py
@@ -31,6 +31,7 @@
 import subprocess
 import sys
 import tempfile
+import threading
 import time
 
 import constants
@@ -904,18 +905,50 @@ class ManagedProcessTests(test_utils.TestBase):
     POPEN_CALL = (
         collections.namedtuple('POPEN_CALL', ['program_args', 'kwargs']))
 
+    def assert_proc_was_managed_as_expected(
+            self, logs, pid,
+            manager_should_have_sent_terminate_signal=True,
+            manager_should_have_sent_kill_signal=False):
+        """Asserts that the process ended as expected.
+
+        Args:
+            logs: list(str). The logs emitted during the process's lifetime.
+            pid: int. The process ID to inspect.
+            manager_should_have_sent_terminate_signal: bool. Whether the manager
+                should have sent a terminate signal to the process.
+            manager_should_have_sent_kill_signal: bool. Whether the manager
+                should have sent a kill signal to the process.
+        """
+        proc_pattern = r'Process\((name=\'python\', )?pid=%d\)' % (pid,)
+
+        expected_patterns = []
+        if manager_should_have_sent_terminate_signal:
+            expected_patterns.append(r'Terminating %s\.\.\.' % proc_pattern)
+        if manager_should_have_sent_kill_signal:
+            expected_patterns.append(r'Forced to kill %s!' % proc_pattern)
+        else:
+            expected_patterns.append(r'%s has ended\.' % proc_pattern)
+
+        logs_with_pid = [msg for msg in logs if re.search(proc_pattern, msg)]
+        if expected_patterns and not logs_with_pid:
+            self.fail(msg='%r has no match in logs=%r' % (proc_pattern, logs))
+
+        self.assert_matches_regexps(logs_with_pid, expected_patterns)
+
     @contextlib.contextmanager
     def _swap_popen(
-            self, make_processes_unresponsive=False, num_children=0):
+            self, make_procs_unresponsive=False, num_children=0, delay_secs=30):
         """Returns values for inspecting and mocking calls to psutil.Popen.
 
         Args:
-            make_processes_unresponsive: bool. Whether the processes created by
+            make_procs_unresponsive: bool. Whether the processes created by
                 the mock will stall when asked to terminate. Processes will
                 always terminate within ~1 minute regardless of this choice.
             num_children: int. The number of child processes the process created
                 by the mock should create. Children inherit the same termination
                 behavior.
+            delay_secs: int. The number of seconds before the process ends
+                naturally.
 
         Returns:
             Context manager. A context manager in which calls to psutil.Popen
@@ -951,11 +984,11 @@ def popen_mock(program_args, **kwargs):
                 if os.fork() == 0:
                     break
 
-            if make_processes_unresponsive:
+            if make_procs_unresponsive:
                 # Register an unresponsive function as the SIGTERM handler.
-                signal.signal(signal.SIGTERM, lambda *_: time.sleep(30))
+                signal.signal(signal.SIGTERM, lambda *_: time.sleep(delay_secs))
 
-            time.sleep(30)
+            time.sleep(delay_secs)
             sys.exit()
 
         with self.swap(psutil, 'Popen', popen_mock):
@@ -974,10 +1007,10 @@ def test_concats_command_args_when_shell_is_true(self):
             logs = stack.enter_context(self.capture_logging())
             popen_calls = stack.enter_context(self._swap_popen())
 
-            stack.enter_context(
+            proc = stack.enter_context(
                 common.managed_process(['a', 1], shell=True, timeout_secs=10))
 
-        self.assertEqual(logs, [])
+        self.assert_proc_was_managed_as_expected(logs, proc.pid)
         self.assertEqual(popen_calls, [self.POPEN_CALL('a 1', {'shell': True})])
 
     def test_passes_command_args_as_list_of_strings_when_shell_is_false(self):
@@ -985,10 +1018,10 @@ def test_passes_command_args_as_list_of_strings_when_shell_is_false(self):
             logs = stack.enter_context(self.capture_logging())
             popen_calls = stack.enter_context(self._swap_popen())
 
-            stack.enter_context(
+            proc = stack.enter_context(
                 common.managed_process(['a', 1], shell=False, timeout_secs=10))
 
-        self.assertEqual(logs, [])
+        self.assert_proc_was_managed_as_expected(logs, proc.pid)
         self.assertEqual(
             popen_calls, [self.POPEN_CALL(['a', '1'], {'shell': False})])
 
@@ -997,10 +1030,10 @@ def test_filters_empty_strings_from_command_args_when_shell_is_true(self):
             logs = stack.enter_context(self.capture_logging())
             popen_calls = stack.enter_context(self._swap_popen())
 
-            stack.enter_context(common.managed_process(
+            proc = stack.enter_context(common.managed_process(
                 ['', 'a', '', 1], shell=True, timeout_secs=10))
 
-        self.assertEqual(logs, [])
+        self.assert_proc_was_managed_as_expected(logs, proc.pid)
         self.assertEqual(popen_calls, [self.POPEN_CALL('a 1', {'shell': True})])
 
     def test_filters_empty_strings_from_command_args_when_shell_is_false(self):
@@ -1008,38 +1041,93 @@ def test_filters_empty_strings_from_command_args_when_shell_is_false(self):
             logs = stack.enter_context(self.capture_logging())
             popen_calls = stack.enter_context(self._swap_popen())
 
-            stack.enter_context(common.managed_process(
+            proc = stack.enter_context(common.managed_process(
                 ['', 'a', '', 1], shell=False, timeout_secs=10))
 
-        self.assertEqual(logs, [])
+        self.assert_proc_was_managed_as_expected(logs, proc.pid)
         self.assertEqual(
             popen_calls, [self.POPEN_CALL(['a', '1'], {'shell': False})])
 
     def test_reports_killed_processes_as_warnings(self):
         with contextlib2.ExitStack() as stack:
             logs = stack.enter_context(self.capture_logging())
-            stack.enter_context(self._swap_popen(
-                make_processes_unresponsive=True))
+            stack.enter_context(self._swap_popen(make_procs_unresponsive=True))
+
+            proc = stack.enter_context(
+                common.managed_process(['a'], timeout_secs=10))
+
+        self.assert_proc_was_managed_as_expected(
+            logs, proc.pid,
+            manager_should_have_sent_terminate_signal=True,
+            manager_should_have_sent_kill_signal=True)
+
+    def test_terminates_child_processes(self):
+        with contextlib2.ExitStack() as stack:
+            logs = stack.enter_context(self.capture_logging())
+            stack.enter_context(self._swap_popen(num_children=3))
 
             proc = stack.enter_context(
                 common.managed_process(['a'], timeout_secs=10))
-            pid = proc.pid
+            pids = [c.pid for c in proc.children()] + [proc.pid]
 
-        self.assertEqual(logs, ['Process killed (pid=%d)' % pid])
+        self.assertEqual(len(set(pids)), 4)
+        for pid in pids:
+            self.assert_proc_was_managed_as_expected(logs, pid)
 
     def test_kills_child_processes(self):
         with contextlib2.ExitStack() as stack:
             logs = stack.enter_context(self.capture_logging())
             stack.enter_context(self._swap_popen(
-                make_processes_unresponsive=True, num_children=3))
+                num_children=3, make_procs_unresponsive=True))
 
             proc = stack.enter_context(
                 common.managed_process(['a'], timeout_secs=10))
             pids = [c.pid for c in proc.children()] + [proc.pid]
 
-        self.assertEqual(len(pids), 4)
-        self.assertItemsEqual(
-            logs, ['Process killed (pid=%d)' % p for p in pids])
+        self.assertEqual(len(set(pids)), 4)
+        for pid in pids:
+            self.assert_proc_was_managed_as_expected(
+                logs, pid,
+                manager_should_have_sent_terminate_signal=True,
+                manager_should_have_sent_kill_signal=True)
+
+    def test_respects_processes_that_are_killed_early(self):
+        with contextlib2.ExitStack() as stack:
+            logs = stack.enter_context(self.capture_logging())
+            stack.enter_context(self._swap_popen())
+
+            proc = stack.enter_context(common.managed_process(
+                ['a'], timeout_secs=10))
+            time.sleep(1)
+            proc.kill()
+            proc.wait()
+
+        self.assert_proc_was_managed_as_expected(
+            logs, proc.pid,
+            manager_should_have_sent_terminate_signal=False)
+
+    def test_respects_processes_that_are_killed_after_delay(self):
+        with contextlib2.ExitStack() as stack:
+            logs = stack.enter_context(self.capture_logging())
+            stack.enter_context(self._swap_popen(make_procs_unresponsive=True))
+
+            proc = stack.enter_context(common.managed_process(
+                ['a'], timeout_secs=10))
+
+            def _kill_after_delay():
+                """Kills the targeted process after a short delay."""
+                time.sleep(5)
+                proc.kill()
+
+            assassin_thread = threading.Thread(target=_kill_after_delay)
+            assassin_thread.start()
+
+        assassin_thread.join()
+
+        self.assert_proc_was_managed_as_expected(
+            logs, proc.pid,
+            manager_should_have_sent_terminate_signal=True,
+            manager_should_have_sent_kill_signal=False)
 
     def test_managed_firebase_emulator(self):
         os.environ['GCLOUD_PROJECT'] = 'foo'