Repository: google/mobly

Issue:
`utils._assert_subprocess_running` throws `ValueError`
If the process died, we try to call `p.communicate` to get info from the process.
Then `communicate` would sometimes fail with ValueError saying it's trying to do I/O operation on a closed file.


Discussion:
Stacktrace
```
I 01:49:36:820 UTC Traceback (most recent call last):
I 01:49:36:820 UTC   File "mobly/base_test.py", line 355, in exec_one_test
I 01:49:36:820 UTC     self._teardown_test(test_name)
I 01:49:36:820 UTC   File "mobly/base_test.py", line 218, in _teardown_test
I 01:49:36:820 UTC     self.teardown_test()
I 01:49:36:821 UTC   File "...", line 226, in teardown_test
I 01:49:36:821 UTC     self._StopAdbLogAndCreateExcerpt(self.dut_b)
I 01:49:36:821 UTC   File "...", line 201, in _StopAdbLogAndCreateExcerpt
I 01:49:36:821 UTC     device.stop_adb_logcat()
I 01:49:36:821 UTC   File "mobly/controllers/android_device.py", line 844, in stop_adb_logcat
I 01:49:36:821 UTC     utils.stop_standing_subprocess(self._adb_logcat_process)
I 01:49:36:821 UTC   File "mobly/utils.py", line 356, in stop_standing_subprocess
I 01:49:36:821 UTC     _assert_subprocess_running(proc)
I 01:49:36:822 UTC   File "mobly/utils.py", line 293, in _assert_subprocess_running
I 01:49:36:822 UTC     out, err = proc.communicate()
I 01:49:36:822 UTC   File "v2_7/python27.zip.tmp/subprocess.py", line 958, in communicate
I 01:49:36:822 UTC     stdout, stderr = self._communicate(input, endtime, timeout)
I 01:49:36:822 UTC   File "v2_7/python27.zip.tmp/subprocess.py", line 1744, in _communicate
I 01:49:36:822 UTC     orig_timeout)
I 01:49:36:822 UTC   File "v2_7/python27.zip.tmp/subprocess.py", line 1800, in _communicate_with_poll
I 01:49:36:822 UTC     register_and_append(self.stdout, select_POLLIN_POLLPRI)
I 01:49:36:822 UTC   File "v2_7/python27.zip.tmp/subprocess.py", line 1779, in register_and_append
I 01:49:36:823 UTC     poller.register(file_obj.fileno(), eventmask)
I 01:49:36:823 UTC ValueError: I/O operation on closed file
```
This might be caused by `start_standing_process` doing this `proc.stdin.close()`
`_assert_subprocess_running` adds no value for our tests, and it keeps throwing errors when it doesn't need to. I'm gonna take it out
The original idea was that we'll do a check to make sure the subprocess was indeed started in `start_standing_subprocess`. But the implementation is flimsy and unreliable.

To fix all the issues:
* Remove `_assert_subprocess_running`.
* If users really want to do this check, they can wrap use `psutil.Process(pid)` object, which has a lot of useful methods like `is_running`.
* Better error msg for `stop_standing_subprocess`

This also gets rid of the unit tests that do blind waits.

Resolving patch:
diff --git a/mobly/utils.py b/mobly/utils.py
--- a/mobly/utils.py
+++ b/mobly/utils.py
@@ -273,29 +273,13 @@ def concurrent_exec(func, param_list):
             try:
                 return_vals.append(future.result())
             except Exception as exc:
-                print("{} generated an exception: {}".format(
+                logging.exception("{} generated an exception: {}".format(
                     params, traceback.format_exc()))
                 return_vals.append(exc)
         return return_vals
 
 
-def _assert_subprocess_running(proc):
-    """Checks if a subprocess has terminated on its own.
-
-    Args:
-        proc: A subprocess returned by subprocess.Popen.
-
-    Raises:
-        Error is raised if the subprocess has stopped.
-    """
-    ret = proc.poll()
-    if ret is not None:
-        out, err = proc.communicate()
-        raise Error("Process %d has terminated. ret: %d, stderr: %s,"
-                    " stdout: %s" % (proc.pid, ret, err, out))
-
-
-def start_standing_subprocess(cmd, check_health_delay=0, shell=False):
+def start_standing_subprocess(cmd, shell=False):
     """Starts a long-running subprocess.
 
     This is not a blocking call and the subprocess started by it should be
@@ -304,36 +288,27 @@ def start_standing_subprocess(cmd, check_health_delay=0, shell=False):
     For short-running commands, you should use subprocess.check_call, which
     blocks.
 
-    You can specify a health check after the subprocess is started to make sure
-    it did not stop prematurely.
-
     Args:
         cmd: string, the command to start the subprocess with.
-        check_health_delay: float, the number of seconds to wait after the
-                            subprocess starts to check its health. Default is 0,
-                            which means no check.
         shell: bool, True to run this command through the system shell,
             False to invoke it directly. See subprocess.Proc() docs.
 
     Returns:
         The subprocess that was started.
     """
-    logging.debug('Start standing subprocess with cmd: %s', cmd)
+    logging.debug('Starting standing subprocess with: %s', cmd)
     proc = subprocess.Popen(
         cmd,
         stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         shell=shell)
-    logging.debug('Start standing subprocess with cmd: %s', cmd)
     # Leaving stdin open causes problems for input, e.g. breaking the
     # code.inspect() shell (http://stackoverflow.com/a/25512460/1612937), so
     # explicitly close it assuming it is not needed for standing subprocesses.
     proc.stdin.close()
     proc.stdin = None
-    if check_health_delay > 0:
-        time.sleep(check_health_delay)
-        _assert_subprocess_running(proc)
+    logging.debug('Started standing subprocess %d', proc.pid)
     return proc
 
 
@@ -352,10 +327,9 @@ def stop_standing_subprocess(proc, kill_signal=signal.SIGTERM):
         Error: if the subprocess could not be stopped.
     """
     pid = proc.pid
-    logging.debug('Stop standing subprocess %d', pid)
-    _assert_subprocess_running(proc)
+    logging.debug('Stopping standing subprocess %d', pid)
     process = psutil.Process(pid)
-    success = True
+    failed = []
     try:
         children = process.children(recursive=True)
     except AttributeError:
@@ -365,18 +339,25 @@ def stop_standing_subprocess(proc, kill_signal=signal.SIGTERM):
         try:
             child.kill()
             child.wait(timeout=10)
+        except psutil.NoSuchProcess:
+            # Ignore if the child process has already terminated.
+            pass
         except:
-            success = False
+            failed.append(child.pid)
             logging.exception('Failed to kill standing subprocess %d',
                               child.pid)
     try:
         process.kill()
         process.wait(timeout=10)
+    except psutil.NoSuchProcess:
+        # Ignore if the process has already terminated.
+        pass
     except:
-        success = False
+        failed.append(pid)
         logging.exception('Failed to kill standing subprocess %d', pid)
-    if not success:
-        raise Error('Some standing subprocess failed to die')
+    if failed:
+        raise Error('Failed to kill standing subprocesses: %s' % failed)
+    logging.debug('Stopped standing subprocess %d', pid)
 
 
 def wait_for_standing_subprocess(proc, timeout=None):


Test patch:
diff --git a/tests/mobly/utils_test.py b/tests/mobly/utils_test.py
--- a/tests/mobly/utils_test.py
+++ b/tests/mobly/utils_test.py
@@ -18,6 +18,7 @@
 from future.tests.base import unittest
 
 import portpicker
+import psutil
 from mobly import utils
 
 MOCK_AVAILABLE_PORT = 5
@@ -27,21 +28,17 @@ class UtilsTest(unittest.TestCase):
     """This test class has unit tests for the implementation of everything
     under mobly.utils.
     """
+
     def test_start_standing_subproc(self):
-        with self.assertRaisesRegex(utils.Error, 'Process .* has terminated'):
-            utils.start_standing_subprocess(
-                ['sleep', '0'], check_health_delay=0.5)
+        p = utils.start_standing_subprocess(['sleep', '1'])
+        p1 = psutil.Process(p.pid)
+        self.assertTrue(p1.is_running())
 
     def test_stop_standing_subproc(self):
-        p = utils.start_standing_subprocess(['sleep', '5'])
+        p = utils.start_standing_subprocess(['sleep', '4'])
+        p1 = psutil.Process(p.pid)
         utils.stop_standing_subprocess(p)
-        self.assertIsNotNone(p.poll())
-
-    def test_stop_standing_subproc_already_dead(self):
-        p = utils.start_standing_subprocess(['sleep', '0'])
-        time.sleep(0.5)
-        with self.assertRaisesRegex(utils.Error, 'Process .* has terminated'):
-            utils.stop_standing_subprocess(p)
+        self.assertFalse(p1.is_running())
 
     @mock.patch(
         'mobly.controllers.android_device_lib.adb.list_occupied_adb_ports')