Repository: pantsbuild/pants

Issue:
PytestRunIntegrationTest fails 50% of the time in OSX CI.
The failure is in `test_pytest_run_timeout_cant_terminate` and looks like:

```
==================== FAILURES ====================
PytestRunIntegrationTest.test_pytest_run_timeout_cant_terminate 

self = <pants_test.backend.python.tasks.test_pytest_run_integration.PytestRunIntegrationTest testMethod=test_pytest_run_timeout_cant_terminate>

 def test_pytest_run_timeout_cant_terminate(self):
   start = time.time()
   terminate_wait = 2
   pants_run = self.run_pants(['clean-all',
                               'test.pytest',
                               '--test-pytest-coverage=1',
                               '--test-pytest-timeout-terminate-wait=%d' % terminate_wait,
                               '--timeout-default=1',
                               'testprojects/tests/python/pants/timeout:ignores_terminate'])
   end = time.time()
   self.assert_failure(pants_run)

   # Ensure that the failure took less than 20 seconds to run.
   self.assertLess(end - start, 20)

   # Ensure that a warning about coverage reporting was emitted.
   self.assertIn("No .coverage file was found! Skipping coverage reporting", pants_run.stderr_data)

   # Ensure that the timeout message triggered.
   self.assertIn("FAILURE: Timeout of 1 seconds reached", pants_run.stdout_data)

   # Ensure that the warning about killing triggered.
   self.assertIn("WARN] Timed out test did not terminate gracefully after {} seconds, "
>                   "killing...".format(terminate_wait), pants_run.stderr_data)
E     AssertionError: u'WARN] Timed out test did not terminate gracefully after 2 seconds, killing...' not found in u'WARN] No .coverage file was found! Skipping coverage reporting.\n'

tests/python/pants_test/backend/python/tasks/test_pytest_run_integration.py:67: AssertionError
```

A full, typical failure log from the [Travis-CI job](https://travis-ci.org/pantsbuild-osx/pants/) is [here](https://github.com/pantsbuild/pants/files/232483/travis-ci.pantsbuild-osx_pants.failure.log.txt).



Discussion:
I can actually get this and or `test_pytest_run_timeout_fails` to fail on my machine maybe 25% of the time.

I am working on this one. Are you replicating with ci.sh or just ./pants test ? I have not yet been able to replicate locally.

I'm replicating with just `./pants test tests/python/pants_test/backend/python/tasks/:pytest_run_integration -- -v` - and its closer to a 90% failure rate.

But just by inspection these are a super-fragile pair of tests.  Its not surprising they flake.
I'm going to send up an RB to apply `@unittest.skip` to right the OSX CI ship in the short term to buy you time to fix this right.

@sameerparekh - if you accept your invitation to the "Contributors" team I'll assign you this issue.

The skips are in master @ https://github.com/pantsbuild/pants/commit/3db4ccead7f275e3fa9f283a523257951a072c16

Noting the flake is also seen here in jenkins on linux: http://jenkins.pantsbuild.org/job/pants_ci.integration_tests/227/SHARD=5/console

@jsirois invite accepted, thanks.


Resolving patch:
diff --git a/src/python/pants/task/testrunner_task_mixin.py b/src/python/pants/task/testrunner_task_mixin.py
--- a/src/python/pants/task/testrunner_task_mixin.py
+++ b/src/python/pants/task/testrunner_task_mixin.py
@@ -206,6 +206,9 @@ def kill_if_not_terminated():
     try:
       return process_handler.wait(timeout=timeout)
     except subprocess.TimeoutExpired as e:
+      # Since we no longer surface the actual underlying exception, we log.error here
+      # to ensure the output indicates why the test has suddenly failed.
+      self.context.log.error('FAILURE: Timeout of {} seconds reached.'.format(timeout))
       raise ErrorWhileTesting(str(e), failed_targets=test_targets)
     finally:
       maybe_terminate(wait_time=self.get_options().timeout_terminate_wait)


Test patch:
diff --git a/testprojects/tests/python/pants/timeout/test_exceeds_timeout.py b/testprojects/tests/python/pants/timeout/test_exceeds_timeout.py
--- a/testprojects/tests/python/pants/timeout/test_exceeds_timeout.py
+++ b/testprojects/tests/python/pants/timeout/test_exceeds_timeout.py
@@ -9,4 +9,4 @@ def test_within_timeout():
 
 
 def test_exceeds_timeout():
-  time.sleep(120)
+  time.sleep(1000)
diff --git a/testprojects/tests/python/pants/timeout/test_ignores_terminate.py b/testprojects/tests/python/pants/timeout/test_ignores_terminate.py
--- a/testprojects/tests/python/pants/timeout/test_ignores_terminate.py
+++ b/testprojects/tests/python/pants/timeout/test_ignores_terminate.py
@@ -11,7 +11,7 @@ def signal_term_handler(signal, frame):
     pass
 
   signal.signal(signal.SIGTERM, signal_term_handler)
-  time.sleep(120)
+  time.sleep(1000)
 
   # We need a second sleep because the SIGTERM will interrupt the first sleep.
-  time.sleep(120)
+  time.sleep(1000)
diff --git a/tests/python/pants_test/backend/jvm/tasks/BUILD b/tests/python/pants_test/backend/jvm/tasks/BUILD
--- a/tests/python/pants_test/backend/jvm/tasks/BUILD
+++ b/tests/python/pants_test/backend/jvm/tasks/BUILD
@@ -411,7 +411,7 @@ python_tests(
     'tests/python/pants_test:int-test',
   ],
   tags = {'integration'},
-  timeout = 240,
+  timeout = 600
 )
 
 python_tests(
diff --git a/tests/python/pants_test/backend/python/tasks2/BUILD b/tests/python/pants_test/backend/python/tasks2/BUILD
--- a/tests/python/pants_test/backend/python/tasks2/BUILD
+++ b/tests/python/pants_test/backend/python/tasks2/BUILD
@@ -39,5 +39,6 @@ python_tests(
     'src/python/pants/util:process_handler',
     'tests/python/pants_test:int-test',
   ],
-  tags = {'integration'},
+  tags={'integration'},
+  timeout=600
 )
diff --git a/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py b/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py
--- a/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py
+++ b/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py
@@ -7,7 +7,6 @@
 
 import os
 import time
-import unittest
 
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
@@ -28,52 +27,51 @@ def test_pytest_run_conftest_succeeds(self):
                                 'testprojects/tests/python/pants/conf_test'])
     self.assert_success(pants_run)
 
-  @unittest.skip('https://github.com/pantsbuild/pants/issues/3255')
   def test_pytest_run_timeout_fails(self):
     start = time.time()
     pants_run = self.run_pants(['clean-all',
                                 'test.pytest',
                                 '--test-pytest-coverage=auto',
                                 '--test-pytest-options=-k exceeds_timeout',
-                                '--timeout-default=1',
+                                '--test-pytest-timeout-default=1',
+                                '--cache-test-pytest-ignore',
                                 'testprojects/tests/python/pants/timeout:exceeds_timeout'])
     end = time.time()
     self.assert_failure(pants_run)
 
-    # Ensure that the failure took less than 20 seconds to run.
-    self.assertLess(end - start, 20)
+    # Ensure that the failure took less than 100 seconds to run to allow for test overhead.
+    self.assertLess(end - start, 100)
 
     # Ensure that a warning about coverage reporting was emitted.
-    self.assertIn("No .coverage file was found! Skipping coverage reporting", pants_run.stderr_data)
+    self.assertIn("No .coverage file was found! Skipping coverage reporting", pants_run.stdout_data)
 
     # Ensure that the timeout message triggered.
-    self.assertIn(" timed out after 1 seconds", pants_run.stdout_data)
+    self.assertIn("FAILURE: Timeout of 1 seconds reached.", pants_run.stdout_data)
 
-  @unittest.skip('https://github.com/pantsbuild/pants/issues/3255')
   def test_pytest_run_timeout_cant_terminate(self):
     start = time.time()
-    terminate_wait = 2
     pants_run = self.run_pants(['clean-all',
                                 'test.pytest',
+                                '--test-pytest-timeout-terminate-wait=2',
+                                '--test-pytest-timeout-default=1',
                                 '--test-pytest-coverage=auto',
-                                '--test-pytest-timeout-terminate-wait=%d' % terminate_wait,
-                                '--timeout-default=1',
+                                '--cache-test-pytest-ignore',
                                 'testprojects/tests/python/pants/timeout:ignores_terminate'])
     end = time.time()
     self.assert_failure(pants_run)
 
-    # Ensure that the failure took less than 20 seconds to run.
-    self.assertLess(end - start, 20)
+    # Ensure that the failure took less than 100 seconds to run to allow for test overhead.
+    self.assertLess(end - start, 100)
 
     # Ensure that a warning about coverage reporting was emitted.
-    self.assertIn("No .coverage file was found! Skipping coverage reporting", pants_run.stderr_data)
+    self.assertIn("No .coverage file was found! Skipping coverage reporting", pants_run.stdout_data)
 
     # Ensure that the timeout message triggered.
-    self.assertIn("FAILURE: Timeout of 1 seconds reached", pants_run.stdout_data)
+    self.assertIn("FAILURE: Timeout of 1 seconds reached.", pants_run.stdout_data)
 
     # Ensure that the warning about killing triggered.
-    self.assertIn("WARN] Timed out test did not terminate gracefully after {} seconds, "
-                  "killing...".format(terminate_wait), pants_run.stderr_data)
+    self.assertIn("WARN] Timed out test did not terminate gracefully after 2 seconds, "
+                  "killing...", pants_run.stderr_data)
 
   def test_pytest_explicit_coverage(self):
     with temporary_dir(cleanup=False) as coverage_dir: