Repository: pantsbuild/pants

Issue:
make error messages in concurrent work in jvm compile execution_graph.py respect --print-exception-stacktrace
If an error occurs while processing the result of e.g. a zinc compile, the error message will be shown alone, without a corresponding exception type, line number, or stacktrace. If we add `log.debug(traceback.format_exc())` at line 267 of `jvm_compile`'s `execution_graph.py`, then [if a zinc job(s) fails](https://github.com/pantsbuild/pants/blob/130a937e8d4dbb3084775b7c627acf3f4b31c398/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py#L326), we get to see the entire stacktrace as well. This makes errors that bubble up from invoking the `work()` method much easier to diagnose and fix.


Discussion:
Here's what I think would be a good solution
- pass a boolean flag to exec graph that's set from `print-exception-stacktrace` for printing stack traces
- collect the error traceback at 267
- if the flag is enabled, print the collected trace back inside the failure handling conditional on 327 in addition to printing the value.
- if the flag isn't enabled, at least print the exception type

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py b/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py
@@ -5,6 +5,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import queue
+import sys
 import threading
 import traceback
 from builtins import map, object, str
@@ -154,11 +155,12 @@ class ExecutionGraph(object):
   global execution graph.
   """
 
-  def __init__(self, job_list):
+  def __init__(self, job_list, print_stack_trace):
     """
 
     :param job_list Job: list of Jobs to schedule and run.
     """
+    self._print_stack_trace = print_stack_trace
     self._dependencies = defaultdict(list)
     self._dependees = defaultdict(list)
     self._jobs = {}
@@ -264,8 +266,9 @@ def worker(worker_key, work):
         try:
           work()
           result = (worker_key, SUCCESSFUL, None)
-        except Exception as e:
-          result = (worker_key, FAILED, e)
+        except Exception:
+          _, exc_value, exc_traceback = sys.exc_info()
+          result = (worker_key, FAILED, (exc_value, traceback.format_tb(exc_traceback)))
         finished_queue.put(result)
         jobs_in_flight.decrement()
 
@@ -325,7 +328,10 @@ def submit_jobs(job_keys):
 
         # Log success or failure for this job.
         if result_status is FAILED:
-          log.error("{} failed: {}".format(finished_key, value))
+          exception, tb = value
+          log.error("{} failed: {}".format(finished_key, exception))
+          if self._print_stack_trace:
+            log.error('Traceback:\n{}'.format('\n'.join(tb)))
         else:
           log.debug("{} finished with status {}".format(finished_key, result_status))
     except ExecutionFailure:
diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
--- a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
+++ b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
@@ -442,7 +442,7 @@ def do_compile(self, invalidation_check, compile_contexts):
                                      invalid_targets,
                                      invalidation_check.invalid_vts)
 
-    exec_graph = ExecutionGraph(jobs)
+    exec_graph = ExecutionGraph(jobs, self.get_options().print_exception_stacktrace)
     try:
       exec_graph.execute(worker_pool, self.context.log)
     except ExecutionFailure as e:


Test patch:
diff --git a/tests/python/pants_test/tasks/test_execution_graph.py b/tests/python/pants_test/tasks/test_execution_graph.py
--- a/tests/python/pants_test/tasks/test_execution_graph.py
+++ b/tests/python/pants_test/tasks/test_execution_graph.py
@@ -4,7 +4,9 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import re
 import unittest
+from collections import defaultdict
 
 from pants.backend.jvm.tasks.jvm_compile.execution_graph import (ExecutionFailure, ExecutionGraph,
                                                                  Job, JobExistsError,
@@ -27,6 +29,17 @@ def debug(self, msg):
     print(msg)
 
 
+class CapturingLogger(object):
+
+  log_entries = defaultdict(list)
+
+  def error(self, msg):
+    self.log_entries['error'].append(msg)
+
+  def debug(self, msg):
+    self.log_entries['debug'].append(msg)
+
+
 def passing_fn():
   pass
 
@@ -35,6 +48,10 @@ def raising_fn():
   raise Exception("I'm an error")
 
 
+def raising_wrapper():
+  raising_fn()
+
+
 class ExecutionGraphTest(unittest.TestCase):
 
   def setUp(self):
@@ -51,7 +68,7 @@ def recording_fn():
     return Job(name, recording_fn, dependencies, size, on_success, on_failure)
 
   def test_single_job(self):
-    exec_graph = ExecutionGraph([self.job("A", passing_fn, [])])
+    exec_graph = ExecutionGraph([self.job("A", passing_fn, [])], False)
 
     self.execute(exec_graph)
 
@@ -59,7 +76,7 @@ def test_single_job(self):
 
   def test_single_dependency(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["B"]),
-                                 self.job("B", passing_fn, [])])
+                                 self.job("B", passing_fn, [])], False)
     self.execute(exec_graph)
 
     self.assertEqual(self.jobs_run, ["B", "A"])
@@ -67,7 +84,7 @@ def test_single_dependency(self):
   def test_simple_binary_tree(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["B", "C"]),
                                  self.job("B", passing_fn, []),
-                                 self.job("C", passing_fn, [])])
+                                 self.job("C", passing_fn, [])], False)
     self.execute(exec_graph)
 
     self.assertEqual(self.jobs_run, ["B", "C", "A"])
@@ -75,7 +92,7 @@ def test_simple_binary_tree(self):
   def test_simple_linear_dependencies(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["B"]),
                                  self.job("B", passing_fn, ["C"]),
-                                 self.job("C", passing_fn, [])])
+                                 self.job("C", passing_fn, [])], False)
 
     self.execute(exec_graph)
 
@@ -84,7 +101,7 @@ def test_simple_linear_dependencies(self):
   def test_simple_unconnected(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, []),
                                  self.job("B", passing_fn, []),
-    ])
+    ], False)
 
     self.execute(exec_graph)
 
@@ -94,7 +111,7 @@ def test_simple_unconnected_tree(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["B"]),
                                  self.job("B", passing_fn, []),
                                  self.job("C", passing_fn, []),
-    ])
+    ], False)
 
     self.execute(exec_graph)
 
@@ -104,14 +121,14 @@ def test_dependee_depends_on_dependency_of_its_dependency(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["B", "C"]),
                                  self.job("B", passing_fn, ["C"]),
                                  self.job("C", passing_fn, []),
-    ])
+    ], False)
 
     self.execute(exec_graph)
 
     self.assertEqual(["C", "B", "A"], self.jobs_run)
 
   def test_one_failure_raises_exception(self):
-    exec_graph = ExecutionGraph([self.job("A", raising_fn, [])])
+    exec_graph = ExecutionGraph([self.job("A", raising_fn, [])], False)
     with self.assertRaises(ExecutionFailure) as cm:
       self.execute(exec_graph)
 
@@ -119,7 +136,7 @@ def test_one_failure_raises_exception(self):
 
   def test_failure_of_dependency_does_not_run_dependents(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["F"]),
-                                 self.job("F", raising_fn, [])])
+                                 self.job("F", raising_fn, [])], False)
     with self.assertRaises(ExecutionFailure) as cm:
       self.execute(exec_graph)
 
@@ -129,7 +146,7 @@ def test_failure_of_dependency_does_not_run_dependents(self):
   def test_failure_of_dependency_does_not_run_second_order_dependents(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, ["B"]),
                                  self.job("B", passing_fn, ["F"]),
-                                 self.job("F", raising_fn, [])])
+                                 self.job("F", raising_fn, [])], False)
     with self.assertRaises(ExecutionFailure) as cm:
       self.execute(exec_graph)
 
@@ -140,7 +157,7 @@ def test_failure_of_one_leg_of_tree_does_not_cancel_other(self):
     # TODO do we want this behavior, or do we want to fail fast on the first failed job?
     exec_graph = ExecutionGraph([self.job("B", passing_fn, []),
                                  self.job("F", raising_fn, ["B"]),
-                                 self.job("A", passing_fn, ["B"])])
+                                 self.job("A", passing_fn, ["B"])], False)
     with self.assertRaises(ExecutionFailure) as cm:
       self.execute(exec_graph)
 
@@ -149,7 +166,7 @@ def test_failure_of_one_leg_of_tree_does_not_cancel_other(self):
 
   def test_failure_of_disconnected_job_does_not_cancel_non_dependents(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, []),
-                                 self.job("F", raising_fn, [])])
+                                 self.job("F", raising_fn, [])], False)
     with self.assertRaises(ExecutionFailure):
       self.execute(exec_graph)
 
@@ -158,7 +175,7 @@ def test_failure_of_disconnected_job_does_not_cancel_non_dependents(self):
   def test_cycle_in_graph_causes_failure(self):
     with self.assertRaises(NoRootJobError) as cm:
       ExecutionGraph([self.job("A", passing_fn, ["B"]),
-                      self.job("B", passing_fn, ["A"])])
+                      self.job("B", passing_fn, ["A"])], False)
 
     self.assertEqual(
       "Unexecutable graph: All scheduled jobs have dependencies. "
@@ -168,12 +185,12 @@ def test_cycle_in_graph_causes_failure(self):
   def test_non_existent_dependency_causes_failure(self):
     with self.assertRaises(UnknownJobError) as cm:
       ExecutionGraph([self.job("A", passing_fn, []),
-                      self.job("B", passing_fn, ["Z"])])
+                      self.job("B", passing_fn, ["Z"])], False)
 
     self.assertEqual("Unexecutable graph: Undefined dependencies u'Z'", str(cm.exception))
 
   def test_on_success_callback_raises_error(self):
-    exec_graph = ExecutionGraph([self.job("A", passing_fn, [], on_success=raising_fn)])
+    exec_graph = ExecutionGraph([self.job("A", passing_fn, [], on_success=raising_fn)], False)
 
     with self.assertRaises(ExecutionFailure) as cm:
       self.execute(exec_graph)
@@ -181,7 +198,7 @@ def test_on_success_callback_raises_error(self):
     self.assertEqual("Error in on_success for A: I'm an error", str(cm.exception))
 
   def test_on_failure_callback_raises_error(self):
-    exec_graph = ExecutionGraph([self.job("A", raising_fn, [], on_failure=raising_fn)])
+    exec_graph = ExecutionGraph([self.job("A", raising_fn, [], on_failure=raising_fn)], False)
 
     with self.assertRaises(ExecutionFailure) as cm:
       self.execute(exec_graph)
@@ -191,7 +208,7 @@ def test_on_failure_callback_raises_error(self):
   def test_same_key_scheduled_twice_is_error(self):
     with self.assertRaises(JobExistsError) as cm:
       ExecutionGraph([self.job("Same", passing_fn, []),
-                      self.job("Same", passing_fn, [])])
+                      self.job("Same", passing_fn, [])], False)
 
     self.assertEqual("Unexecutable graph: Job already scheduled u'Same'", str(cm.exception))
 
@@ -199,7 +216,7 @@ def test_priorities_for_chain_of_jobs(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, [], 8),
                                  self.job("B", passing_fn, ["A"], 4),
                                  self.job("C", passing_fn, ["B"], 2),
-                                 self.job("D", passing_fn, ["C"], 1)])
+                                 self.job("D", passing_fn, ["C"], 1)], False)
     self.assertEqual(exec_graph._job_priority, {"A": 15, "B": 7, "C": 3, "D": 1})
     self.execute(exec_graph)
     self.assertEqual(self.jobs_run, ["A", "B", "C", "D"])
@@ -207,7 +224,7 @@ def test_priorities_for_chain_of_jobs(self):
   def test_priorities_for_fork(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, [], 4),
                                  self.job("B", passing_fn, ["A"], 2),
-                                 self.job("C", passing_fn, ["A"], 1)])
+                                 self.job("C", passing_fn, ["A"], 1)], False)
     self.assertEqual(exec_graph._job_priority, {"A": 6, "B": 2, "C": 1})
     self.execute(exec_graph)
     self.assertEqual(self.jobs_run, ["A", "B", "C"])
@@ -215,7 +232,7 @@ def test_priorities_for_fork(self):
   def test_priorities_for_mirrored_fork(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, [], 4),
                                  self.job("B", passing_fn, ["A"], 1),
-                                 self.job("C", passing_fn, ["A"], 2)])
+                                 self.job("C", passing_fn, ["A"], 2)], False)
     self.assertEqual(exec_graph._job_priority, {"A": 6, "B": 1, "C": 2})
     self.execute(exec_graph)
     self.assertEqual(self.jobs_run, ["A", "C", "B"])
@@ -224,7 +241,7 @@ def test_priorities_for_diamond(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, [], 8),
                                  self.job("B", passing_fn, ["A"], 4),
                                  self.job("C", passing_fn, ["A"], 2),
-                                 self.job("D", passing_fn, ["B", "C"], 1)])
+                                 self.job("D", passing_fn, ["B", "C"], 1)], False)
     self.assertEqual(exec_graph._job_priority, {"A": 13, "B": 5, "C": 3, "D": 1})
     self.execute(exec_graph)
     self.assertEqual(self.jobs_run, ["A", "B", "C", "D"])
@@ -233,7 +250,7 @@ def test_priorities_for_mirrored_diamond(self):
     exec_graph = ExecutionGraph([self.job("A", passing_fn, [], 8),
                                  self.job("B", passing_fn, ["A"], 2),
                                  self.job("C", passing_fn, ["A"], 4),
-                                 self.job("D", passing_fn, ["B", "C"], 1)])
+                                 self.job("D", passing_fn, ["B", "C"], 1)], False)
     self.assertEqual(exec_graph._job_priority, {"A": 13, "B": 3, "C": 5, "D": 1})
     self.execute(exec_graph)
     self.assertEqual(self.jobs_run, ["A", "C", "B", "D"])
@@ -243,7 +260,7 @@ def test_priorities_for_skewed_diamond(self):
                                  self.job("B", passing_fn, ["A"], 2),
                                  self.job("C", passing_fn, ["B"], 4),
                                  self.job("D", passing_fn, ["A"], 8),
-                                 self.job("E", passing_fn, ["C", "D"], 16)])
+                                 self.job("E", passing_fn, ["C", "D"], 16)], False)
     self.assertEqual(exec_graph._job_priority, {"A": 25, "B": 22, "C": 20, "D": 24, "E": 16})
     self.execute(exec_graph)
     self.assertEqual(self.jobs_run, ["A", "D", "B", "C", "E"])
@@ -264,10 +281,24 @@ def my_job(name, result_fn, deps):
                                  my_job('B2', passing_fn, ['A']),
                                  my_job('C1', passing_fn, ['B1', 'B2']),
                                  my_job('C2', passing_fn, ['B1', 'B2']),
-                                 my_job('E', passing_fn, ['C2'])])
+                                 my_job('E', passing_fn, ['C2'])], False)
 
     with self.assertRaises(ExecutionFailure):
       self.execute(exec_graph)
 
     self.assertEqual(self.jobs_run, ['A'])
     self.assertEqual(failures, ['A', 'B1', 'B2', 'C1', 'C2', 'E'])
+
+  def test_dumps_stack_trace(self):
+    graph = ExecutionGraph([self.job('A', raising_wrapper, [])], True)
+    capturing_logger = CapturingLogger()
+    with self.assertRaises(ExecutionFailure):
+      graph.execute(ImmediatelyExecutingPool(), capturing_logger)
+    error_logs = capturing_logger.log_entries['error']
+    self.assertEquals(2, len(error_logs), msg='Wanted one error log, got: {}'.format(error_logs))
+    self.assertEquals("A failed: I'm an error", error_logs[0])
+    regex = re.compile(
+      "Traceback:.*in raising_wrapper.*raise Exception\\(\"I'm an error\"\\)",
+      re.DOTALL,
+    )
+    self.assertRegexpMatches(error_logs[1], regex)