Repository: asottile/git-code-debt

Issue:
Multiprocessing eats tracebacks
Multiprocessing eats the traceback from errors raised in debt metrics which makes it look like the underlying issue might be in git-code-debt and not our metric:

I have redacted some things that are env specific for reasons.
Here is the tb with multiprocessing:
```(venv) master ✗ 🦄  make generate
venv/bin/git-code-debt-generate \
	<codebase> \
	database.db \
	code_debt_metrics
Cloning into '/tmp/tmp8XDrXvtemp-repo'...
done.
Traceback (most recent call last):
  File "venv/bin/git-code-debt-generate", line 11, in <module>
    sys.exit(main())
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 181, in main
    args.skip_default_metrics,
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 98, in load_data
    commits, pool.imap(_get_metrics_inner, mp_args),
  File "/usr/lib/python2.7/multiprocessing/pool.py", line 668, in next
    raise value
UnicodeDecodeError: 'ascii' codec can't decode byte 0xd5 in position 41: ordinal not in range(128)
Makefile:26: recipe for target 'generate' failed
make: *** [generate] Error 1
```

Here is the tb when `pool.imap` is replaced with `map` in `generate.py`:
```(venv) master ✗ 🍪  make generate
venv/bin/git-code-debt-generate \
	<codebase> \
	database.db \
	code_debt_metrics
Cloning into '/tmp/tmpxGxnB9temp-repo'...
done.
Traceback (most recent call last):
  File "venv/bin/git-code-debt-generate", line 11, in <module>
    sys.exit(main())
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 181, in main
    args.skip_default_metrics,
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 98, in load_data
    commits, map(_get_metrics_inner, mp_args),
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 54, in _get_metrics_inner
    return get_metrics(commit, diff, metric_parsers)
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 40, in get_metrics
    return list(get_all_metrics(file_diff_stats))
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/generate.py", line 35, in get_all_metrics
    commit, file_diff_stats,
  File "/code-debt/venv/local/lib/python2.7/site-packages/git_code_debt/metrics/base.py", line 39, in get_metrics_from_stat
    if self.line_matches_metric(line, file_diff_stat):
  File "/code-debt/venv/local/lib/python2.7/site-packages/code_debt_metrics/<metric>.py", line 29, in line_matches_metric
    return '<thing> = True' in line
UnicodeDecodeError: 'ascii' codec can't decode byte 0xd5 in position 41: ordinal not in range(128)
Makefile:26: recipe for target 'generate' failed
make: *** [generate] Error 1
```

Comments:

Patch:
diff --git a/git_code_debt/generate.py b/git_code_debt/generate.py
index cbc7de4..270966a 100644
--- a/git_code_debt/generate.py
+++ b/git_code_debt/generate.py
@@ -55,11 +55,19 @@ def _get_metrics_inner(m_args):
     return get_metrics(commit, diff, metric_parsers)
 
 
+def mapper(jobs):
+    if jobs == 1:
+        return map
+    else:
+        return multiprocessing.Pool(jobs).imap
+
+
 def load_data(
         database_file,
         repo,
         package_names,
         skip_defaults,
+        jobs,
 ):
     metric_parsers = get_metric_parsers_from_args(package_names, skip_defaults)
 
@@ -94,9 +102,9 @@ def load_data(
                 itertools.repeat(repo_parser),
                 itertools.repeat(metric_parsers),
             )
-            pool = multiprocessing.pool.Pool(15)
+            do_map = mapper(jobs)
             for commit, metrics in six.moves.zip(
-                    commits, pool.imap(_get_metrics_inner, mp_args),
+                    commits, do_map(_get_metrics_inner, mp_args),
             ):
                 increment_metric_values(metric_values, metrics)
                 insert_metric_values(
@@ -152,6 +160,9 @@ def get_options_from_config(config_filename):
 def main(argv=None):
     parser = argparse.ArgumentParser()
     options.add_generate_config_filename(parser)
+    parser.add_argument(
+        '-j', '--jobs', type=int, default=multiprocessing.cpu_count(),
+    )
     parsed_args = parser.parse_args(argv)
     args = get_options_from_config(parsed_args.config_filename)
 
@@ -163,6 +174,7 @@ def main(argv=None):
         args.repo,
         args.metric_package_names,
         args.skip_default_metrics,
+        parsed_args.jobs,
     )
 
 


Test patch:
diff --git a/tests/generate_test.py b/tests/generate_test.py
index b99b85a..2f05015 100644
--- a/tests/generate_test.py
+++ b/tests/generate_test.py
@@ -17,6 +17,7 @@ from git_code_debt.generate import get_metric_ids
 from git_code_debt.generate import get_options_from_config
 from git_code_debt.generate import increment_metric_values
 from git_code_debt.generate import main
+from git_code_debt.generate import mapper
 from git_code_debt.generate import populate_metric_ids
 from git_code_debt.metric import Metric
 from git_code_debt.metrics.lines import LinesOfCodeParser
@@ -58,6 +59,16 @@ def test_get_metrics_inner_nth_commit(cloneable_with_commits):
         assert Metric(name='TotalLinesOfCode', value=2) in metrics
 
 
+def square(x):
+    return x * x
+
+
+@pytest.mark.parametrize('jobs', (1, 4))
+def test_mapper(jobs):
+    ret = tuple(mapper(jobs)(square, (3, 5, 9)))
+    assert ret == (9, 25, 81)
+
+
 def test_generate_integration(sandbox, cloneable):
     main(('-C', sandbox.gen_config(repo=cloneable)))