Repository: dbt-labs/dbt-core

Issue:
Add performance profiling to dbt
## Feature

### Feature description
Application performance is important, but it can be difficult to pin down the performance impact of code changes. Profiling will help us benchmark dbt runtimes, and then evaluate the performance impact of prospective code changes.

I'm sure there are lots of low-hanging fruit, as we haven't dedicated a ton of brain cycles to performance recently. Adding profiling is the first step in any prospective performance optimization.

### Who will this benefit?
Folks who use dbt; folks who have important things to do with their time; [whiterose](http://mrrobot.wikia.com/wiki/Whiterose)


Discussion:

Resolving patch:
diff --git a/dbt/main.py b/dbt/main.py
--- a/dbt/main.py
+++ b/dbt/main.py
@@ -25,6 +25,7 @@
 import dbt.ui.printer
 import dbt.compat
 import dbt.deprecations
+import dbt.profiler
 
 from dbt.utils import ExitCodes
 from dbt.config import Project, RuntimeConfig, DbtProjectError, \
@@ -112,27 +113,36 @@ def handle(args):
 
 def handle_and_check(args):
     parsed = parse_args(args)
-    # this needs to happen after args are parsed so we can determine the
-    # correct profiles.yml file
-    profile_config = read_config(parsed.profiles_dir)
-    if not send_anonymous_usage_stats(profile_config):
-        dbt.tracking.do_not_track()
-    else:
-        dbt.tracking.initialize_tracking()
+    profiler_enabled = False
+
+    if parsed.record_timing_info:
+        profiler_enabled = True
+
+    with dbt.profiler.profiler(
+        enable=profiler_enabled,
+        outfile=parsed.record_timing_info
+    ):
+        # this needs to happen after args are parsed so we can determine the
+        # correct profiles.yml file
+        profile_config = read_config(parsed.profiles_dir)
+        if not send_anonymous_usage_stats(profile_config):
+            dbt.tracking.do_not_track()
+        else:
+            dbt.tracking.initialize_tracking()
 
-    if colorize_output(profile_config):
-        dbt.ui.printer.use_colors()
+        if colorize_output(profile_config):
+            dbt.ui.printer.use_colors()
 
-    reset_adapters()
+        reset_adapters()
 
-    try:
-        task, res = run_from_args(parsed)
-    finally:
-        dbt.tracking.flush()
+        try:
+            task, res = run_from_args(parsed)
+        finally:
+            dbt.tracking.flush()
 
-    success = task.interpret_results(res)
+        success = task.interpret_results(res)
 
-    return res, success
+        return res, success
 
 
 def get_nearest_project_dir():
@@ -290,6 +300,18 @@ def parse_args(args):
         action='dbtversion',
         help="Show version information")
 
+    p.add_argument(
+        '-r',
+        '--record-timing-info',
+        default=None,
+        type=str,
+        help="""
+        When this option is passed, dbt will output low-level timing
+        stats to the specified file. Example:
+        `--record-timing-info output.profile`
+        """
+    )
+
     p.add_argument(
         '-d',
         '--debug',
@@ -304,6 +326,17 @@ def parse_args(args):
         help='''Run schema validations at runtime. This will surface
         bugs in dbt, but may incur a performance penalty.''')
 
+    # if set, run dbt in single-threaded mode: thread count is ignored, and
+    # calls go through `map` instead of the thread pool. This is useful for
+    # getting performance information about aspects of dbt that normally run in
+    # a thread, as the profiler ignores child threads. Users should really
+    # never use this.
+    p.add_argument(
+        '--single-threaded',
+        action='store_true',
+        help=argparse.SUPPRESS,
+    )
+
     subs = p.add_subparsers(title="Available sub-commands")
 
     base_subparser = argparse.ArgumentParser(add_help=False)
diff --git a/dbt/profiler.py b/dbt/profiler.py
new file mode 100644
--- /dev/null
+++ b/dbt/profiler.py
@@ -0,0 +1,19 @@
+from contextlib import contextmanager
+from cProfile import Profile
+from pstats import Stats
+
+
+@contextmanager
+def profiler(enable, outfile):
+    try:
+        if enable:
+            profiler = Profile()
+            profiler.enable()
+
+        yield
+    finally:
+        if enable:
+            profiler.disable()
+            stats = Stats(profiler)
+            stats.sort_stats('tottime')
+            stats.dump_stats(outfile)
diff --git a/dbt/runner.py b/dbt/runner.py
--- a/dbt/runner.py
+++ b/dbt/runner.py
@@ -94,6 +94,18 @@ def get_relevant_runners(self, node_runners, node_subset):
                 runners.append(node_runners[unique_id])
         return runners
 
+    def map_run(self, pool, args):
+        """If the caller has passed the magic 'single-threaded' flag, use map()
+        instead of the pool.imap_unordered. The single-threaded flag is
+        intended for gathering more useful performance information about what
+        happens beneath `call_runner`, since python's default profiling tools
+        ignore child threads.
+        """
+        if self.config.args.single_threaded:
+            return map(self.call_runner, args)
+        else:
+            return pool.imap_unordered(self.call_runner, args)
+
     def execute_nodes(self, linker, Runner, manifest, node_dependency_list):
         adapter = get_adapter(self.config)
 
@@ -121,7 +133,7 @@ def execute_nodes(self, linker, Runner, manifest, node_dependency_list):
                 })
 
             try:
-                for result in pool.imap_unordered(self.call_runner, args_list):
+                for result in self.map_run(pool, args_list):
                     is_ephemeral = Runner.is_ephemeral_model(result.node)
                     if not is_ephemeral:
                         node_results.append(result)


Test patch:
diff --git a/test/integration/base.py b/test/integration/base.py
--- a/test/integration/base.py
+++ b/test/integration/base.py
@@ -35,11 +35,13 @@ def __init__(self):
         self.full_refresh = False
         self.models = None
         self.exclude = None
+        self.single_threaded = False
 
 
 class TestArgs(object):
     def __init__(self, kwargs):
         self.which = 'run'
+        self.single_threaded = False
         self.__dict__.update(kwargs)