Repository: dbt-labs/dbt-core

Issue:
Support model selection syntax for sources
## Feature
dbt should support running dependents of 'sources' from the model selection syntax, as described at the bottom of #814 

### Feature description
Currently sources don't actually participate in the graph at all, so that's the first thing to do. Once that's done, add a new selector filter and implementation.

### Who will this benefit?
This will help flesh out the sources feature.


Discussion:
We have a decent mechanism in place for handling custom "selector filters" [here](https://github.com/fishtown-analytics/dbt/blob/dev/stephen-girard/core/dbt/graph/selector.py#L162). While it's not super difficult to just keep adding entries to this dict, I'm super open to doing this differently if there's some better mechanism you can think of.

One thing to note: We support model selection in the dbt docs site, which means that I have to re-implement any model selection logic in JS :p. I don't _love_ that long-term, but I think it's ok for now.

Lmk if there's anything in here that I can clarify!

Resolving patch:
diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py
--- a/core/dbt/compilation.py
+++ b/core/dbt/compilation.py
@@ -172,7 +172,6 @@ def link_node(self, linker, node, manifest):
                 linker.dependency(
                     node.unique_id,
                     (manifest.nodes.get(dependency).unique_id))
-
             else:
                 dbt.exceptions.dependency_not_found(node, dependency)
 
diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py
--- a/core/dbt/graph/selector.py
+++ b/core/dbt/graph/selector.py
@@ -15,6 +15,7 @@
 class SELECTOR_FILTERS(object):
     FQN = 'fqn'
     TAG = 'tag'
+    SOURCE = 'source'
 
 
 def split_specs(node_specs):
@@ -135,6 +136,27 @@ def __init__(self, linker, manifest):
         self.linker = linker
         self.manifest = manifest
 
+    def _node_iterator(self, graph, exclude, include):
+        for node in graph.nodes():
+            real_node = self.manifest.nodes[node]
+            if include is not None and real_node.resource_type not in include:
+                continue
+            if exclude is not None and real_node.resource_type in exclude:
+                continue
+            yield node, real_node
+
+    def parsed_nodes(self, graph):
+        return self._node_iterator(
+            graph,
+            exclude=(NodeType.Source,),
+            include=None)
+
+    def source_nodes(self, graph):
+        return self._node_iterator(
+            graph,
+            exclude=None,
+            include=(NodeType.Source,))
+
     def get_nodes_by_qualified_name(self, graph, qualified_name_selector):
         """Yield all nodes in the graph that match the qualified_name_selector.
 
@@ -142,23 +164,35 @@ def get_nodes_by_qualified_name(self, graph, qualified_name_selector):
         """
         qualified_name = qualified_name_selector.split(".")
         package_names = get_package_names(graph)
-        for node in graph.nodes():
-            real_node = self.manifest.nodes[node]
-            if real_node.resource_type == NodeType.Source:
-                continue
-            fqn_ish = real_node.fqn
-            if _node_is_match(qualified_name, package_names, fqn_ish):
+        for node, real_node in self.parsed_nodes(graph):
+            if _node_is_match(qualified_name, package_names, real_node.fqn):
                 yield node
 
     def get_nodes_by_tag(self, graph, tag_name):
         """ yields nodes from graph that have the specified tag """
-        for node in graph.nodes():
-            real_node = self.manifest.nodes[node]
-            if real_node.resource_type == NodeType.Source:
-                continue
-            tags = real_node.tags
+        for node, real_node in self.parsed_nodes(graph):
+            if tag_name in real_node.tags:
+                yield node
 
-            if tag_name in tags:
+    def get_nodes_by_source(self, graph, source_full_name):
+        """yields nodes from graph are the specified source."""
+        parts = source_full_name.split('.')
+        if len(parts) == 1:
+            target_source, target_table = parts[0], None
+        elif len(parts) == 2:
+            target_source, target_table = parts
+        else:  # len(parts) > 2 or len(parts) == 0
+            msg = (
+                'Invalid source selector value "{}". Sources must be of the '
+                'form `${{source_name}}` or '
+                '`${{source_name}}.${{target_name}}`'
+            ).format(source_full_name)
+            raise dbt.exceptions.RuntimeException(msg)
+
+        for node, real_node in self.source_nodes(graph):
+            if real_node.source_name != target_source:
+                continue
+            if target_table is None or target_table == real_node.name:
                 yield node
 
     def get_nodes_from_spec(self, graph, spec):
@@ -168,6 +202,7 @@ def get_nodes_from_spec(self, graph, spec):
         filter_map = {
             SELECTOR_FILTERS.FQN: self.get_nodes_by_qualified_name,
             SELECTOR_FILTERS.TAG: self.get_nodes_by_tag,
+            SELECTOR_FILTERS.SOURCE: self.get_nodes_by_source,
         }
 
         node_filter = spec['filter']
@@ -209,7 +244,11 @@ def get_nodes_from_spec(self, graph, spec):
                            NodeType.Test]
             test_nodes.update(child_tests)
 
-        return model_nodes | test_nodes
+        # filter out all actual source nodes now that we've calculated any
+        # children specifiers
+        source_nodes = set(uid for uid, _ in self.source_nodes(graph))
+
+        return (model_nodes | test_nodes).difference(source_nodes)
 
     def select_nodes(self, graph, raw_include_specs, raw_exclude_specs):
         selected_nodes = set()
@@ -232,14 +271,17 @@ def select_nodes(self, graph, raw_include_specs, raw_exclude_specs):
 
         return selected_nodes
 
-    def get_valid_nodes(self, graph):
-        valid = []
-        for node_name in graph.nodes():
-            node = self.manifest.nodes[node_name]
+    def _is_graph_member(self, node_name):
+        node = self.manifest.nodes[node_name]
+        if node.resource_type == NodeType.Source:
+            return True
+        return not node.get('empty') and is_enabled(node)
 
-            if not node.get('empty') and is_enabled(node):
-                valid.append(node_name)
-        return valid
+    def get_valid_nodes(self, graph):
+        return [
+            node_name for node_name in graph.nodes()
+            if self._is_graph_member(node_name)
+        ]
 
     def get_selected(self, include, exclude, resource_types, tags):
         graph = self.linker.graph


Test patch:
diff --git a/test/integration/042_sources_test/models/multi_source_model.sql b/test/integration/042_sources_test/models/multi_source_model.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/042_sources_test/models/multi_source_model.sql
@@ -0,0 +1,2 @@
+select * from {{ source('test_source', 'other_test_table')}}
+	join {{ source('other_source', 'test_table')}} using (id)
diff --git a/test/integration/042_sources_test/models/nonsource_descendant.sql b/test/integration/042_sources_test/models/nonsource_descendant.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/042_sources_test/models/nonsource_descendant.sql
@@ -0,0 +1 @@
+select * from {{ schema }}.source
diff --git a/test/integration/042_sources_test/models/schema.yml b/test/integration/042_sources_test/models/schema.yml
--- a/test/integration/042_sources_test/models/schema.yml
+++ b/test/integration/042_sources_test/models/schema.yml
@@ -37,3 +37,9 @@ sources:
               column_name: favorite_color
               to: ref('descendant_model')
               field: favorite_color
+      - name: other_test_table
+        sql_table_name: "{{ var('test_run_schema') }}.other_table"
+  - name: other_source
+    tables:
+      - name: test_table
+        sql_table_name: "{{ var('test_run_schema') }}.other_source_table"
diff --git a/test/integration/042_sources_test/source.sql b/test/integration/042_sources_test/source.sql
--- a/test/integration/042_sources_test/source.sql
+++ b/test/integration/042_sources_test/source.sql
@@ -111,3 +111,43 @@ VALUES
     ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'),
     ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'),
     ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39');
+
+create table {schema}.other_table (
+    id INTEGER,
+    first_name VARCHAR(11)
+);
+
+
+INSERT INTO {schema}.other_table
+    (id, first_name)
+VALUES
+    (1, 'Larry'),
+    (2, 'Curly'),
+    (3, 'Moe');
+
+create table {schema}.other_source_table (
+    id INTEGER,
+    color VARCHAR(10)
+);
+
+
+INSERT INTO {schema}.other_source_table
+    (id, color)
+VALUES
+    (1, 'blue'),
+    (2, 'red'),
+    (3, 'green');
+
+
+create table {schema}.expected_multi_source(
+    id INTEGER,
+    first_name VARCHAR(11),
+    color VARCHAR(10)
+);
+
+INSERT INTO {schema}.expected_multi_source
+    (id, first_name, color)
+VALUES
+    (1, 'Larry', 'blue'),
+    (2, 'Curly', 'red'),
+    (3, 'Moe', 'green');
diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py
--- a/test/integration/042_sources_test/test_sources.py
+++ b/test/integration/042_sources_test/test_sources.py
@@ -11,21 +11,78 @@ def schema(self):
     def models(self):
         return "test/integration/042_sources_test/models"
 
-    @use_profile('postgres')
-    def test_basic_source_def(self):
+    def setUp(self):
+        super(TestSources, self).setUp()
         os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE'] = 'test_run_schema'
         self.run_sql_file('test/integration/042_sources_test/source.sql',
                           kwargs={'schema':self.unique_schema()})
-        self.run_dbt([
+
+    def tearDown(self):
+        del os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE']
+        super(TestSources, self).tearDown()
+
+    def run_dbt_with_vars(self, cmd, *args, **kwargs):
+        cmd.extend(['--vars',
+                    '{{test_run_schema: {}}}'.format(self.unique_schema())])
+        return self.run_dbt(cmd, *args, **kwargs)
+
+    @use_profile('postgres')
+    def test_basic_source_def(self):
+        results = self.run_dbt_with_vars(['run'])
+        self.assertEqual(len(results), 3)
+        self.assertManyTablesEqual(
+            ['source', 'descendant_model', 'nonsource_descendant'],
+            ['expected_multi_source', 'multi_source_model'])
+        results = self.run_dbt_with_vars(['test'])
+        self.assertEqual(len(results), 4)
+
+    @use_profile('postgres')
+    def test_source_selector(self):
+        # only one of our models explicitly depends upon a source
+        results = self.run_dbt_with_vars([
             'run',
-            '--vars',
-            '{{test_run_schema: {}}}'.format(self.unique_schema())
+            '--models',
+            'source:test_source.test_table+'
         ])
+        self.assertEqual(len(results), 1)
         self.assertTablesEqual('source', 'descendant_model')
-        results = self.run_dbt([
+        self.assertTableDoesNotExist('nonsource_descendant')
+        self.assertTableDoesNotExist('multi_source_model')
+        results = self.run_dbt_with_vars([
             'test',
-            '--vars',
-            '{{test_run_schema: {}}}'.format(self.unique_schema())
+            '--models',
+            'source:test_source.test_table+'
         ])
         self.assertEqual(len(results), 4)
-        del os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE']
+
+    @use_profile('postgres')
+    def test_empty_source_def(self):
+        # sources themselves can never be selected, so nothing should be run
+        results = self.run_dbt_with_vars([
+            'run',
+            '--models',
+            'source:test_source.test_table'
+        ])
+        self.assertTableDoesNotExist('nonsource_descendant')
+        self.assertTableDoesNotExist('multi_source_model')
+        self.assertTableDoesNotExist('descendant_model')
+        self.assertEqual(len(results), 0)
+
+    @use_profile('postgres')
+    def test_source_only_def(self):
+        results = self.run_dbt_with_vars([
+            'run', '--models', 'source:other_source+'
+        ])
+        self.assertEqual(len(results), 1)
+        self.assertTablesEqual('expected_multi_source', 'multi_source_model')
+        self.assertTableDoesNotExist('nonsource_descendant')
+        self.assertTableDoesNotExist('descendant_model')
+
+        results = self.run_dbt_with_vars([
+            'run', '--models', 'source:test_source+'
+        ])
+        self.assertEqual(len(results), 2)
+        self.assertManyTablesEqual(
+            ['source', 'descendant_model'],
+            ['expected_multi_source', 'multi_source_model'])
+        self.assertTableDoesNotExist('nonsource_descendant')