Repository: dbt-labs/dbt-core

Issue:
support clustering on BigQuery
## Feature

### Feature description
BigQuery recently announced support for table clustering. With this new feature, a list of up to four "clustering columns" can be supplied to a `create table as (...)` statement. BigQuery will use the clustering columns to colocate similar data. This should speed up filters and aggregations, and BigQuery can both skip scanning whole blocks of data, as well as avoid shuffling data between nodes for certain aggregations. Docs: https://cloud.google.com/bigquery/docs/creating-clustered-tables

### Who will this benefit?
This feature will benefit users of BigQuery. It will work in concert with table partitioning.

### Proposed Implementation
This feature can be implemented in the BigQuery implementation of [create_table_as()](https://github.com/fishtown-analytics/dbt/blob/development/dbt/include/global_project/macros/adapters/bigquery.sql#L14). It's implementation will look similar to the `partition_by` implementation.



Discussion:

Resolving patch:
diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py
--- a/dbt/adapters/bigquery/impl.py
+++ b/dbt/adapters/bigquery/impl.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 from contextlib import contextmanager
+import copy
 
 import dbt.compat
 import dbt.deprecations
@@ -698,7 +699,8 @@ def _get_stats_column_names(cls):
         columns of data.
         """
         columns = []
-        stats = ('num_bytes', 'num_rows', 'location', 'partitioning_type')
+        stats = ('num_bytes', 'num_rows', 'location', 'partitioning_type',
+                 'clustering_fields')
         stat_components = ('label', 'value', 'description', 'include')
         for stat_id in stats:
             for stat_component in stat_components:
@@ -711,6 +713,11 @@ def _get_stats_columns(cls, table, relation_type):
         column names/values.
         """
         column_names = cls._get_stats_column_names()
+
+        # agate does not handle the array of column names gracefully
+        clustering_value = None
+        if table.clustering_fields is not None:
+            clustering_value = ','.join(table.clustering_fields)
         # cast num_bytes/num_rows to str before they get to agate, or else
         # agate will incorrectly decide they are booleans.
         column_values = (
@@ -733,6 +740,11 @@ def _get_stats_columns(cls, table, relation_type):
             table.partitioning_type,
             'The partitioning type used for this table',
             relation_type == 'table',
+
+            'Clustering Fields',
+            clustering_value,
+            'The clustering fields for this table',
+            relation_type == 'table',
         )
         return zip(column_names, column_values)
 
@@ -772,6 +784,8 @@ def get_catalog(cls, profile, project_cfg, manifest):
                 table = client.get_table(table_ref)
 
                 flattened = cls._flat_columns_in_table(table)
+                relation_stats = dict(cls._get_stats_columns(table,
+                                                             relation.type))
 
                 for index, column in enumerate(flattened, start=1):
                     column_data = (
@@ -786,8 +800,7 @@ def get_catalog(cls, profile, project_cfg, manifest):
                         None,
                     )
                     column_dict = dict(zip(column_names, column_data))
-                    column_dict.update(cls._get_stats_columns(table,
-                                                              relation.type))
+                    column_dict.update(copy.deepcopy(relation_stats))
 
                     columns.append(column_dict)
 


Test patch:
diff --git a/test/integration/022_bigquery_test/models/clustered_model.sql b/test/integration/022_bigquery_test/models/clustered_model.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/022_bigquery_test/models/clustered_model.sql
@@ -0,0 +1,10 @@
+
+{{
+	config(
+		materialized = "table",
+		partition_by = "updated_at",
+		cluster_by = "dupe",
+	)
+}}
+
+select * from {{ ref('view_model') }}
diff --git a/test/integration/022_bigquery_test/models/partitioned_model.sql b/test/integration/022_bigquery_test/models/partitioned_model.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/022_bigquery_test/models/partitioned_model.sql
@@ -0,0 +1,9 @@
+
+{{
+	config(
+		materialized = "table",
+		partition_by = "updated_at",
+	)
+}}
+
+select * from {{ ref('view_model') }}
diff --git a/test/integration/022_bigquery_test/test_simple_bigquery_view.py b/test/integration/022_bigquery_test/test_simple_bigquery_view.py
--- a/test/integration/022_bigquery_test/test_simple_bigquery_view.py
+++ b/test/integration/022_bigquery_test/test_simple_bigquery_view.py
@@ -24,12 +24,11 @@ def profile_config(self):
 
     @use_profile('bigquery')
     def test__bigquery_simple_run(self):
-        self.use_default_project()
         # make sure seed works twice. Full-refresh is a no-op
         self.run_dbt(['seed'])
         self.run_dbt(['seed', '--full-refresh'])
         results = self.run_dbt()
-        self.assertEqual(len(results), 2)
+        self.assertEqual(len(results), 4)
 
         # The 'dupe' model should fail, but all others should pass
         test_results = self.run_dbt(['test'], expect_pass=False)
@@ -49,10 +48,9 @@ def test__bigquery_simple_run(self):
 
     @use_profile('bigquery')
     def test__bigquery_exists_non_destructive(self):
-        self.use_default_project()
         self.run_dbt(['seed'])
         # first run dbt. this should work
-        self.assertEqual(len(self.run_dbt()), 2)
+        self.assertEqual(len(self.run_dbt()), 4)
         # then run dbt with --non-destructive. The view should still exist.
         self.run_dbt(['run', '--non-destructive'])
         # The 'dupe' model should fail, but all others should pass
diff --git a/test/integration/029_docs_generate_tests/bq_models/clustered.sql b/test/integration/029_docs_generate_tests/bq_models/clustered.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/029_docs_generate_tests/bq_models/clustered.sql
@@ -0,0 +1,9 @@
+{{
+    config(
+        materialized='table',
+        partition_by='updated_at',
+        cluster_by=['first_name']
+    )
+}}
+
+select id,first_name,email,ip_address,DATE(updated_at) as updated_at from {{ ref('seed') }}
diff --git a/test/integration/029_docs_generate_tests/bq_models/seed.sql b/test/integration/029_docs_generate_tests/bq_models/nested_table.sql
similarity index 72%
rename from test/integration/029_docs_generate_tests/bq_models/seed.sql
rename to test/integration/029_docs_generate_tests/bq_models/nested_table.sql
--- a/test/integration/029_docs_generate_tests/bq_models/seed.sql
+++ b/test/integration/029_docs_generate_tests/bq_models/nested_table.sql
@@ -1,3 +1,9 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
+
 select
     1 as field_1,
     2 as field_2,
diff --git a/test/integration/029_docs_generate_tests/bq_models/model.sql b/test/integration/029_docs_generate_tests/bq_models/nested_view.sql
similarity index 56%
rename from test/integration/029_docs_generate_tests/bq_models/model.sql
rename to test/integration/029_docs_generate_tests/bq_models/nested_view.sql
--- a/test/integration/029_docs_generate_tests/bq_models/model.sql
+++ b/test/integration/029_docs_generate_tests/bq_models/nested_view.sql
@@ -4,4 +4,4 @@
     )
 }}
 
-select * from {{ ref('seed') }}
+select * from {{ ref('nested_table') }}
diff --git a/test/integration/029_docs_generate_tests/bq_models/schema.yml b/test/integration/029_docs_generate_tests/bq_models/schema.yml
--- a/test/integration/029_docs_generate_tests/bq_models/schema.yml
+++ b/test/integration/029_docs_generate_tests/bq_models/schema.yml
@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: model
+  - name: nested_view
     description: "The test model"
     columns:
       - name: field_1
@@ -14,3 +14,17 @@ models:
         description: The first nested field
       - name: nested_field.field_5
         description: The second nested field
+  - name: clustered
+    description: "A clustered and partitioned copy of the test model"
+    columns:
+      - name: id
+        description: The user id
+      - name: first_name
+        description: The user's name
+      - name: email
+        description: The user's email
+      - name: ip_address
+        description: The user's IP address
+      - name: updated_at
+        description: When the user was updated
+
diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
--- a/test/integration/029_docs_generate_tests/test_docs_generate.py
+++ b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -183,7 +183,7 @@ def _snowflake_stats(self):
             },
         }
 
-    def _bigquery_stats(self, is_table):
+    def _bigquery_stats(self, is_table, partition=None, cluster=None):
         stats = {
             'has_stats': {
                 'id': 'has_stats',
@@ -219,10 +219,17 @@ def _bigquery_stats(self, is_table):
                 'partitioning_type': {
                     'id': 'partitioning_type',
                     'label': 'Partitioning Type',
-                    'value': None,
+                    'value': partition,
                     'description': 'The partitioning type used for this table',
                     'include': True,
                 },
+                'clustering_fields': {
+                    'id': 'clustering_fields',
+                    'label': 'Clustering Fields',
+                    'value': cluster,
+                    'description': 'The clustering fields for this table',
+                    'include': True,
+                },
             })
         return stats
 
@@ -424,11 +431,48 @@ def expected_bigquery_catalog(self):
             seed_stats=self._bigquery_stats(True),
         )
 
-    def expected_bigquery_nested_catalog(self):
+    @staticmethod
+    def _clustered_bigquery_columns(update_type):
+        return {
+            'id': {
+                'comment': None,
+                'index': 1,
+                'name': 'id',
+                'type': 'INT64',
+            },
+            'first_name': {
+                'comment': None,
+                'index': 2,
+                'name': 'first_name',
+                'type': 'STRING',
+            },
+            'email': {
+                'comment': None,
+                'index': 3,
+                'name': 'email',
+                'type': 'STRING',
+            },
+            'ip_address': {
+                'comment': None,
+                'index': 4,
+                'name': 'ip_address',
+                'type': 'STRING',
+            },
+            'updated_at': {
+                'comment': None,
+                'index': 5,
+                'name': 'updated_at',
+                'type': update_type,
+            },
+        }
+
+    def expected_bigquery_complex_catalog(self):
         my_schema_name = self.unique_schema()
         role = self.get_role()
-        stats = self._bigquery_stats(False)
-        expected_cols = {
+        table_stats = self._bigquery_stats(True)
+        clustering_stats = self._bigquery_stats(True, partition='DAY',
+                                                cluster='first_name')
+        nesting_columns = {
             'field_1': {
                 "name": "field_1",
                 "index": 1,
@@ -460,30 +504,55 @@ def expected_bigquery_nested_catalog(self):
                 "comment": None
             }
         }
+
         return {
-            "model.test.model": {
-                'unique_id': 'model.test.model',
+            'model.test.clustered': {
+                'unique_id': 'model.test.clustered',
+                'metadata': {
+                    'comment': None,
+                    'name': 'clustered',
+                    'owner': None,
+                    'schema': my_schema_name,
+                    'type': 'table'
+                },
+                'stats': clustering_stats,
+                'columns': self._clustered_bigquery_columns('DATE'),
+            },
+            'seed.test.seed': {
+                'unique_id': 'seed.test.seed',
+                'metadata': {
+                    'comment': None,
+                    'name': 'seed',
+                    'owner': None,
+                    'schema': my_schema_name,
+                    'type': 'table',
+                },
+                'stats': table_stats,
+                'columns': self._clustered_bigquery_columns('DATETIME'),
+            },
+            "model.test.nested_view": {
+                'unique_id': 'model.test.nested_view',
                 "metadata": {
                     "schema": my_schema_name,
-                    "name": "model",
+                    "name": "nested_view",
                     "type": "view",
                     "owner": role,
                     "comment": None
                 },
-                'stats': stats,
-                "columns": expected_cols
+                'stats': self._bigquery_stats(False),
+                "columns": nesting_columns,
             },
-            "model.test.seed": {
-                'unique_id': 'model.test.seed',
+            "model.test.nested_table": {
+                'unique_id': 'model.test.nested_table',
                 "metadata": {
                     "schema": my_schema_name,
-                    "name": "seed",
-                    "type": "view",
+                    "name": "nested_table",
+                    "type": "table",
                     "owner": role,
                     "comment": None
                 },
-                'stats': stats,
-                "columns": expected_cols
+                'stats': table_stats,
+                "columns": nesting_columns,
             }
         }
 
@@ -1100,14 +1169,68 @@ def expected_postgres_references_manifest(self):
             },
         }
 
-    def expected_bigquery_nested_manifest(self):
-        model_sql_path = self.dir('bq_models/model.sql')
-        seed_sql_path = self.dir('bq_models/seed.sql')
+    def expected_bigquery_complex_manifest(self):
+        nested_view_sql_path = self.dir('bq_models/nested_view.sql')
+        nested_table_sql_path = self.dir('bq_models/nested_table.sql')
+        clustered_sql_path = self.dir('bq_models/clustered.sql')
         my_schema_name = self.unique_schema()
         return {
             'nodes': {
-               'model.test.model': {
-                    'alias': 'model',
+                'model.test.clustered': {
+                    'alias': 'clustered',
+                    'config': {
+                        'cluster_by': ['first_name'],
+                        'column_types': {},
+                        'enabled': True,
+                        'materialized': 'table',
+                        'partition_by': 'updated_at',
+                        'post-hook': [],
+                        'pre-hook': [],
+                        'quoting': {},
+                        'vars': {}
+                    },
+                    'depends_on': {'macros': [], 'nodes': ['seed.test.seed']},
+                    'empty': False,
+                    'fqn': ['test', 'clustered'],
+                    'name': 'clustered',
+                    'original_file_path': clustered_sql_path,
+                    'package_name': 'test',
+                    'path': 'clustered.sql',
+                    'raw_sql': open(clustered_sql_path).read().rstrip('\n'),
+                    'refs': [['seed']],
+                    'resource_type': 'model',
+                    'root_path': os.getcwd(),
+                    'schema': my_schema_name,
+                    'tags': [],
+                    'unique_id': 'model.test.clustered',
+                    'columns': {
+                        'email': {
+                            'description': "The user's email",
+                            'name': 'email'
+                        },
+                        'first_name': {
+                            'description': "The user's name",
+                            'name': 'first_name'
+                        },
+                        'id': {
+                            'description': 'The user id',
+                            'name': 'id'
+                        },
+                        'ip_address': {
+                            'description': "The user's IP address",
+                            'name': 'ip_address'
+                        },
+                        'updated_at': {
+                            'description': 'When the user was updated',
+                            'name': 'updated_at'
+                        },
+                    },
+                    'description': 'A clustered and partitioned copy of the test model',
+                    'patch_path': self.dir('bq_models/schema.yml'),
+                    'docrefs': [],
+                },
+                'model.test.nested_view': {
+                    'alias': 'nested_view',
                     'config': {
                         'column_types': {},
                         'enabled': True,
@@ -1119,21 +1242,21 @@ def expected_bigquery_nested_manifest(self):
                     },
                     'depends_on': {
                         'macros': [],
-                        'nodes': ['model.test.seed']
+                        'nodes': ['model.test.nested_table']
                     },
                     'empty': False,
-                    'fqn': ['test', 'model'],
-                    'name': 'model',
-                    'original_file_path': model_sql_path,
+                    'fqn': ['test', 'nested_view'],
+                    'name': 'nested_view',
+                    'original_file_path': nested_view_sql_path,
                     'package_name': 'test',
-                    'path': 'model.sql',
-                    'raw_sql': open(model_sql_path).read().rstrip('\n'),
-                    'refs': [['seed']],
+                    'path': 'nested_view.sql',
+                    'raw_sql': open(nested_view_sql_path).read().rstrip('\n'),
+                    'refs': [['nested_table']],
                     'resource_type': 'model',
                     'root_path': os.getcwd(),
                     'schema': my_schema_name,
                     'tags': [],
-                    'unique_id': 'model.test.model',
+                    'unique_id': 'model.test.nested_view',
                     'columns': {
                         'field_1': {
                             'name': 'field_1',
@@ -1160,12 +1283,12 @@ def expected_bigquery_nested_manifest(self):
                     'patch_path': self.dir('bq_models/schema.yml'),
                     'docrefs': [],
                 },
-                'model.test.seed': {
-                    'alias': 'seed',
+                'model.test.nested_table': {
+                    'alias': 'nested_table',
                     'config': {
                         'column_types': {},
                         'enabled': True,
-                        'materialized': 'view',
+                        'materialized': 'table',
                         'post-hook': [],
                         'pre-hook': [],
                         'quoting': {},
@@ -1176,29 +1299,64 @@ def expected_bigquery_nested_manifest(self):
                         'nodes': []
                     },
                     'empty': False,
-                    'fqn': ['test', 'seed'],
-                    'name': 'seed',
-                    'original_file_path': seed_sql_path,
+                    'fqn': ['test', 'nested_table'],
+                    'name': 'nested_table',
+                    'original_file_path': nested_table_sql_path,
                     'package_name': 'test',
-                    'path': 'seed.sql',
-                    'raw_sql': open(seed_sql_path).read().rstrip('\n'),
+                    'path': 'nested_table.sql',
+                    'raw_sql': open(nested_table_sql_path).read().rstrip('\n'),
                     'refs': [],
                     'resource_type': 'model',
                     'root_path': os.getcwd(),
                     'schema': my_schema_name,
                     'tags': [],
-                    'unique_id': 'model.test.seed',
+                    'unique_id': 'model.test.nested_table',
                     'columns': {},
                     'description': '',
-                }
+                },
+                'seed.test.seed': {
+                    'path': 'seed.csv',
+                    'name': 'seed',
+                    'root_path': os.getcwd(),
+                    'resource_type': 'seed',
+                    'raw_sql': '-- csv --',
+                    'package_name': 'test',
+                    'original_file_path': self.dir('seed/seed.csv'),
+                    'refs': [],
+                    'depends_on': {
+                        'nodes': [],
+                        'macros': [],
+                    },
+                    'unique_id': 'seed.test.seed',
+                    'empty': False,
+                    'fqn': ['test', 'seed'],
+                    'tags': [],
+                    'config': {
+                        'enabled': True,
+                        'materialized': 'seed',
+                        'pre-hook': [],
+                        'post-hook': [],
+                        'vars': {},
+                        'column_types': {},
+                        'quoting': {},
+                    },
+                    'schema': my_schema_name,
+                    'alias': 'seed',
+                    'columns': {},
+                    'description': '',
+                },
             },
             'child_map': {
-                'model.test.model': [],
-                'model.test.seed': ['model.test.model']
+                'model.test.clustered': [],
+                'model.test.nested_table': ['model.test.nested_view'],
+                'model.test.nested_view': [],
+                'seed.test.seed': ['model.test.clustered']
             },
             'parent_map': {
-                'model.test.model': ['model.test.seed'],
-                'model.test.seed': []
+                'model.test.clustered': ['seed.test.seed'],
+                'seed.test.seed': [],
+                'model.test.nested_table': [],
+                'model.test.nested_view': ['model.test.nested_table'],
             },
             'docs': {
                 'dbt.__overview__': ANY,
@@ -1878,17 +2036,14 @@ def test__bigquery__run_and_generate(self):
         self.verify_run_results(self.expected_run_results())
 
     @use_profile('bigquery')
-    def test__bigquery__nested_models(self):
-        self.use_default_project({'source-paths': [self.dir('bq_models')]})
-
-        self.assertEqual(len(self.run_dbt()), 2)
-        os.remove(os.path.normpath('target/manifest.json'))
-        os.remove(os.path.normpath('target/run_results.json'))
-        self.generate_start_time = datetime.utcnow()
-        self.run_dbt(['docs', 'generate'])
+    def test__bigquery__complex_models(self):
+        self.run_and_generate(
+            extra={'source-paths': [self.dir('bq_models')]},
+            model_count=3
+        )
 
-        self.verify_catalog(self.expected_bigquery_nested_catalog())
-        self.verify_manifest(self.expected_bigquery_nested_manifest())
+        self.verify_catalog(self.expected_bigquery_complex_catalog())
+        self.verify_manifest(self.expected_bigquery_complex_manifest())
 
     @use_profile('redshift')
     def test__redshift__run_and_generate(self):