Repository: dbt-labs/dbt-core

Issue:
Support persist docs on BigQuery: columns 
Child issue of https://github.com/fishtown-analytics/dbt/issues/1573

To do:
 - Support persisting docs for columns in tables and views

If the project specifies a column that does not exist in the destination table, the `comment on column` statement may fail. This is the desired behavior.

BigQuery does not support adding column-level descriptions in DDL unless all columns in the relation are specified. Instead, we should pursue the following approach:

1. Create a relation without column comments
2. Fetch the relation schema from the BigQuery API
3. Update the relation schema with descriptions from the corresponding .yml file
4. Save the relation schema to persist these changes back to BigQuery


Discussion:
Just like in the generated docs, could the column descriptions also include the schema tests? We are using the schema tests as documentation as well as tests, and I think it would be helpful to show whether a column is UNIQUE, NOT NULL, etc in BigQuery too.
I think I'd recommend using the dbt documentation website for that purpose. I'm super on board with persisting column descriptions into the database, but I'd hesitate to add a bunch of free-form text generated by dbt at the same time.

What I _can_ imagine is providing tools to generate a list of tests for a column as a string, then include that string in the stated column description. I'm also open to figuring out a way to add this to every column comment so that it happens automatically, but that's out of scope for the feature described in this issue IMO
In case it helps, here's the code we use for this:

```python
from typing import Dict, List, Sequence

from google.cloud import bigquery
from google.cloud.bigquery.schema import SchemaField

def update_schema_documentation(
    client: bigquery.Client,
    table_name: str,
    table_description: str,
    field_descriptions: Dict[str, str],
) -> None:  # pragma: no cover
    """Update the documentation for a BigQuery table.

    Known limitations:

    - Does not support nested field definitions

    Feel free to fix these limitations!
    """
    table = client.get_table(table_name)
    table.description = table_description
    table.schema = _document_schema(table_name, table.schema, field_descriptions)
    client.update_table(table, ["description", "schema"])


def _document_schema(
    table_name: str, schema: Sequence[SchemaField], field_descriptions: Dict[str, str]
) -> List[SchemaField]:
    """Create a documented version of the given BigQuery schema."""
    existing_fields = set()
    new_schema = []
    for field in schema:
        description = field_descriptions.get(field.name)
        new_schema.append(
            SchemaField(
                name=field.name,
                field_type=field.field_type,
                mode=field.mode,
                description=description,
                fields=field.fields,
            )
        )
        existing_fields.add(field.name)
    undescribed_fields = existing_fields - set(field_descriptions)
    # TODO: Also raise an exception if we have described fields that don't exist.
    if undescribed_fields:
        # TODO: Raise a more specific exception so we don't have to pass table_name in.
        raise RuntimeError(f"Unexpected fields defined in {table_name}: {undescribed_fields}")
    return new_schema
```

Tests:

```python
def test_document_schema_undocumented_fields() -> None:
    # _document_schema raises RuntimeError if we are missing documentation for
    # a field.
    with pytest.raises(RuntimeError):
        _document_schema("foo", [SchemaField("name", "INTEGER")], {})


def test_document_schema_empty() -> None:
    # We successfully document an empty schema by returning the same thing.
    assert _document_schema("foo", [], {}) == []


def test_document_schema_adds_docs() -> None:
    # _document_schema returns a new schema with the given descriptions added.
    new_schema = _document_schema(
        "table",
        [SchemaField("foo", "INTEGER"), SchemaField("bar", "STRING")],
        {"foo": "Full of fooness", "bar": "As you'd expect"},
    )
    assert new_schema == [
        SchemaField("foo", "INTEGER", "NULLABLE", "Full of fooness"),
        SchemaField("bar", "STRING", "NULLABLE", "As you'd expect"),
    ]
```

thanks @jml - that's super helpful! Are you able to submit a PR with a change like this? It would be super welcomed!
I'd genuinely love to, but I doubt I'll have an opportunity to do so any time soon. 
I need this feature, so I'll take a stab at it. But I'm new to the codebase so might be slow... if anybody else wants to work on this please do!

Resolving patch:
diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py
--- a/core/dbt/context/providers.py
+++ b/core/dbt/context/providers.py
@@ -210,6 +210,12 @@ def require(self, name, validator=None):
     def get(self, name, validator=None, default=None):
         return ''
 
+    def persist_relation_docs(self) -> bool:
+        return False
+
+    def persist_column_docs(self) -> bool:
+        return False
+
 
 class RuntimeConfigObject(Config):
     def __init__(
@@ -249,6 +255,24 @@ def get(self, name, validator=None, default=None):
 
         return to_return
 
+    def persist_relation_docs(self) -> bool:
+        persist_docs = self.get('persist_docs', default={})
+        if not isinstance(persist_docs, dict):
+            raise_compiler_error(
+                f"Invalid value provided for 'persist_docs'. Expected dict "
+                f"but received {type(persist_docs)}")
+
+        return persist_docs.get('relation', False)
+
+    def persist_column_docs(self) -> bool:
+        persist_docs = self.get('persist_docs', default={})
+        if not isinstance(persist_docs, dict):
+            raise_compiler_error(
+                f"Invalid value provided for 'persist_docs'. Expected dict "
+                f"but received {type(persist_docs)}")
+
+        return persist_docs.get('columns', False)
+
 
 # `adapter` implementations
 class ParseDatabaseWrapper(BaseDatabaseWrapper):
diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py
--- a/plugins/bigquery/dbt/adapters/bigquery/impl.py
+++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py
@@ -28,8 +28,20 @@
 import google.cloud.exceptions
 import google.cloud.bigquery
 
+from google.cloud.bigquery import SchemaField
+
 import time
 import agate
+import json
+
+
+def sql_escape(string):
+    if not isinstance(string, str):
+        dbt.exceptions.raise_compiler_exception(
+            f'cannot escape a non-string: {string}'
+        )
+
+    return json.dumps(string)[1:-1]
 
 
 @dataclass
@@ -320,16 +332,14 @@ def _get_dbt_columns_from_bq_table(self, table) -> List[BigQueryColumn]:
 
     def _agate_to_schema(
         self, agate_table: agate.Table, column_override: Dict[str, str]
-    ) -> List[google.cloud.bigquery.SchemaField]:
+    ) -> List[SchemaField]:
         """Convert agate.Table with column names to a list of bigquery schemas.
         """
         bq_schema = []
         for idx, col_name in enumerate(agate_table.column_names):
             inferred_type = self.convert_agate_type(agate_table, idx)
             type_ = column_override.get(col_name, inferred_type)
-            bq_schema.append(
-                google.cloud.bigquery.SchemaField(col_name, type_)
-            )
+            bq_schema.append(SchemaField(col_name, type_))
         return bq_schema
 
     def _materialize_as_view(self, model: Dict[str, Any]) -> str:
@@ -545,6 +555,33 @@ def parse_partition_by(
         """
         return PartitionConfig.parse(raw_partition_by)
 
+    def get_table_ref_from_relation(self, conn, relation):
+        return self.connections.table_ref(relation.database,
+                                          relation.schema,
+                                          relation.identifier,
+                                          conn)
+
+    @available.parse_none
+    def update_column_descriptions(self, relation, columns):
+        if len(columns) == 0:
+            return
+
+        conn = self.connections.get_thread_connection()
+        table_ref = self.get_table_ref_from_relation(conn, relation)
+        table = conn.handle.get_table(table_ref)
+
+        new_schema = []
+        for column in table.schema:
+            if column.name in columns:
+                column_config = columns[column.name]
+                column_dict = column.to_api_repr()
+                column_dict['description'] = column_config.get('description')
+                column = SchemaField.from_api_repr(column_dict)
+            new_schema.append(column)
+
+        new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema)
+        conn.handle.update_table(new_table, ['schema'])
+
     @available.parse_none
     def alter_table_add_columns(self, relation, columns):
 
@@ -614,3 +651,25 @@ def _get_cache_schemas(
                     .format(database, candidate.schema)
                 )
         return result
+
+    @available.parse(lambda *a, **k: {})
+    def get_table_options(
+        self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool
+    ) -> Dict[str, Any]:
+        opts = {}
+        if temporary:
+            expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'
+            opts['expiration_timestamp'] = expiration
+
+        if config.persist_relation_docs() and 'description' in node:
+            description = sql_escape(node['description'])
+            opts['description'] = '"""{}"""'.format(description)
+
+        if config.get('kms_key_name') is not None:
+            opts['kms_key_name'] = "'{}'".format(config.get('kms_key_name'))
+
+        if config.get('labels'):
+            labels = config.get('labels', {})
+            opts['labels'] = list(labels.items())
+
+        return opts


Test patch:
diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py
--- a/test/integration/060_persist_docs_tests/test_persist_docs.py
+++ b/test/integration/060_persist_docs_tests/test_persist_docs.py
@@ -122,3 +122,30 @@ def test_redshift_late_binding_view(self):
 
         view_node = catalog_data['nodes']['model.test.view_model']
         self._assert_has_view_comments(view_node, False, False)
+
+
+class TestPersistDocsSimple(BasePersistDocsTest):
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'models': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                    'view_model': {
+                        'bind': False,
+                    }
+                }
+            }
+        }
+
+    @use_profile('snowflake')
+    def test_snowflake_persist_docs(self):
+        self.run_dbt()
+
+    @use_profile('bigquery')
+    def test_bigquery_persist_docs(self):
+        self.run_dbt()