Repository: dbt-labs/dbt-core

Issue:
Support persist docs on Postgres and Redshift: Relations and columns
Child issue of https://github.com/fishtown-analytics/dbt/issues/1573

Notes:
 - We can render multiline comments using [this string escaping syntax](https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS)

To do:
 - Support persisting docs for relations  (tables and views)
 - Support persisting docs for columns in tables and views

Syntax:
```
comment on table {{ relation }} is $__dbt_comment__${{ comment }}$__dbt_comment__$;

comment on column {{ relation }}.{{ column }} is $__dbt_comment__${{ comment }}$__dbt_comment__$
```

If the project specifies a column that does not exist in the destination table, the `comment on column` statement may fail. This is the desired behavior.


Discussion:

Resolving patch:
diff --git a/core/dbt/clients/_jinja_blocks.py b/core/dbt/clients/_jinja_blocks.py
--- a/core/dbt/clients/_jinja_blocks.py
+++ b/core/dbt/clients/_jinja_blocks.py
@@ -92,6 +92,18 @@ def __init__(self, data):
         self._parenthesis_stack = []
         self.pos = 0
 
+    def linepos(self, end=None) -> str:
+        """Given an absolute position in the input data, return a pair of
+        line number + relative position to the start of the line.
+        """
+        end_val: int = self.pos if end is None else end
+        data = self.data[:end_val]
+        # if not found, rfind returns -1, and -1+1=0, which is perfect!
+        last_line_start = data.rfind('\n') + 1
+        # it's easy to forget this, but line numbers are 1-indexed
+        line_number = data.count('\n') + 1
+        return f'{line_number}:{end_val - last_line_start}'
+
     def advance(self, new_position):
         self.pos = new_position
 
@@ -320,20 +332,28 @@ def find_blocks(self, allowed_blocks=None, collect_raw_data=True):
                     dbt.exceptions.raise_compiler_error((
                         'Got an unexpected control flow end tag, got {} but '
                         'never saw a preceeding {} (@ {})'
-                    ).format(tag.block_type_name, expected, tag.start))
+                    ).format(
+                        tag.block_type_name,
+                        expected,
+                        self.tag_parser.linepos(tag.start)
+                    ))
                 expected = _CONTROL_FLOW_TAGS[found]
                 if expected != tag.block_type_name:
                     dbt.exceptions.raise_compiler_error((
                         'Got an unexpected control flow end tag, got {} but '
                         'expected {} next (@ {})'
-                    ).format(tag.block_type_name, expected, tag.start))
+                    ).format(
+                        tag.block_type_name,
+                        expected,
+                        self.tag_parser.linepos(tag.start)
+                    ))
 
             if tag.block_type_name in allowed_blocks:
                 if self.stack:
                     dbt.exceptions.raise_compiler_error((
                         'Got a block definition inside control flow at {}. '
                         'All dbt block definitions must be at the top level'
-                    ).format(tag.start))
+                    ).format(self.tag_parser.linepos(tag.start)))
                 if self.current is not None:
                     dbt.exceptions.raise_compiler_error(
                         duplicate_tags.format(outer=self.current, inner=tag)
diff --git a/core/dbt/parser/macros.py b/core/dbt/parser/macros.py
--- a/core/dbt/parser/macros.py
+++ b/core/dbt/parser/macros.py
@@ -48,22 +48,26 @@ def parse_macro(
     def parse_unparsed_macros(
         self, base_node: UnparsedMacro
     ) -> Iterable[ParsedMacro]:
-        blocks: List[jinja.BlockTag] = [
-            t for t in
-            jinja.extract_toplevel_blocks(
-                base_node.raw_sql,
-                allowed_blocks={'macro', 'materialization'},
-                collect_raw_data=False,
-            )
-            if isinstance(t, jinja.BlockTag)
-        ]
+        try:
+            blocks: List[jinja.BlockTag] = [
+                t for t in
+                jinja.extract_toplevel_blocks(
+                    base_node.raw_sql,
+                    allowed_blocks={'macro', 'materialization'},
+                    collect_raw_data=False,
+                )
+                if isinstance(t, jinja.BlockTag)
+            ]
+        except CompilationException as exc:
+            exc.add_node(base_node)
+            raise
 
         for block in blocks:
             try:
                 ast = jinja.parse(block.full_block)
             except CompilationException as e:
                 e.add_node(base_node)
-                raise e
+                raise
 
             macro_nodes = list(ast.find_all(jinja2.nodes.Macro))
 


Test patch:
diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
--- a/test/integration/029_docs_generate_tests/test_docs_generate.py
+++ b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -708,7 +708,7 @@ def expected_bigquery_complex_catalog(self):
     def expected_redshift_catalog(self):
         return self._expected_catalog(
             id_type='integer',
-            text_type='character varying',
+            text_type=AnyStringWith('character varying'),
             time_type='timestamp without time zone',
             view_type='VIEW',
             table_type='BASE TABLE',
@@ -787,19 +787,19 @@ def expected_redshift_incremental_catalog(self):
                         'first_name': {
                             'name': 'first_name',
                             'index': 2,
-                            'type': 'character varying',
+                            'type': 'character varying(5)',
                             'comment': None,
                         },
                         'email': {
                             'name': 'email',
                             'index': 3,
-                            'type': 'character varying',
+                            'type': 'character varying(23)',
                             'comment': None,
                         },
                         'ip_address': {
                             'name': 'ip_address',
                             'index': 4,
-                            'type': 'character varying',
+                            'type': 'character varying(14)',
                             'comment': None,
                         },
                         'updated_at': {
diff --git a/test/integration/060_persist_docs_tests/models/my_fun_docs.md b/test/integration/060_persist_docs_tests/models/my_fun_docs.md
new file mode 100644
--- /dev/null
+++ b/test/integration/060_persist_docs_tests/models/my_fun_docs.md
@@ -0,0 +1,10 @@
+{% docs my_fun_doc %}
+name Column description "with double quotes"
+and with 'single  quotes' as welll as other;
+'''abc123'''
+reserved -- characters
+--
+/* comment */
+Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+
+{% enddocs %}
diff --git a/test/integration/060_persist_docs_tests/models/schema.yml b/test/integration/060_persist_docs_tests/models/schema.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/060_persist_docs_tests/models/schema.yml
@@ -0,0 +1,45 @@
+version: 2
+
+models:
+  - name: table_model
+    description: |
+      Table model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+      - name: name
+        description: |
+          Some stuff here and then a call to
+          {{ doc('my_fun_doc')}}
+  - name: view_model
+    description: |
+      View model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
diff --git a/test/integration/060_persist_docs_tests/models/table_model.sql b/test/integration/060_persist_docs_tests/models/table_model.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/060_persist_docs_tests/models/table_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='table') }}
+select 1 as id, 'Joe' as name
diff --git a/test/integration/060_persist_docs_tests/models/view_model.sql b/test/integration/060_persist_docs_tests/models/view_model.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/060_persist_docs_tests/models/view_model.sql
@@ -0,0 +1,2 @@
+{{ config(materialized='view') }}
+select 2 as id, 'Bob' as name
diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py
new file mode 100644
--- /dev/null
+++ b/test/integration/060_persist_docs_tests/test_persist_docs.py
@@ -0,0 +1,124 @@
+from test.integration.base import DBTIntegrationTest, use_profile
+
+import json
+
+
+class BasePersistDocsTest(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "persist_docs_060"
+
+    @property
+    def models(self):
+        return "models"
+
+    def _assert_common_comments(self, *comments):
+        for comment in comments:
+            assert '"with double quotes"' in comment
+            assert """'''abc123'''""" in comment
+            assert '\n' in comment
+            assert 'Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting' in comment
+            assert '/* comment */' in comment
+            assert '--\n' in comment
+
+    def _assert_has_table_comments(self, table_node):
+        table_comment = table_node['metadata']['comment']
+        assert table_comment.startswith('Table model description')
+
+        table_id_comment = table_node['columns']['id']['comment']
+        assert table_id_comment.startswith('id Column description')
+
+        table_name_comment = table_node['columns']['name']['comment']
+        assert table_name_comment.startswith('Some stuff here and then a call to')
+
+        self._assert_common_comments(
+            table_comment, table_id_comment, table_name_comment
+        )
+
+    def _assert_has_view_comments(self, view_node, has_node_comments=True, has_column_comments=True):
+        view_comment = view_node['metadata']['comment']
+        if has_node_comments:
+            assert view_comment.startswith('View model description')
+            self._assert_common_comments(view_comment)
+        else:
+            assert view_comment is None
+
+        view_id_comment = view_node['columns']['id']['comment']
+        if has_column_comments:
+            assert view_id_comment.startswith('id Column description')
+            self._assert_common_comments(view_id_comment)
+        else:
+            assert view_id_comment is None
+
+        view_name_comment = view_node['columns']['name']['comment']
+        assert view_name_comment is None
+
+
+class TestPersistDocs(BasePersistDocsTest):
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'models': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                }
+            }
+        }
+
+    def run_has_comments_pglike(self):
+        self.run_dbt()
+        self.run_dbt(['docs', 'generate'])
+        with open('target/catalog.json') as fp:
+            catalog_data = json.load(fp)
+        assert 'nodes' in catalog_data
+        assert len(catalog_data['nodes']) == 2
+        table_node = catalog_data['nodes']['model.test.table_model']
+        view_node = self._assert_has_table_comments(table_node)
+
+        view_node = catalog_data['nodes']['model.test.view_model']
+        self._assert_has_view_comments(view_node)
+
+    @use_profile('postgres')
+    def test_postgres_comments(self):
+        self.run_has_comments_pglike()
+
+    @use_profile('redshift')
+    def test_redshift_comments(self):
+        self.run_has_comments_pglike()
+
+
+class TestPersistDocsLateBinding(BasePersistDocsTest):
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'models': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                    'view_model': {
+                        'bind': False,
+                    }
+                }
+            }
+        }
+
+    @use_profile('redshift')
+    def test_redshift_late_binding_view(self):
+        self.run_dbt()
+        self.run_dbt(['docs', 'generate'])
+        with open('target/catalog.json') as fp:
+            catalog_data = json.load(fp)
+        assert 'nodes' in catalog_data
+        assert len(catalog_data['nodes']) == 2
+        table_node = catalog_data['nodes']['model.test.table_model']
+        view_node = self._assert_has_table_comments(table_node)
+
+        view_node = catalog_data['nodes']['model.test.view_model']
+        self._assert_has_view_comments(view_node, False, False)
diff --git a/test/unit/test_jinja.py b/test/unit/test_jinja.py
--- a/test/unit/test_jinja.py
+++ b/test/unit/test_jinja.py
@@ -313,13 +313,19 @@ def test_endif(self):
         body = '{% snapshot foo %}select * from thing{% endsnapshot%}{% endif %}'
         with self.assertRaises(CompilationException) as err:
             extract_toplevel_blocks(body)
-        self.assertIn('Got an unexpected control flow end tag, got endif but never saw a preceeding if (@ 53)', str(err.exception))
+        self.assertIn('Got an unexpected control flow end tag, got endif but never saw a preceeding if (@ 1:53)', str(err.exception))
 
     def test_if_endfor(self):
         body = '{% if x %}...{% endfor %}{% endif %}'
         with self.assertRaises(CompilationException) as err:
             extract_toplevel_blocks(body)
-        self.assertIn('Got an unexpected control flow end tag, got endfor but expected endif next (@ 13)', str(err.exception))
+        self.assertIn('Got an unexpected control flow end tag, got endfor but expected endif next (@ 1:13)', str(err.exception))
+
+    def test_if_endfor_newlines(self):
+        body = '{% if x %}\n    ...\n    {% endfor %}\n{% endif %}'
+        with self.assertRaises(CompilationException) as err:
+            extract_toplevel_blocks(body)
+        self.assertIn('Got an unexpected control flow end tag, got endfor but expected endif next (@ 3:4)', str(err.exception))
 
 
 bar_block = '''{% mytype bar %}