Repository: dbt-labs/dbt-core

Issue:
Update create_adapter_plugins.py
Improvements to [`create_adapter_plugins.py`](https://github.com/fishtown-analytics/dbt/blob/dev/kiyoshi-kuromiya/core/scripts/create_adapter_plugins.py). Some of these are things we should do now, some are things we should seek to automate for future releases.

### Versioning
- [ ] Bump [default core version](https://github.com/fishtown-analytics/dbt/blob/30f473a2b1e261e3f4b712564df854f8a95134f3/core/scripts/create_adapter_plugins.py#L235) to `0.19.0-rc1`, add to `.bumpversion.cfg`
- [ ] Create file `dbt/adapters/[adapter]/__version__.py`, to enable reporting from `dbt --version`
- [ ] Codify our recommended practice: plugin versions should match dbt versions, with additional suffixes as needed. (E.g. [here's how dbt-spark does it](https://github.com/fishtown-analytics/dbt-spark/blob/2f7c2dcb419b66d463bb83212287bd29280d6964/setup.py#L15-L38).) Change default `package-version` from `0.0.1` to match the core version, and add setup functions to effectuate.

### Testing
- [ ] Testing: Add scaffolding for `test/[adapter].dbtspec`, modeled off the [postgres spec](https://github.com/fishtown-analytics/dbt-adapter-tests/blob/master/specs/postgres.dbtspec)
- [ ] Add scaffolding for `dbt/include/[adapter]/sample_profiles.yml`, which dbt will use to populate `~/.dbt/profiles.yml` if a user installs the plugin and runs `dbt init --adapter [adapter]` for the first time. (We should make this work if they just run `dbt init`, too: https://github.com/fishtown-analytics/dbt/issues/2814)
- [ ] Should the script accept/expect a `connection_keys` argument? If we did, we could use that to populate both of the above. Currently, [we expect the user to add in these fields](https://docs.getdbt.com/docs/contributing/building-a-new-adapter/#the-credentials-class) after running the scaffolding script.

### Future: dbt Cloud

Eventually, we may want community plugin maintainers of community plugins to include:
- "blessed" dockerfiles of runtime environments
- JSONSchema of connection profile spec (same info as above)

This will enable us to support the plugin in dbt Cloud. We're still a ways away from that, but it's worth thinking about now in terms of the scaffolding we provide.


Discussion:

Resolving patch:
diff --git a/core/scripts/create_adapter_plugins.py b/core/scripts/create_adapter_plugins.py
--- a/core/scripts/create_adapter_plugins.py
+++ b/core/scripts/create_adapter_plugins.py
@@ -14,12 +14,12 @@
 
 SETUP_PY_TEMPLATE = '''
 #!/usr/bin/env python
-from setuptools import find_packages
-from setuptools import setup
+from setuptools import find_namespace_packages, setup
 
 package_name = "dbt-{adapter}"
+# make sure this always matches dbt/adapters/{adapter}/__version__.py
 package_version = "{version}"
-description = """The {adapter} adapter plugin for dbt (data build tool)"""
+description = """The {adapter} adapter plugin for dbt"""
 
 setup(
     name=package_name,
@@ -29,18 +29,18 @@
     author={author_name},
     author_email={author_email},
     url={url},
-    packages=find_packages(),
-    package_data={{
-        'dbt': [
-{package_data}
-        ]
-    }},
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
+    include_package_data=True,
     install_requires=[
         "{dbt_core_str}",{dependencies}
     ]
 )
 '''.lstrip()
 
+
+MANIFEST_IN_TEMPLATE = "recursive-include dbt/include *.sql *.yml *.md"
+
+
 ADAPTER_INIT_TEMPLATE = '''
 from dbt.adapters.{adapter}.connections import {title_adapter}ConnectionManager
 from dbt.adapters.{adapter}.connections import {title_adapter}Credentials
@@ -115,6 +115,44 @@ class {title_adapter}Adapter({adapter_cls}):
 """.lstrip()
 
 
+SAMPLE_PROFILE_TEMPLATE = '''
+default:
+  outputs:
+    dev:
+      type: {adapter}
+      # Add sample credentials here, like:
+      # host: <host>
+      # port: <port_num>
+      # username: <user>
+      # password: <pass>
+  target: dev
+'''
+
+
+DBTSPEC_TEMPLATE = '''
+# See https://github.com/fishtown-analytics/dbt-adapter-tests
+# for installation and use
+
+target:
+  type: {adapter}
+  # Add CI credentials here, like:
+  # host: localhost
+  # port: 5432
+  # username: root
+  # password: pass
+sequences:
+  test_dbt_empty: empty
+  test_dbt_base: base
+  test_dbt_ephemeral: ephemeral
+  test_dbt_incremental: incremental
+  test_dbt_snapshot_strategy_timestamp: snapshot_strategy_timestamp
+  test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols
+  test_dbt_data_test: data_test
+  test_dbt_schema_test: schema_test
+  test_dbt_ephemeral_data_tests: data_test_ephemeral_models
+'''
+
+
 class Builder:
     def __init__(self, args):
         self.args = args
@@ -131,6 +169,7 @@ def go(self):
         self.write_setup()
         self.write_adapter()
         self.write_include()
+        self.write_test_spec()
 
     def include_paths(self):
         return [
@@ -162,9 +201,9 @@ def write_setup(self):
             url=self.args.url,
             dbt_core_str=dbt_core_str,
             dependencies=self.args.dependency,
-            package_data=package_data,
         )
         self.dest_path('setup.py').write_text(setup_py_contents)
+        self.dest_path('MANIFEST.in').write_text(MANIFEST_IN_TEMPLATE)
 
     def _make_adapter_kwargs(self):
         if self.args.sql:
@@ -197,10 +236,12 @@ def write_adapter(self):
             adapter=self.adapter,
             title_adapter=self.args.title_case
         )
+        version_text = f'{self.args.package_version}'
         connections_text = ADAPTER_CONNECTIONS_TEMPLATE.format(**kwargs)
         impl_text = ADAPTER_IMPL_TEMPLATE.format(**kwargs)
 
         (adapters_dest / '__init__.py').write_text(init_text)
+        (adapters_dest / '__version__.py').write_text(version_text)
         (adapters_dest / 'connections.py').write_text(connections_text)
         (adapters_dest / 'impl.py').write_text(impl_text)
 
@@ -214,15 +255,26 @@ def write_include(self):
             adapter=self.adapter,
             version=self.args.project_version,
         )
+        sample_profiles_text = SAMPLE_PROFILE_TEMPLATE.format(
+            adapter=self.adapter
+        )
         catalog_macro_text = CATALOG_MACRO_TEMPLATE.format(
             adapter=self.adapter
         )
 
         (include_dest / '__init__.py').write_text(INCLUDE_INIT_TEXT)
         (include_dest / 'dbt_project.yml').write_text(dbt_project_text)
+        (include_dest / 'sample_profiles.yml').write_text(sample_profiles_text)
         # make sure something satisfies the 'include/macros/*.sql' in setup.py
         (macros_dest / 'catalog.sql').write_text(catalog_macro_text)
 
+    def write_test_spec(self):
+        test_dest = self.dest_path('test')
+        test_dest.mkdir(parents=True, exist_ok=True)
+        spec_file = f'{self.adapter}.dbtspec'
+        spec_text = DBTSPEC_TEMPLATE.format(adapter=self.adapter)
+        (test_dest / spec_file).write_text(spec_text)
+
 
 def parse_args(argv=None):
     if argv is None:
@@ -232,12 +284,12 @@ def parse_args(argv=None):
     parser.add_argument('adapter')
     parser.add_argument('--title-case', '-t', default=None)
     parser.add_argument('--dependency', action='append')
-    parser.add_argument('--dbt-core-version', default='0.16.1rc1')
+    parser.add_argument('--dbt-core-version', default='0.20.0rc2')
     parser.add_argument('--email')
     parser.add_argument('--author')
     parser.add_argument('--url')
     parser.add_argument('--sql', action='store_true')
-    parser.add_argument('--package-version', default='0.0.1')
+    parser.add_argument('--package-version', default='0.20.0rc2')
     parser.add_argument('--project-version', default='1.0')
     parser.add_argument(
         '--no-dependency', action='store_false', dest='set_dependency'


Test patch: