Repository: dbt-labs/dbt-core (Python)

Issue:
[ADAP-974][CT-3230] [Bug] `type` should not be set to `null` for dynamic tables and materialized view
### Is this a new bug in dbt-core?

- [X] I believe this is a new bug in dbt-core
- [X] I have searched the existing issues, and I could not find an existing issue for this bug

### Summary

A succinct summary by @jtcohen6:

_if table be static
type table - pragmatic
if table dynamic
null type - panic!_

### Current Behavior

This is showing up in `catalog.json` with materialized views / dynamic tables: there are model nodes that have `metadata.type`: `null`.

### Expected Behavior

Table `type` should be a required field that is never `null`.

We should create some kind of integration test to validate that the value is not null.

### Steps To Reproduce

Run `dbt docs generate` within a dbt project that contains a model materialized as a dynamic table (Snowflake) or materialized view (other adapters). Examine `catalog.json` and see that node > metadata > type is `null`.

### Relevant log output

`catalog.json` versus the expected JSON schema

#### `catalog.json`
<img width="317" alt="image" src="https://github.com/dbt-labs/dbt-core/assets/44704949/041cd982-ad3e-4e14-bd63-0b5d22fde3f6">

#### expected JSON schema
<img width="160" alt="image" src="https://github.com/dbt-labs/dbt-core/assets/44704949/ae0bce1c-5c05-4ba3-98ad-8710033cdc21">

### Environment

```markdown
- OS:
- Python:
- dbt:
```


### Which database adapter are you using with dbt?

dbt-snowflake is where this was noticed, but we expect this to affect other adapters as well.

### Additional Context

Internal slack thread:
https://dbt-labs.slack.com/archives/C05FWBP9X1U/p1697228694650389

### Related issues
- https://github.com/dbt-labs/dbt-core/issues/8864
- https://github.com/dbt-labs/dbt-bigquery/issues/995
- https://github.com/dbt-labs/dbt-redshift/issues/652
- https://github.com/dbt-labs/dbt-snowflake/issues/817
- dbt-spark: N/A (?)



Fix (patch):
diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py
index 50c05b0f199..f9a69ad386c 100644
--- a/dbt/adapters/postgres/impl.py
+++ b/dbt/adapters/postgres/impl.py
@@ -53,6 +53,10 @@ def date_function(cls):
     def get_status(cls, cursor):
         return cursor.statusmessage
 
+    @classmethod
+    def get_credentials(cls, credentials):
+        return credentials
+
     @classmethod
     def open_connection(cls, connection):
         if connection.get('state') == 'open':
@@ -61,8 +65,10 @@ def open_connection(cls, connection):
 
         result = connection.copy()
 
+        base_credentials = connection.get('credentials', {})
+        credentials = cls.get_credentials(base_credentials.copy())
+
         try:
-            credentials = connection.get('credentials', {})
             handle = psycopg2.connect(
                 dbname=credentials.get('dbname'),
                 user=credentials.get('user'),
diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py
index 311e09d3923..319edf2bf45 100644
--- a/dbt/adapters/redshift/impl.py
+++ b/dbt/adapters/redshift/impl.py
@@ -2,13 +2,13 @@
 
 from dbt.adapters.postgres import PostgresAdapter
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
-
+import dbt.exceptions
+import boto3
 
 drop_lock = multiprocessing.Lock()
 
 
 class RedshiftAdapter(PostgresAdapter):
-
     @classmethod
     def type(cls):
         return 'redshift'
@@ -17,6 +17,69 @@ def type(cls):
     def date_function(cls):
         return 'getdate()'
 
+    @classmethod
+    def fetch_cluster_credentials(cls, db_user, db_name, cluster_id,
+                                  duration_s):
+        """Fetches temporary login credentials from AWS. The specified user
+        must already exist in the database, or else an error will occur"""
+        boto_client = boto3.client('redshift')
+
+        try:
+            return boto_client.get_cluster_credentials(
+                DbUser=db_user,
+                DbName=db_name,
+                ClusterIdentifier=cluster_id,
+                DurationSeconds=duration_s,
+                AutoCreate=False)
+
+        except boto_client.exceptions.ClientError as e:
+            raise dbt.exceptions.FailedToConnectException(
+                    "Unable to get temporary Redshift cluster credentials: "
+                    "{}".format(e))
+
+    @classmethod
+    def get_tmp_iam_cluster_credentials(cls, credentials):
+        cluster_id = credentials.get('cluster_id')
+
+        # default via:
+        # boto3.readthedocs.io/en/latest/reference/services/redshift.html
+        iam_duration_s = credentials.get('iam_duration_seconds', 900)
+
+        if not cluster_id:
+            raise dbt.exceptions.FailedToConnectException(
+                    "'cluster_id' must be provided in profile if IAM "
+                    "authentication method selected")
+
+        cluster_creds = cls.fetch_cluster_credentials(
+            credentials.get('user'),
+            credentials.get('dbname'),
+            credentials.get('cluster_id'),
+            iam_duration_s,
+        )
+
+        # replace username and password with temporary redshift credentials
+        return dbt.utils.merge(credentials, {
+            'user': cluster_creds.get('DbUser'),
+            'pass': cluster_creds.get('DbPassword')
+        })
+
+    @classmethod
+    def get_credentials(cls, credentials):
+        method = credentials.get('method')
+
+        # Support missing 'method' for backwards compatibility
+        if method == 'database' or method is None:
+            logger.debug("Connecting to Redshift using 'database' credentials")
+            return credentials
+
+        elif method == 'iam':
+            logger.debug("Connecting to Redshift using 'IAM' credentials")
+            return cls.get_tmp_iam_cluster_credentials(credentials)
+
+        else:
+            raise dbt.exceptions.FailedToConnectException(
+                    "Invalid 'method' in profile: '{}'".format(method))
+
     @classmethod
     def _get_columns_in_table_sql(cls, schema_name, table_name, database):
         # Redshift doesn't support cross-database queries,
@@ -27,7 +90,7 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database):
             table_schema_filter = '1=1'
         else:
             table_schema_filter = "table_schema = '{schema_name}'".format(
-                    schema_name=schema_name)
+                schema_name=schema_name)
 
         sql = """
             with bound_views as (
diff --git a/dbt/contracts/connection.py b/dbt/contracts/connection.py
index 1ac374ea095..2183c5d891b 100644
--- a/dbt/contracts/connection.py
+++ b/dbt/contracts/connection.py
@@ -36,6 +36,61 @@
     'required': ['dbname', 'host', 'user', 'pass', 'port', 'schema'],
 }
 
+REDSHIFT_CREDENTIALS_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'method': {
+            'enum': ['database', 'iam'],
+            'description': (
+                'database: use user/pass creds; iam: use temporary creds'
+            ),
+        },
+        'dbname': {
+            'type': 'string',
+        },
+        'host': {
+            'type': 'string',
+        },
+        'user': {
+            'type': 'string',
+        },
+        'pass': {
+            'type': 'string',
+        },
+        'port': {
+            'oneOf': [
+                {
+                    'type': 'integer',
+                    'minimum': 0,
+                    'maximum': 65535,
+                },
+                {
+                    'type': 'string'
+                },
+            ],
+        },
+        'schema': {
+            'type': 'string',
+        },
+        'cluster_id': {
+            'type': 'string',
+            'description': (
+                'If using IAM auth, the name of the cluster'
+            )
+        },
+        'iam_duration_seconds': {
+            'type': 'integer',
+            'minimum': 900,
+            'maximum': 3600,
+            'description': (
+                'If using IAM auth, the ttl for the temporary credentials'
+            )
+        },
+        'required': ['dbname', 'host', 'user', 'port', 'schema']
+    }
+}
+
 SNOWFLAKE_CREDENTIALS_CONTRACT = {
     'type': 'object',
     'additionalProperties': False,
@@ -113,11 +168,11 @@
         },
         'credentials': {
             'description': (
-                'The credentials object here should match the connection '
-                'type. Redshift uses the Postgres connection model.'
+                'The credentials object here should match the connection type.'
             ),
-            'oneOf': [
+            'anyOf': [
                 POSTGRES_CREDENTIALS_CONTRACT,
+                REDSHIFT_CREDENTIALS_CONTRACT,
                 SNOWFLAKE_CREDENTIALS_CONTRACT,
                 BIGQUERY_CREDENTIALS_CONTRACT,
             ],
@@ -133,6 +188,10 @@ class PostgresCredentials(APIObject):
     SCHEMA = POSTGRES_CREDENTIALS_CONTRACT
 
 
+class RedshiftCredentials(APIObject):
+    SCHEMA = REDSHIFT_CREDENTIALS_CONTRACT
+
+
 class SnowflakeCredentials(APIObject):
     SCHEMA = SNOWFLAKE_CREDENTIALS_CONTRACT
 
@@ -143,7 +202,7 @@ class BigQueryCredentials(APIObject):
 
 CREDENTIALS_MAPPING = {
     'postgres': PostgresCredentials,
-    'redshift': PostgresCredentials,
+    'redshift': RedshiftCredentials,
     'snowflake': SnowflakeCredentials,
     'bigquery': BigQueryCredentials,
 }
diff --git a/requirements.txt b/requirements.txt
index 5c2e7de65d2..5b59f615305 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,3 +13,4 @@ google-cloud-bigquery==0.29.0
 requests>=2.18.0
 agate>=1.6,<2
 jsonschema==2.6.0
+boto3>=1.6.23
diff --git a/setup.py b/setup.py
index 04c4c443fe1..b6e69f6749e 100644
--- a/setup.py
+++ b/setup.py
@@ -52,5 +52,6 @@ def read(fname):
         'google-cloud-bigquery==0.29.0',
         'agate>=1.6,<2',
         'jsonschema==2.6.0',
+        'boto3>=1.6.23'
     ]
 )