Repository: dbt-labs/dbt-core

Issue:
DBT Archive Failing in BiqQuery
## Issue

Currently the DBT archive feature uses [temporary archive](https://github.com/fishtown-analytics/dbt/blob/dev/wilt-chamberlain/plugins/bigquery/dbt/adapters/bigquery/impl.py#L337-L351) features in bigquery. When tables are "large enough" then this fails due to [Maximum response size](https://cloud.google.com/bigquery/quotas#queries). 

If we can write to a table in BQ instead of a temporary table, then delete it this may solve the issue, as `The maximum response size is unlimited when writing large query results to a destination table.`


### Issue description

```
>> dbt archive --target prod
Running with dbt=0.13.0
WARNING: Configuration paths exist in your dbt_project.yml file which do not apply to any resources.
Found 61 models, 844 tests, 1 archives, 0 analyses, 111 macros, 0 operations, 10 seed files, 0 sources

13:56:09 | Concurrency: 8 threads (target='prod')
13:56:09 |
13:56:09 | 1 of 1 START archive project_name.dataset_source_name.table_name --> project_name.dataset_archive_name.table_name [RUN]
13:57:07 | 1 of 1 ERROR archiving dataset_source_name.table_name --> dataset_archive_name.table_name [ERROR in 57.37s]
13:57:07 |
13:57:07 | Finished running 1 archives in 59.96s.

Completed with 1 errors:

Database Error in archive table_name (dbt_project.yml)
  Response too large to return. Consider setting allowLargeResults to true in your job configuration. For more information, see https://cloud.google.com/bigquery/troubleshooting-errors

Done. PASS=0 ERROR=1 SKIP=0 TOTAL=1
```

### Results



### System information
The output of `dbt --version`:
```
installed version: 0.13.0
   latest version: 0.13.0

Up to date!
```

The operating system you're running on: Mac OS X 10.14.4

The python version you're using (probably the output of `python --version`): Python 3.7.1

### Steps to reproduce

DBT archive on a table > ~280 gigs within BigQuery.

## Feature

### Feature description
DBT archives could also benefit from Partitioning / Clustering in BiqQuery. This will improve cost for querying and archiving in the future.

### Who will this benefit?

DBT archive adoption for bigquery users.


Discussion:

Resolving patch:
diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py
--- a/core/dbt/adapters/base/impl.py
+++ b/core/dbt/adapters/base/impl.py
@@ -387,6 +387,7 @@ def list_schemas(self, database):
             '`list_schemas` is not implemented for this adapter!'
         )
 
+    @available.parse(lambda *a, **k: False)
     def check_schema_exists(self, database, schema):
         """Check if a schema exists.
 
@@ -584,7 +585,14 @@ def valid_archive_target(self, relation):
             dbt.exceptions.raise_compiler_error(msg)
 
     @available.parse_none
-    def expand_target_column_types(self, temp_table, to_relation):
+    def expand_target_column_types(self, from_relation, to_relation):
+        if not isinstance(from_relation, self.Relation):
+            dbt.exceptions.invalid_type_error(
+                method_name='expand_target_column_types',
+                arg_name='from_relation',
+                got_value=from_relation,
+                expected_type=self.Relation)
+
         if not isinstance(to_relation, self.Relation):
             dbt.exceptions.invalid_type_error(
                 method_name='expand_target_column_types',
@@ -592,14 +600,7 @@ def expand_target_column_types(self, temp_table, to_relation):
                 got_value=to_relation,
                 expected_type=self.Relation)
 
-        goal = self.Relation.create(
-            database=None,
-            schema=None,
-            identifier=temp_table,
-            type='table',
-            quote_policy=self.config.quoting
-        )
-        self.expand_column_types(goal, to_relation)
+        self.expand_column_types(from_relation, to_relation)
 
     def list_relations(self, database, schema):
         if self._schema_is_cached(database, schema):
diff --git a/core/dbt/parser/archives.py b/core/dbt/parser/archives.py
--- a/core/dbt/parser/archives.py
+++ b/core/dbt/parser/archives.py
@@ -11,6 +11,19 @@
 import os
 
 
+def set_archive_attributes(node):
+    config_keys = {
+        'target_database': 'database',
+        'target_schema': 'schema'
+    }
+
+    for config_key, node_key in config_keys.items():
+        if config_key in node.config:
+            setattr(node, node_key, node.config[config_key])
+
+    return node
+
+
 class ArchiveParser(MacrosKnownParser):
     @classmethod
     def parse_archives_from_project(cls, config):
@@ -87,12 +100,14 @@ def load_and_parse(self):
                                       archive.package_name,
                                       archive.name)
 
-            to_return[node_path] = self.parse_node(
+            parsed_node = self.parse_node(
                 archive,
                 node_path,
                 self.all_projects.get(archive.package_name),
                 archive_config=archive_config)
 
+            to_return[node_path] = set_archive_attributes(parsed_node)
+
         return to_return
 
 
@@ -138,7 +153,9 @@ def get_fqn(cls, node, package_project_config, extra=[]):
     def validate_archives(node):
         if node.resource_type == NodeType.Archive:
             try:
-                return ParsedArchiveNode(**node.to_shallow_dict())
+                parsed_node = ParsedArchiveNode(**node.to_shallow_dict())
+                return set_archive_attributes(parsed_node)
+
             except dbt.exceptions.JSONValidationException as exc:
                 raise dbt.exceptions.CompilationException(str(exc), node)
         else:
diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py
--- a/plugins/bigquery/dbt/adapters/bigquery/impl.py
+++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py
@@ -343,22 +343,6 @@ def execute_model(self, model, materialization, sql_override=None,
 
         return res
 
-    @available.parse(_stub_relation)
-    def create_temporary_table(self, sql, **kwargs):
-        # BQ queries always return a temp table with their results
-        query_job, _ = self.connections.raw_execute(sql)
-        bq_table = query_job.destination
-
-        return self.Relation.create(
-            database=bq_table.project,
-            schema=bq_table.dataset_id,
-            identifier=bq_table.table_id,
-            quote_policy={
-                'schema': True,
-                'identifier': True
-            },
-            type=BigQueryRelation.Table)
-
     @available.parse_none
     def alter_table_add_columns(self, relation, columns):
 


Test patch:
diff --git a/test/integration/004_simple_archive_test/seed.sql b/test/integration/004_simple_archive_test/seed.sql
--- a/test/integration/004_simple_archive_test/seed.sql
+++ b/test/integration/004_simple_archive_test/seed.sql
@@ -1,4 +1,4 @@
-    create table {database}.{schema}.seed (
+create table {database}.{schema}.seed (
 	id INTEGER,
 	first_name VARCHAR(50),
 	last_name VARCHAR(50),
@@ -20,7 +20,7 @@ create table {database}.{schema}.archive_expected (
 	updated_at TIMESTAMP WITHOUT TIME ZONE,
 	dbt_valid_from TIMESTAMP WITHOUT TIME ZONE,
 	dbt_valid_to   TIMESTAMP WITHOUT TIME ZONE,
-	dbt_scd_id     VARCHAR(256),
+	dbt_scd_id     VARCHAR(32),
 	dbt_updated_at TIMESTAMP WITHOUT TIME ZONE
 );
 
@@ -79,8 +79,6 @@ select
     md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id
 from {database}.{schema}.seed;
 
-
-
 create table {database}.{schema}.archive_castillo_expected (
     id INTEGER,
     first_name VARCHAR(50),
@@ -93,8 +91,9 @@ create table {database}.{schema}.archive_castillo_expected (
     updated_at TIMESTAMP WITHOUT TIME ZONE,
     dbt_valid_from TIMESTAMP WITHOUT TIME ZONE,
     dbt_valid_to   TIMESTAMP WITHOUT TIME ZONE,
-    dbt_scd_id     VARCHAR(256),
+    dbt_scd_id     VARCHAR(32),
     dbt_updated_at TIMESTAMP WITHOUT TIME ZONE
+
 );
 
 -- one entry
@@ -139,7 +138,7 @@ create table {database}.{schema}.archive_alvarez_expected (
     updated_at TIMESTAMP WITHOUT TIME ZONE,
     dbt_valid_from TIMESTAMP WITHOUT TIME ZONE,
     dbt_valid_to   TIMESTAMP WITHOUT TIME ZONE,
-    dbt_scd_id     VARCHAR(256),
+    dbt_scd_id     VARCHAR(32),
     dbt_updated_at TIMESTAMP WITHOUT TIME ZONE
 );
 
@@ -185,7 +184,7 @@ create table {database}.{schema}.archive_kelly_expected (
     updated_at TIMESTAMP WITHOUT TIME ZONE,
     dbt_valid_from TIMESTAMP WITHOUT TIME ZONE,
     dbt_valid_to   TIMESTAMP WITHOUT TIME ZONE,
-    dbt_scd_id     VARCHAR(256),
+    dbt_scd_id     VARCHAR(32),
     dbt_updated_at TIMESTAMP WITHOUT TIME ZONE
 );
 
diff --git a/test/integration/004_simple_archive_test/test-archives-bq/archive.sql b/test/integration/004_simple_archive_test/test-archives-bq/archive.sql
--- a/test/integration/004_simple_archive_test/test-archives-bq/archive.sql
+++ b/test/integration/004_simple_archive_test/test-archives-bq/archive.sql
@@ -9,6 +9,6 @@
             updated_at='updated_at',
         )
     }}
-    select * from `{{database}}`.`{{schema}}`.seed
+    select * from `{{target.database}}`.`{{schema}}`.seed
 
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql b/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql
--- a/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql
+++ b/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql
@@ -7,6 +7,6 @@
             updated_at='updated_at',
         )
     }}
-    select * from {{database}}.{{schema}}.seed
+    select * from {{target.database}}.{{schema}}.seed
 
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test-archives-longtext/longtext.sql b/test/integration/004_simple_archive_test/test-archives-longtext/longtext.sql
--- a/test/integration/004_simple_archive_test/test-archives-longtext/longtext.sql
+++ b/test/integration/004_simple_archive_test/test-archives-longtext/longtext.sql
@@ -8,5 +8,5 @@
             updated_at='updated_at',
         )
     }}
-    select * from {{database}}.{{schema}}.super_long
+    select * from {{target.database}}.{{schema}}.super_long
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test-archives-pg/archive.sql b/test/integration/004_simple_archive_test/test-archives-pg/archive.sql
--- a/test/integration/004_simple_archive_test/test-archives-pg/archive.sql
+++ b/test/integration/004_simple_archive_test/test-archives-pg/archive.sql
@@ -3,12 +3,12 @@
     {{
         config(
             target_database=var('target_database', database),
-            target_schema=schema,
+            target_schema=var('target_schema', schema),
             unique_key='id || ' ~ "'-'" ~ ' || first_name',
             strategy='timestamp',
             updated_at='updated_at',
         )
     }}
-    select * from {{database}}.{{schema}}.seed
+    select * from {{target.database}}.{{target.schema}}.seed
 
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test-archives-select/archives.sql b/test/integration/004_simple_archive_test/test-archives-select/archives.sql
--- a/test/integration/004_simple_archive_test/test-archives-select/archives.sql
+++ b/test/integration/004_simple_archive_test/test-archives-select/archives.sql
@@ -9,7 +9,7 @@
             updated_at='updated_at',
         )
     }}
-    select * from {{database}}.{{schema}}.seed where last_name = 'Castillo'
+    select * from {{target.database}}.{{schema}}.seed where last_name = 'Castillo'
 
 {% endarchive %}
 
@@ -24,7 +24,7 @@
             updated_at='updated_at',
         )
     }}
-    select * from {{database}}.{{schema}}.seed where last_name = 'Alvarez'
+    select * from {{target.database}}.{{schema}}.seed where last_name = 'Alvarez'
 
 {% endarchive %}
 
@@ -40,6 +40,6 @@
             updated_at='updated_at',
         )
     }}
-    select * from {{database}}.{{schema}}.seed where last_name = 'Kelly'
+    select * from {{target.database}}.{{schema}}.seed where last_name = 'Kelly'
 
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test-check-col-archives-bq/archive.sql b/test/integration/004_simple_archive_test/test-check-col-archives-bq/archive.sql
--- a/test/integration/004_simple_archive_test/test-check-col-archives-bq/archive.sql
+++ b/test/integration/004_simple_archive_test/test-check-col-archives-bq/archive.sql
@@ -8,7 +8,7 @@
             check_cols=('email',),
         )
     }}
-    select * from `{{database}}`.`{{schema}}`.seed
+    select * from `{{target.database}}`.`{{schema}}`.seed
 {% endarchive %}
 
 
@@ -23,5 +23,5 @@
             check_cols='all',
         )
     }}
-    select * from `{{database}}`.`{{schema}}`.seed
+    select * from `{{target.database}}`.`{{schema}}`.seed
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql b/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql
--- a/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql
+++ b/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql
@@ -9,7 +9,7 @@
             check_cols=['email'],
         )
     }}
-    select * from {{database}}.{{schema}}.seed
+    select * from {{target.database}}.{{schema}}.seed
 
 {% endarchive %}
 
@@ -24,5 +24,5 @@
             check_cols='all',
         )
     }}
-    select * from {{database}}.{{schema}}.seed
+    select * from {{target.database}}.{{schema}}.seed
 {% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py
--- a/test/integration/004_simple_archive_test/test_simple_archive.py
+++ b/test/integration/004_simple_archive_test/test_simple_archive.py
@@ -382,6 +382,41 @@ def run_archive(self):
         return self.run_dbt(['archive', '--vars', '{{"target_database": {}}}'.format(self.alternative_database)])
 
 
+class TestCrossSchemaArchiveFiles(DBTIntegrationTest):
+    NUM_ARCHIVE_MODELS = 1
+
+    @property
+    def schema(self):
+        return "simple_archive_004"
+
+    @property
+    def models(self):
+        return "test/integration/004_simple_archive_test/models"
+
+    @property
+    def project_config(self):
+        paths = ['test/integration/004_simple_archive_test/test-archives-pg']
+        return {
+            'archive-paths': paths,
+        }
+
+    def target_schema(self):
+        return "{}_archived".format(self.unique_schema())
+
+    def run_archive(self):
+        return self.run_dbt(['archive', '--vars', '{{"target_schema": {}}}'.format(self.target_schema())])
+
+    @use_profile('postgres')
+    def test__postgres__cross_schema_archive(self):
+        self.run_sql_file('test/integration/004_simple_archive_test/seed_pg.sql')
+
+        results = self.run_archive()
+        self.assertEqual(len(results),  self.NUM_ARCHIVE_MODELS)
+
+        results = self.run_dbt(['run', '--vars', '{{"target_schema": {}}}'.format(self.target_schema())])
+        self.assertEqual(len(results), 1)
+
+
 class TestBadArchive(DBTIntegrationTest):
     @property
     def schema(self):
diff --git a/test/integration/033_event_tracking_test/test_events.py b/test/integration/033_event_tracking_test/test_events.py
--- a/test/integration/033_event_tracking_test/test_events.py
+++ b/test/integration/033_event_tracking_test/test_events.py
@@ -610,7 +610,7 @@ def test__event_tracking_archive(self):
                 model_id='3cdcd0fef985948fd33af308468da3b9',
                 index=1,
                 total=1,
-                status='INSERT 0 1',
+                status='SELECT 1',
                 materialization='archive'
             ),
             self.build_context('archive', 'end', result_type='ok')