Repository: ibis-project/ibis

Issue:
CI: Bigquery backend failing to load data
@tswast can you have a look at [this CI error](https://github.com/ibis-project/ibis/runs/984884965#step:4:1418) please?

Not sure what's the problem, I'd say that I'm calling the loading of data for bigquery in the same exact way as it was before.


Discussion:
Oops. Looks like I never added the schema file to the testing data repo. https://github.com/ibis-project/testing-data/pull/3

Alternatively, I could create the equivalent DDL statements and put a bigquery.sql file in https://github.com/ibis-project/ibis/tree/master/ci/schema Would you prefer that?
Not so familiar with that part. Would be having a `bigquery.sql` file the approach used by the other backends? I'd prefer that if that's the case.
Yes, that seems to be what the other backends are doing

Resolving patch:
diff --git a/ci/datamgr.py b/ci/datamgr.py
--- a/ci/datamgr.py
+++ b/ci/datamgr.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 import io
-import json
 import logging
 import os
 import warnings
@@ -216,6 +215,7 @@ def parquet(tables, data_directory, ignore_missing_dependency, **params):
     '--schema',
     type=click.File('rt'),
     default=str(SCRIPT_DIR / 'schema' / 'postgresql.sql'),
+    help='Path to SQL file that initializes the database via DDL.',
 )
 @click.option('-t', '--tables', multiple=True, default=TEST_TABLES + ['geo'])
 @click.option('-d', '--data-directory', default=DATA_DIR)
@@ -304,6 +304,7 @@ def postgres(schema, tables, data_directory, psql_path, plpython, **params):
     '--schema',
     type=click.File('rt'),
     default=str(SCRIPT_DIR / 'schema' / 'sqlite.sql'),
+    help='Path to SQL file that initializes the database via DDL.',
 )
 @click.option('-t', '--tables', multiple=True, default=TEST_TABLES)
 @click.option('-d', '--data-directory', default=DATA_DIR)
@@ -334,6 +335,7 @@ def sqlite(database, schema, tables, data_directory, **params):
     '--schema',
     type=click.File('rt'),
     default=str(SCRIPT_DIR / 'schema' / 'omniscidb.sql'),
+    help='Path to SQL file that initializes the database via DDL.',
 )
 @click.option('-t', '--tables', multiple=True, default=TEST_TABLES + ['geo'])
 @click.option('-d', '--data-directory', default=DATA_DIR)
@@ -428,6 +430,7 @@ def omniscidb(schema, tables, data_directory, **params):
     '--schema',
     type=click.File('rt'),
     default=str(SCRIPT_DIR / 'schema' / 'mysql.sql'),
+    help='Path to SQL file that initializes the database via DDL.',
 )
 @click.option('-t', '--tables', multiple=True, default=TEST_TABLES)
 @click.option('-d', '--data-directory', default=DATA_DIR)
@@ -453,6 +456,7 @@ def mysql(schema, tables, data_directory, **params):
     '--schema',
     type=click.File('rt'),
     default=str(SCRIPT_DIR / 'schema' / 'clickhouse.sql'),
+    help='Path to SQL file that initializes the database via DDL.',
 )
 @click.option('-t', '--tables', multiple=True, default=TEST_TABLES)
 @click.option('-d', '--data-directory', default=DATA_DIR)
@@ -477,9 +481,16 @@ def clickhouse(schema, tables, data_directory, **params):
 
 
 @cli.command()
+@click.option(
+    '-S',
+    '--schema',
+    type=click.File('rt'),
+    default=str(SCRIPT_DIR / 'schema' / 'bigquery.sql'),
+    help='Path to SQL file that initializes the database via DDL.',
+)
 @click.option('-d', '--data-directory', default=DATA_DIR)
 @click.option('-i', '--ignore-missing-dependency', is_flag=True, default=False)
-def bigquery(data_directory, ignore_missing_dependency, **params):
+def bigquery(schema, data_directory, ignore_missing_dependency, **params):
     try:
         import google.api_core.exceptions
         from google.cloud import bigquery
@@ -501,24 +512,17 @@ def bigquery(data_directory, ignore_missing_dependency, **params):
     except google.api_core.exceptions.Conflict:
         pass  # Skip if already created.
 
-    # Set up main data table.
+    # Set up main data tables.
+    job = bqclient.query(schema.read())
+    job.result()
+    if job.error_result:
+        raise click.ClickException(str(job.error_result))
+
+    # Load main data table.
     data_directory = Path(data_directory)
     functional_alltypes_path = data_directory / 'functional_alltypes.csv'
-    functional_alltypes_schema = []
-    schema_path = data_directory / 'functional_alltypes_bigquery_schema.json'
-    with open(str(schema_path)) as schemafile:
-        schema_json = json.load(schemafile)
-        for field in schema_json:
-            functional_alltypes_schema.append(
-                bigquery.SchemaField.from_api_repr(field)
-            )
     load_config = bigquery.LoadJobConfig()
-    load_config.write_disposition = 'WRITE_TRUNCATE'
     load_config.skip_leading_rows = 1  # skip the header row.
-    load_config.schema = functional_alltypes_schema
-
-    # Load main data table.
-    functional_alltypes_schema = []
     with open(str(functional_alltypes_path), 'rb') as csvfile:
         job = bqclient.load_table_from_file(
             csvfile,
@@ -530,9 +534,7 @@ def bigquery(data_directory, ignore_missing_dependency, **params):
             raise click.ClickException(str(job.error_result))
 
     # Load an ingestion time partitioned table.
-    functional_alltypes_path = data_directory / 'functional_alltypes.csv'
     with open(str(functional_alltypes_path), 'rb') as csvfile:
-        load_config.time_partitioning = bigquery.TimePartitioning()
         job = bqclient.load_table_from_file(
             csvfile,
             testing_dataset.table('functional_alltypes_parted'),
@@ -592,7 +594,7 @@ def bigquery(data_directory, ignore_missing_dependency, **params):
     bqclient.create_table(numeric_table, exists_ok=True)
 
     df = pd.read_csv(
-        str(data_directory / 'functional_alltypes.csv'),
+        str(functional_alltypes_path),
         usecols=['string_col', 'double_col'],
         header=0,
     )


Test patch: