Repository: pandas-dev/pandas

Issue:
DOC: track status for stale comment in gbq docs
https://github.com/pydata/pandas/pull/5179 introduced the following warning

```
As of 10/10/13, there is a bug in Google’s API preventing result sets from being 
larger than 100,000 rows. A patch is scheduled for the week of 10/14/13.
```

@jacobschaer, is this still valid?



Discussion:
The patch was released, though a new bug has similar limitations. I'll double check the status of this.

As long as that comment has an expected expiry date, keeping this open.


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -669,47 +669,43 @@ def to_dict(self, outtype='dict'):
         else:  # pragma: no cover
             raise ValueError("outtype %s not understood" % outtype)
 
-    def to_gbq(self, destination_table, schema=None, col_order=None,
-               if_exists='fail', **kwargs):
+    def to_gbq(self, destination_table, project_id=None, chunksize=10000,
+               verbose=True, reauth=False):
         """Write a DataFrame to a Google BigQuery table.
 
-        If the table exists, the DataFrame will be appended. If not, a new
-        table will be created, in which case the schema will have to be
-        specified. By default, rows will be written in the order they appear
-        in the DataFrame, though the user may specify an alternative order.
+        THIS IS AN EXPERIMENTAL LIBRARY
+
+        If the table exists, the dataframe will be written to the table using
+        the defined table schema and column types. For simplicity, this method
+        uses the Google BigQuery streaming API. The to_gbq method chunks data
+        into a default chunk size of 10,000. Failures return the complete error
+        response which can be quite long depending on the size of the insert. 
+        There are several important limitations of the Google streaming API 
+        which are detailed at:
+        https://developers.google.com/bigquery/streaming-data-into-bigquery.
 
         Parameters
-        ---------------
+        ----------
+        dataframe : DataFrame
+            DataFrame to be written
         destination_table : string
-             name of table to be written, in the form 'dataset.tablename'
-        schema : sequence (optional)
-             list of column types in order for data to be inserted, e.g.
-             ['INTEGER', 'TIMESTAMP', 'BOOLEAN']
-        col_order : sequence (optional)
-             order which columns are to be inserted, e.g. ['primary_key',
-             'birthday', 'username']
-        if_exists : {'fail', 'replace', 'append'} (optional)
-            - fail: If table exists, do nothing.
-            - replace: If table exists, drop it, recreate it, and insert data.
-            - append: If table exists, insert data. Create if does not exist.
-        kwargs are passed to the Client constructor
-
-        Raises
-        ------
-        SchemaMissing :
-            Raised if the 'if_exists' parameter is set to 'replace', but no
-            schema is specified
-        TableExists :
-            Raised if the specified 'destination_table' exists but the
-            'if_exists' parameter is set to 'fail' (the default)
-        InvalidSchema :
-            Raised if the 'schema' parameter does not match the provided
-            DataFrame
+            Name of table to be written, in the form 'dataset.tablename'
+        project_id : str
+            Google BigQuery Account project ID.
+        chunksize : int (default 10000)
+            Number of rows to be inserted in each chunk from the dataframe.
+        verbose : boolean (default True)
+            Show percentage complete
+        reauth : boolean (default False)
+            Force Google BigQuery to reauthenticate the user. This is useful
+            if multiple accounts are used.
+
         """
 
         from pandas.io import gbq
-        return gbq.to_gbq(self, destination_table, schema=None, col_order=None,
-                          if_exists='fail', **kwargs)
+        return gbq.to_gbq(self, destination_table, project_id=project_id,
+                          chunksize=chunksize, verbose=verbose,
+                          reauth=reauth)
 
     @classmethod
     def from_records(cls, data, index=None, exclude=None, columns=None,
diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py
--- a/pandas/io/gbq.py
+++ b/pandas/io/gbq.py
@@ -1,68 +1,104 @@
-"""
-Pandas module to interface with Google BigQuery.
-"""
-import os
-import sys
-import tempfile
-import csv
-import logging
 from datetime import datetime
-import pkg_resources
-from distutils.version import LooseVersion
-from pandas.compat import u
+import json
+import logging
+import sys
+from time import sleep
+import uuid
 
-import pandas as pd
 import numpy as np
+import pkg_resources
 
-from pandas.core.common import PandasError
-from pandas.core.frame import DataFrame
+from distutils.version import LooseVersion
+from pandas import compat
+from pandas.core.api import DataFrame
 from pandas.tools.merge import concat
+from pandas.core.common import PandasError
 
-try:
-    import bq
-    import bigquery_client
-    import gflags as flags
-    _BQ_INSTALLED = True
 
-    _BQ_VERSION = pkg_resources.get_distribution('bigquery').version
-    if LooseVersion(_BQ_VERSION) >= '2.0.17':
-        _BQ_VALID_VERSION = True
-    else:
-        _BQ_VALID_VERSION = False
+_GOOGLE_API_CLIENT_INSTALLED = False
+_GOOGLE_API_CLIENT_VALID_VERSION = False
+_GOOGLE_FLAGS_INSTALLED = False
+_GOOGLE_FLAGS_VALID_VERSION = False
+_HTTPLIB2_INSTALLED = False
 
-except ImportError:
-    _BQ_INSTALLED = False
+if not compat.PY3:
+    
+    try:
+        from apiclient.discovery import build
+        from apiclient.http import MediaFileUpload
+        from apiclient.errors import HttpError
 
+        from oauth2client.client import OAuth2WebServerFlow
+        from oauth2client.client import AccessTokenRefreshError
+        from oauth2client.client import flow_from_clientsecrets
+        from oauth2client.file import Storage
+        from oauth2client.tools import run
+        _GOOGLE_API_CLIENT_INSTALLED=True
+        _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
 
-# Setup the logger
-logger = logging.getLogger('pandas.io.gbq')
+        if LooseVersion(_GOOGLE_API_CLIENT_VERSION >= '1.2.0'):
+            _GOOGLE_API_CLIENT_VALID_VERSION = True
+
+    except ImportError:
+        _GOOGLE_API_CLIENT_INSTALLED = False
+
+
+    try:
+        import gflags as flags
+        _GOOGLE_FLAGS_INSTALLED = True
 
-# These are some custom exceptions that the
-# to_gbq() method can throw
+        _GOOGLE_FLAGS_VERSION = pkg_resources.get_distribution('python-gflags').version
 
+        if LooseVersion(_GOOGLE_FLAGS_VERSION >= '2.0.0'):
+            _GOOGLE_FLAGS_VALID_VERSION = True
 
-class SchemaMissing(PandasError, IOError):
+    except ImportError:
+        _GOOGLE_FLAGS_INSTALLED = False
+
+    try:
+        import httplib2
+        _HTTPLIB2_INSTALLED = True
+    except ImportError:
+        _HTTPLIB2_INSTALLED = False
+
+
+logger = logging.getLogger('pandas.io.gbq')
+logger.setLevel(logging.ERROR)
+
+class InvalidPageToken(PandasError, IOError):
+    """
+    Raised when Google BigQuery fails to return,
+    or returns a duplicate page token.
+    """
+    pass
+
+class InvalidQueryException(PandasError, IOError):
     """
-    Raised when attempting to write a DataFrame to
-    a new table in Google BigQuery without specifying
-    a schema describing the DataFrame.
+    Raised when a malformed query is given to read_gbq.
     """
     pass
 
+class AccessDeniedException(PandasError, IOError):
+    """
+    Raised when invalid credentials are provided, or tokens have expired.
+    """
+    pass
 
-class InvalidSchema(PandasError, IOError):
+class NotFoundException(PandasError, IOError):
     """
-    Raised when attempting to write a DataFrame to
-    Google BigQuery with an invalid table schema.
+    Raised when the project_id/table provided in the query could not be found.
     """
     pass
 
+class TermsOfServiceNotAcceptedException(PandasError, IOError):
+    """
+    Raised when the terms of service were not accepted or have been unaccepted.
+    """
+    pass
 
-class TableExistsFail(PandasError, IOError):
+class UnknownGBQException(PandasError, IOError):
     """
-    Raised when attempting to write a DataFrame to
-    an existing Google BigQuery table without specifying
-    that a replace/update action be taken.
+    Raised when an unrecognized Google API Error occurs.
     """
     pass
 
@@ -75,253 +111,263 @@ class InvalidColumnOrder(PandasError, IOError):
     """
     pass
 
+class GbqConnector:
+    def __init__(self, project_id, reauth=False):
+        self.project_id     = project_id
+        self.reauth         = reauth
+        self.credentials    = self.get_credentials()
+        self.service        = self.get_service(self.credentials)
+
+    def get_credentials(self):
+        flow = OAuth2WebServerFlow(client_id='495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd.apps.googleusercontent.com',
+                                   client_secret='kOc9wMptUtxkcIFbtZCcrEAc',
+                                   scope='https://www.googleapis.com/auth/bigquery',
+                                   redirect_uri='urn:ietf:wg:oauth:2.0:oob')
+
+        storage = Storage('bigquery_credentials.dat')
+        credentials = storage.get()
+
+        if credentials is None or credentials.invalid or self.reauth:
+            credentials = run(flow, storage)
+
+        return credentials
+
+    def get_service(self, credentials):
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+        bigquery_service = build('bigquery', 'v2', http=http)
+
+        return bigquery_service
+
+    def run_query(self, query):
+        job_collection = self.service.jobs()
+        job_data = {
+            'configuration': {
+                'query': {
+                    'query': query
+                    #'allowLargeResults', 'createDisposition', 'preserveNulls', destinationTable, useQueryCache
+                }
+            }
+        }
+
+        try:
+            query_reply = job_collection.insert(projectId=self.project_id,
+                                           body=job_data).execute()
+            status = query_reply['status']
+        except AccessTokenRefreshError:
+            raise AccessDeniedException("The credentials have been revoked or expired, please re-run"
+                "the application to re-authorize")
+        except HttpError as ex:
+            status = json.loads(ex.content)['error']
+
+
+        errors = status.get('errors', None)
+
+        if errors:
+            reasons = [error['reason'] for error in errors]
+            if 'accessDenied' in reasons:
+                raise AccessDeniedException
+            if 'invalidQuery' in reasons:
+                raise InvalidQueryException
+            if 'notFound' in reasons:
+                raise NotFoundException
+            if 'termsOfServiceNotAccepted' in reasons:
+                raise TermsOfServiceNotAcceptedException
+            else:
+                raise UnknownGBQException(errors)
+
+        job_reference = query_reply['jobReference']
+
+        while(not 'jobComplete' in query_reply):
+            print('Job not yet complete...')
+            query_reply = job_collection.getQueryResults(
+                            projectId=job_reference['projectId'],
+                            jobId=job_reference['jobId']).execute()
+
+        total_rows = int(query_reply['totalRows'])
+        result_pages = list()
+        seen_page_tokens = list()
+        current_row = 0
+        #Only read schema on first page
+        schema = query_reply['schema']
+
+        # Loop through each page of data
+        while('rows' in query_reply and current_row < total_rows):
+            page = query_reply['rows']
+            result_pages.append(page)
+            current_row += len(page)
+            page_token = query_reply.get('pageToken', None)
+
+            if not page_token and current_row < total_rows:
+                raise InvalidPageToken("Required pageToken was missing. Recieved {0} of {1} rows".format(current_row,total_rows))
+
+            elif page_token in seen_page_tokens:
+                raise InvalidPageToken("A duplicate pageToken was returned")
+
+            seen_page_tokens.append(page_token)
+            query_reply = job_collection.getQueryResults(
+                            projectId = job_reference['projectId'],
+                            jobId = job_reference['jobId'],
+                            pageToken = page_token).execute()
+
+        if (current_row < total_rows):
+            raise InvalidPageToken()
+
+        return schema, result_pages
+
+    def load_data(self, dataframe, dataset_id, table_id, chunksize, verbose):
+        job_id = uuid.uuid4().hex
+        rows = []
+        remaining_rows = len(dataframe)
+
+        if verbose:
+            total_rows = remaining_rows
+            sys.stdout.write("\n\n")
+            sys.stdout.flush()
+
+        for index, row in dataframe.reset_index(drop=True).iterrows():
+            row_dict = dict()
+            row_dict['json'] = json.loads(row.to_json(force_ascii = False,
+                                                      date_unit = 's',
+                                                      date_format = 'iso'))
+            row_dict['insertId'] = job_id + str(index)
+            rows.append(row_dict)
+            remaining_rows -= 1
+
+            if (len(rows) % chunksize == 0) or (remaining_rows == 0):
+                if verbose:
+                    sys.stdout.write("\rStreaming Insert is {0}% Complete".format(((total_rows - remaining_rows) * 100) / total_rows))
+                    sys.stdout.flush()
+
+                body = {'rows': rows}
+                response = self.service.tabledata().insertAll(
+                                                        projectId = self.project_id,
+                                                        datasetId = dataset_id,
+                                                        tableId = table_id,
+                                                        body = body).execute()
+                if 'insertErrors' in response:
+                    raise UnknownGBQException(response)
+
+                sleep(1) # Maintains the inserts "per second" rate per API
+                rows = []
+
+        if verbose:
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+
+def _parse_data(schema, rows):
+    # see: http://pandas.pydata.org/pandas-docs/dev/missing_data.html#missing-data-casting-rules-and-indexing
+    dtype_map = {'INTEGER': np.dtype(float),
+                 'FLOAT': np.dtype(float),
+                 'TIMESTAMP': 'M8[ns]'}     # This seems to be buggy without
+                                            # nanosecond indicator
 
-def _authenticate():
-    """
-    For testing, we abstract the authentication to BigQuery API.
-    Presently this is implemented using the bq.py Client.Get()
-    method. Any exceptions raised are considered fatal, so we
-    do not process them.
+    fields = schema['fields']
+    col_types = [field['type'] for field in fields]
+    col_names = [field['name'].encode('ascii', 'ignore') for field in fields]
+    col_dtypes = [dtype_map.get(field['type'], object) for field in fields]
+    page_array = np.zeros((len(rows),),
+                          dtype=zip(col_names, col_dtypes))
 
-    Returns
-    -------
-    BigqueryClient : Configured connection to Google BigQuery
-    """
-    return bq.Client.Get()
+    for row_num, raw_row in enumerate(rows):
+        entries = raw_row.get('f', [])
+        for col_num, field_type in enumerate(col_types):
+            field_value = _parse_entry(entries[col_num].get('v', ''),
+                                       field_type)
+            page_array[row_num][col_num] = field_value
 
+    return DataFrame(page_array)
 
 def _parse_entry(field_value, field_type):
-    """
-    Given a value and the corresponding BigQuery data type,
-    perform any operations needed and return in a format
-    appropriate for a numpy record dictionary
-
-    Parameters
-    ----------
-    field_value : Source object to be transformed
-    field_type : String representation of Google BigQuery
-                 data type (per schema)
-
-    Returns
-    -------
-    field_value : object or primitive of type corresponding
-                  to field_type
-    """
-
-    # Avoid any casting problems
     if field_value is None or field_value == 'null':
         return None
     if field_type == 'INTEGER' or field_type == 'FLOAT':
-        field_value = float(field_value)
+        return float(field_value)
     elif field_type == 'TIMESTAMP':
         timestamp = datetime.utcfromtimestamp(float(field_value))
-        field_value = np.datetime64(timestamp)
+        return np.datetime64(timestamp)
     elif field_type == 'BOOLEAN':
-        field_value = field_value == 'true'
-    elif field_type == 'STRING':
-        field_value = field_value
-    else:
-        field_value = str(field_value)
+        return field_value == 'true'
     return field_value
 
+def _test_imports():
+    _GOOGLE_API_CLIENT_INSTALLED
+    _GOOGLE_API_CLIENT_VALID_VERSION
+    _GOOGLE_FLAGS_INSTALLED
+    _GOOGLE_FLAGS_VALID_VERSION
+    _HTTPLIB2_INSTALLED
 
-def _parse_page(raw_page, col_names, col_types, col_dtypes):
-    """
-    Given a list of rows produced by the client.apiclient.tabledata().list(),
-    build a numpy array with proper dtypes and column names as specified
-    by the arguments.
+    if compat.PY3:
+        raise NotImplementedError("Google's libraries do not support Python 3 yet")
 
-    Parameters
-    ----------
-    raw_page : Resulting list of rows from a page retrieved via
-        bigquery API
-        client.apiclient.tabledata().list().execute()['rows']
-    col_names: An ordered list of names for the columns
-    col_types: String representation of the BigQuery DataType for that
-        column
-    col_dtypes: Target numpy.dtype for the column
+    if not _GOOGLE_API_CLIENT_INSTALLED:
+        raise ImportError('Could not import Google API Client.')
 
-    Returns
-    -------
-    page_array : numpy record array corresponding
-        to the page data
-    """
+    if not _GOOGLE_FLAGS_INSTALLED:
+        raise ImportError('Could not import Google Command Line Flags Module.')
 
-    # Should be at most 100,000 per the API, but this could
-    # be increased in the future. Should only be less than
-    # this for the last page to reduce API calls
-    page_row_count = len(raw_page)
+    if not _GOOGLE_API_CLIENT_VALID_VERSION:
+        raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google "
+                          "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION)
 
-    # Place to hold the results for a page of data
-    page_array = np.zeros((page_row_count,), dtype=zip(col_names, col_dtypes))
-    for row_num, raw_row in enumerate(raw_page):
-        entries = raw_row.get('f', [])
-         # Iterate over each entry - setting proper field types
-        for col_num, field_type in enumerate(col_types):
-            # Process the field's types using schema
-            field_value = _parse_entry(entries[col_num].get('v', ''),
-                                       field_type)
-            # Fill the value into the final array
-            page_array[row_num][col_num] = field_value
+    if not _GOOGLE_FLAGS_VALID_VERSION:
+        raise ImportError("pandas requires python-gflags >= 2.0.0 for Google "
+                          "BigQuery support, current version " + _GOOGLE_FLAGS_VERSION)
 
-    return page_array
+    if not _HTTPLIB2_INSTALLED:
+        raise ImportError("pandas requires httplib2 for Google BigQuery support")
 
+def read_gbq(query, project_id = None, index_col=None, col_order=None, reauth=False):
+    """Load data from Google BigQuery.
 
-def _parse_data(client, job, index_col=None, col_order=None):
-    """
-    Iterate through the query results and piece together the
-    final DataFrame. Builds a DataFrame for each page of
-    results, then concatenates them together when finished.
-    To save memory, we use numpy record arrays to build these
-    DataFrames.
+    THIS IS AN EXPERIMENTAL LIBRARY
+
+    The main method a user calls to execute a Query in Google BigQuery and read results
+    into a pandas DataFrame using the v2 Google API client for Python.  Documentation for
+    the API is available at https://developers.google.com/api-client-library/python/.
+    Authentication to the Google BigQuery service is via OAuth 2.0 using the product name
+    'pandas GBQ'.
 
     Parameters
     ----------
-    client: An instance of bq.Client
-    job: An array containing the job info for a completed query
-    index_col: str (optional)
+    query : str
+        SQL-Like Query to return data values
+    project_id : str
+        Google BigQuery Account project ID.
+    index_col : str (optional)
         Name of result column to use for index in results DataFrame
-    col_order: list() (optional)
+    col_order : list(str) (optional)
         List of BigQuery column names in the desired order for results
         DataFrame
+    reauth : boolean (default False)
+        Force Google BigQuery to reauthenticate the user. This is useful
+        if multiple accounts are used.
 
     Returns
     -------
-    df: pandas DataFrame
+    df: DataFrame
         DataFrame representing results of query
 
-    Raises:
-    ------
-    InvalidColumnOrder:
-        Raised if 'col_order' parameter doesn't match returned DataFrame
-    BigqueryError:
-        Raised by bigquery_client if a Google API error is encountered
-
-
-    Notes:
-    -----
-    This script relies on Google being consistent with their
-    pagination API. We are using the most flexible iteration method
-    that we could find in the bq.py/bigquery_client.py API's, but
-    these have undergone large amounts of change recently.
     """
 
-    # dtype Map -
-    # see: http://pandas.pydata.org/pandas-docs/dev/missing_data.html#missing-data-casting-rules-and-indexing
-    dtype_map = {'INTEGER': np.dtype(float),
-                 'FLOAT': np.dtype(float),
-                 'TIMESTAMP': 'M8[ns]'}     # This seems to be buggy without
-                                            # nanosecond indicator
-
-    # We first need the schema to get information about the columns of
-    # our dataframe.
-
-    table_dict = job['configuration']['query']['destinationTable']
-    fields = client.GetTableSchema(table_dict)['fields']
-
-    # Get the schema into a format useable to create our
-    # dataframe
-    col_dtypes = []
-    col_types = []
-    col_names = []
-
-    # TODO: Do this in one clean step
-    for field in fields:
-        col_types.append(field['type'])
-        # Note the encoding... numpy doesn't like titles that are UTF8, which
-        # is the return type from the API
-        col_names.append(field['name'].encode('ascii', 'ignore'))
-        # Note, it would be nice to use 'str' types, but BigQuery doesn't have
-        # a fixed length in mind - just maxes out at 64k
-        col_dtypes.append(dtype_map.get(field['type'], object))
-
-    # How many columns are there
-    num_columns = len(col_names)
-
-    # Iterate over the result rows.
-    # Since Google's API now requires pagination of results,
-    # we do that here. The following is repurposed from
-    # bigquery_client.py  :: Client._JobTableReader._ReadOnePage
-
-    # TODO: Enable Reading From Table,
-    # see Client._TableTableReader._ReadOnePage
-
-    # Initially, no page token is set
-    page_token = None
-
-    # This number is the current max results per page
-    max_rows = bigquery_client._MAX_ROWS_PER_REQUEST
-
-    # How many rows in result set? Initialize to max_rows
-    total_rows = max_rows
-
-    # This is the starting row for a particular page...
-    # is ignored if page_token is present, though
-    # it may be useful if we wish to implement SQL like LIMITs
-    # with minimums
-    start_row = 0
-
-    # Keep our page DataFrames until the end when we concatenate them
-    dataframe_list = list()
-
-    current_job = job['jobReference']
-
-    # Iterate over all rows
-    while start_row < total_rows:
-        # Setup the parameters for getQueryResults() API Call
-        kwds = dict(current_job)
-        kwds['maxResults'] = max_rows
-        # Sets the timeout to 0 because we assume the table is already ready.
-        # This is because our previous call to Query() is synchronous
-        # and will block until it's actually done
-        kwds['timeoutMs'] = 0
-        # Use start row if there's no page_token ... in other words, the
-        # user requested to start somewhere other than the beginning...
-        # presently this is not a parameter to read_gbq(), but it will be
-        # added eventually.
-        if page_token:
-            kwds['pageToken'] = page_token
-        else:
-            kwds['startIndex'] = start_row
-        data = client.apiclient.jobs().getQueryResults(**kwds).execute()
-        if not data['jobComplete']:
-            raise bigquery_client.BigqueryError('Job was not completed, or was invalid')
-
-        # How many rows are there across all pages?
-        # Note: This is presently the only reason we don't just use
-        # _ReadOnePage() directly
-        total_rows = int(data['totalRows'])
-
-        page_token = data.get('pageToken', None)
-        raw_page = data.get('rows', [])
-        page_array = _parse_page(raw_page, col_names, col_types, col_dtypes)
-
-        start_row += len(raw_page)
-        if total_rows > 0:
-            completed = (100 * start_row) / total_rows
-            logger.info('Remaining Rows: ' + str(total_rows - start_row) + '('
-                        + str(completed) + '% Complete)')
-        else:
-            logger.info('No Rows')
+    _test_imports()
 
-        dataframe_list.append(DataFrame(page_array))
+    if not project_id:
+        raise TypeError("Missing required parameter: project_id")
 
-        # Did we get enough rows? Note: gbq.py stopped checking for this
-        # but we felt it was still a good idea.
-        if not page_token and not raw_page and start_row != total_rows:
-            raise bigquery_client.BigqueryInterfaceError(
-                'Not enough rows returned by server. Expected: {0} Rows, But '
-                'Received {1}'.format(total_rows, start_row)
-            )
+    connector = GbqConnector(project_id, reauth = reauth)
+    schema, pages = connector.run_query(query)
+    dataframe_list = []
+    while len(pages) > 0:
+        page = pages.pop()
+        dataframe_list.append(_parse_data(schema, page))
 
-    # Build final dataframe
-    final_df = concat(dataframe_list, ignore_index=True)
+    final_df = concat(dataframe_list, ignore_index = True)
 
     # Reindex the DataFrame on the provided column
     if index_col is not None:
-        if index_col in col_names:
-            final_df.set_index(index_col, inplace=True)
-            col_names.remove(index_col)
+        if index_col in final_df.columns:
+            final_df.set_index(index_col, inplace = True)
         else:
             raise InvalidColumnOrder(
                 'Index column "{0}" does not exist in DataFrame.'
@@ -330,7 +376,7 @@ def _parse_data(client, job, index_col=None, col_order=None):
 
     # Change the order of columns in the DataFrame based on provided list
     if col_order is not None:
-        if sorted(col_order) == sorted(col_names):
+        if sorted(col_order) == sorted(final_df.columns):
             final_df = final_df[col_order]
         else:
             raise InvalidColumnOrder(
@@ -343,188 +389,47 @@ def _parse_data(client, job, index_col=None, col_order=None):
     final_df._data = final_df._data.downcast(dtypes='infer')
     return final_df
 
-
-def to_gbq(dataframe, destination_table, schema=None, col_order=None,
-           if_exists='fail', **kwargs):
+def to_gbq(dataframe, destination_table, project_id=None, chunksize=10000,
+           verbose=True, reauth=False):
     """Write a DataFrame to a Google BigQuery table.
 
     THIS IS AN EXPERIMENTAL LIBRARY
 
-    If the table exists, the DataFrame will be appended. If not, a new table
-    will be created, in which case the schema will have to be specified. By
-    default, rows will be written in the order they appear in the DataFrame,
-    though the user may specify an alternative order.
+    If the table exists, the dataframe will be written to the table using
+    the defined table schema and column types. For simplicity, this method
+    uses the Google BigQuery streaming API. The to_gbq method chunks data
+    into a default chunk size of 10,000. Failures return the complete error
+    response which can be quite long depending on the size of the insert. 
+    There are several important limitations of the Google streaming API 
+    which are detailed at:
+    https://developers.google.com/bigquery/streaming-data-into-bigquery.
 
     Parameters
     ----------
     dataframe : DataFrame
         DataFrame to be written
     destination_table : string
-         name of table to be written, in the form 'dataset.tablename'
-    schema : sequence (optional)
-         list of column types in order for data to be inserted,
-         e.g. ['INTEGER', 'TIMESTAMP', 'BOOLEAN']
-    col_order : sequence (optional)
-         order which columns are to be inserted,
-         e.g. ['primary_key', 'birthday', 'username']
-    if_exists : {'fail', 'replace', 'append'} (optional)
-        - fail: If table exists, do nothing.
-        - replace: If table exists, drop it, recreate it, and insert data.
-        - append: If table exists, insert data. Create if does not exist.
-    kwargs are passed to the Client constructor
-
-    Raises
-    ------
-    SchemaMissing :
-        Raised if the 'if_exists' parameter is set to 'replace', but no schema
-        is specified
-    TableExists :
-        Raised if the specified 'destination_table' exists but the 'if_exists'
-        parameter is set to 'fail' (the default)
-    InvalidSchema :
-        Raised if the 'schema' parameter does not match the provided DataFrame
-    """
-
-    if not _BQ_INSTALLED:
-        if sys.version_info >= (3, 0):
-            raise NotImplementedError('gbq module does not support Python 3 '
-                                      'yet')
-        else:
-            raise ImportError('Could not import Google BigQuery Client.')
-
-    if not _BQ_VALID_VERSION:
-        raise ImportError("pandas requires bigquery >= 2.0.17 for Google "
-                          "BigQuery support, current version " + _BQ_VERSION)
-
-    ALLOWED_TYPES = ['STRING', 'INTEGER', 'FLOAT', 'BOOLEAN', 'TIMESTAMP',
-                     'RECORD']
-
-    if if_exists == 'replace' and schema is None:
-        raise SchemaMissing('Cannot replace a table without specifying the '
-                            'data schema')
-    else:
-        client = _authenticate()
-        table_reference = client.GetTableReference(destination_table)
-        if client.TableExists(table_reference):
-            if if_exists == 'fail':
-                raise TableExistsFail('Cannot overwrite existing tables if '
-                                      '\'if_exists="fail"\'')
-            else:
-                # Build up a string representation of the
-                # table's schema. Since the table already
-                # exists, we ask ask the API for it, which
-                # is returned in a list of dictionaries
-                # describing column data. Iterate over these
-                # and build up a string of form:
-                # "col_name1 : col_type1, col_name2 : col_type2..."
-                schema_full = client.GetTableSchema(
-                    dict(table_reference)
-                )['fields']
-                schema = ''
-                for count, row in enumerate(schema_full):
-                    if count > 0:
-                        schema += ', '
-                    schema += row['name'] + ':' + row['type']
-        else:
-            logger.info('Creating New Table')
-            if schema is None:
-                raise SchemaMissing('Cannot create a new table without '
-                                    'specifying the data schema')
-            else:
-                columns = dataframe.columns
-                if len(schema) != len(columns):
-                    raise InvalidSchema('Incorrect number of columns in '
-                                        'schema')
-                else:
-                    schema_string = ''
-                    for count, name in enumerate(columns):
-                        if count > 0:
-                            schema_string += ', '
-                        column_type = schema[count].upper()
-                        if column_type in ALLOWED_TYPES:
-                            schema_string += name + ':' + schema[count].lower()
-                        else:
-                            raise InvalidSchema('Invalid Type: ' + column_type
-                                                + ". Must be one of: " +
-                                                str(ALLOWED_TYPES))
-                    schema = schema_string
-
-    opts = kwargs
-    opts['sync'] = True
-    opts['skip_leading_rows'] = 1
-    opts['encoding'] = 'UTF-8'
-    opts['max_bad_records'] = 0
-
-    # See: https://developers.google.com/bigquery/docs/reference/v2/jobs
-    if if_exists == 'replace':
-        opts['write_disposition'] = 'WRITE_TRUNCATE'
-    elif if_exists == 'append':
-        opts['write_disposition'] = 'WRITE_APPEND'
-
-    with tempfile.NamedTemporaryFile() as csv_file:
-        dataframe.to_csv(csv_file.name, index=False, encoding='utf-8')
-        job = client.Load(table_reference, csv_file.name, schema=schema,
-                          **opts)
-
-
-def read_gbq(query, project_id=None, destination_table=None, index_col=None,
-             col_order=None, **kwargs):
-    """Load data from Google BigQuery.
-
-    THIS IS AN EXPERIMENTAL LIBRARY
-
-    The main method a user calls to load data from Google BigQuery into a
-    pandas DataFrame. This is a simple wrapper for Google's bq.py and
-    bigquery_client.py, which we use to get the source data. Because of this,
-    this script respects the user's bq settings file, '~/.bigqueryrc', if it
-    exists. Such a file can be generated using 'bq init'. Further, additional
-    parameters for the query can be specified as either ``**kwds`` in the
-    command, or using FLAGS provided in the 'gflags' module. Particular options
-    can be found in bigquery_client.py.
-
-    Parameters
-    ----------
-    query : str
-        SQL-Like Query to return data values
-    project_id : str (optional)
-        Google BigQuery Account project ID. Optional, since it may be
-        located in ~/.bigqueryrc
-    index_col : str (optional)
-        Name of result column to use for index in results DataFrame
-    col_order : list(str) (optional)
-        List of BigQuery column names in the desired order for results
-        DataFrame
-    destination_table : string (optional)
-        If provided, send the results to the given table.
-    **kwargs :
-        To be passed to bq.Client.Create(). Particularly: 'trace',
-        'sync', 'api', 'api_version'
-
-    Returns
-    -------
-    df: DataFrame
-        DataFrame representing results of query
+        Name of table to be written, in the form 'dataset.tablename'
+    project_id : str
+        Google BigQuery Account project ID.
+    chunksize : int (default 10000)
+        Number of rows to be inserted in each chunk from the dataframe.
+    verbose : boolean (default True)
+        Show percentage complete
+    reauth : boolean (default False)
+        Force Google BigQuery to reauthenticate the user. This is useful
+        if multiple accounts are used.
 
     """
-    if not _BQ_INSTALLED:
-        if sys.version_info >= (3, 0):
-            raise NotImplementedError('gbq module does not support Python 3 '
-                                      'yet')
-        else:
-            raise ImportError('Could not import Google BigQuery Client.')
-
-    if not _BQ_VALID_VERSION:
-        raise ImportError('pandas requires bigquery >= 2.0.17 for Google '
-                          'BigQuery support, current version ' + _BQ_VERSION)
+    _test_imports()
 
-    query_args = kwargs
-    query_args['project_id'] = project_id
-    query_args['query'] = query
-    query_args['destination_table'] = destination_table
-    query_args['sync'] = True
+    if not project_id:
+        raise TypeError("Missing required parameter: project_id")
 
-    client = _authenticate()
+    if not '.' in destination_table:
+        raise NotFoundException("Invalid Table Name. Should be of the form 'datasetId.tableId' ")
 
-    job = client.Query(**query_args)
+    connector = GbqConnector(project_id, reauth = reauth)
+    dataset_id, table_id = destination_table.rsplit('.',1)
 
-    return _parse_data(client, job, index_col=index_col, col_order=col_order)
+    connector.load_data(dataframe, dataset_id, table_id, chunksize, verbose)
diff --git a/pandas/util/print_versions.py b/pandas/util/print_versions.py
--- a/pandas/util/print_versions.py
+++ b/pandas/util/print_versions.py
@@ -82,7 +82,7 @@ def show_versions(as_json=False):
         ("lxml", lambda mod: mod.etree.__version__),
         ("bs4", lambda mod: mod.__version__),
         ("html5lib", lambda mod: mod.__version__),
-        ("bq", lambda mod: mod._VersionNumber()),
+        ("httplib2", lambda mod: mod.__version__),
         ("apiclient", lambda mod: mod.__version__),
         ("rpy2", lambda mod: mod.__version__),
         ("sqlalchemy", lambda mod: mod.__version__),


Test patch:
diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py
--- a/pandas/io/tests/test_gbq.py
+++ b/pandas/io/tests/test_gbq.py
@@ -1,474 +1,290 @@
 import ast
+import datetime
+import json
 import nose
 import os
+import pytz
 import shutil
 import subprocess
+import sys
+import platform
+from time import sleep
 
 import numpy as np
 
+from pandas import NaT
+from pandas.compat import u
+from pandas.core.frame import DataFrame
 import pandas.io.gbq as gbq
 import pandas.util.testing as tm
 
-from pandas.core.frame import DataFrame
-from pandas.util.testing import with_connectivity_check
-from pandas.compat import u
-from pandas import NaT
+PROJECT_ID = None
 
+VERSION = platform.python_version()
 
-try:
-    import bq
-    import bigquery_client
-    import gflags as flags
-except ImportError:
-    raise nose.SkipTest
-
-####################################################################################
-# Fake Google BigQuery Client
-
-class FakeClient:
-    def __init__(self):
-        self.apiclient = FakeApiClient()
-    def GetTableSchema(self,table_dict):
-        retval =  {'fields': [
-                    {'type': 'STRING', 'name': 'corpus', 'mode': 'NULLABLE'},
-                    {'type': 'INTEGER', 'name': 'corpus_date', 'mode': 'NULLABLE'},
-                    {'type': 'STRING', 'name': 'word', 'mode': 'NULLABLE'},
-                    {'type': 'INTEGER', 'name': 'word_count', 'mode': 'NULLABLE'}
-                  ]}
-        return retval
-
-# Fake Google BigQuery API Client
-class FakeApiClient:
-    def __init__(self):
-        self._fakejobs = FakeJobs()
-
-
-    def jobs(self):
-        return self._fakejobs
-
-class FakeJobs:
-    def __init__(self):
-        self._fakequeryresults = FakeResults()
-
-    def getQueryResults(self, job_id=None, project_id=None,
-                        max_results=None, timeout_ms=None, **kwargs):
-        return self._fakequeryresults
-
-class FakeResults:
-    def execute(self):
-        return {'rows': [  {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'brave'}, {'v': '3'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'attended'}, {'v': '1'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'treason'}, {'v': '1'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'islanders'}, {'v': '1'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'heed'}, {'v': '3'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'alehouse'}, {'v': '1'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'corrigible'}, {'v': '1'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'brawl'}, {'v': '2'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': "'"}, {'v': '17'}]},
-                            {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'troubled'}, {'v': '1'}]}
-                        ],
-                'kind': 'bigquery#tableDataList',
-                'etag': '"4PTsVxg68bQkQs1RJ1Ndewqkgg4/hoRHzb4qfhJAIa2mEewC-jhs9Bg"',
-                'totalRows': '10',
-                'jobComplete' : True}
-
-####################################################################################
-
-class TestGbq(tm.TestCase):
+def missing_bq():
+    try:
+        subprocess.call('bq')
+        return False
+    except OSError:
+        return True
+
+def test_requirements():
+    try:
+        gbq._test_imports()
+    except (ImportError, NotImplementedError) as import_exception:
+        raise nose.SkipTest(import_exception)
+
+class TestGBQConnectorIntegration(tm.TestCase):
     def setUp(self):
-        with open(self.fake_job_path, 'r') as fin:
-            self.fake_job = ast.literal_eval(fin.read())
-
-        self.test_data_small = [{'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'brave'}, {'v': '3'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'attended'}, {'v': '1'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'treason'}, {'v': '1'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'islanders'}, {'v': '1'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'heed'}, {'v': '3'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'alehouse'}, {'v': '1'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'corrigible'}, {'v': '1'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'brawl'}, {'v': '2'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': "'"}, {'v': '17'}]},
-                {'f': [{'v': 'othello'}, {'v': '1603'}, {'v': 'troubled'},
-                {'v': '1'}]}]
-
-        self.correct_data_small = np.array(
-            [('othello', 1603, 'brave', 3),
-             ('othello', 1603, 'attended', 1),
-             ('othello', 1603, 'treason', 1),
-             ('othello', 1603, 'islanders', 1),
-             ('othello', 1603, 'heed', 3),
-             ('othello', 1603, 'alehouse', 1),
-             ('othello', 1603, 'corrigible', 1),
-             ('othello', 1603, 'brawl', 2),
-             ('othello', 1603, "'", 17),
-             ('othello', 1603, 'troubled', 1)
-            ],
-            dtype=[('corpus', 'S16'),
-                   ('corpus_date', '<i8'),
-                   ('word', 'S16'),
-                   ('word_count', '<i8')]
-        )
+        test_requirements()
 
-        self.correct_test_datatype = DataFrame(
-            {'VALID_STRING' : ['PI'],
-             'EMPTY_STRING' :  [""],
-             'NULL_STRING' : [None],
-             'VALID_INTEGER' : [3],
-             'NULL_INTEGER' : [np.nan],
-             'VALID_FLOAT' : [3.141592653589793],
-             'NULL_FLOAT' : [np.nan],
-             'UNIX_EPOCH' : [np.datetime64('1970-01-01T00:00:00.000000Z')],
-             'VALID_TIMESTAMP' : [np.datetime64('2004-09-15T05:00:00.000000Z')],
-             'NULL_TIMESTAMP' :[NaT],
-             'TRUE_BOOLEAN' : [True],
-             'FALSE_BOOLEAN' : [False],
-             'NULL_BOOLEAN' : [None]
-            }
-        )[['VALID_STRING',
-          'EMPTY_STRING',
-          'NULL_STRING',
-          'VALID_INTEGER',
-          'NULL_INTEGER',
-          'VALID_FLOAT',
-          'NULL_FLOAT',
-          'UNIX_EPOCH',
-          'VALID_TIMESTAMP',
-          'NULL_TIMESTAMP',
-          'TRUE_BOOLEAN',
-          'FALSE_BOOLEAN',
-          'NULL_BOOLEAN']]
+        if not PROJECT_ID:
+            raise nose.SkipTest("Cannot run integration tests without a project id")
+        
+        self.sut = gbq.GbqConnector(PROJECT_ID)
 
-    @classmethod
-    def setUpClass(cls):
-        # Integration tests require a valid bigquery token
-        # be present in the user's home directory. This
-        # can be generated with 'bq init' in the command line
-        super(TestGbq, cls).setUpClass()
-        cls.dirpath = tm.get_data_path()
-        home = os.path.expanduser("~")
-        cls.bq_token = os.path.join(home, '.bigquery.v2.token')
-        cls.fake_job_path = os.path.join(cls.dirpath, 'gbq_fake_job.txt')
-
-        # If we're using a valid token, make a test dataset
-        # Note, dataset functionality is beyond the scope
-        # of the module under test, so we rely on the command
-        # line utility for this.
-        if os.path.exists(cls.bq_token):
-            subprocess.call(['bq','mk', '-d', 'pandas_testing_dataset'])
+    def test_should_be_able_to_make_a_connector(self):
+        self.assertTrue(self.sut is not None, 'Could not create a GbqConnector')
 
-    @classmethod
-    def tearDownClass(cls):
-        super(TestGbq, cls).tearDownClass()
+    def test_should_be_able_to_get_valid_credentials(self):
+        credentials = self.sut.get_credentials()
+        self.assertFalse(credentials.invalid, 'Returned credentials invalid')
 
-        # If we're using a valid token, remove the test dataset
-        # created.
-        if os.path.exists(cls.bq_token):
-            subprocess.call(['bq', 'rm', '-r', '-f', '-d', 'pandas_testing_dataset'])
+    def test_should_be_able_to_get_a_bigquery_service(self):
+        credentials = self.sut.get_credentials()
+        bigquery_service = self.sut.get_service(credentials)
+        self.assertTrue(bigquery_service is not None, 'No service returned')
 
-    @with_connectivity_check
-    def test_valid_authentication(self):
-        # If the user has a token file, they should recieve a client from gbq._authenticate
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
+    def test_should_be_able_to_get_schema_from_query(self):
+        schema, pages = self.sut.run_query('SELECT 1')
+        self.assertTrue(schema is not None)
 
-        self.assertTrue(gbq._authenticate is not None, 'Authentication To GBQ Failed')
+    def test_should_be_able_to_get_results_from_query(self):
+        schema, pages = self.sut.run_query('SELECT 1')
+        self.assertTrue(pages is not None)
 
-    @with_connectivity_check
-    def test_malformed_query(self):
-        # If the user has a connection file, performing an invalid query should raise an error
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-        else:
-            self.assertRaises(bigquery_client.BigqueryInvalidQueryError,
-                              gbq.read_gbq, "SELCET * FORM [publicdata:samples.shakespeare]")
-
-    def test_type_conversion(self):
-        # All BigQuery Types should be cast into appropriate numpy types
-        sample_input = [('1.095292800E9', 'TIMESTAMP'),
-                 ('false', 'BOOLEAN'),
-                 ('2', 'INTEGER'),
-                 ('3.14159', 'FLOAT'),
-                 ('Hello World', 'STRING')]
-        actual_output = [gbq._parse_entry(result[0],result[1]) for result in sample_input]
-        sample_output = [np.datetime64('2004-09-16T00:00:00.000000Z'),
-                  np.bool(False),
-                  np.int('2'),
-                  np.float('3.14159'),
-                  u('Hello World')]
-        self.assertEqual(actual_output, sample_output, 'A format conversion failed')
-
-    @with_connectivity_check
-    def test_unicode_string_conversion(self):
-        # Strings from BigQuery Should be converted to UTF-8 properly
-
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
+class TestReadGBQUnitTests(tm.TestCase):
+    def setUp(self):
+        test_requirements()
 
-        correct_test_datatype = DataFrame(
-            {'UNICODE_STRING' : [u("\xe9\xfc")]}
-        )
+    def test_should_return_bigquery_integers_as_python_floats(self):
+        result = gbq._parse_entry(1, 'INTEGER')
+        tm.assert_equal(result, float(1))
 
-        query = """SELECT '\xc3\xa9\xc3\xbc' as UNICODE_STRING"""
+    def test_should_return_bigquery_floats_as_python_floats(self):
+        result = gbq._parse_entry(1, 'FLOAT')
+        tm.assert_equal(result, float(1))
 
-        client = gbq._authenticate()
-        a = gbq.read_gbq(query)
-        tm.assert_frame_equal(a, correct_test_datatype)
+    def test_should_return_bigquery_timestamps_as_numpy_datetime(self):
+        result = gbq._parse_entry('0e9', 'TIMESTAMP')
+        tm.assert_equal(result, np.datetime64('1970-01-01T00:00:00Z'))
 
+    def test_should_return_bigquery_booleans_as_python_booleans(self):
+        result = gbq._parse_entry('false', 'BOOLEAN')
+        tm.assert_equal(result, False)
 
+    def test_should_return_bigquery_strings_as_python_strings(self):
+        result = gbq._parse_entry('STRING', 'STRING')
+        tm.assert_equal(result, 'STRING')
 
-    def test_data_small(self):
-        # Parsing a fixed page of data should return the proper fixed np.array()
-        result_frame = gbq._parse_page(self.test_data_small,
-                                        ['corpus','corpus_date','word','word_count'],
-                                        ['STRING','INTEGER','STRING','INTEGER'],
-                                        [object,np.dtype(int),object,np.dtype(int)]
-                                      )
-        tm.assert_frame_equal(DataFrame(result_frame), DataFrame(self.correct_data_small),
-                        'An element in the result DataFrame didn\'t match the sample set')
+    def test_to_gbq_should_fail_if_invalid_table_name_passed(self):
+        with tm.assertRaises(gbq.NotFoundException):
+            gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234")
+
+    def test_to_gbq_with_no_project_id_given_should_fail(self):
+        with tm.assertRaises(TypeError):
+            gbq.to_gbq(DataFrame(), 'dataset.tablename')
+
+    def test_read_gbq_with_no_project_id_given_should_fail(self):
+        with tm.assertRaises(TypeError):
+            gbq.read_gbq('SELECT "1" as NUMBER_1')
+
+    def test_that_parse_data_works_properly(self):
+        test_schema = {'fields': [{'mode': 'NULLABLE',
+                  'name': 'VALID_STRING',
+                  'type': 'STRING'}]}
+        test_page = [{'f': [{'v': 'PI'}]}]
+
+        test_output = gbq._parse_data(test_schema, test_page)
+        correct_output = DataFrame({'VALID_STRING' : ['PI']})
+        tm.assert_frame_equal(test_output, correct_output)
+
+class TestReadGBQIntegration(tm.TestCase):
+    def setUp(self):
+        test_requirements()
+
+        if not PROJECT_ID:
+            raise nose.SkipTest("Cannot run integration tests without a project id")
+
+    def test_should_properly_handle_valid_strings(self):
+        query = 'SELECT "PI" as VALID_STRING'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'VALID_STRING' : ['PI']}))
+
+    def test_should_properly_handle_empty_strings(self):
+        query = 'SELECT "" as EMPTY_STRING'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'EMPTY_STRING' : [""]}))
+
+    def test_should_properly_handle_null_strings(self):
+        query = 'SELECT STRING(NULL) as NULL_STRING'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'NULL_STRING' : [None]}))
+
+    def test_should_properly_handle_valid_integers(self):
+        query = 'SELECT INTEGER(3) as VALID_INTEGER'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'VALID_INTEGER' : [3]}))
+
+    def test_should_properly_handle_null_integers(self):
+        query = 'SELECT INTEGER(NULL) as NULL_INTEGER'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'NULL_INTEGER' : [np.nan]}))
+
+    def test_should_properly_handle_valid_floats(self):
+        query = 'SELECT PI() as VALID_FLOAT'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'VALID_FLOAT' : [3.141592653589793]}))
+
+    def test_should_properly_handle_null_floats(self):
+        query = 'SELECT FLOAT(NULL) as NULL_FLOAT'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'NULL_FLOAT' : [np.nan]}))
+
+    def test_should_properly_handle_timestamp_unix_epoch(self):
+        query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") as UNIX_EPOCH'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'UNIX_EPOCH' : [np.datetime64('1970-01-01T00:00:00.000000Z')]}))
+
+    def test_should_properly_handle_arbitrary_timestamp(self):
+        query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") as VALID_TIMESTAMP'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'VALID_TIMESTAMP' : [np.datetime64('2004-09-15T05:00:00.000000Z')]}))
+
+    def test_should_properly_handle_null_timestamp(self):
+        query = 'SELECT TIMESTAMP(NULL) as NULL_TIMESTAMP'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'NULL_TIMESTAMP' :[NaT]}))
+
+    def test_should_properly_handle_true_boolean(self):
+        query = 'SELECT BOOLEAN(TRUE) as TRUE_BOOLEAN'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'TRUE_BOOLEAN' : [True]}))
+
+    def test_should_properly_handle_false_boolean(self):
+        query = 'SELECT BOOLEAN(FALSE) as FALSE_BOOLEAN'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'FALSE_BOOLEAN' : [False]}))
+
+    def test_should_properly_handle_null_boolean(self):
+        query = 'SELECT BOOLEAN(NULL) as NULL_BOOLEAN'
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, DataFrame({'NULL_BOOLEAN' : [None]}))
+
+    def test_unicode_string_conversion_and_normalization(self):
+        correct_test_datatype = DataFrame(
+            {'UNICODE_STRING' : [u("\xe9\xfc")]}
+        )
+
+        query = 'SELECT "\xc3\xa9\xc3\xbc" as UNICODE_STRING'
+
+        df = gbq.read_gbq(query, project_id=PROJECT_ID)
+        tm.assert_frame_equal(df, correct_test_datatype)
 
     def test_index_column(self):
-        # A user should be able to specify an index column for return
-        result_frame = gbq._parse_data(FakeClient(), self.fake_job, index_col='word')
-        correct_frame = DataFrame(self.correct_data_small)
-        correct_frame.set_index('word', inplace=True)
-        self.assertTrue(result_frame.index.name == correct_frame.index.name)
+        query = "SELECT 'a' as STRING_1, 'b' as STRING_2"
+        result_frame = gbq.read_gbq(query, project_id=PROJECT_ID, index_col="STRING_1")
+        correct_frame = DataFrame({'STRING_1' : ['a'], 'STRING_2' : ['b']}).set_index("STRING_1")
+        tm.assert_equal(result_frame.index.name, correct_frame.index.name)
 
     def test_column_order(self):
-        # A User should be able to specify the order in which columns are returned in the dataframe
-        col_order = ['corpus_date', 'word_count', 'corpus', 'word']
-        result_frame = gbq._parse_data(FakeClient(), self.fake_job, col_order=col_order)
-        tm.assert_index_equal(result_frame.columns, DataFrame(self.correct_data_small)[col_order].columns)
+        query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3"
+        col_order = ['STRING_3', 'STRING_1', 'STRING_2']
+        result_frame = gbq.read_gbq(query, project_id=PROJECT_ID, col_order=col_order)
+        correct_frame = DataFrame({'STRING_1' : ['a'], 'STRING_2' : ['b'], 'STRING_3' : ['c']})[col_order]
+        tm.assert_frame_equal(result_frame, correct_frame)
 
     def test_column_order_plus_index(self):
-        # A User should be able to specify an index and the order of THE REMAINING columns.. they should be notified
-        # if they screw up
-        col_order = ['corpus_date', 'word', 'corpus']
-        result_frame = gbq._parse_data(FakeClient(), self.fake_job, index_col='word_count', col_order=col_order)
-        correct_frame_small = DataFrame(self.correct_data_small)
-        correct_frame_small.set_index('word_count',inplace=True)
-        correct_frame_small = DataFrame(correct_frame_small)[col_order]
-        tm.assert_index_equal(result_frame.columns, correct_frame_small.columns)
-
-    @with_connectivity_check
-    def test_download_dataset_larger_than_100k_rows(self):
+        query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3"
+        col_order = ['STRING_3', 'STRING_2']
+        result_frame = gbq.read_gbq(query, project_id=PROJECT_ID, index_col='STRING_1', col_order=col_order)
+        correct_frame = DataFrame({'STRING_1' : ['a'], 'STRING_2' : ['b'], 'STRING_3' : ['c']})
+        correct_frame.set_index('STRING_1', inplace=True)
+        correct_frame = correct_frame[col_order]
+        tm.assert_frame_equal(result_frame, correct_frame)
+
+    def test_malformed_query(self):
+        with tm.assertRaises(gbq.InvalidQueryException):
+            gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", project_id=PROJECT_ID)
+
+    def test_bad_project_id(self):
+        with tm.assertRaises(gbq.NotFoundException):
+            gbq.read_gbq("SELECT 1", project_id='001')
+
+    def test_bad_table_name(self):
+        with tm.assertRaises(gbq.NotFoundException):
+            gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", project_id=PROJECT_ID)
+
+    def test_download_dataset_larger_than_200k_rows(self):
         # Test for known BigQuery bug in datasets larger than 100k rows
         # http://stackoverflow.com/questions/19145587/bq-py-not-paging-results
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        client = gbq._authenticate()
-        a = gbq.read_gbq("SELECT id, FROM [publicdata:samples.wikipedia] LIMIT 100005")
-        self.assertTrue(len(a) == 100005)
-
-    @with_connectivity_check
-    def test_download_all_data_types(self):
-        # Test that all available data types from BigQuery (as of now)
-        # are handled properly
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        query = """SELECT "PI" as VALID_STRING,
-                   "" as EMPTY_STRING,
-                   STRING(NULL) as NULL_STRING,
-                   INTEGER(3) as VALID_INTEGER,
-                   INTEGER(NULL) as NULL_INTEGER,
-                   PI() as VALID_FLOAT,
-                   FLOAT(NULL) as NULL_FLOAT,
-                   TIMESTAMP("1970-01-01 00:00:00") as UNIX_EPOCH,
-                   TIMESTAMP("2004-09-15 05:00:00") as VALID_TIMESTAMP,
-                   TIMESTAMP(NULL) as NULL_TIMESTAMP,
-                   BOOLEAN(TRUE) as TRUE_BOOLEAN,
-                   BOOLEAN(FALSE) as FALSE_BOOLEAN,
-                   BOOLEAN(NULL) as NULL_BOOLEAN"""
-
-        client = gbq._authenticate()
-        a = gbq.read_gbq(query, col_order = ['VALID_STRING',
-                                             'EMPTY_STRING',
-                                             'NULL_STRING',
-                                             'VALID_INTEGER',
-                                             'NULL_INTEGER',
-                                             'VALID_FLOAT',
-                                             'NULL_FLOAT',
-                                             'UNIX_EPOCH',
-                                             'VALID_TIMESTAMP',
-                                             'NULL_TIMESTAMP',
-                                             'TRUE_BOOLEAN',
-                                             'FALSE_BOOLEAN',
-                                             'NULL_BOOLEAN'])
-
-        tm.assert_frame_equal(a, self.correct_test_datatype)
-
-    @with_connectivity_check
-    def test_table_exists(self):
-        # Given a table name in the format {dataset}.{tablename}, if a table exists,
-        # the GetTableReference should accurately indicate this.
-        # This could possibly change in future implementations of bq,
-        # but it is the simplest way to provide users with appropriate
-        # error messages regarding schemas.
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        client = gbq._authenticate()
-        table_reference = client.GetTableReference("publicdata:samples.shakespeare")
-        self.assertTrue(client.TableExists(table_reference))
-
-    @with_connectivity_check
-    def test_table__not_exists(self):
-        # Test the inverse of `test_table_exists`
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        client = gbq._authenticate()
-        table_reference = client.GetTableReference("publicdata:samples.does_not_exist")
-        self.assertFalse(client.TableExists(table_reference))
-
-    @with_connectivity_check
-    def test_upload_new_table_schema_error(self):
-        # Attempting to upload to a non-existent table without a schema should fail
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        df = DataFrame(self.correct_data_small)
-        with self.assertRaises(gbq.SchemaMissing):
-            gbq.to_gbq(df, 'pandas_testing_dataset.test_database', schema=None, col_order=None, if_exists='fail')
-
-    @with_connectivity_check
-    def test_upload_replace_schema_error(self):
-        # Attempting to replace an existing table without specifying a schema should fail
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        df = DataFrame(self.correct_data_small)
-        with self.assertRaises(gbq.SchemaMissing):
-            gbq.to_gbq(df, 'pandas_testing_dataset.test_database', schema=None, col_order=None, if_exists='replace')
-
-    @with_connectivity_check
-    def test_upload_public_data_error(self):
-        # Attempting to upload to a public, read-only, dataset should fail
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        array = [['TESTING_GBQ', 999999999, 'hi', 0, True, 9999999999, '00.000.00.000', 1, 'hola',
-                 99999999, False, False, 1, 'Jedi', 11210]]
-        df = DataFrame(array)
-        with self.assertRaises(bigquery_client.BigqueryServiceError):
-            gbq.to_gbq(df, 'publicdata:samples.wikipedia', schema=None, col_order=None, if_exists='append')
-
-    @with_connectivity_check
-    def test_upload_new_table(self):
-        # Attempting to upload to a new table with valid data and a valid schema should succeed
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        schema = ['STRING', 'INTEGER', 'STRING', 'INTEGER', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER',
-                  'STRING', 'INTEGER', 'BOOLEAN', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER']
-
-        array = [['TESTING_GBQ', 999999999, 'hi', 0, True, 9999999999, '00.000.00.000', 1, 'hola',
-                 99999999, False, False, 1, 'Jedi', 11210]]
-        df = DataFrame(array, columns=['title','id','language','wp_namespace','is_redirect','revision_id',
-                                       'contributor_ip','contributor_id','contributor_username','timestamp',
-                                       'is_minor','is_bot','reversion_id','comment','num_characters'])
-        gbq.to_gbq(df, 'pandas_testing_dataset.test_data2', schema=schema, col_order=None, if_exists='append')
-        a = gbq.read_gbq("SELECT * FROM pandas_testing_dataset.test_data2")
-        self.assertTrue((a == df).all().all())
-
-    @with_connectivity_check
-    def test_upload_bad_data_table(self):
-        # Attempting to upload data that does not match schema should fail
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        schema = ['STRING', 'INTEGER', 'STRING', 'INTEGER', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER',
-                  'STRING', 'INTEGER', 'BOOLEAN', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER']
-
-        array = [['TESTING_GBQ\',', False, 'hi', 0, True, 'STRING IN INTEGER', '00.000.00.000', 1, 'hola',
-                 99999999, -100, 1000, 1, 'Jedi', 11210]]
-        df = DataFrame(array, columns=['title','id','language','wp_namespace','is_redirect','revision_id',
-                                       'contributor_ip','contributor_id','contributor_username','timestamp',
-                                       'is_minor','is_bot','reversion_id','comment','num_characters'])
-        with self.assertRaises(bigquery_client.BigqueryServiceError):
-            gbq.to_gbq(df, 'pandas_testing_dataset.test_data1', schema=schema, col_order=None, if_exists='append')
-
-    @with_connectivity_check
-    def test_invalid_column_name_schema(self):
-        # Specifying a schema that contains an invalid column name should fail
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        schema = ['INCORRECT']
-        df = DataFrame([[1]],columns=['fake'])
-        with self.assertRaises(gbq.InvalidSchema):
-            gbq.to_gbq(df, 'pandas_testing_dataset.test_data', schema=schema, col_order=None, if_exists='append')
-
-    @with_connectivity_check
-    def test_invalid_number_of_columns_schema(self):
-        # Specifying a schema that does not have same shape as dataframe should fail
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        schema = ['INTEGER']
-        df = DataFrame([[1, 'STRING']],columns=['fake','fake'])
-        with self.assertRaises(gbq.InvalidSchema):
-            gbq.to_gbq(df, 'pandas_testing_dataset.test_data4', schema=schema, col_order=None, if_exists='append')
-
-    @with_connectivity_check
-    def test_upload_fail_if_exists(self):
-        # Attempting to upload to a new table with valid data and a valid schema should succeed
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        schema = ['STRING', 'INTEGER', 'STRING', 'INTEGER', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER',
-                  'STRING', 'INTEGER', 'BOOLEAN', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER']
-
-        array = [['TESTING_GBQ', 999999999, 'hi', 0, True, 9999999999, '00.000.00.000', 1, 'hola',
-                 99999999, False, False, 1, 'Jedi', 11210]]
-        df = DataFrame(array, columns=['title','id','language','wp_namespace','is_redirect','revision_id',
-                                       'contributor_ip','contributor_id','contributor_username','timestamp',
-                                       'is_minor','is_bot','reversion_id','comment','num_characters'])
-        gbq.to_gbq(df, 'pandas_testing_dataset.test_data3', schema=schema, col_order=None, if_exists='fail')
-
-        with self.assertRaises(gbq.TableExistsFail):
-            gbq.to_gbq(df, 'pandas_testing_dataset.test_data3', schema=schema, col_order=None, if_exists='fail')
-
-    @with_connectivity_check
-    def test_upload_replace(self):
-        # Attempting to overwrite an existing table with valid data and a valid schema should succeed
-        if not os.path.exists(self.bq_token):
-            raise nose.SkipTest('Skipped because authentication information is not available.')
-
-        schema = ['STRING', 'INTEGER', 'STRING', 'INTEGER', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER',
-                  'STRING', 'INTEGER', 'BOOLEAN', 'BOOLEAN',
-                  'INTEGER', 'STRING', 'INTEGER']
-
-        # Setup an existing table
-        array1 = [['', 1, '', 1, False, 1, '00.111.00.111', 1, 'hola',
-                 1, True, True, 1, 'Sith', 1]]
-        df1 = DataFrame(array1, columns=['title','id','language','wp_namespace','is_redirect','revision_id',
-                                       'contributor_ip','contributor_id','contributor_username','timestamp',
-                                       'is_minor','is_bot','reversion_id','comment','num_characters'])
-        gbq.to_gbq(df1, 'pandas_testing_dataset.test_data5', schema=schema, col_order=None, if_exists='fail')
-
-        array2 = [['TESTING_GBQ', 999999999, 'hi', 0, True, 9999999999, '00.000.00.000', 1, 'hola',
-                 99999999, False, False, 1, 'Jedi', 11210]]
-
-        # Overwrite the existing table with different data
-        df2 = DataFrame(array2, columns=['title','id','language','wp_namespace','is_redirect','revision_id',
-                                       'contributor_ip','contributor_id','contributor_username','timestamp',
-                                       'is_minor','is_bot','reversion_id','comment','num_characters'])
-        gbq.to_gbq(df2, 'pandas_testing_dataset.test_data5', schema=schema, col_order=None, if_exists='replace')
-
-        # Read the table and confirm the new data is all that is there
-        a = gbq.read_gbq("SELECT * FROM pandas_testing_dataset.test_data5")
-        self.assertTrue((a == df2).all().all())
+        df = gbq.read_gbq("SELECT id FROM [publicdata:samples.wikipedia] GROUP EACH BY id ORDER BY id ASC LIMIT 200005", project_id=PROJECT_ID)
+        self.assertEqual(len(df.drop_duplicates()), 200005)
 
+class TestToGBQIntegration(tm.TestCase):
+    # This class requires bq.py to be installed for setup/teardown. 
+    # It will also need to be preconfigured with a default dataset,
+    # so, be sure to `bq init` in terminal before running.
+
+    def setUp(self):
+        test_requirements()
+
+        if not PROJECT_ID:
+            raise nose.SkipTest("Cannot run integration tests without a project id")
+        if missing_bq():
+            raise nose.SkipTest("Cannot run to_gbq tests without bq command line client")
+
+    @classmethod
+    def setUpClass(cls):
+        if PROJECT_ID and not missing_bq():
+            subprocess.call(['bq','mk','pydata_pandas_bq_testing'])
+            subprocess.call(['bq','mk','pydata_pandas_bq_testing.new_test','bools:BOOLEAN,flts:FLOAT,ints:INTEGER,strs:STRING,times:TIMESTAMP'])
+
+    def test_upload_data(self):
+        test_size = 1000001
+        #create df to test for all BQ datatypes except RECORD
+        bools = np.random.randint(2, size=(1,test_size)).astype(bool)
+        flts = np.random.randn(1,test_size)
+        ints = np.random.randint(1,10, size=(1,test_size))
+        strs = np.random.randint(1,10, size=(1,test_size)).astype(str)
+        times = [datetime.datetime.now(pytz.timezone('US/Arizona')) for t in xrange(test_size)]
+        df = DataFrame({'bools':bools[0], 'flts':flts[0], 'ints':ints[0], 'strs':strs[0], 'times':times[0]}, index=range(test_size))
+        gbq.to_gbq(df,"pydata_pandas_bq_testing.new_test", project_id=PROJECT_ID, chunksize=10000)
+        sleep(60) # <- Curses Google!!!
+
+        result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM pydata_pandas_bq_testing.new_test", project_id=PROJECT_ID)
+        self.assertEqual(result['NUM_ROWS'][0], test_size)
+
+    def test_google_upload_errors_should_raise_exception(self):
+        test_timestamp = datetime.datetime.now(pytz.timezone('US/Arizona'))
+        bad_df = DataFrame( {'bools': [False, False],
+                             'flts': [0.0,1.0],
+                             'ints': [0,'1'],
+                             'strs': ['a', 1],
+                             'times': [test_timestamp, test_timestamp]
+                             }, index=range(2))
+        with tm.assertRaises(gbq.UnknownGBQException):
+            gbq.to_gbq(bad_df, 'pydata_pandas_bq_testing.new_test', project_id = PROJECT_ID)
+
+
+    @classmethod
+    def tearDownClass(cls):
+        if PROJECT_ID and not missing_bq():
+            subprocess.call(['bq','rm','-f','pydata_pandas_bq_testing.new_test'])
+            subprocess.call(['bq','rm','-f','pydata_pandas_bq_testing'])
 
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)
+