Repository: ibis-project/ibis

Issue:
bigquery connect doesn't use project_id if no credentials are passed
`ibis.bigquery.connect` takes an arg `project_id` of type `Optional[str]=None`. It also takes an optional `credentials` arg. The way [the code](https://github.com/ibis-project/ibis/blob/master/ibis/backends/bigquery/__init__.py#L86) is written, if no credentials are passed, the user-passed `project_id` is thrown away in favor of the [user's default project](https://pydata-google-auth.readthedocs.io/en/latest/api.html#pydata_google_auth.default). I don't have a BQ account myself, but I think that some logic is needed to check if the user passes an explicit `project_id` and use that even if they pass `None` for `credentials`.


Discussion:

Resolving patch:
diff --git a/ibis/backends/bigquery/__init__.py b/ibis/backends/bigquery/__init__.py
--- a/ibis/backends/bigquery/__init__.py
+++ b/ibis/backends/bigquery/__init__.py
@@ -84,17 +84,21 @@ def connect(
     BigQueryClient
 
     """
+    default_project_id = None
+
     if credentials is None:
         credentials_cache = pydata_google_auth.cache.ReadWriteCredentialsCache(
             filename="ibis.json"
         )
-        credentials, project_id = pydata_google_auth.default(
+        credentials, default_project_id = pydata_google_auth.default(
             SCOPES,
             client_id=CLIENT_ID,
             client_secret=CLIENT_SECRET,
             credentials_cache=credentials_cache,
         )
 
+    project_id = project_id or default_project_id
+
     return BigQueryClient(
         project_id,
         dataset_id=dataset_id,
diff --git a/ibis/backends/bigquery/client.py b/ibis/backends/bigquery/client.py
--- a/ibis/backends/bigquery/client.py
+++ b/ibis/backends/bigquery/client.py
@@ -529,7 +529,7 @@ def exists_table(self, name, database=None):
 
     def list_tables(self, like=None, database=None):
         project, dataset = self._parse_project_and_dataset(database)
-        dataset_ref = self.client.dataset(dataset, project=project)
+        dataset_ref = bq.DatasetReference(project, dataset)
         result = [
             table.table_id for table in self.client.list_tables(dataset_ref)
         ]


Test patch:
diff --git a/ibis/backends/bigquery/tests/conftest.py b/ibis/backends/bigquery/tests/conftest.py
--- a/ibis/backends/bigquery/tests/conftest.py
+++ b/ibis/backends/bigquery/tests/conftest.py
@@ -15,8 +15,7 @@
 DATASET_ID = 'testing'
 
 
-def connect(project_id, dataset_id, application_name=None):
-    ga = pytest.importorskip('google.auth')
+def _credentials():
     service_account = pytest.importorskip('google.oauth2.service_account')
     google_application_credentials = os.environ.get(
         "GOOGLE_APPLICATION_CREDENTIALS", None
@@ -38,22 +37,9 @@ def connect(project_id, dataset_id, application_name=None):
             )
         )
 
-    skip_message = (
-        'No BigQuery credentials found using project_id={}, '
-        'dataset_id={}. Skipping BigQuery tests.'
-    ).format(project_id, dataset_id)
-    credentials = service_account.Credentials.from_service_account_file(
+    return service_account.Credentials.from_service_account_file(
         google_application_credentials
     )
-    try:
-        return ibis.bigquery.connect(
-            project_id,
-            dataset_id,
-            credentials=credentials,
-            application_name=application_name,
-        )
-    except ga.exceptions.DefaultCredentialsError:
-        pytest.skip(skip_message)
 
 
 class TestConf(UnorderedComparator, BackendTest, RoundAwayFromZero):
@@ -63,6 +49,7 @@ class TestConf(UnorderedComparator, BackendTest, RoundAwayFromZero):
 
     @staticmethod
     def connect(data_directory: Path) -> ibis.client.Client:
+        bq = pytest.importorskip('ibis.backends.bigquery')
         project_id = os.environ.get('GOOGLE_BIGQUERY_PROJECT_ID')
         if project_id is None:
             pytest.skip(
@@ -73,7 +60,11 @@ def connect(data_directory: Path) -> ibis.client.Client:
             pytest.skip(
                 'Environment variable GOOGLE_BIGQUERY_PROJECT_ID is empty'
             )
-        return connect(project_id, dataset_id='testing')
+        return bq.connect(
+            project_id=project_id,
+            dataset_id=DATASET_ID,
+            credentials=_credentials(),
+        )
 
     @property
     def batting(self) -> ir.TableExpr:
@@ -90,13 +81,24 @@ def project_id():
 
 
 @pytest.fixture(scope='session')
-def client():
-    return connect(PROJECT_ID, DATASET_ID)
+def credentials():
+    return _credentials()
+
+
+@pytest.fixture(scope='session')
+def client(credentials, project_id):
+    bq = pytest.importorskip('ibis.backends.bigquery')
+    return bq.connect(
+        project_id=project_id, dataset_id=DATASET_ID, credentials=credentials,
+    )
 
 
 @pytest.fixture(scope='session')
-def client2():
-    return connect(PROJECT_ID, DATASET_ID)
+def client2(credentials, project_id):
+    bq = pytest.importorskip('ibis.backends.bigquery')
+    return bq.connect(
+        project_id=project_id, dataset_id=DATASET_ID, credentials=credentials,
+    )
 
 
 @pytest.fixture(scope='session')
@@ -130,5 +132,10 @@ def numeric_table(client):
 
 
 @pytest.fixture(scope='session')
-def public():
-    return connect(PROJECT_ID, dataset_id='bigquery-public-data.stackoverflow')
+def public(project_id, credentials):
+    bq = pytest.importorskip('ibis.backends.bigquery')
+    return bq.connect(
+        project_id=project_id,
+        dataset_id='bigquery-public-data.stackoverflow',
+        credentials=credentials,
+    )
diff --git a/ibis/backends/bigquery/tests/test_client.py b/ibis/backends/bigquery/tests/test_client.py
--- a/ibis/backends/bigquery/tests/test_client.py
+++ b/ibis/backends/bigquery/tests/test_client.py
@@ -1,7 +1,6 @@
 import collections
 import datetime
 import decimal
-from unittest import mock
 
 import numpy as np
 import pandas as pd
@@ -18,8 +17,7 @@
 ga = pytest.importorskip('google.auth')
 exceptions = pytest.importorskip('google.api_core.exceptions')
 
-from .client import bigquery_param  # noqa: E402, isort:skip
-from .tests.conftest import connect  # noqa: E402, isort:skip
+from ..client import bigquery_param  # noqa: E402, isort:skip
 
 
 def test_table(alltypes):
@@ -542,11 +540,6 @@ def test_exists_database_different_project(client, name, expected):
     assert client.exists_database(name) is expected
 
 
-def test_repeated_project_name(project_id):
-    con = connect(project_id, dataset_id='{}.testing'.format(project_id))
-    assert 'functional_alltypes' in con.list_tables()
-
-
 def test_multiple_project_queries(client):
     so = client.table(
         'posts_questions', database='bigquery-public-data.stackoverflow'
@@ -721,23 +714,3 @@ def test_approx_median(alltypes):
     # Since 6 and 7 are right on the edge for median in the range of months
     # (1-12), accept either for the approximate function.
     assert result in (6, 7)
-
-
-def test_client_without_dataset(project_id):
-    con = connect(project_id, dataset_id=None)
-    with pytest.raises(ValueError, match="Unable to determine BigQuery"):
-        con.list_tables()
-
-
-def test_client_sets_user_agent(project_id, monkeypatch):
-    mock_client = mock.create_autospec(bq.Client)
-    monkeypatch.setattr(bq, 'Client', mock_client)
-    connect(
-        project_id,
-        dataset_id='bigquery-public-data.stackoverflow',
-        application_name='my-great-app/0.7.0',
-    )
-    info = mock_client.call_args[1]['client_info']
-    user_agent = info.to_user_agent()
-    assert ' ibis/{}'.format(ibis.__version__) in user_agent
-    assert 'my-great-app/0.7.0 ' in user_agent
diff --git a/ibis/backends/bigquery/tests/test_connect.py b/ibis/backends/bigquery/tests/test_connect.py
new file mode 100644
--- /dev/null
+++ b/ibis/backends/bigquery/tests/test_connect.py
@@ -0,0 +1,56 @@
+from unittest import mock
+
+import pytest
+
+import ibis
+
+pytestmark = pytest.mark.bigquery
+bq = pytest.importorskip('google.cloud.bigquery')
+bq_backend = pytest.importorskip('ibis.backends.bigquery')
+pydata_google_auth = pytest.importorskip('pydata_google_auth')
+
+
+def test_repeated_project_name(project_id, credentials):
+    con = bq_backend.connect(
+        project_id=project_id,
+        dataset_id='{}.testing'.format(project_id),
+        credentials=credentials,
+    )
+    assert 'functional_alltypes' in con.list_tables()
+
+
+def test_project_id_different_from_default_credentials(monkeypatch):
+    auth = pytest.importorskip('google.auth.credentials')
+    creds = mock.create_autospec(auth.Credentials)
+
+    def mock_credentials(*args, **kwargs):
+        return creds, 'default-project-id'
+
+    monkeypatch.setattr(pydata_google_auth, 'default', mock_credentials)
+    con = bq_backend.connect(project_id='explicit-project-id',)
+    assert con.billing_project == 'explicit-project-id'
+
+
+def test_without_dataset(project_id, credentials):
+    con = bq_backend.connect(
+        project_id=project_id, dataset_id=None, credentials=credentials,
+    )
+    with pytest.raises(ValueError, match="Unable to determine BigQuery"):
+        con.list_tables()
+
+
+def test_application_name_sets_user_agent(
+    project_id, credentials, monkeypatch
+):
+    mock_client = mock.create_autospec(bq.Client)
+    monkeypatch.setattr(bq, 'Client', mock_client)
+    bq_backend.connect(
+        project_id=project_id,
+        dataset_id='bigquery-public-data.stackoverflow',
+        application_name='my-great-app/0.7.0',
+        credentials=credentials,
+    )
+    info = mock_client.call_args[1]['client_info']
+    user_agent = info.to_user_agent()
+    assert ' ibis/{}'.format(ibis.__version__) in user_agent
+    assert 'my-great-app/0.7.0 ' in user_agent
diff --git a/ibis/backends/bigquery/tests/test_datatypes.py b/ibis/backends/bigquery/tests/test_datatypes.py
--- a/ibis/backends/bigquery/tests/test_datatypes.py
+++ b/ibis/backends/bigquery/tests/test_datatypes.py
@@ -4,7 +4,7 @@
 
 import ibis.expr.datatypes as dt
 
-from .datatypes import (
+from ..datatypes import (
     TypeTranslationContext,
     UDFContext,
     ibis_type_to_bigquery_type,
diff --git a/ibis/backends/bigquery/tests/udf/test_core.py b/ibis/backends/bigquery/tests/udf/test_core.py
--- a/ibis/backends/bigquery/tests/udf/test_core.py
+++ b/ibis/backends/bigquery/tests/udf/test_core.py
@@ -6,7 +6,7 @@
 
 from ibis.compat import PY38
 
-from .udf.core import PythonToJavaScriptTranslator, SymbolTable
+from ...udf.core import PythonToJavaScriptTranslator, SymbolTable
 
 if PY38:
     # ref: https://github.com/ibis-project/ibis/issues/2098
diff --git a/ibis/backends/bigquery/tests/udf/test_find.py b/ibis/backends/bigquery/tests/udf/test_find.py
--- a/ibis/backends/bigquery/tests/udf/test_find.py
+++ b/ibis/backends/bigquery/tests/udf/test_find.py
@@ -5,7 +5,7 @@
 from ibis.compat import PY38
 from ibis.util import is_iterable
 
-from .udf.find import find_names
+from ...udf.find import find_names
 
 if PY38:
     # ref: https://github.com/ibis-project/ibis/issues/2098
diff --git a/ibis/backends/bigquery/tests/udf/test_udf_execute.py b/ibis/backends/bigquery/tests/udf/test_udf_execute.py
--- a/ibis/backends/bigquery/tests/udf/test_udf_execute.py
+++ b/ibis/backends/bigquery/tests/udf/test_udf_execute.py
@@ -9,7 +9,7 @@
 import ibis.expr.datatypes as dt
 from ibis.compat import PY38
 
-from . import udf  # noqa: E402
+from ... import udf  # noqa: E402
 
 pytest.importorskip('google.cloud.bigquery')