Repository: CartoDB/cartoframes
Pull request #1311 (2019-12-03 20:51:33)

DO info
Fix:
- the cartoframes part of https://github.com/CartoDB/cartodb-platform/issues/5896
- https://github.com/CartoDB/cartoframes/issues/1309
- https://github.com/CartoDB/cartoframes/issues/1314

**Note:** it needs a new version of `carto-python`: https://github.com/CartoDB/carto-python/pull/155

### Acceptance:

- [x] Step 1: using user without any dataset or geography subscribed:
  - [x]  enrich private dataset returns the not subscribed error (not other different: https://github.com/CartoDB/cartoframes/issues/1314)
  - [x] download public dataset works
  - [x] download public geography works
  - [x] download private data fails (again: returning the right error)
  - [x] Enrich public dataset works


- [ ] Step 2:
  - [x] subscribe dataset and geo works
  - [x] Download private dataset works
  - [x] Download private geography works
  - [x] Enrich private dataset
  - [ ] enrich with project quotas to 0 should fail


- [ ] Step 3:
  - [ ] Download using file_path parameter




Patch:
diff --git a/cartoframes/auth/credentials.py b/cartoframes/auth/credentials.py
index 1577042c..19c15472 100644
--- a/cartoframes/auth/credentials.py
+++ b/cartoframes/auth/credentials.py
@@ -210,18 +210,15 @@ class Credentials(object):
         except OSError:
             warnings.warn('No credential file found at {}.'.format(path_to_remove))
 
-    def get_do_token(self):
-        """Returns the Data Observatory v2 token"""
+    def get_do_credentials(self):
+        """Returns the Data Observatory v2 credentials"""
 
         do_token_manager = DoTokenManager(self.get_api_key_auth_client())
-        token = do_token_manager.get()
-        if not token:
+        do_credentials = do_token_manager.get()
+        if not do_credentials:
             raise CartoException('Authentication error: do you have permissions to access Data Observatory v2?')
 
-        return token.access_token
-
-    def get_do_user_dataset(self):
-        return self._username.replace('-', '_')
+        return do_credentials
 
     def get_api_key_auth_client(self):
         if not self._api_key_auth_client:
diff --git a/cartoframes/data/clients/bigquery_client.py b/cartoframes/data/clients/bigquery_client.py
index abeb4566..a59a2bf1 100644
--- a/cartoframes/data/clients/bigquery_client.py
+++ b/cartoframes/data/clients/bigquery_client.py
@@ -21,7 +21,7 @@ def refresh_clients(func):
         try:
             return func(self, *args, **kwargs)
         except RefreshError:
-            self.bq_client, self.gcs_client = self._init_clients()
+            self._init_clients()
             try:
                 return func(self, *args, **kwargs)
             except RefreshError:
@@ -32,31 +32,42 @@ def refresh_clients(func):
 
 class BigQueryClient(object):
 
-    def __init__(self, project, credentials):
-        self._project = project
+    def __init__(self, credentials):
         self._credentials = credentials or get_default_credentials()
-        self._bucket = 'carto-do-{username}'.format(username=self._credentials.username)
-        self.bq_client, self.gcs_client = self._init_clients()
+        self.bq_client = None
+        self.gcs_client = None
+
+        self.bq_public_project = None
+        self.bq_project = None
+        self.bq_dataset = None
+        self.instant_licensing = None
+        self._gcs_bucket = None
+
+        self._init_clients()
 
     def _init_clients(self):
-        google_credentials = GoogleCredentials(self._credentials.get_do_token())
+        do_credentials = self._credentials.get_do_credentials()
+        google_credentials = GoogleCredentials(do_credentials.access_token)
 
-        bq_client = bigquery.Client(
-            project=self._project,
+        self.bq_client = bigquery.Client(
+            project=do_credentials.gcp_execution_project,
             credentials=google_credentials)
 
-        gcs_client = storage.Client(
-            project=self._project,
+        self.gcs_client = storage.Client(
+            project=do_credentials.bq_project,
             credentials=google_credentials
         )
 
-        return bq_client, gcs_client
+        self.bq_public_project = do_credentials.bq_public_project
+        self.bq_project = do_credentials.bq_project
+        self.bq_dataset = do_credentials.bq_dataset
+        self.instant_licensing = do_credentials.instant_licensing
+        self._gcs_bucket = do_credentials.gcs_bucket
 
     @refresh_clients
-    def upload_dataframe(self, dataframe, schema, tablename, project, dataset):
-
+    def upload_dataframe(self, dataframe, schema, tablename):
         # Upload file to Google Cloud Storage
-        bucket = self.gcs_client.bucket(self._bucket)
+        bucket = self.gcs_client.get_bucket(self._gcs_bucket)
         blob = bucket.blob(tablename, chunk_size=_GCS_CHUNK_SIZE)
         dataframe.to_csv(tablename, index=False, header=False)
         try:
@@ -65,14 +76,14 @@ class BigQueryClient(object):
             os.remove(tablename)
 
         # Import from GCS To BigQuery
-        dataset_ref = self.bq_client.dataset(dataset, project=project)
+        dataset_ref = self.bq_client.dataset(self.bq_dataset, project=self.bq_project)
         table_ref = dataset_ref.table(tablename)
         schema_wrapped = [bigquery.SchemaField(column, dtype) for column, dtype in schema.items()]
 
         job_config = bigquery.LoadJobConfig()
         job_config.schema = schema_wrapped
         job_config.source_format = bigquery.SourceFormat.CSV
-        uri = 'gs://{bucket}/{tablename}'.format(bucket=self._bucket, tablename=tablename)
+        uri = 'gs://{bucket}/{tablename}'.format(bucket=self._gcs_bucket, tablename=tablename)
 
         job = self.bq_client.load_table_from_uri(
             uri, table_ref, job_config=job_config
@@ -93,8 +104,8 @@ class BigQueryClient(object):
         table_info = self.get_table(project, dataset, table)
         return [field.name for field in table_info.schema]
 
-    def download_to_file(self, project, dataset, table, limit=None, offset=None,
-                         file_path=None, fail_if_exists=False, progress_bar=True):
+    def download_to_file(self, project, dataset, table, file_path=None, limit=None, offset=None,
+                         fail_if_exists=False, progress_bar=True):
         if not file_path:
             file_name = '{}.{}.{}.csv'.format(project, dataset, table)
             file_path = os.path.join(_USER_CONFIG_DIR, file_name)
diff --git a/cartoframes/data/observatory/catalog/dataset.py b/cartoframes/data/observatory/catalog/dataset.py
index df97607f..2c4a2967 100644
--- a/cartoframes/data/observatory/catalog/dataset.py
+++ b/cartoframes/data/observatory/catalog/dataset.py
@@ -366,14 +366,15 @@ class Dataset(CatalogEntity):
 
         return cls._entity_repo.get_all(filters, credentials)
 
-    def download(self, credentials=None):
+    def download(self, credentials=None, file_path=None):
         """Download dataset data as a local file. You need Data Observatory enabled in your CARTO
         account, please contact us at support@carto.com for more information.
 
-        For premium geographies (those with `is_public_data` set to False), you need a subscription to the geography.
+        For premium datasets (those with `is_public_data` set to False), you need a subscription to the dataset.
         Check the subscription guides for more information.
 
         Args:
+            file_path (str, optional): the file path where save the dataset
             credentials (:py:class:`Credentials <cartoframes.auth.Credentials>`, optional):
                 credentials of CARTO user account. If not provided,
                 a default credentials (if set with :py:meth:`set_default_credentials
@@ -388,7 +389,7 @@ class Dataset(CatalogEntity):
         if not self._is_subscribed(credentials):
             raise CartoException('You are not subscribed to this Dataset yet. Please, use the subscribe method first.')
 
-        return self._download(credentials)
+        return self._download(credentials, file_path)
 
     @classmethod
     def get_datasets_spatial_filtered(cls, filter_dataset):
@@ -487,7 +488,7 @@ class Dataset(CatalogEntity):
 
         datasets = Dataset.get_all({}, _credentials)
 
-        return self in datasets
+        return datasets is not None and self in datasets
 
     def _get_summary_data(self):
         data = self.data.get('summary_json')
@@ -497,3 +498,6 @@ class Dataset(CatalogEntity):
         else:
             log.info('Summary information is not available')
             return None
+
+    def __str__(self):
+        return "<Dataset.get('{}')>".format(self._get_print_id())
diff --git a/cartoframes/data/observatory/catalog/entity.py b/cartoframes/data/observatory/catalog/entity.py
index 9165a917..a44b6dfc 100644
--- a/cartoframes/data/observatory/catalog/entity.py
+++ b/cartoframes/data/observatory/catalog/entity.py
@@ -1,5 +1,4 @@
 import pandas as pd
-from warnings import warn
 
 from google.api_core.exceptions import NotFound
 
@@ -7,6 +6,7 @@ from carto.exceptions import CartoException
 
 from ...clients.bigquery_client import BigQueryClient
 from ....auth import Credentials, defaults
+from ....core.logger import log
 
 try:
     from abc import ABC
@@ -14,7 +14,6 @@ except ImportError:
     from abc import ABCMeta
     ABC = ABCMeta('ABC', (object,), {'__slots__': ()})
 
-_WORKING_PROJECT = 'carto-do-customers'
 _PLATFORM_BQ = 'bq'
 
 
@@ -115,24 +114,28 @@ class CatalogEntity(ABC):
 
         return self.id
 
-    def _download(self, credentials=None):
+    def _download(self, credentials=None, file_path=None):
         if not self._is_available_in('bq'):
             raise CartoException('{} is not ready for Download. Please, contact us for more information.'.format(self))
 
         credentials = self._get_credentials(credentials)
-        user_dataset = credentials.get_do_user_dataset()
-        bq_client = _get_bigquery_client(_WORKING_PROJECT, credentials)
+        bq_client = _get_bigquery_client(credentials)
 
-        project, dataset, table = self.id.split('.')
-        view = 'view_{}_{}'.format(dataset.replace('-', '_'), table)
+        full_remote_table_name = self._get_remote_full_table_name(
+            bq_client.bq_project,
+            bq_client.bq_dataset,
+            bq_client.bq_public_project
+        )
+
+        project, dataset, table = full_remote_table_name.split('.')
 
         try:
-            file_path = bq_client.download_to_file(_WORKING_PROJECT, user_dataset, view)
+            file_path = bq_client.download_to_file(project, dataset, table, file_path)
         except NotFound:
             raise CartoException('You have not purchased the dataset `{}` yet'.format(self.id))
 
-        warn('Data saved: {}.'.format(file_path))
-        warn("To read it you can do: `pandas.read_csv('{}')`.".format(file_path))
+        log.info('Data saved: {}.'.format(file_path))
+        log.info("To read it you can do: `pandas.read_csv('{}')`.".format(file_path))
 
         return file_path
 
@@ -147,9 +150,21 @@ class CatalogEntity(ABC):
 
         return _credentials
 
+    def _get_remote_full_table_name(self, user_project, user_dataset, public_project):
+        project, dataset, table = self.id.split('.')
+
+        if project != public_project:
+            return '{project}.{dataset}.{table_name}'.format(
+                project=user_project,
+                dataset=user_dataset,
+                table_name='view_{}_{}'.format(dataset, table)
+            )
+        else:
+            return self.id
+
 
-def _get_bigquery_client(project, credentials):
-    return BigQueryClient(project, credentials)
+def _get_bigquery_client(credentials):
+    return BigQueryClient(credentials)
 
 
 def is_slug_value(id_value):
diff --git a/cartoframes/data/observatory/catalog/geography.py b/cartoframes/data/observatory/catalog/geography.py
index 4c0961d7..0033fa10 100644
--- a/cartoframes/data/observatory/catalog/geography.py
+++ b/cartoframes/data/observatory/catalog/geography.py
@@ -200,14 +200,15 @@ class Geography(CatalogEntity):
 
         return cls._entity_repo.get_all(filters, credentials)
 
-    def download(self, credentials=None):
-        """Download Geography data as a pandas DataFrame locally. You need Data Observatory enabled in your CARTO
+    def download(self, credentials=None, file_path=None):
+        """Download geography data as a local file. You need Data Observatory enabled in your CARTO
         account, please contact us at support@carto.com for more information.
 
         For premium geographies (those with `is_public_data` set to False), you need a subscription to the geography.
         Check the subscription guides for more information.
 
         Args:
+            file_path (str, optional): the file path where save the dataset
             credentials (:py:class:`Credentials <cartoframes.auth.Credentials>`, optional):
                 credentials of CARTO user account. If not provided,
                 a default credentials (if set with :py:meth:`set_default_credentials
@@ -223,7 +224,7 @@ class Geography(CatalogEntity):
             raise CartoException('You are not subscribed to this Geography yet. Please, use the subscribe method '
                                  'first.')
 
-        return self._download(credentials)
+        return self._download(credentials, file_path)
 
     def subscribe(self, credentials=None):
         """Subscribe to a Geography. You need Data Observatory enabled in your CARTO account, please contact us at
@@ -292,4 +293,4 @@ class Geography(CatalogEntity):
 
         geographies = Geography.get_all({}, _credentials)
 
-        return self in geographies
+        return geographies is not None and self in geographies
diff --git a/cartoframes/data/observatory/enrichment/enrichment_service.py b/cartoframes/data/observatory/enrichment/enrichment_service.py
index f2b5f06b..c74def40 100644
--- a/cartoframes/data/observatory/enrichment/enrichment_service.py
+++ b/cartoframes/data/observatory/enrichment/enrichment_service.py
@@ -15,9 +15,6 @@ from ....utils.geom_utils import to_geojson
 
 _ENRICHMENT_ID = 'enrichment_id'
 _GEOJSON_COLUMN = '__geojson_geom'
-_DEFAULT_PROJECT = 'carto-do'
-_WORKING_PROJECT = 'carto-do-customers'
-_PUBLIC_PROJECT = 'carto-do-public-data'
 
 AGGREGATION_DEFAULT = 'default'
 AGGREGATION_NONE = 'none'
@@ -67,12 +64,12 @@ class EnrichmentService(object):
 
     def __init__(self, credentials=None):
         self.credentials = credentials = credentials or get_default_credentials()
-        self.user_dataset = self.credentials.get_do_user_dataset()
-        self.bq_client = bigquery_client.BigQueryClient(_WORKING_PROJECT, credentials)
+        self.bq_client = bigquery_client.BigQueryClient(credentials)
+        self.bq_dataset = self.bq_client.bq_dataset
+        self.bq_project = self.bq_client.bq_project
+        self.bq_public_project = self.bq_client.bq_public_project
         self.enrichment_id = _ENRICHMENT_ID
         self.geojson_column = _GEOJSON_COLUMN
-        self.working_project = _WORKING_PROJECT
-        self.public_project = _PUBLIC_PROJECT
 
     def _execute_enrichment(self, queries, cartodataframe):
         dfs_enriched = list()
@@ -117,9 +114,7 @@ class EnrichmentService(object):
         self.bq_client.upload_dataframe(
             dataframe=bq_dataframe,
             schema=schema,
-            tablename=tablename,
-            project=self.working_project,
-            dataset=self.user_dataset
+            tablename=tablename
         )
 
     def _get_tables_metadata(self, variables):
@@ -141,7 +136,7 @@ class EnrichmentService(object):
         return tables_metadata
 
     def __get_enrichment_table(self, variable):
-        if variable.project_name != self.public_project:
+        if variable.project_name != self.bq_public_project:
             return 'view_{dataset}_{table}'.format(
                 dataset=variable.schema_name,
                 table=variable.dataset_name
@@ -150,10 +145,10 @@ class EnrichmentService(object):
             return variable.dataset_name
 
     def __get_dataset(self, variable, table_name):
-        if variable.project_name != self.public_project:
+        if variable.project_name != self.bq_public_project:
             return '{project}.{dataset}.{table_name}'.format(
-                project=self.working_project,
-                dataset=self.user_dataset,
+                project=self.bq_project,
+                dataset=self.bq_dataset,
                 table_name=table_name
             )
         else:
@@ -163,41 +158,41 @@ class EnrichmentService(object):
         geography_id = Dataset.get(variable.dataset).geography
         _, dataset_geo_table, geo_table = geography_id.split('.')
 
-        if variable.project_name != self.public_project:
+        if variable.project_name != self.bq_public_project:
             return '{project}.{dataset}.view_{dataset_geo_table}_{geo_table}'.format(
-                project=self.working_project,
-                dataset=self.user_dataset,
+                project=self.bq_project,
+                dataset=self.bq_dataset,
                 dataset_geo_table=dataset_geo_table,
                 geo_table=geo_table
             )
         else:
             return '{project}.{dataset}.{geo_table}'.format(
-                project=self.public_project,
+                project=self.bq_public_project,
                 dataset=dataset_geo_table,
                 geo_table=geo_table
             )
 
     def __get_project(self, variable):
-        project = self.public_project
+        project = self.bq_public_project
 
-        if variable.project_name != self.public_project:
-            project = self.working_project
+        if variable.project_name != self.bq_public_project:
+            project = self.bq_project
 
         return project
 
     def _get_points_enrichment_sql(self, temp_table_name, variables, filters):
         tables_metadata = self._get_tables_metadata(variables).items()
 
-        return [self._build_points_query(table, metadata, temp_table_name, filters)
-                for table, metadata in tables_metadata]
+        return [self._build_points_query(metadata, temp_table_name, filters)
+                for _, metadata in tables_metadata]
 
-    def _build_points_query(self, table, metadata, temp_table_name, filters):
+    def _build_points_query(self, metadata, temp_table_name, filters):
         variables = ['enrichment_table.{}'.format(variable.column_name) for variable in metadata['variables']]
         enrichment_dataset = metadata['dataset']
         enrichment_geo_table = metadata['geo_table']
         data_table = '{project}.{user_dataset}.{temp_table_name}'.format(
-            project=self.working_project,
-            user_dataset=self.user_dataset,
+            project=self.bq_project,
+            user_dataset=self.bq_dataset,
             temp_table_name=temp_table_name
         )
 
@@ -217,23 +212,22 @@ class EnrichmentService(object):
             enrichment_geo_table=enrichment_geo_table,
             enrichment_id=self.enrichment_id,
             where=self._build_where_clausule(filters),
-            data_table=data_table,
-            table=table
+            data_table=data_table
         )
 
     def _get_polygon_enrichment_sql(self, temp_table_name, variables, filters, aggregation):
         tables_metadata = self._get_tables_metadata(variables).items()
 
-        return [self._build_polygons_query(table, metadata, temp_table_name, filters, aggregation)
-                for table, metadata in tables_metadata]
+        return [self._build_polygons_query(metadata, temp_table_name, filters, aggregation)
+                for _, metadata in tables_metadata]
 
-    def _build_polygons_query(self, table, metadata, temp_table_name, filters, aggregation):
+    def _build_polygons_query(self, metadata, temp_table_name, filters, aggregation):
         variables = metadata['variables']
         enrichment_dataset = metadata['dataset']
         enrichment_geo_table = metadata['geo_table']
         data_table = '{project}.{user_dataset}.{temp_table_name}'.format(
-            project=self.working_project,
-            user_dataset=self.user_dataset,
+            project=self.bq_project,
+            user_dataset=self.bq_dataset,
             temp_table_name=temp_table_name
         )
 
diff --git a/setup.py b/setup.py
index 5ce1c38c..f5390199 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ def get_version():
 
 REQUIRES = [
     'appdirs>=1.4.3,<2.0',
-    'carto>=1.8.1,<2.0',
+    'carto>=1.8.2,<2.0',
     'jinja2>=2.10.1,<3.0',
     'geopandas>=0.6.0,<1.0',
     'tqdm>=4.32.1,<5.0',


Test patch:
diff --git a/tests/e2e/data/client/test_bigquery_client.py b/tests/e2e/data/client/test_bigquery_client.py
index 779961d6..f77fff51 100644
--- a/tests/e2e/data/client/test_bigquery_client.py
+++ b/tests/e2e/data/client/test_bigquery_client.py
@@ -65,7 +65,7 @@ class TestBigQueryClient(unittest.TestCase):
             os.remove(self.file_path)
 
     def test_instantiation(self):
-        bq_client = BigQueryClient(_WORKING_PROJECT, self.credentials)
+        bq_client = BigQueryClient(self.credentials)
         assert isinstance(bq_client, BigQueryClient)
 
     def test_refresh_token_raises_cartoexception(self):
@@ -73,7 +73,7 @@ class TestBigQueryClient(unittest.TestCase):
         original_query_method = BigQueryClient.query
         bigquery.Client.query = refresh_token_checker.query_raiser
 
-        bq_client = BigQueryClient(_WORKING_PROJECT, self.credentials)
+        bq_client = BigQueryClient(self.credentials)
         with pytest.raises(CartoException):
             bq_client.query('select * from')
 
@@ -85,7 +85,7 @@ class TestBigQueryClient(unittest.TestCase):
         original_query_method = BigQueryClient.query
         bigquery.Client.query = refresh_token_checker.query_raiser
 
-        bq_client = BigQueryClient(_WORKING_PROJECT, self.credentials)
+        bq_client = BigQueryClient(self.credentials)
         response = bq_client.query('select * from')
         assert response == expected_response
 
@@ -105,7 +105,7 @@ class TestBigQueryClient(unittest.TestCase):
         table = 'fake_table'
         file_path = self.file_path
 
-        bq_client = BigQueryClient(project, self.credentials)
+        bq_client = BigQueryClient(self.credentials)
         bq_client.download_to_file(project, dataset, table, file_path=file_path, progress_bar=False)
 
         self.assertTrue(os.path.isfile(file_path))
@@ -128,7 +128,7 @@ class TestBigQueryClient(unittest.TestCase):
         table = 'fake_table'
         file_path = self.file_path
 
-        bq_client = BigQueryClient(project, self.credentials)
+        bq_client = BigQueryClient(self.credentials)
 
         with open(file_path, 'w'):
             with self.assertRaises(CartoException):
diff --git a/tests/unit/auth/test_credentials.py b/tests/unit/auth/test_credentials.py
index e03d0473..193531b7 100644
--- a/tests/unit/auth/test_credentials.py
+++ b/tests/unit/auth/test_credentials.py
@@ -143,7 +143,7 @@ class TestCredentials(object):
         credentials.get_api_key_auth_client()
         assert credentials._api_key_auth_client is not None
 
-    def test_get_do_token(self, mocker):
+    def test_get_do_credentials(self, mocker):
         access_token = '1234'
 
         class Token:
@@ -153,9 +153,9 @@ class TestCredentials(object):
         mocker.patch('carto.do_token.DoTokenManager.get', return_value=Token())
 
         credentials = Credentials(self.username, self.api_key)
-        token = credentials.get_do_token()
+        do_credentials = credentials.get_do_credentials()
 
-        assert token == access_token
+        assert do_credentials.access_token == access_token
 
 
 class TestCredentialsFromFile(object):
diff --git a/tests/unit/data/observatory/catalog/mocks.py b/tests/unit/data/observatory/catalog/mocks.py
index c77ee347..bea3160e 100644
--- a/tests/unit/data/observatory/catalog/mocks.py
+++ b/tests/unit/data/observatory/catalog/mocks.py
@@ -2,7 +2,12 @@ class BigQueryClientMock(object):
     def __init__(self, response):
         self.response = response
 
-    def download_to_file(self, _1, _2, _3):
+        self.bq_public_project = 'public_data_project'
+        self.bq_project = 'user_data_project'
+        self.bq_dataset = 'username'
+        self._gcs_bucket = 'bucket_name'
+
+    def download_to_file(self, _1, _2, _3, _4):
         if isinstance(self.response, Exception):
             raise self.response
         else:
diff --git a/tests/unit/data/observatory/catalog/test_dataset.py b/tests/unit/data/observatory/catalog/test_dataset.py
index 24c14100..caf40929 100644
--- a/tests/unit/data/observatory/catalog/test_dataset.py
+++ b/tests/unit/data/observatory/catalog/test_dataset.py
@@ -171,7 +171,7 @@ class TestDataset(object):
         dataset_str = str(dataset)
 
         # Then
-        assert dataset_str == 'Dataset({dict_str})'.format(dict_str=str(db_dataset1))
+        assert dataset_str == "<Dataset.get('{id}')>".format(id=db_dataset1['slug'])
 
     def test_summary_values(self):
         # Given
diff --git a/tests/unit/data/observatory/enrichment/test_enrichment_service.py b/tests/unit/data/observatory/enrichment/test_enrichment_service.py
index 9c20df48..c612b436 100644
--- a/tests/unit/data/observatory/enrichment/test_enrichment_service.py
+++ b/tests/unit/data/observatory/enrichment/test_enrichment_service.py
@@ -1,3 +1,4 @@
+from google.cloud import bigquery, storage
 import pytest
 import pandas as pd
 from shapely.geometry.point import Point
@@ -19,16 +20,37 @@ try:
 except ImportError:
     from mock import Mock, patch
 
+_WORKING_PROJECT = 'carto-do-customers'
+_PUBLIC_PROJECT = 'carto-do-public-data'
+
+
+class DoCredentials:
+    def __init__(self, public_data_project, user_data_project, access_token='access_token', instant_licensing=False,
+                 execution_project='execution_project', dataset='username', bucket='bucket'):
+        self.access_token = access_token
+        self.gcp_execution_project = execution_project
+        self.bq_public_project = public_data_project
+        self.bq_project = user_data_project
+        self.bq_dataset = dataset
+        self.gcs_bucket = bucket
+        self.instant_licensing = instant_licensing
+
 
 class TestEnrichmentService(object):
     def setup_method(self):
-        self.original_init_clients = BigQueryClient._init_clients
-        BigQueryClient._init_clients = Mock(return_value=(True, True))
+        self.original_bigquery_Client = bigquery.Client
+        bigquery.Client = Mock(return_value=True)
+        self.original_storage_Client = storage.Client
+        storage.Client = Mock(return_value=True)
+        self.original_get_do_credentials = Credentials.get_do_credentials
+        Credentials.get_do_credentials = Mock(return_value=DoCredentials(_PUBLIC_PROJECT, _WORKING_PROJECT))
         self.credentials = Credentials('username', 'apikey')
 
     def teardown_method(self):
+        bigquery.Client = self.original_bigquery_Client
+        storage.Client = self.original_storage_Client
+        Credentials.get_do_credentials = self.original_get_do_credentials
         self.credentials = None
-        BigQueryClient._init_clients = self.original_init_clients
 
     def test_prepare_data_no_geom(self):
         geom_column = 'the_geom'
@@ -83,7 +105,6 @@ class TestEnrichmentService(object):
 
     def test_upload_data(self):
         geom_column = 'the_geom'
-        expected_project = 'carto-do-customers'
         user_dataset = 'test_dataset'
 
         point = Point(1, 1)
@@ -99,13 +120,11 @@ class TestEnrichmentService(object):
             columns=[_ENRICHMENT_ID, _GEOJSON_COLUMN])
 
         # mock
-        def assert_upload_data(_, dataframe, schema, tablename, project, dataset):
+        def assert_upload_data(_, dataframe, schema, tablename):
             assert dataframe.equals(expected_cdf)
             assert schema == expected_schema
             assert isinstance(tablename, str) and len(tablename) > 0
-            assert project == expected_project
             assert tablename == user_dataset
-            assert dataset == 'username'
 
         enrichment_service = EnrichmentService(credentials=self.credentials)
         original = BigQueryClient.upload_dataframe
@@ -116,7 +135,6 @@ class TestEnrichmentService(object):
 
     def test_upload_data_null_geometries(self):
         geom_column = 'the_geom'
-        expected_project = 'carto-do-customers'
         user_dataset = 'test_dataset'
 
         point = Point(1, 1)
@@ -135,13 +153,11 @@ class TestEnrichmentService(object):
             columns=[_ENRICHMENT_ID, _GEOJSON_COLUMN])
 
         # mock
-        def assert_upload_data(_, dataframe, schema, tablename, project, dataset):
+        def assert_upload_data(_, dataframe, schema, tablename):
             assert dataframe.equals(expected_cdf)
             assert schema == expected_schema
             assert isinstance(tablename, str) and len(tablename) > 0
-            assert project == expected_project
             assert tablename == user_dataset
-            assert dataset == 'username'
 
         enrichment_service = EnrichmentService(credentials=self.credentials)
         original = BigQueryClient.upload_dataframe
diff --git a/tests/unit/data/observatory/enrichment/test_points_enrichment.py b/tests/unit/data/observatory/enrichment/test_points_enrichment.py
index 780aa697..b769e63e 100644
--- a/tests/unit/data/observatory/enrichment/test_points_enrichment.py
+++ b/tests/unit/data/observatory/enrichment/test_points_enrichment.py
@@ -1,5 +1,6 @@
+from google.cloud import bigquery, storage
+
 from cartoframes.auth import Credentials
-from cartoframes.data.clients.bigquery_client import BigQueryClient
 from cartoframes.data.observatory import Enrichment, Variable, Dataset, VariableFilter
 from cartoframes.data.observatory.enrichment.enrichment_service import _GEOJSON_COLUMN
 
@@ -8,6 +9,21 @@ try:
 except ImportError:
     from mock import Mock, patch
 
+_WORKING_PROJECT = 'carto-do-customers'
+_PUBLIC_PROJECT = 'carto-do-public-data'
+
+
+class DoCredentials:
+    def __init__(self, public_data_project, user_data_project, access_token='access_token', instant_licensing=False,
+                 execution_project='execution_project', dataset='username', bucket='bucket'):
+        self.access_token = access_token
+        self.gcp_execution_project = execution_project
+        self.bq_public_project = public_data_project
+        self.bq_project = user_data_project
+        self.bq_dataset = dataset
+        self.gcs_bucket = bucket
+        self.instant_licensing = instant_licensing
+
 
 class CatalogEntityWithGeographyMock:
     def __init__(self, geography):
@@ -16,17 +32,20 @@ class CatalogEntityWithGeographyMock:
 
 class TestPointsEnrichment(object):
     def setup_method(self):
-        self.original_init_clients = BigQueryClient._init_clients
-        BigQueryClient._init_clients = Mock(return_value=(True, True))
+        self.original_bigquery_Client = bigquery.Client
+        bigquery.Client = Mock(return_value=True)
+        self.original_storage_Client = storage.Client
+        storage.Client = Mock(return_value=True)
+        self.original_get_do_credentials = Credentials.get_do_credentials
+        Credentials.get_do_credentials = Mock(return_value=DoCredentials(_PUBLIC_PROJECT, _WORKING_PROJECT))
         self.username = 'username'
         self.apikey = 'apikey'
         self.credentials = Credentials(self.username, self.apikey)
 
     def teardown_method(self):
-        self.credentials = None
-        self.apikey = None
-        self.username = None
-        BigQueryClient._init_clients = self.original_init_clients
+        bigquery.Client = self.original_bigquery_Client
+        storage.Client = self.original_storage_Client
+        Credentials.get_do_credentials = self.original_get_do_credentials
 
     @patch.object(Dataset, 'get')
     def test_enrichment_query_by_points_one_variable(self, dataset_get_mock):
diff --git a/tests/unit/data/observatory/enrichment/test_polygon_enrichment.py b/tests/unit/data/observatory/enrichment/test_polygon_enrichment.py
index a80f08e2..a5413f9d 100644
--- a/tests/unit/data/observatory/enrichment/test_polygon_enrichment.py
+++ b/tests/unit/data/observatory/enrichment/test_polygon_enrichment.py
@@ -1,14 +1,30 @@
+from google.cloud import bigquery, storage
+
 from cartoframes.auth import Credentials
-from cartoframes.data.clients.bigquery_client import BigQueryClient
 from cartoframes.data.observatory import Enrichment, Variable, Dataset, VariableFilter
-from cartoframes.data.observatory.enrichment.enrichment_service import _WORKING_PROJECT, _PUBLIC_PROJECT, \
-    AGGREGATION_DEFAULT, AGGREGATION_NONE, prepare_variables, _GEOJSON_COLUMN
+from cartoframes.data.observatory.enrichment.enrichment_service import AGGREGATION_DEFAULT, AGGREGATION_NONE, \
+    prepare_variables, _GEOJSON_COLUMN
 
 try:
     from unittest.mock import Mock, patch
 except ImportError:
     from mock import Mock, patch
 
+_WORKING_PROJECT = 'carto-do-customers'
+_PUBLIC_PROJECT = 'carto-do-public-data'
+
+
+class DoCredentials:
+    def __init__(self, public_data_project, user_data_project, access_token='access_token', instant_licensing=False,
+                 execution_project='execution_project', dataset='username', bucket='bucket'):
+        self.access_token = access_token
+        self.gcp_execution_project = execution_project
+        self.bq_public_project = public_data_project
+        self.bq_project = user_data_project
+        self.bq_dataset = dataset
+        self.gcs_bucket = bucket
+        self.instant_licensing = instant_licensing
+
 
 class CatalogEntityWithGeographyMock:
     def __init__(self, geography):
@@ -17,15 +33,21 @@ class CatalogEntityWithGeographyMock:
 
 class TestPolygonEnrichment(object):
     def setup_method(self):
-        self.original_init_clients = BigQueryClient._init_clients
-        BigQueryClient._init_clients = Mock(return_value=(True, True))
+        self.original_bigquery_Client = bigquery.Client
+        bigquery.Client = Mock(return_value=True)
+        self.original_storage_Client = storage.Client
+        storage.Client = Mock(return_value=True)
+        self.original_get_do_credentials = Credentials.get_do_credentials
+        Credentials.get_do_credentials = Mock(return_value=DoCredentials(_PUBLIC_PROJECT, _WORKING_PROJECT))
         self.username = 'username'
         self.apikey = 'apikey'
         self.credentials = Credentials(self.username, self.apikey)
 
     def teardown_method(self):
+        bigquery.Client = self.original_bigquery_Client
+        storage.Client = self.original_storage_Client
+        Credentials.get_do_credentials = self.original_get_do_credentials
         self.credentials = None
-        BigQueryClient._init_clients = self.original_init_clients
 
     @patch('cartoframes.data.observatory.enrichment.enrichment_service._is_available_in_bq')
     @patch.object(Dataset, 'get')