Repository: googleapis/python-storage

Issue:
"Anonymous credentials cannot be refreshed" with fake-gcs-server
  - OS type and version: Ubuntu 18.04
  - Python version: 3.7.6
  - pip version: 20.0.2
  - `google-cloud-storage` version: 1.27.0

I'm trying to use [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) for local GCS testing.

Basic client setup works fine:
```python
>>> creds = AnonymousCredentials() 
>>> options = ClientOptions(api_endpoint="http://127.0.0.1:4443") 
>>> client = gcs.Client(credentials=creds, client_options=options, project='') 
```

I can list buckets and blobs:
```python
>>> print(list(client.list_buckets()))                                                                                                                                      
[<Bucket: bucket1>, <Bucket: bucket2>]
>>> print(list(client.list_blobs('bucket1')))                                                                                                                               
[<Blob: bucket1, hello.txt, 1586453246295809>,
 <Blob: bucket1, subdir/toto.txt, 1586453246295810>]
>>> a, b = client.list_blobs('bucket1')                                                                                                                                     
>>> a.size                                                                                                                                                                  
548
>>> a.name                                                                                                                                                                  
'hello.txt'
>>> a.id                                                                                                                                                                    
'bucket1/hello.txt'
```

`fake-gcs-server` doesn't need any credentials, so reading the file should just work. But for some reason it doesn't:
```python
>>> a.download_as_string()                                                                                                                                                  
Traceback (most recent call last):
  File "<ipython-input-19-72ec41b6efbf>", line 1, in <module>
    a.download_as_string()
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 879, in download_as_string
    raw_download=raw_download,
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 796, in download_to_file
    transport, file_obj, download_url, headers, start, end, raw_download
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 720, in _do_download
    download.consume(transport)
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/resumable_media/requests/download.py", line 151, in consume
    result = _helpers.http_request(transport, method, url, **request_kwargs)
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/resumable_media/requests/_helpers.py", line 136, in http_request
    return _helpers.wait_and_retry(func, RequestsMixin._get_status_code, retry_strategy)
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/resumable_media/_helpers.py", line 150, in wait_and_retry
    response = func()
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/auth/transport/requests.py", line 482, in request
    self.credentials.refresh(auth_request)
  File "/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/site-packages/google/auth/credentials.py", line 148, in refresh
    raise ValueError("Anonymous credentials cannot be refreshed.")
ValueError: Anonymous credentials cannot be refreshed.
```


Discussion:
HemangChothani: @frand still we can refactor, need your suggestion.

-  We can remove [_get_transport_method()](https://github.com/googleapis/python-storage/blob/cecc7ac95a00ab1437a24bee44f8b6c5b8a7189d/google/cloud/storage/blob.py#L638) from blob class as we are calling [_require_client()](https://github.com/googleapis/python-storage/pull/107/files#diff-5d3277a5f0f072a447a1eb89f9fa1ae0R796) method just above  _get_transport_method() call, so no need of that method and we can also directly pass client._http.

OR

- We can modify return type of [_get_transport_method](https://github.com/googleapis/python-storage/blob/cecc7ac95a00ab1437a24bee44f8b6c5b8a7189d/google/cloud/storage/blob.py#L638) method, as of now it returns transport object but also need to return client object as well, so no need to call [_require_client()](https://github.com/googleapis/python-storage/pull/107/files#diff-5d3277a5f0f072a447a1eb89f9fa1ae0R796) in the respective method where we are using `*_URL_TEMPLATE` global variables. 
frankyn: @HemangChothani asked @crwilcox to take another pass to confirm.

Patch:
diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py
index 4200c9d..d341661 100644
--- a/google/cloud/storage/blob.py
+++ b/google/cloud/storage/blob.py
@@ -54,7 +54,6 @@ from google.cloud._helpers import _bytes_to_unicode
 from google.cloud._helpers import _rfc3339_to_datetime
 from google.cloud._helpers import _to_bytes
 from google.cloud.exceptions import NotFound
-from google.cloud.storage._helpers import _get_storage_host
 from google.cloud.storage._helpers import _PropertyMixin
 from google.cloud.storage._helpers import _scalar_property
 from google.cloud.storage._helpers import _convert_to_timestamp
@@ -70,12 +69,11 @@ from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS
 from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS
 from google.cloud.storage.constants import STANDARD_STORAGE_CLASS
 
-_STORAGE_HOST = _get_storage_host()
 
 _API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
 _DEFAULT_CONTENT_TYPE = u"application/octet-stream"
-_DOWNLOAD_URL_TEMPLATE = _STORAGE_HOST + u"/download/storage/v1{path}?alt=media"
-_BASE_UPLOAD_TEMPLATE = _STORAGE_HOST + u"/upload/storage/v1{bucket_path}/o?uploadType="
+_DOWNLOAD_URL_TEMPLATE = u"{hostname}/download/storage/v1{path}?alt=media"
+_BASE_UPLOAD_TEMPLATE = u"{hostname}/upload/storage/v1{bucket_path}/o?uploadType="
 _MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"multipart"
 _RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"resumable"
 # NOTE: "acl" is also writeable but we defer ACL management to
@@ -650,19 +648,24 @@ class Blob(_PropertyMixin):
         client = self._require_client(client)
         return client._http
 
-    def _get_download_url(self):
+    def _get_download_url(self, client):
         """Get the download URL for the current blob.
 
         If the ``media_link`` has been loaded, it will be used, otherwise
         the URL will be constructed from the current blob's path (and possibly
         generation) to avoid a round trip.
 
+        :type client: :class:`~google.cloud.storage.client.Client`
+        :param client: The client to use.
+
         :rtype: str
         :returns: The download URL for the current blob.
         """
         name_value_pairs = []
         if self.media_link is None:
-            base_url = _DOWNLOAD_URL_TEMPLATE.format(path=self.path)
+            base_url = _DOWNLOAD_URL_TEMPLATE.format(
+                hostname=client._connection.API_BASE_URL, path=self.path
+            )
             if self.generation is not None:
                 name_value_pairs.append(("generation", "{:d}".format(self.generation)))
         else:
@@ -790,7 +793,8 @@ class Blob(_PropertyMixin):
 
         :raises: :class:`google.cloud.exceptions.NotFound`
         """
-        download_url = self._get_download_url()
+        client = self._require_client(client)
+        download_url = self._get_download_url(client)
         headers = _get_encryption_headers(self._encryption_key)
         headers["accept-encoding"] = "gzip"
 
@@ -1055,7 +1059,9 @@ class Blob(_PropertyMixin):
         info = self._get_upload_arguments(content_type)
         headers, object_metadata, content_type = info
 
-        base_url = _MULTIPART_URL_TEMPLATE.format(bucket_path=self.bucket.path)
+        base_url = _MULTIPART_URL_TEMPLATE.format(
+            hostname=self.client._connection.API_BASE_URL, bucket_path=self.bucket.path
+        )
         name_value_pairs = []
 
         if self.user_project is not None:
@@ -1190,7 +1196,9 @@ class Blob(_PropertyMixin):
         if extra_headers is not None:
             headers.update(extra_headers)
 
-        base_url = _RESUMABLE_URL_TEMPLATE.format(bucket_path=self.bucket.path)
+        base_url = _RESUMABLE_URL_TEMPLATE.format(
+            hostname=self.client._connection.API_BASE_URL, bucket_path=self.bucket.path
+        )
         name_value_pairs = []
 
         if self.user_project is not None:


Test patch:
diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py
index 4ae0e21..bb1aa11 100644
--- a/tests/unit/test_blob.py
+++ b/tests/unit/test_blob.py
@@ -751,7 +751,10 @@ class Test_Blob(unittest.TestCase):
         # Set the media link on the blob
         blob._properties["mediaLink"] = media_link
 
-        download_url = blob._get_download_url()
+        client = mock.Mock(_connection=_Connection)
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
+        download_url = blob._get_download_url(client)
+
         self.assertEqual(download_url, media_link)
 
     def test__get_download_url_with_media_link_w_user_project(self):
@@ -763,7 +766,10 @@ class Test_Blob(unittest.TestCase):
         # Set the media link on the blob
         blob._properties["mediaLink"] = media_link
 
-        download_url = blob._get_download_url()
+        client = mock.Mock(_connection=_Connection)
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
+        download_url = blob._get_download_url(client)
+
         self.assertEqual(
             download_url, "{}?userProject={}".format(media_link, user_project)
         )
@@ -774,7 +780,9 @@ class Test_Blob(unittest.TestCase):
         blob = self._make_one(blob_name, bucket=bucket)
 
         self.assertIsNone(blob.media_link)
-        download_url = blob._get_download_url()
+        client = mock.Mock(_connection=_Connection)
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
+        download_url = blob._get_download_url(client)
         expected_url = (
             "https://storage.googleapis.com/download/storage/v1/b/"
             "buhkit/o/bzzz-fly.txt?alt=media"
@@ -790,7 +798,9 @@ class Test_Blob(unittest.TestCase):
         blob._properties["generation"] = str(generation)
 
         self.assertIsNone(blob.media_link)
-        download_url = blob._get_download_url()
+        client = mock.Mock(_connection=_Connection)
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
+        download_url = blob._get_download_url(client)
         expected_url = (
             "https://storage.googleapis.com/download/storage/v1/b/"
             "fictional/o/pretend.txt?alt=media&generation=1493058489532987"
@@ -804,7 +814,9 @@ class Test_Blob(unittest.TestCase):
         blob = self._make_one(blob_name, bucket=bucket)
 
         self.assertIsNone(blob.media_link)
-        download_url = blob._get_download_url()
+        client = mock.Mock(_connection=_Connection)
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
+        download_url = blob._get_download_url(client)
         expected_url = (
             "https://storage.googleapis.com/download/storage/v1/b/"
             "fictional/o/pretend.txt?alt=media&userProject={}".format(user_project)
@@ -823,7 +835,9 @@ class Test_Blob(unittest.TestCase):
         blob = self._make_one(blob_name, bucket=bucket, kms_key_name=kms_resource)
 
         self.assertIsNone(blob.media_link)
-        download_url = blob._get_download_url()
+        client = mock.Mock(_connection=_Connection)
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
+        download_url = blob._get_download_url(client)
         expected_url = (
             "https://storage.googleapis.com/download/storage/v1/b/"
             "buhkit/o/bzzz-fly.txt?alt=media"
@@ -1003,7 +1017,8 @@ class Test_Blob(unittest.TestCase):
 
     def test_download_to_file_wo_media_link(self):
         blob_name = "blob-name"
-        client = mock.Mock(spec=[u"_http"])
+        client = mock.Mock(_connection=_Connection, spec=[u"_http"])
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
         bucket = _Bucket(client)
         blob = self._make_one(blob_name, bucket=bucket)
         blob._do_download = mock.Mock()
@@ -1319,7 +1334,8 @@ class Test_Blob(unittest.TestCase):
         transport = self._mock_transport(http_client.OK, {})
 
         # Create some mock arguments.
-        client = mock.Mock(_http=transport, spec=["_http"])
+        client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"])
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
         data = b"data here hear hier"
         stream = io.BytesIO(data)
         content_type = u"application/xml"
@@ -1485,7 +1501,8 @@ class Test_Blob(unittest.TestCase):
         transport = self._mock_transport(http_client.OK, response_headers)
 
         # Create some mock arguments and call the method under test.
-        client = mock.Mock(_http=transport, spec=[u"_http"])
+        client = mock.Mock(_http=transport, _connection=_Connection, spec=[u"_http"])
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
         data = b"hello hallo halo hi-low"
         stream = io.BytesIO(data)
         content_type = u"text/plain"
@@ -1772,7 +1789,8 @@ class Test_Blob(unittest.TestCase):
         )
 
         # Create some mock arguments and call the method under test.
-        client = mock.Mock(_http=transport, spec=["_http"])
+        client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"])
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
         stream = io.BytesIO(data)
         content_type = u"text/html"
         response = blob._do_resumable_upload(
@@ -2114,7 +2132,8 @@ class Test_Blob(unittest.TestCase):
         # Create some mock arguments and call the method under test.
         content_type = u"text/plain"
         size = 10000
-        client = mock.Mock(_http=transport, spec=[u"_http"])
+        client = mock.Mock(_http=transport, _connection=_Connection, spec=[u"_http"])
+        client._connection.API_BASE_URL = "https://storage.googleapis.com"
         new_url = blob.create_resumable_upload_session(
             content_type=content_type, size=size, origin=origin, client=client
         )
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index 63eeb37..c3a3935 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -137,18 +137,16 @@ class TestClient(unittest.TestCase):
         )
 
     def test_ctor_w_client_options_dict(self):
-
         PROJECT = "PROJECT"
         credentials = _make_credentials()
-        client_options = {"api_endpoint": "https://www.foo-googleapis.com"}
+        api_endpoint = "https://www.foo-googleapis.com"
+        client_options = {"api_endpoint": api_endpoint}
 
         client = self._make_one(
             project=PROJECT, credentials=credentials, client_options=client_options
         )
 
-        self.assertEqual(
-            client._connection.API_BASE_URL, "https://www.foo-googleapis.com"
-        )
+        self.assertEqual(client._connection.API_BASE_URL, api_endpoint)
 
     def test_ctor_w_client_options_object(self):
         from google.api_core.client_options import ClientOptions