Repository: googleapis/google-auth-library-python

Issue:
Use extra for asyncio dependencies
Hello!  The latest release for this library pulls in aiohttp and its dependencies unconditionally, which adds non-trivial burden to projects that don’t need it.  Would you consider using a packaging extra so that people can opt-in?
TODO: undo pin of 'aiohttp' once 'aioresponses' releases a fix
Environment details

  - OS: $ sw_vers
ProductName:    Mac OS X
ProductVersion: 10.14.6
BuildVersion:   18G6020

  - Python version: 3.6, 3.7, 3.8
  - pip version: pip 20.2.4
  - `google-auth` version: 5906c8583ca351b5385a079a30521a9a8a0c7c59

#### Steps to reproduce

  1. nox -s unit


There are 9 tests that fail, all with the same error:

`TypeError: __init__() missing 1 required positional argument: 'limit'`


```
====================================================== short test summary info =======================================================
FAILED tests_async/transport/test_aiohttp_requests.py::TestCombinedResponse::test_content_compressed - TypeError: __init__() missin...
FAILED tests_async/transport/test_aiohttp_requests.py::TestResponse::test_headers_prop - TypeError: __init__() missing 1 required p...
FAILED tests_async/transport/test_aiohttp_requests.py::TestResponse::test_status_prop - TypeError: __init__() missing 1 required po...
FAILED tests_async/transport/test_aiohttp_requests.py::TestAuthorizedSession::test_request - TypeError: __init__() missing 1 requir...
FAILED tests_async/transport/test_aiohttp_requests.py::TestAuthorizedSession::test_ctx - TypeError: __init__() missing 1 required p...
FAILED tests_async/transport/test_aiohttp_requests.py::TestAuthorizedSession::test_http_headers - TypeError: __init__() missing 1 r...
FAILED tests_async/transport/test_aiohttp_requests.py::TestAuthorizedSession::test_regexp_example - TypeError: __init__() missing 1...
FAILED tests_async/transport/test_aiohttp_requests.py::TestAuthorizedSession::test_request_no_refresh - TypeError: __init__() missi...
FAILED tests_async/transport/test_aiohttp_requests.py::TestAuthorizedSession::test_request_refresh - TypeError: __init__() missing ...
============================================ 9 failed, 609 passed, 12 warnings in 33.41s =============================================
```

Here is the traceback for one of the failing tests:


```
____________________________________________ TestCombinedResponse.test_content_compressed ____________________________________________

self = <tests_async.transport.test_aiohttp_requests.TestCombinedResponse object at 0x108803160>
urllib3_mock = <function decompress at 0x10880a820>

    @mock.patch(
        "google.auth.transport._aiohttp_requests.urllib3.response.MultiDecoder.decompress",
        return_value="decompressed",
        autospec=True,
    )
    @pytest.mark.asyncio
    async def test_content_compressed(self, urllib3_mock):
        rm = core.RequestMatch(
            "url", headers={"Content-Encoding": "gzip"}, payload="compressed"
        )
>       response = await rm.build_response(core.URL("url"))

tests_async/transport/test_aiohttp_requests.py:72: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../.virtualenv/google-auth-library-python/lib/python3.8/site-packages/aioresponses/core.py:192: in build_response
    resp = self._build_response(
../../../.virtualenv/google-auth-library-python/lib/python3.8/site-packages/aioresponses/core.py:173: in _build_response
    resp.content = stream_reader_factory(loop)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

loop = <Mock id='4437587472'>

    def stream_reader_factory(  # noqa
        loop: 'Optional[asyncio.AbstractEventLoop]' = None
    ):
        protocol = ResponseHandler(loop=loop)
>       return StreamReader(protocol, loop=loop)
E       TypeError: __init__() missing 1 required positional argument: 'limit'

../../../.virtualenv/google-auth-library-python/lib/python3.8/site-packages/aioresponses/compat.py:48: TypeError
========================================================== warnings summary ==========================================================
```

The root cause is a change in aiohttp version 3.7.0 which was released a few hours ago. The signature for StreamReader has changed, making the optional argument `limit` a required argument.

https://github.com/aio-libs/aiohttp/blob/56e78836aa7c67292ace9e256711699d51d57285/aiohttp/streams.py#L106

This change breaks aioresponses:

https://github.com/pnuckowski/aioresponses/blob/e61977f42a0164e0c572031dfb18ae95ba198df0/aioresponses/compat.py#L44


Add support for Python 3.9



Discussion:

Resolving patch:
diff --git a/synth.py b/synth.py
--- a/synth.py
+++ b/synth.py
@@ -10,8 +10,8 @@
 s.move(
     templated_files / ".kokoro",
     excludes=[
-        ".kokoro/continuous/common.cfg",
-        ".kokoro/presubmit/common.cfg",
-        ".kokoro/build.sh",
+        "continuous/common.cfg",
+        "presubmit/common.cfg",
+        "build.sh",
     ],
 )  # just move kokoro configs


Test patch:
diff --git a/system_tests/__init__.py b/system_tests/__init__.py
new file mode 100644
diff --git a/system_tests/noxfile.py b/system_tests/noxfile.py
--- a/system_tests/noxfile.py
+++ b/system_tests/noxfile.py
@@ -30,7 +30,7 @@
 import py.path
 
 HERE = os.path.abspath(os.path.dirname(__file__))
-LIBRARY_DIR = os.path.join(HERE, "..")
+LIBRARY_DIR = os.path.abspath(os.path.dirname(HERE))
 DATA_DIR = os.path.join(HERE, "data")
 SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, "service_account.json")
 AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, "authorized_user.json")
@@ -169,7 +169,7 @@ def configure_cloud_sdk(session, application_default_credentials, project=False)
 # Test sesssions
 
 TEST_DEPENDENCIES_ASYNC = ["aiohttp", "pytest-asyncio", "nest-asyncio"]
-TEST_DEPENDENCIES_SYNC = ["pytest", "requests"]
+TEST_DEPENDENCIES_SYNC = ["pytest", "requests", "mock"]
 PYTHON_VERSIONS_ASYNC = ["3.7"]
 PYTHON_VERSIONS_SYNC = ["2.7", "3.7"]
 
@@ -249,6 +249,7 @@ def app_engine(session):
         session.log("Skipping App Engine tests.")
         return
 
+    session.install(LIBRARY_DIR)
     # Unlike the default tests above, the App Engine system test require a
     # 'real' gcloud sdk installation that is configured to deploy to an
     # app engine project.
@@ -269,9 +270,8 @@ def app_engine(session):
     application_url = GAE_APP_URL_TMPL.format(GAE_TEST_APP_SERVICE, project_id)
 
     # Vendor in the test application's dependencies
-    session.chdir(os.path.join(HERE, "../app_engine_test_app"))
+    session.chdir(os.path.join(HERE, "system_tests_sync/app_engine_test_app"))
     session.install(*TEST_DEPENDENCIES_SYNC)
-    session.install(LIBRARY_DIR)
     session.run(
         "pip", "install", "--target", "lib", "-r", "requirements.txt", silent=True
     )
@@ -288,7 +288,7 @@ def app_engine(session):
 @nox.session(python=PYTHON_VERSIONS_SYNC)
 def grpc(session):
     session.install(LIBRARY_DIR)
-    session.install(*TEST_DEPENDENCIES_SYNC, "google-cloud-pubsub==1.0.0")
+    session.install(*TEST_DEPENDENCIES_SYNC, "google-cloud-pubsub==1.7.0")
     session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE
     session.run("pytest", "system_tests_sync/test_grpc.py")
 
diff --git a/system_tests/secrets.tar.enc b/system_tests/secrets.tar.enc
new file mode 100644
Binary files /dev/null and b/system_tests/secrets.tar.enc differ
diff --git a/system_tests/system_tests_async/__init__.py b/system_tests/system_tests_async/__init__.py
new file mode 100644
diff --git a/system_tests/system_tests_async/conftest.py b/system_tests/system_tests_async/conftest.py
--- a/system_tests/system_tests_async/conftest.py
+++ b/system_tests/system_tests_async/conftest.py
@@ -71,7 +71,8 @@ async def _token_info(access_token=None, id_token=None):
         url = _helpers.update_query(sync_conftest.TOKEN_INFO_URL, query_params)
 
         response = await http_request(url=url, method="GET")
-        data = await response.data.read()
+
+        data = await response.content()
 
         return json.loads(data.decode("utf-8"))
 
diff --git a/system_tests/system_tests_async/test_default.py b/system_tests/system_tests_async/test_default.py
--- a/system_tests/system_tests_async/test_default.py
+++ b/system_tests/system_tests_async/test_default.py
@@ -15,14 +15,13 @@
 import os
 import pytest
 
-import google.auth
+from google.auth import _default_async
 
 EXPECT_PROJECT_ID = os.environ.get("EXPECT_PROJECT_ID")
 
 @pytest.mark.asyncio
 async def test_application_default_credentials(verify_refresh):
-    credentials, project_id = google.auth.default_async()
-    #breakpoint()
+    credentials, project_id = _default_async.default_async()
 
     if EXPECT_PROJECT_ID is not None:
         assert project_id is not None
diff --git a/system_tests/system_tests_sync/app_engine_test_app/requirements.txt b/system_tests/system_tests_sync/app_engine_test_app/requirements.txt
--- a/system_tests/system_tests_sync/app_engine_test_app/requirements.txt
+++ b/system_tests/system_tests_sync/app_engine_test_app/requirements.txt
@@ -1,3 +1,3 @@
 urllib3
 # Relative path to google-auth-python's source.
-../..
+../../..
diff --git a/system_tests/system_tests_sync/secrets.tar.enc b/system_tests/system_tests_sync/secrets.tar.enc
Binary files a/system_tests/system_tests_sync/secrets.tar.enc and b/system_tests/system_tests_sync/secrets.tar.enc differ
diff --git a/system_tests/system_tests_sync/test_grpc.py b/system_tests/system_tests_sync/test_grpc.py
--- a/system_tests/system_tests_sync/test_grpc.py
+++ b/system_tests/system_tests_sync/test_grpc.py
@@ -17,8 +17,6 @@
 import google.auth.jwt
 import google.auth.transport.grpc
 from google.cloud import pubsub_v1
-from google.cloud.pubsub_v1.gapic import publisher_client
-from google.cloud.pubsub_v1.gapic.transports import publisher_grpc_transport
 
 
 def test_grpc_request_with_regular_credentials(http_request):
@@ -27,13 +25,8 @@ def test_grpc_request_with_regular_credentials(http_request):
         credentials, ["https://www.googleapis.com/auth/pubsub"]
     )
 
-    transport = publisher_grpc_transport.PublisherGrpcTransport(
-        address=publisher_client.PublisherClient.SERVICE_ADDRESS,
-        credentials=credentials,
-    )
-
     # Create a pub/sub client.
-    client = pubsub_v1.PublisherClient(transport=transport)
+    client = pubsub_v1.PublisherClient(credentials=credentials)
 
     # list the topics and drain the iterator to test that an authorized API
     # call works.
@@ -48,13 +41,8 @@ def test_grpc_request_with_jwt_credentials():
         credentials, audience=audience
     )
 
-    transport = publisher_grpc_transport.PublisherGrpcTransport(
-        address=publisher_client.PublisherClient.SERVICE_ADDRESS,
-        credentials=credentials,
-    )
-
     # Create a pub/sub client.
-    client = pubsub_v1.PublisherClient(transport=transport)
+    client = pubsub_v1.PublisherClient(credentials=credentials)
 
     # list the topics and drain the iterator to test that an authorized API
     # call works.
@@ -68,13 +56,8 @@ def test_grpc_request_with_on_demand_jwt_credentials():
         credentials
     )
 
-    transport = publisher_grpc_transport.PublisherGrpcTransport(
-        address=publisher_client.PublisherClient.SERVICE_ADDRESS,
-        credentials=credentials,
-    )
-
     # Create a pub/sub client.
-    client = pubsub_v1.PublisherClient(transport=transport)
+    client = pubsub_v1.PublisherClient(credentials=credentials)
 
     # list the topics and drain the iterator to test that an authorized API
     # call works.
diff --git a/system_tests/system_tests_sync/test_mtls_http.py b/system_tests/system_tests_sync/test_mtls_http.py
--- a/system_tests/system_tests_sync/test_mtls_http.py
+++ b/system_tests/system_tests_sync/test_mtls_http.py
@@ -13,8 +13,11 @@
 # limitations under the License.
 
 import json
-from os import path
+import mock
+import os
 import time
+from os import path
+
 
 import google.auth
 import google.auth.credentials