Repository: urllib3/urllib3

Issue:
contrib.appengine.AppEngineManager
This is currently _very_ rough and early. I want to make sure I'm on the right track before continuing.

Outstanding questions:
1. What functionality should be covered by the connection pool? (retries, etc.)
2. Should I just inherit from TestConnectionPool and null out the tests that aren't applicable to GAE instead of duplicating code?
3. How should this connection class be wired in? On GAE Sandbox environments w/o sockets, I assume this should be default.
4. What exact edge cases should this class be aware; e.g, where does URLFetch behavior differ from what's expected of httplib/sockets? Do the tests already cover this?

[_shazow edit: Fixes #664]_



Discussion:

Resolving patch:
diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py
--- a/dummyserver/handlers.py
+++ b/dummyserver/handlers.py
@@ -73,6 +73,10 @@ def options(self):
         """ Handle OPTIONS requests """
         self._call_method()
 
+    def head(self):
+        """ Handle HEAD requests """
+        self._call_method()
+
     def _call_method(self):
         """ Call the correct method in this class based on the incoming URI """
         req = self.request
@@ -232,6 +236,13 @@ def chunked_gzip(self, request):
 
         return Response(chunks, headers=[('Content-Encoding', 'gzip')])
 
+    def nbytes(self, request):
+        length = int(request.params.get('length'))
+        data = b'1' * length
+        return Response(
+            data,
+            headers=[('Content-Type', 'application/octet-stream')])
+
     def shutdown(self, request):
         sys.exit()
 
diff --git a/urllib3/contrib/appengine.py b/urllib3/contrib/appengine.py
new file mode 100644
--- /dev/null
+++ b/urllib3/contrib/appengine.py
@@ -0,0 +1,222 @@
+import logging
+import os
+import warnings
+
+from ..exceptions import (
+    HTTPError,
+    HTTPWarning,
+    MaxRetryError,
+    ProtocolError,
+    TimeoutError,
+    SSLError
+)
+
+from ..packages.six import BytesIO
+from ..request import RequestMethods
+from ..response import HTTPResponse
+from ..util.timeout import Timeout
+from ..util.retry import Retry
+
+try:
+    from google.appengine.api import urlfetch
+except ImportError:
+    urlfetch = None
+
+
+log = logging.getLogger(__name__)
+
+
+class AppEnginePlatformWarning(HTTPWarning):
+    pass
+
+
+class AppEnginePlatformError(HTTPError):
+    pass
+
+
+class AppEngineManager(RequestMethods):
+    """
+    Connection manager for Google App Engine sandbox applications.
+
+    This manager uses the URLFetch service directly instead of using the
+    emulated httplib, and is subject to URLFetch limitations as described in
+    the App Engine documentation here:
+
+        https://cloud.google.com/appengine/docs/python/urlfetch
+
+    Notably it will raise an AppEnginePlatformError if:
+        * URLFetch is not available.
+        * If you attempt to use this on GAEv2 (Managed VMs), as full socket
+          support is available.
+        * If a request size is more than 10 megabytes.
+        * If a response size is more than 32 megabtyes.
+        * If you use an unsupported request method such as OPTIONS.
+
+    Beyond those cases, it will raise normal urllib3 errors.
+    """
+
+    def __init__(self, headers=None, retries=None, validate_certificate=True):
+        if not urlfetch:
+            raise AppEnginePlatformError(
+                "URLFetch is not available in this environment.")
+
+        if is_prod_appengine_v2():
+            raise AppEnginePlatformError(
+                "Use normal urllib3.PoolManager instead of AppEngineManager"
+                "on Managed VMs, as using URLFetch is not necessary in "
+                "this environment.")
+
+        warnings.warn(
+            "urllib3 is using URLFetch on Google App Engine sandbox instead "
+            "of sockets. To use sockets directly instead of URLFetch see "
+            "https://urllib3.readthedocs.org/en/latest/contrib.html.",
+            AppEnginePlatformWarning)
+
+        RequestMethods.__init__(self, headers)
+        self.validate_certificate = validate_certificate
+
+        self.retries = retries or Retry.DEFAULT
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # Return False to re-raise any potential exceptions
+        return False
+
+    def urlopen(self, method, url, body=None, headers=None,
+                retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT,
+                **response_kw):
+
+        retries = self._get_retries(retries, redirect)
+
+        try:
+            response = urlfetch.fetch(
+                url,
+                payload=body,
+                method=method,
+                headers=headers or {},
+                allow_truncated=False,
+                follow_redirects=(
+                    redirect and
+                    retries.redirect != 0 and
+                    retries.total),
+                deadline=self._get_absolute_timeout(timeout),
+                validate_certificate=self.validate_certificate,
+            )
+        except urlfetch.DeadlineExceededError as e:
+            raise TimeoutError(self, e)
+
+        except urlfetch.InvalidURLError as e:
+            if 'too large' in e.message:
+                raise AppEnginePlatformError(
+                    "URLFetch request too large, URLFetch only "
+                    "supports requests up to 10mb in size.", e)
+            raise ProtocolError(e)
+
+        except urlfetch.DownloadError as e:
+            if 'Too many redirects' in e.message:
+                raise MaxRetryError(self, url, reason=e)
+            raise ProtocolError(e)
+
+        except urlfetch.ResponseTooLargeError as e:
+            raise AppEnginePlatformError(
+                "URLFetch response too large, URLFetch only supports"
+                "responses up to 32mb in size.", e)
+
+        except urlfetch.SSLCertificateError as e:
+            raise SSLError(e)
+
+        except urlfetch.InvalidMethodError as e:
+            raise AppEnginePlatformError(
+                "URLFetch does not support method: %s" % method, e)
+
+        http_response = self._urlfetch_response_to_http_response(
+            response, **response_kw)
+
+        # Check for redirect response
+        if (http_response.get_redirect_location() and
+                retries.raise_on_redirect and redirect):
+            raise MaxRetryError(self, url, "too many redirects")
+
+        # Check if we should retry the HTTP response.
+        if retries.is_forced_retry(method, status_code=http_response.status):
+            retries = retries.increment(
+                method, url, response=http_response, _pool=self)
+            log.info("Forced retry: %s" % url)
+            retries.sleep()
+            return self.urlopen(
+                method, url,
+                body=body, headers=headers,
+                retries=retries, redirect=redirect,
+                timeout=timeout, **response_kw)
+
+        return http_response
+
+    def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw):
+
+        if is_prod_appengine_v1():
+            # Production GAE handles deflate encoding automatically, but does
+            # not remove the encoding header.
+            content_encoding = urlfetch_resp.headers.get('content-encoding')
+
+            if content_encoding == 'deflate':
+                del urlfetch_resp.headers['content-encoding']
+
+        return HTTPResponse(
+            # In order for decoding to work, we must present the content as
+            # a file-like object.
+            body=BytesIO(urlfetch_resp.content),
+            headers=urlfetch_resp.headers,
+            status=urlfetch_resp.status_code,
+            **response_kw
+        )
+
+    def _get_absolute_timeout(self, timeout):
+        if timeout is Timeout.DEFAULT_TIMEOUT:
+            return 5  # 5s is the default timeout for URLFetch.
+        if isinstance(timeout, Timeout):
+            if not timeout.read is timeout.connect:
+                warnings.warn(
+                    "URLFetch does not support granular timeout settings, "
+                    "reverting to total timeout.", AppEnginePlatformWarning)
+            return timeout.total
+        return timeout
+
+    def _get_retries(self, retries, redirect):
+        if not isinstance(retries, Retry):
+            retries = Retry.from_int(
+                retries, redirect=redirect, default=self.retries)
+
+        if retries.connect or retries.read or retries.redirect:
+            warnings.warn(
+                "URLFetch only supports total retries and does not "
+                "recognize connect, read, or redirect retry parameters.",
+                AppEnginePlatformWarning)
+
+        return retries
+
+
+def is_appengine():
+    return (is_local_appengine() or
+            is_prod_appengine_v1() or
+            is_prod_appengine_v2())
+
+
+def is_appengine_sandbox():
+    return is_appengine() and not is_prod_appengine_v2()
+
+
+def is_local_appengine():
+    return ('APPENGINE_RUNTIME' in os.environ and
+            'Development/' in os.environ['SERVER_SOFTWARE'])
+
+
+def is_prod_appengine_v1():
+    return ('APPENGINE_RUNTIME' in os.environ and
+            'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
+            not is_prod_appengine_v2())
+
+
+def is_prod_appengine_v2():
+    return os.environ.get('GAE_VM', False) == 'true'


Test patch:
diff --git a/test/contrib/test_gae_manager.py b/test/contrib/test_gae_manager.py
new file mode 100644
--- /dev/null
+++ b/test/contrib/test_gae_manager.py
@@ -0,0 +1,185 @@
+import unittest
+
+from dummyserver.testcase import HTTPSDummyServerTestCase
+from nose.plugins.skip import SkipTest
+
+try:
+    from google.appengine.api import urlfetch
+    (urlfetch)
+except ImportError:
+    raise SkipTest("App Engine SDK not available.")
+
+from urllib3.contrib.appengine import AppEngineManager, AppEnginePlatformError
+from urllib3.exceptions import (
+    TimeoutError,
+    ProtocolError,
+    SSLError)
+from urllib3.util.url import Url
+from urllib3.util.retry import Retry
+
+from test.with_dummyserver.test_connectionpool import (
+    TestConnectionPool, TestRetry)
+
+
+# Prevent nose from running these test.
+TestConnectionPool.__test__ = False
+TestRetry.__test__ = False
+
+
+# This class is used so we can re-use the tests from the connection pool.
+# It proxies all requests to the manager.
+class MockPool(object):
+    def __init__(self, host, port, manager, scheme='http'):
+        self.host = host
+        self.port = port
+        self.manager = manager
+        self.scheme = scheme
+
+    def request(self, method, url, *args, **kwargs):
+        url = self._absolute_url(url)
+        return self.manager.request(method, url, *args, **kwargs)
+
+    def urlopen(self, method, url, *args, **kwargs):
+        url = self._absolute_url(url)
+        return self.manager.urlopen(method, url, *args, **kwargs)
+
+    def _absolute_url(self, path):
+        return Url(
+            scheme=self.scheme,
+            host=self.host,
+            port=self.port,
+            path=path).url
+
+
+# Note that this doesn't run in the sandbox, it only runs with the URLFetch
+# API stub enabled. There's no need to enable the sandbox as we know for a fact
+# that URLFetch is used by the connection manager.
+class TestGAEConnectionManager(TestConnectionPool):
+    __test__ = True
+
+    # Magic class variable that tells NoseGAE to enable the URLFetch stub.
+    nosegae_urlfetch = True
+
+    def setUp(self):
+        self.manager = AppEngineManager()
+        self.pool = MockPool(self.host, self.port, self.manager)
+
+    # Tests specific to AppEngineManager
+
+    def test_exceptions(self):
+        # DeadlineExceededError -> TimeoutError
+        self.assertRaises(
+            TimeoutError,
+            self.pool.request,
+            'GET',
+            '/sleep?seconds=0.005',
+            timeout=0.001)
+
+        # InvalidURLError -> ProtocolError
+        self.assertRaises(
+            ProtocolError,
+            self.manager.request,
+            'GET',
+            'ftp://invalid/url')
+
+        # DownloadError -> ProtocolError
+        self.assertRaises(
+            ProtocolError,
+            self.manager.request,
+            'GET',
+            'http://0.0.0.0')
+
+        # ResponseTooLargeError -> AppEnginePlatformError
+        self.assertRaises(
+            AppEnginePlatformError,
+            self.pool.request,
+            'GET',
+            '/nbytes?length=33554433')  # One byte over 32 megabtyes.
+
+        # URLFetch reports the request too large error as a InvalidURLError,
+        # which maps to a AppEnginePlatformError.
+        body = b'1' * 10485761  # One byte over 10 megabytes.
+        self.assertRaises(
+            AppEnginePlatformError,
+            self.manager.request,
+            'POST',
+            '/',
+            body=body)
+
+    # Re-used tests below this line.
+    # Subsumed tests
+    test_timeout_float = None  # Covered by test_exceptions.
+
+    # Non-applicable tests
+    test_conn_closed = None
+    test_nagle = None
+    test_socket_options = None
+    test_disable_default_socket_options = None
+    test_defaults_are_applied = None
+    test_tunnel = None
+    test_keepalive = None
+    test_keepalive_close = None
+    test_connection_count = None
+    test_connection_count_bigpool = None
+    test_for_double_release = None
+    test_release_conn_parameter = None
+    test_stream_keepalive = None
+    test_cleanup_on_connection_error = None
+
+    # Tests that should likely be modified for appengine specific stuff
+    test_timeout = None
+    test_connect_timeout = None
+    test_connection_error_retries = None
+    test_total_timeout = None
+    test_none_total_applies_connect = None
+    test_timeout_success = None
+    test_source_address_error = None
+    test_bad_connect = None
+    test_partial_response = None
+    test_dns_error = None
+
+
+class TestGAEConnectionManagerWithSSL(HTTPSDummyServerTestCase):
+    nosegae_urlfetch = True
+
+    def setUp(self):
+        self.manager = AppEngineManager()
+        self.pool = MockPool(self.host, self.port, self.manager, 'https')
+
+    def test_exceptions(self):
+        # SSLCertificateError -> SSLError
+        # SSLError is raised with dummyserver because URLFetch doesn't allow
+        # self-signed certs.
+        self.assertRaises(
+            SSLError,
+            self.pool.request,
+            'GET',
+            '/')
+
+
+class TestGAERetry(TestRetry):
+    __test__ = True
+
+    # Magic class variable that tells NoseGAE to enable the URLFetch stub.
+    nosegae_urlfetch = True
+
+    def setUp(self):
+        self.manager = AppEngineManager()
+        self.pool = MockPool(self.host, self.port, self.manager)
+
+    def test_default_method_whitelist_retried(self):
+        """ urllib3 should retry methods in the default method whitelist """
+        retry = Retry(total=1, status_forcelist=[418])
+        # Use HEAD instead of OPTIONS, as URLFetch doesn't support OPTIONS
+        resp = self.pool.request(
+            'HEAD', '/successful_retry',
+            headers={'test-name': 'test_default_whitelist'},
+            retries=retry)
+        self.assertEqual(resp.status, 200)
+
+    #test_max_retry = None
+    #test_disabled_retry = None
+
+
+if __name__ == '__main__':
+    unittest.main()