Repository: pydap/pydap
Pull request #99 (2017-05-10 14:48:18)

Allow redirects even without credentials
This PR fixes issue #98. It solves it by allowing redirects even when no ``session`` is passed to the ``GET`` command. I've also added some testing to prevent regressions.

laliberte: Tests have passed, I'm merging. I've fixed some of things that you found confusing.

Patch:
diff --git a/setup.py b/setup.py
index a639316..e348e37 100644
--- a/setup.py
+++ b/setup.py
@@ -45,6 +45,7 @@ testing_extras = tests_require + [
     'pytest-cov',
     'pytest-attrib',
     'mock',
+    'requests-mock',
     'requests'
 ]
 
diff --git a/src/pydap/net.py b/src/pydap/net.py
index c9d0f7f..b4ac512 100644
--- a/src/pydap/net.py
+++ b/src/pydap/net.py
@@ -1,7 +1,8 @@
 from webob.request import Request
 from webob.exc import HTTPError
 from contextlib import closing
-from requests.exceptions import MissingSchema
+import requests
+from requests.exceptions import MissingSchema, InvalidSchema
 
 from six.moves.urllib.parse import urlsplit, urlunsplit
 
@@ -23,7 +24,8 @@ def GET(url, application=None, session=None):
 
 
 def raise_for_status(response):
-    if response.status_code >= 400:
+    # Raise error if status is above 300:
+    if response.status_code >= 300:
         raise HTTPError(
             detail=response.status,
             headers=response.headers,
@@ -56,24 +58,32 @@ def create_request(url, session=None):
         # adjust the cookies as needed. We can then use the final url and
         # the final cookies to set up a webob Request object that will
         # be guaranteed to have all the needed credentials:
-        try:
-            # Use session to follow redirects:
-            with closing(session.head(url)) as head:
-                req = Request.blank(head.url)
-
-                # Get cookies from head:
-                cookies_dict = head.cookies.get_dict()
-
-                # Set request cookies to the head cookies:
-                req.headers['Cookie'] = ','.join(name + '=' +
-                                                 cookies_dict[name]
-                                                 for name in cookies_dict)
-                # Set the headers to the session headers:
-                for item in head.request.headers:
-                    req.headers[item] = head.request.headers[item]
-                return req
-        except MissingSchema:
-            # Missing schema can occur in tests when the url
-            # is not pointing to any resource. Simply pass.
-            pass
-    return Request.blank(url)
+        return create_request_from_session(url, session)
+    else:
+        # If a session object was not passed, we simply pass a new
+        # requests.Session() object. The requests library allows the
+        # handling of redirects that are not naturally handled by Webob.
+        return create_request_from_session(url, requests.Session())
+
+
+def create_request_from_session(url, session):
+    try:
+        # Use session to follow redirects:
+        with closing(session.head(url, allow_redirects=True)) as head:
+            req = Request.blank(head.url)
+
+            # Get cookies from head:
+            cookies_dict = head.cookies.get_dict()
+
+            # Set request cookies to the head cookies:
+            req.headers['Cookie'] = ','.join(name + '=' +
+                                             cookies_dict[name]
+                                             for name in cookies_dict)
+            # Set the headers to the session headers:
+            for item in head.request.headers:
+                req.headers[item] = head.request.headers[item]
+            return req
+    except (MissingSchema, InvalidSchema):
+        # Missing schema can occur in tests when the url
+        # is not pointing to any resource. Simply pass.
+        return Request.blank(url)


Test patch:
diff --git a/src/pydap/tests/test_net.py b/src/pydap/tests/test_net.py
new file mode 100644
index 0000000..448f67c
--- /dev/null
+++ b/src/pydap/tests/test_net.py
@@ -0,0 +1,43 @@
+"""
+Test the follow redirects and handling of more complex routing situations
+"""
+
+import requests
+from webob.request import Request
+import requests_mock
+from pydap.net import create_request
+
+
+def test_redirect():
+    """ Test that redirection is handled properly """
+    mock_url = 'http://www.test.com/'
+    # mock_url is redirected to https:
+    redirect_url = 'http://www.test2.com/'
+    with requests_mock.Mocker() as m:
+        m.register_uri('HEAD', mock_url, text='resp2', status_code=200)
+        req = create_request(mock_url)
+        assert len(m.request_history) == 1
+        assert isinstance(req, Request)
+        assert req.headers['Host'] == 'www.test.com:80'
+
+    # Without session:
+    with requests_mock.Mocker() as m:
+        m.register_uri('HEAD', mock_url, text='resp1', status_code=301,
+                       headers={'Location': redirect_url})
+        m.register_uri('HEAD', redirect_url, text='resp2', status_code=200)
+        req = create_request(mock_url)
+        # Ensure follow redirect:
+        assert len(m.request_history) == 2
+        assert isinstance(req, Request)
+        assert req.headers['Host'] == 'www.test2.com:80'
+
+    # With session:
+    with requests_mock.Mocker() as m:
+        m.register_uri('HEAD', mock_url, text='resp1', status_code=301,
+                       headers={'Location': redirect_url})
+        m.register_uri('HEAD', redirect_url, text='resp2', status_code=200)
+        req = create_request(mock_url, session=requests.Session())
+        # Ensure follow redirect:
+        assert len(m.request_history) == 2
+        assert isinstance(req, Request)
+        assert req.headers['Host'] == 'www.test2.com:80'