Repository: tornadoweb/tornado

Issue:
tornado.httpclient.HTTPClient.fetch() executes callback on contents of redirect response
Assume the following example of using the HTTPClient to retrieve a file:

``` python
>>> tornado.version
'4.2.1'
>>> try:
...     fh = open('/tmp/foo.msi', 'wb')
...     def handle_chunk(chunk):
...         fh.write(chunk)
...     tornado.httpclient.HTTPClient(max_body_size=107374182400).fetch('http://download.macromedia.com/get/flashplayer/current/licensing/win/install_flash_player_18_plugin.msi', method='GET', streaming_callback=handle_chunk)
... finally:
...     fh.close()
...
HTTPResponse(_body=None,buffer=<_io.BytesIO object at 0x7f933e4a7470>,code=200,effective_url='http://fpdownload.macromedia.com/get/flashplayer/current/licensing/win/install_flash_player_18_plugin.msi',error=None,headers={'Content-Length': '19924480', 'Accept-Ranges': 'bytes', 'Server': 'Apache', 'Last-Modified': 'Fri, 07 Aug 2015 12:22:31 GMT', 'Connection': 'close', 'Etag': '"1300600-51cb7b0998fc0"', 'Date': 'Mon, 14 Sep 2015 17:24:22 GMT', 'Content-Type': 'application/x-msi'},reason='OK',request=<tornado.httpclient.HTTPRequest object at 0x7f933e196250>,request_time=4.273449897766113,time_info={})
>>> with open('/tmp/foo.msi', 'rb') as fh:
...     first_288_bytes = fh.read(288)
...
>>> first_288_bytes
'<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\n<html><head>\n<title>302 Found</title>\n</head><body>\n<h1>Found</h1>\n<p>The document has moved <a href="http://fpdownload.macromedia.com/get/flashplayer/current/licensing/win/install_flash_player_18_plugin.msi">here</a>.</p>\n</body></html>'
```

This results in a corrupted file, as the HTTP response from the 302 redirect is written to the destination file via the callback.

It's entirely possible that I am doing something wrong, but this looks like a bug. I can conceive of no way to reliably determine that the callback is being executed on a 302 redirect response, since the raw data from the chunk of the file is what is passed to the callback function, leaving nothing from the HTTPResponse object to be examined in order to determine whether or not to write the chunk to the destination file.



Discussion:
Hmm, good point. In your example `fetch()` still returns a response, and you could delete the file after the fact if `response.code != 200` (and try again with `response.headers['Location']` if `code==302`). 

To detect this case before writing to disk, you need to use `header_callback` in addition to `streaming_callback`. Before `streaming_callback` gets any data, `header_callback` will get all the headers, including the first line like `HTTP/1.1 302 Redirect`. Unfortunately this is cumbersome; you have to parse the headers (there are functions in tornado.httputil to help but you still have to know to parse the first one separately), and there's no way to bail out early so you can start the next request.

The cleanest thing to do is probably to give the option of an `HTTPMessageDelegate`-based interface instead of `streaming_callback` and `header_callback`.

Hmm... Could I not just raise an exception in the header_callback function and catch it? Or would that not have an effect due to the asynchronous nature of tornado? Pardon my ignorance, but I'm not very familiar with tornado itself.


Resolving patch:
diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py
--- a/tornado/simple_httpclient.py
+++ b/tornado/simple_httpclient.py
@@ -462,9 +462,12 @@ def headers_received(self, first_line, headers):
         if self.request.expect_100_continue and first_line.code == 100:
             self._write_body(False)
             return
-        self.headers = headers
         self.code = first_line.code
         self.reason = first_line.reason
+        self.headers = headers
+
+        if self._should_follow_redirect():
+            return
 
         if self.request.header_callback is not None:
             # Reassemble the start line.
@@ -473,14 +476,17 @@ def headers_received(self, first_line, headers):
                 self.request.header_callback("%s: %s\r\n" % (k, v))
             self.request.header_callback('\r\n')
 
+    def _should_follow_redirect(self):
+        return (self.request.follow_redirects and
+                self.request.max_redirects > 0 and
+                self.code in (301, 302, 303, 307))
+
     def finish(self):
         data = b''.join(self.chunks)
         self._remove_timeout()
         original_request = getattr(self.request, "original_request",
                                    self.request)
-        if (self.request.follow_redirects and
-            self.request.max_redirects > 0 and
-                self.code in (301, 302, 303, 307)):
+        if self._should_follow_redirect():
             assert isinstance(self.request, _RequestProxy)
             new_request = copy.copy(self.request.request)
             new_request.url = urlparse.urljoin(self.request.url,
@@ -527,6 +533,9 @@ def _on_end_request(self):
         self.stream.close()
 
     def data_received(self, chunk):
+        if self._should_follow_redirect():
+            # We're going to follow a redirect so just discard the body.
+            return
         if self.request.streaming_callback is not None:
             self.request.streaming_callback(chunk)
         else:


Test patch:
diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py
--- a/tornado/test/httpclient_test.py
+++ b/tornado/test/httpclient_test.py
@@ -48,6 +48,7 @@ def put(self):
 
 class RedirectHandler(RequestHandler):
     def prepare(self):
+        self.write('redirects can have bodies too')
         self.redirect(self.get_argument("url"),
                       status=int(self.get_argument("status", "302")))
 
diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py
--- a/tornado/test/simple_httpclient_test.py
+++ b/tornado/test/simple_httpclient_test.py
@@ -11,14 +11,15 @@
 import ssl
 import sys
 
+from tornado.escape import to_unicode
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
 from tornado.httputil import HTTPHeaders, ResponseStartLine
 from tornado.ioloop import IOLoop
 from tornado.log import gen_log
 from tornado.netutil import Resolver, bind_sockets
-from tornado.simple_httpclient import SimpleAsyncHTTPClient, _default_ca_certs
-from tornado.test.httpclient_test import ChunkHandler, CountdownHandler, HelloWorldHandler
+from tornado.simple_httpclient import SimpleAsyncHTTPClient
+from tornado.test.httpclient_test import ChunkHandler, CountdownHandler, HelloWorldHandler, RedirectHandler
 from tornado.test import httpclient_test
 from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, ExpectLog
 from tornado.test.util import skipOnTravis, skipIfNoIPv6, refusing_port, unittest
@@ -145,6 +146,7 @@ def get_app(self):
             url("/no_content_length", NoContentLengthHandler),
             url("/echo_post", EchoPostHandler),
             url("/respond_in_prepare", RespondInPrepareHandler),
+            url("/redirect", RedirectHandler),
         ], gzip=True)
 
     def test_singleton(self):
@@ -416,6 +418,24 @@ def body_producer(write):
                               expect_100_continue=True)
         self.assertEqual(response.code, 403)
 
+    def test_streaming_follow_redirects(self):
+        # When following redirects, header and streaming callbacks
+        # should only be called for the final result.
+        # TODO(bdarnell): this test belongs in httpclient_test instead of
+        # simple_httpclient_test, but it fails with the version of libcurl
+        # available on travis-ci. Move it when that has been upgraded
+        # or we have a better framework to skip tests based on curl version.
+        headers = []
+        chunks = []
+        self.fetch("/redirect?url=/hello",
+                   header_callback=headers.append,
+                   streaming_callback=chunks.append)
+        chunks = list(map(to_unicode, chunks))
+        self.assertEqual(chunks, ['Hello world!'])
+        # Make sure we only got one set of headers.
+        num_start_lines = len([h for h in headers if h.startswith("HTTP/")])
+        self.assertEqual(num_start_lines, 1)
+
 
 class SimpleHTTPClientTestCase(SimpleHTTPClientTestMixin, AsyncHTTPTestCase):
     def setUp(self):