Repository: tornadoweb/tornado

Issue:
release: 4.5.3
The following PRs have been nominated for inclusion in a 4.5.3 release:

- #2225 
- #2020 
- #2236 (fixes issue introduced by #2225)
- #1949 (fixes tests for tornado_http2)

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 4b2ac2bc..e3b33c04 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 # https://travis-ci.org/tornadoweb/tornado
 language: python
 python:
-    - 2.7.8
     - 2.7
     - pypy
     - 3.3
@@ -9,7 +8,7 @@ python:
     - 3.5
     - 3.6
     - nightly
-    - pypy3
+    - pypy3.5-5.8.0
 
 install:
     - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install futures mock monotonic trollius; fi
diff --git a/docs/releases.rst b/docs/releases.rst
index 3a9ef777..128c7603 100644
--- a/docs/releases.rst
+++ b/docs/releases.rst
@@ -4,6 +4,7 @@ Release notes
 .. toctree::
    :maxdepth: 2
 
+   releases/v4.5.3
    releases/v4.5.2
    releases/v4.5.1
    releases/v4.5.0
diff --git a/docs/releases/v4.5.3.rst b/docs/releases/v4.5.3.rst
new file mode 100644
index 00000000..b1102459
--- /dev/null
+++ b/docs/releases/v4.5.3.rst
@@ -0,0 +1,49 @@
+What's new in Tornado 4.5.2
+===========================
+
+Aug 27, 2017
+------------
+
+`tornado.curl_httpclient`
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Improved debug logging on Python 3.
+
+`tornado.httpserver`
+~~~~~~~~~~~~~~~~~~~~
+
+- ``Content-Length`` and ``Transfer-Encoding`` headers are no longer
+  sent with 1xx or 204 responses (this was already true of 304
+  responses).
+- Reading chunked requests no longer leaves the connection in a broken
+  state.
+
+`tornado.iostream`
+~~~~~~~~~~~~~~~~~~
+
+- Writing a `memoryview` can no longer result in "BufferError:
+  Existing exports of data: object cannot be re-sized".
+
+`tornado.options`
+~~~~~~~~~~~~~~~~~
+
+- Duplicate option names are now detected properly whether they use
+  hyphens or underscores.
+
+`tornado.testing`
+~~~~~~~~~~~~~~~~~
+
+- `.AsyncHTTPTestCase.fetch` now uses ``127.0.0.1`` instead of
+  ``localhost``, improving compatibility with systems that have
+  partially-working ipv6 stacks.
+
+`tornado.web`
+~~~~~~~~~~~~~
+
+- It is no longer allowed to send a body with 1xx or 204 responses.
+
+`tornado.websocket`
+~~~~~~~~~~~~~~~~~~~
+
+- Requests with invalid websocket headers now get a response with
+  status code 400 instead of a closed connection.
diff --git a/setup.py b/setup.py
index 66d846be..a1feea67 100644
--- a/setup.py
+++ b/setup.py
@@ -103,7 +103,7 @@ http://api.mongodb.org/python/current/installation.html#osx
 
 kwargs = {}
 
-version = "4.5.2"
+version = "4.5.3"
 
 with open('README.rst') as f:
     kwargs['long_description'] = f.read()
diff --git a/tornado/__init__.py b/tornado/__init__.py
index 3eaa57b8..fa71bf61 100644
--- a/tornado/__init__.py
+++ b/tornado/__init__.py
@@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "4.5.2"
-version_info = (4, 5, 2, 0)
+version = "4.5.3"
+version_info = (4, 5, 3, 0)
diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py
index eef4a17a..8558d65c 100644
--- a/tornado/curl_httpclient.py
+++ b/tornado/curl_httpclient.py
@@ -493,6 +493,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
 
     def _curl_debug(self, debug_type, debug_msg):
         debug_types = ('I', '<', '>', '<', '>')
+        debug_msg = native_str(debug_msg)
         if debug_type == 0:
             curl_log.debug('%s', debug_msg.strip())
         elif debug_type in (1, 2):
diff --git a/tornado/http1connection.py b/tornado/http1connection.py
index 53744ece..32bed6c9 100644
--- a/tornado/http1connection.py
+++ b/tornado/http1connection.py
@@ -349,10 +349,11 @@ class HTTP1Connection(httputil.HTTPConnection):
                 # self._request_start_line.version or
                 # start_line.version?
                 self._request_start_line.version == 'HTTP/1.1' and
-                # 304 responses have no body (not even a zero-length body), and so
-                # should not have either Content-Length or Transfer-Encoding.
-                # headers.
+                # 1xx, 204 and 304 responses have no body (not even a zero-length
+                # body), and so should not have either Content-Length or
+                # Transfer-Encoding headers.
                 start_line.code not in (204, 304) and
+                (start_line.code < 100 or start_line.code >= 200) and
                 # No need to chunk the output if a Content-Length is specified.
                 'Content-Length' not in headers and
                 # Applications are discouraged from touching Transfer-Encoding,
@@ -592,6 +593,9 @@ class HTTP1Connection(httputil.HTTPConnection):
             chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64)
             chunk_len = int(chunk_len.strip(), 16)
             if chunk_len == 0:
+                crlf = yield self.stream.read_bytes(2)
+                if crlf != b'\r\n':
+                    raise httputil.HTTPInputError("improperly terminated chunked request")
                 return
             total_size += chunk_len
             if total_size > self._max_body_size:
diff --git a/tornado/iostream.py b/tornado/iostream.py
index a1619c49..639ed508 100644
--- a/tornado/iostream.py
+++ b/tornado/iostream.py
@@ -1061,7 +1061,12 @@ class IOStream(BaseIOStream):
         return chunk
 
     def write_to_fd(self, data):
-        return self.socket.send(data)
+        try:
+            return self.socket.send(data)
+        finally:
+            # Avoid keeping to data, which can be a memoryview.
+            # See https://github.com/tornadoweb/tornado/pull/2008
+            del data
 
     def connect(self, address, callback=None, server_hostname=None):
         """Connects the socket to a remote address without blocking.
@@ -1471,6 +1476,10 @@ class SSLIOStream(IOStream):
                 # simply return 0 bytes written.
                 return 0
             raise
+        finally:
+            # Avoid keeping to data, which can be a memoryview.
+            # See https://github.com/tornadoweb/tornado/pull/2008
+            del data
 
     def read_from_fd(self):
         if self._ssl_accepting:
@@ -1528,7 +1537,12 @@ class PipeIOStream(BaseIOStream):
         os.close(self.fd)
 
     def write_to_fd(self, data):
-        return os.write(self.fd, data)
+        try:
+            return os.write(self.fd, data)
+        finally:
+            # Avoid keeping to data, which can be a memoryview.
+            # See https://github.com/tornadoweb/tornado/pull/2008
+            del data
 
     def read_from_fd(self):
         try:
diff --git a/tornado/options.py b/tornado/options.py
index 0a72cc65..707fbd35 100644
--- a/tornado/options.py
+++ b/tornado/options.py
@@ -223,9 +223,10 @@ class OptionParser(object):
         override options set earlier on the command line, but can be overridden
         by later flags.
         """
-        if name in self._options:
+        normalized = self._normalize_name(name)
+        if normalized in self._options:
             raise Error("Option %r already defined in %s" %
-                        (name, self._options[name].file_name))
+                        (normalized, self._options[normalized].file_name))
         frame = sys._getframe(0)
         options_file = frame.f_code.co_filename
 
@@ -247,7 +248,6 @@ class OptionParser(object):
             group_name = group
         else:
             group_name = file_name
-        normalized = self._normalize_name(name)
         option = _Option(name, file_name=file_name,
                          default=default, type=type, help=help,
                          metavar=metavar, multiple=multiple,
diff --git a/tornado/web.py b/tornado/web.py
index d79889fa..e8d102b5 100644
--- a/tornado/web.py
+++ b/tornado/web.py
@@ -974,7 +974,8 @@ class RequestHandler(object):
                 if self.check_etag_header():
                     self._write_buffer = []
                     self.set_status(304)
-            if self._status_code in (204, 304):
+            if (self._status_code in (204, 304) or
+                (self._status_code >= 100 and self._status_code < 200)):
                 assert not self._write_buffer, "Cannot send body with %s" % self._status_code
                 self._clear_headers_for_304()
             elif "Content-Length" not in self._headers:
diff --git a/tornado/websocket.py b/tornado/websocket.py
index 69437ee4..0e9d339f 100644
--- a/tornado/websocket.py
+++ b/tornado/websocket.py
@@ -616,6 +616,14 @@ class WebSocketProtocol13(WebSocketProtocol):
     def accept_connection(self):
         try:
             self._handle_websocket_headers()
+        except ValueError:
+            self.handler.set_status(400)
+            log_msg = "Missing/Invalid WebSocket headers"
+            self.handler.finish(log_msg)
+            gen_log.debug(log_msg)
+            return
+
+        try:
             self._accept_connection()
         except ValueError:
             gen_log.debug("Malformed WebSocket request received",


Test patch:
diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py
index 11cb7231..59eb6fd1 100644
--- a/tornado/test/httpserver_test.py
+++ b/tornado/test/httpserver_test.py
@@ -786,9 +786,12 @@ class KeepAliveTest(AsyncHTTPTestCase):
     def test_keepalive_chunked(self):
         self.http_version = b'HTTP/1.0'
         self.connect()
-        self.stream.write(b'POST / HTTP/1.0\r\nConnection: keep-alive\r\n'
+        self.stream.write(b'POST / HTTP/1.0\r\n'
+                          b'Connection: keep-alive\r\n'
                           b'Transfer-Encoding: chunked\r\n'
-                          b'\r\n0\r\n')
+                          b'\r\n'
+                          b'0\r\n'
+                          b'\r\n')
         self.read_response()
         self.assertEqual(self.headers['Connection'], 'Keep-Alive')
         self.stream.write(b'GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n')
diff --git a/tornado/test/iostream_test.py b/tornado/test/iostream_test.py
index 91bc7bf6..56fffe60 100644
--- a/tornado/test/iostream_test.py
+++ b/tornado/test/iostream_test.py
@@ -9,7 +9,7 @@ from tornado.netutil import ssl_wrap_socket
 from tornado.stack_context import NullContext
 from tornado.tcpserver import TCPServer
 from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, bind_unused_port, ExpectLog, gen_test
-from tornado.test.util import unittest, skipIfNonUnix, refusing_port
+from tornado.test.util import unittest, skipIfNonUnix, refusing_port, skipPypy3V58
 from tornado.web import RequestHandler, Application
 import errno
 import logging
@@ -539,6 +539,7 @@ class TestIOStreamMixin(object):
             client.close()
 
     @skipIfNonUnix
+    @skipPypy3V58
     def test_inline_read_error(self):
         # An error on an inline read is raised without logging (on the
         # assumption that it will eventually be noticed or logged further
@@ -557,6 +558,7 @@ class TestIOStreamMixin(object):
             server.close()
             client.close()
 
+    @skipPypy3V58
     def test_async_read_error_logging(self):
         # Socket errors on asynchronous reads should be logged (but only
         # once).
@@ -993,7 +995,7 @@ class TestIOStreamStartTLS(AsyncTestCase):
         server_future = self.server_start_tls(_server_ssl_options())
         client_future = self.client_start_tls(
             ssl.create_default_context(),
-            server_hostname=b'127.0.0.1')
+            server_hostname='127.0.0.1')
         with ExpectLog(gen_log, "SSL Error"):
             with self.assertRaises(ssl.SSLError):
                 # The client fails to connect with an SSL error.
diff --git a/tornado/test/options_test.py b/tornado/test/options_test.py
index bafeea6f..1a0ac8fb 100644
--- a/tornado/test/options_test.py
+++ b/tornado/test/options_test.py
@@ -7,7 +7,7 @@ import sys
 
 from tornado.options import OptionParser, Error
 from tornado.util import basestring_type, PY3
-from tornado.test.util import unittest
+from tornado.test.util import unittest, subTest
 
 if PY3:
     from io import StringIO
@@ -232,6 +232,24 @@ class OptionsTest(unittest.TestCase):
         self.assertRegexpMatches(str(cm.exception),
                                  'Option.*foo.*already defined')
 
+    def test_error_redefine_underscore(self):
+        # Ensure that the dash/underscore normalization doesn't
+        # interfere with the redefinition error.
+        tests = [
+            ('foo-bar', 'foo-bar'),
+            ('foo_bar', 'foo_bar'),
+            ('foo-bar', 'foo_bar'),
+            ('foo_bar', 'foo-bar'),
+            ]
+        for a, b in tests:
+            with subTest(self, a=a, b=b):
+                options = OptionParser()
+                options.define(a)
+                with self.assertRaises(Error) as cm:
+                    options.define(b)
+                self.assertRegexpMatches(str(cm.exception),
+                                         'Option.*foo.bar.*already defined')
+
     def test_dash_underscore_cli(self):
         # Dashes and underscores should be interchangeable.
         for defined_name in ['foo-bar', 'foo_bar']:
diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py
index 02d57c5f..0e75e530 100644
--- a/tornado/test/simple_httpclient_test.py
+++ b/tornado/test/simple_httpclient_test.py
@@ -272,16 +272,9 @@ class SimpleHTTPClientTestMixin(object):
 
     @skipIfNoIPv6
     def test_ipv6(self):
-        try:
-            [sock] = bind_sockets(None, '::1', family=socket.AF_INET6)
-            port = sock.getsockname()[1]
-            self.http_server.add_socket(sock)
-        except socket.gaierror as e:
-            if e.args[0] == socket.EAI_ADDRFAMILY:
-                # python supports ipv6, but it's not configured on the network
-                # interface, so skip this test.
-                return
-            raise
+        [sock] = bind_sockets(None, '::1', family=socket.AF_INET6)
+        port = sock.getsockname()[1]
+        self.http_server.add_socket(sock)
         url = '%s://[::1]:%d/hello' % (self.get_protocol(), port)
 
         # ipv6 is currently enabled by default but can be disabled
@@ -327,7 +320,7 @@ class SimpleHTTPClientTestMixin(object):
         self.assertNotIn("Content-Length", response.headers)
 
     def test_host_header(self):
-        host_re = re.compile(b"^localhost:[0-9]+$")
+        host_re = re.compile(b"^127.0.0.1:[0-9]+$")
         response = self.fetch("/host_echo")
         self.assertTrue(host_re.match(response.body))
 
diff --git a/tornado/test/util.py b/tornado/test/util.py
index 6c032da6..90a9c7b8 100644
--- a/tornado/test/util.py
+++ b/tornado/test/util.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, division, print_function
 
+import contextlib
 import os
 import platform
 import socket
@@ -34,14 +35,39 @@ skipOnAppEngine = unittest.skipIf('APPENGINE_RUNTIME' in os.environ,
 skipIfNoNetwork = unittest.skipIf('NO_NETWORK' in os.environ,
                                   'network access disabled')
 
-skipIfNoIPv6 = unittest.skipIf(not socket.has_ipv6, 'ipv6 support not present')
-
-
 skipBefore33 = unittest.skipIf(sys.version_info < (3, 3), 'PEP 380 (yield from) not available')
 skipBefore35 = unittest.skipIf(sys.version_info < (3, 5), 'PEP 492 (async/await) not available')
 skipNotCPython = unittest.skipIf(platform.python_implementation() != 'CPython',
                                  'Not CPython implementation')
 
+# Used for tests affected by
+# https://bitbucket.org/pypy/pypy/issues/2616/incomplete-error-handling-in
+# TODO: remove this after pypy3 5.8 is obsolete.
+skipPypy3V58 = unittest.skipIf(platform.python_implementation() == 'PyPy' and
+                               sys.version_info > (3,) and
+                               sys.pypy_version_info < (5, 9),
+                               'pypy3 5.8 has buggy ssl module')
+
+
+def _detect_ipv6():
+    if not socket.has_ipv6:
+        # socket.has_ipv6 check reports whether ipv6 was present at compile
+        # time. It's usually true even when ipv6 doesn't work for other reasons.
+        return False
+    sock = None
+    try:
+        sock = socket.socket(socket.AF_INET6)
+        sock.bind(('::1', 0))
+    except socket.error:
+        return False
+    finally:
+        if sock is not None:
+            sock.close()
+    return True
+
+
+skipIfNoIPv6 = unittest.skipIf(not _detect_ipv6(), 'ipv6 support not present')
+
 
 def refusing_port():
     """Returns a local port number that will refuse all connections.
@@ -94,3 +120,15 @@ def is_coverage_running():
         except AttributeError:
             return False
     return mod.startswith('coverage')
+
+
+def subTest(test, *args, **kwargs):
+    """Compatibility shim for unittest.TestCase.subTest.
+
+    Usage: ``with tornado.test.util.subTest(self, x=x):``
+    """
+    try:
+        subTest = test.subTest  # py34+
+    except AttributeError:
+        subTest = contextlib.contextmanager(lambda *a, **kw: (yield))
+    return subTest(*args, **kwargs)
diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py
index d79ea52c..013c2ac2 100644
--- a/tornado/test/web_test.py
+++ b/tornado/test/web_test.py
@@ -356,7 +356,7 @@ class AuthRedirectTest(WebTestCase):
         response = self.wait()
         self.assertEqual(response.code, 302)
         self.assertTrue(re.match(
-            'http://example.com/login\?next=http%3A%2F%2Flocalhost%3A[0-9]+%2Fabsolute',
+            'http://example.com/login\?next=http%3A%2F%2F127.0.0.1%3A[0-9]+%2Fabsolute',
             response.headers['Location']), response.headers['Location'])
 
 
@@ -2134,7 +2134,7 @@ class StreamingRequestBodyTest(WebTestCase):
         stream.write(b"4\r\nqwer\r\n")
         data = yield self.data
         self.assertEquals(data, b"qwer")
-        stream.write(b"0\r\n")
+        stream.write(b"0\r\n\r\n")
         yield self.finished
         data = yield gen.Task(stream.read_until_close)
         # This would ideally use an HTTP1Connection to read the response.
diff --git a/tornado/test/websocket_test.py b/tornado/test/websocket_test.py
index d47a74e6..95a5ecd4 100644
--- a/tornado/test/websocket_test.py
+++ b/tornado/test/websocket_test.py
@@ -189,6 +189,13 @@ class WebSocketTest(WebSocketBaseTestCase):
         response = self.fetch('/echo')
         self.assertEqual(response.code, 400)
 
+    def test_missing_websocket_key(self):
+        response = self.fetch('/echo',
+                              headers={'Connection': 'Upgrade',
+                                       'Upgrade': 'WebSocket',
+                                       'Sec-WebSocket-Version': '13'})
+        self.assertEqual(response.code, 400)
+
     def test_bad_websocket_version(self):
         response = self.fetch('/echo',
                               headers={'Connection': 'Upgrade',
diff --git a/tornado/testing.py b/tornado/testing.py
index 74d04b60..82a3b937 100644
--- a/tornado/testing.py
+++ b/tornado/testing.py
@@ -423,7 +423,7 @@ class AsyncHTTPTestCase(AsyncTestCase):
 
     def get_url(self, path):
         """Returns an absolute url for the given path on the test server."""
-        return '%s://localhost:%s%s' % (self.get_protocol(),
+        return '%s://127.0.0.1:%s%s' % (self.get_protocol(),
                                         self.get_http_port(), path)
 
     def tearDown(self):