Repository: scrapy/scrapy

Issue:
PY3: error decoding Content-Disposition header
This request
```
scrapy shell 'http://npe.com.cn/plus/save_to_doc.php?id=1666'
```

raises this error:

```
Traceback (most recent call last):
  File "/Users/kmike/envs/dl/bin/scrapy", line 9, in <module>
    load_entry_point('Scrapy', 'console_scripts', 'scrapy')()
  File "/Users/kmike/svn/scrapy/scrapy/cmdline.py", line 142, in execute
    _run_print_help(parser, _run_command, cmd, args, opts)
  File "/Users/kmike/svn/scrapy/scrapy/cmdline.py", line 88, in _run_print_help
    func(*a, **kw)
  File "/Users/kmike/svn/scrapy/scrapy/cmdline.py", line 149, in _run_command
    cmd.run(args, opts)
  File "/Users/kmike/svn/scrapy/scrapy/commands/shell.py", line 71, in run
    shell.start(url=url)
  File "/Users/kmike/svn/scrapy/scrapy/shell.py", line 47, in start
    self.fetch(url, spider)
  File "/Users/kmike/svn/scrapy/scrapy/shell.py", line 112, in fetch
    reactor, self._schedule, request, spider)
  File "/Users/kmike/envs/dl/lib/python3.5/site-packages/Twisted-15.5.0-py3.5.egg/twisted/internet/threads.py", line 122, in blockingCallFromThread
    result.raiseException()
  File "/Users/kmike/envs/dl/lib/python3.5/site-packages/Twisted-15.5.0-py3.5.egg/twisted/python/failure.py", line 368, in raiseException
    raise self.value.with_traceback(self.tb)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb8 in position 25: invalid start byte
```

The error points to a wrong location (similar to #1760); the real traceback is

```
Traceback (most recent call last):
  File "/Users/kmike/envs/dl/lib/python3.5/site-packages/Twisted-15.5.0-py3.5.egg/twisted/internet/defer.py", line 1126, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/Users/kmike/envs/dl/lib/python3.5/site-packages/Twisted-15.5.0-py3.5.egg/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/Users/kmike/svn/scrapy/scrapy/core/downloader/middleware.py", line 43, in process_request
    defer.returnValue((yield download_func(request=request,spider=spider)))
  File "/Users/kmike/envs/dl/lib/python3.5/site-packages/Twisted-15.5.0-py3.5.egg/twisted/internet/defer.py", line 588, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/Users/kmike/svn/scrapy/scrapy/core/downloader/handlers/http11.py", line 272, in _cb_bodydone
    respcls = responsetypes.from_args(headers=headers, url=url)
  File "/Users/kmike/svn/scrapy/scrapy/responsetypes.py", line 110, in from_args
    cls = self.from_headers(headers)
  File "/Users/kmike/svn/scrapy/scrapy/responsetypes.py", line 78, in from_headers
    cls = self.from_content_disposition(headers[b'Content-Disposition'])
  File "/Users/kmike/svn/scrapy/scrapy/responsetypes.py", line 62, in from_content_disposition
    filename = to_native_str(content_disposition).split(';')[1].split('=')[1]
  File "/Users/kmike/svn/scrapy/scrapy/utils/python.py", line 129, in to_native_str
    return to_unicode(text, encoding, errors)
  File "/Users/kmike/svn/scrapy/scrapy/utils/python.py", line 107, in to_unicode
    return text.decode(encoding, errors)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb8 in position 25: invalid start byte
```

It looks like Content-Disposition is decoded using utf-8, but the encoding was not UTF-8.

Discussion:

Patch:
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4880cc7b9..c667b141d 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -59,7 +59,8 @@ class ResponseTypes(object):
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition).split(';')[1].split('=')[1]
+            filename = to_native_str(content_disposition,
+                encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
         except IndexError:


Test patch:
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2374d518f..118136ac4 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import unittest
 from scrapy.responsetypes import responsetypes
 
@@ -20,8 +21,14 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_content_disposition(self):
         mappings = [
-            ('attachment; filename="data.xml"', XmlResponse),
-            ('attachment; filename=data.xml', XmlResponse),
+            (b'attachment; filename="data.xml"', XmlResponse),
+            (b'attachment; filename=data.xml', XmlResponse),
+            (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+            (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+            (u'attachment;filename=data高.doc'.encode('gbk'), Response),
+            (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+            (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)