Repository: scrapy/scrapy

Issue:
Big download is not being cancelled properly
I am running scrapy 1.0.1 in my pyhon web crawler and setting the max download size like this: 

`DOWNLOAD_MAXSIZE = 41943040 #40MB`.

When I run into a bigger file, scrappy keeps logging the error: `ERROR: Received (51316256) bytes larger than download max size (41943040)` but never stops. I have looked into the code and saw a call to cancel the processe [here](https://github.com/scrapy/scrapy/blob/1.0.1/scrapy/core/downloader/handlers/http11.py#L306), but in my case is not working, the download keeps going on.

Any thoughts on why this would happen ? It seems to be something wrong on twisted not cancelling the download.

Detailed version info:

```
scrapy version -v
2015-11-25 10:44:54 [scrapy] INFO: Scrapy 1.0.1 started (bot:)
2015-11-25 10:44:54 [scrapy] INFO: Optional features available: ssl, http11
2015-11-25 10:44:54 [scrapy] INFO: Overridden settings: {'COOKIES_DEBUG': True, 'DOWNLOAD_TIMEOUT': 600, 'SPIDER_MODULES': ['spiders'], 'CONCURRENT_REQUESTS': 10, 'RANDOMIZE_DOWNLOAD_DELAY': False, 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', 'RETRY_TIMES': 10, 'BOT_NAME': 'bot', 'DOWNLOAD_MAXSIZE': 41943040, 'USER_AGENT': 'Mozilla/5.0 (X11; Linux i686; rv:23.0) Gecko/20100101 Firefox/23.0', 'NEWSPIDER_MODULE': 'spiders'}
Scrapy  : 1.0.1
lxml    : 3.5.0.0
libxml2 : 2.9.2
Twisted : 15.4.0
Python  : 2.7.10 (default, Aug 13 2015, 12:27:27) - [GCC 4.9.2]
Platform: Linux-3.16.0-38-generic-x86_64-with
```



Discussion:
How do i reproduce this error ?


Resolving patch:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -348,7 +348,8 @@ def _cb_bodyready(self, txresponse, request):
                            {'size': expected_size, 'warnsize': warnsize})
 
         def _cancel(_):
-            txresponse._transport._producer.loseConnection()
+            # Abort connection inmediately.
+            txresponse._transport._producer.abortConnection()
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(_ResponseReader(
@@ -401,6 +402,11 @@ def __init__(self, finished, txresponse, request, maxsize, warnsize,
         self._bytes_received = 0
 
     def dataReceived(self, bodyBytes):
+        # This maybe called several times after cancel was called with buffered
+        # data.
+        if self._finished.called:
+            return
+
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
@@ -409,6 +415,9 @@ def dataReceived(self, bodyBytes):
                          "max size (%(maxsize)s).",
                          {'bytes': self._bytes_received,
                           'maxsize': self._maxsize})
+            # Clear buffer earlier to avoid keeping data in memory for a long
+            # time.
+            self._bodybuf.truncate(0)
             self._finished.cancel()
 
         if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:


Test patch:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -177,6 +177,16 @@ def render(self, request):
         return request.content.read()
 
 
+class LargeChunkedFileResource(resource.Resource):
+    def render(self, request):
+        def response():
+            for i in range(1024):
+                request.write(b"x" * 1024)
+            request.finish()
+        reactor.callLater(0, response)
+        return server.NOT_DONE_YET
+
+
 class HttpTestCase(unittest.TestCase):
 
     scheme = 'http'
@@ -202,6 +212,7 @@ def setUp(self):
         r.putChild(b"broken-chunked", BrokenChunkedResource())
         r.putChild(b"contentlength", ContentLengthHeaderResource())
         r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
+        r.putChild(b"largechunkedfile", LargeChunkedFileResource())
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
@@ -384,6 +395,25 @@ def test_download_with_maxsize(self):
         d = self.download_request(request, Spider('foo', download_maxsize=9))
         yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
+    @defer.inlineCallbacks
+    def test_download_with_maxsize_very_large_file(self):
+        with mock.patch('scrapy.core.downloader.handlers.http11.logger') as logger:
+            request = Request(self.getURL('largechunkedfile'))
+
+            def check(logger):
+                logger.error.assert_called_once_with(mock.ANY, mock.ANY)
+
+            d = self.download_request(request, Spider('foo', download_maxsize=1500))
+            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+            # As the error message is logged in the dataReceived callback, we
+            # have to give a bit of time to the reactor to process the queue
+            # after closing the connection.
+            d = defer.Deferred()
+            d.addCallback(check)
+            reactor.callLater(.1, d.callback, logger)
+            yield d
+
     @defer.inlineCallbacks
     def test_download_with_maxsize_per_req(self):
         meta = {'download_maxsize': 2}