Repository: scrapy/scrapy

Issue:
DOWNLOAD_MAXSIZE logger level shouldn't be Error
When triggered, the `DOWNLOAD_MAXSIZE` setting raises an ERROR level logging event. I don't think it should be at this level. Error is "something went wrong"; I've explicitly told Scrapy I don't want files of this size, so it's not an error to not download them. WARN or even just INFO makes more sense.

I'm logging Error levels and above to stderr so I can keep track of them easily and it's being flooded by these. In fact, apart from a couple of genuine errors, of the ~100,000 requests my spider has made so far, these are the only Errors that are showing up.
[MRG+1] Change DOWNLOAD_MAXSIZE logger level from Error to Warning
This PR based on #3874

Raising error on reaching download maxsize is removed. 
Logger level is replaced to warning.

Closes #3874


Discussion:
Sounds good; pull requests are welcome.


Resolving patch:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -394,13 +394,14 @@ def _cb_bodyready(self, txresponse, request):
         fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
 
         if maxsize and expected_size > maxsize:
-            error_msg = ("Cancelling download of %(url)s: expected response "
-                         "size (%(size)s) larger than download max size (%(maxsize)s).")
-            error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+            warning_msg = ("Cancelling download of %(url)s: expected response "
+                           "size (%(size)s) larger than download max size (%(maxsize)s).")
+            warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+
+            logger.warning(warning_msg, warning_args)
 
-            logger.error(error_msg, error_args)
             txresponse._transport._producer.loseConnection()
-            raise defer.CancelledError(error_msg % error_args)
+            raise defer.CancelledError(warning_msg % warning_args)
 
         if warnsize and expected_size > warnsize:
             logger.warning("Expected response size (%(size)s) larger than "
@@ -523,11 +524,11 @@ def dataReceived(self, bodyBytes):
                 self._finish_response(flags=["download_stopped"], failure=failure)
 
         if self._maxsize and self._bytes_received > self._maxsize:
-            logger.error("Received (%(bytes)s) bytes larger than download "
-                         "max size (%(maxsize)s) in request %(request)s.",
-                         {'bytes': self._bytes_received,
-                          'maxsize': self._maxsize,
-                          'request': self._request})
+            logger.warning("Received (%(bytes)s) bytes larger than download "
+                           "max size (%(maxsize)s) in request %(request)s.",
+                           {'bytes': self._bytes_received,
+                            'maxsize': self._maxsize,
+                            'request': self._request})
             # Clear buffer earlier to avoid keeping data in memory for a long time.
             self._bodybuf.truncate(0)
             self._finished.cancel()


Test patch:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -410,7 +410,7 @@ def test_download_with_maxsize_very_large_file(self):
             request = Request(self.getURL('largechunkedfile'))
 
             def check(logger):
-                logger.error.assert_called_once_with(mock.ANY, mock.ANY)
+                logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
 
             d = self.download_request(request, Spider('foo', download_maxsize=1500))
             yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)