Repository: scrapy/scrapy

Issue:
Add flag to disable Scraped debug messages
## Summary
Add a flag to settings (e.g. `LOG_SCRAPED_ENABLED`) to be able to disable the logging of each scraped Item on `DEBUG` level.

## Motivation
Crawling with Scrapy on log level `DEBUG` provides a lot of useful context during development. The only problem is the "scraped" message (constructed in `scrapy.logformatter.LogFormatter.scraped`) - it obfuscates the log entirely in the command line and can be really frustrating in general. Adding a flag to disable this log message (with a default value so that the current behaviour doesn't change) would be really useful.

## Describe alternatives you've considered
Right now you either put up with it or use `INFO` level instead which makes you lose a lot of useful log entries available in `DEBUG` level.



Discussion:
Seems like a valid request, I have seen my logs flooded with those messages as well.
I _believe_ it shouldn't be too hard to implement, probably just an `if` check around [these lines](https://github.com/scrapy/scrapy/blob/1.7/scrapy/core/scraper.py#L240-L241).
Thanks @elacuesta, that's what I thought as well, I'd be happy to pick this up when I have a bit of time.

Resolving patch:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -233,10 +233,11 @@ def _download(self, request, spider):
         def _on_success(response):
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
-                response.request = request # tie request to response received
+                response.request = request  # tie request to response received
                 logkws = self.logformatter.crawled(request, response, spider)
-                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
-                self.signals.send_catch_log(signal=signals.response_received, \
+                if logkws is not None:
+                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                self.signals.send_catch_log(signal=signals.response_received,
                     response=response, request=request, spider=spider)
             return response
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -225,7 +225,8 @@ def _itemproc_finished(self, output, item, response, spider):
             ex = output.value
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
-                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                if logkws is not None:
+                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
                 return self.signals.send_catch_log_deferred(
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
@@ -238,7 +239,8 @@ def _itemproc_finished(self, output, item, response, spider):
                     spider=spider, failure=output)
         else:
             logkws = self.logformatter.scraped(output, response, spider)
-            logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+            if logkws is not None:
+                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
             return self.signals.send_catch_log_deferred(
                 signal=signals.item_scraped, item=output, response=response,
                 spider=spider)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -29,6 +29,10 @@ class LogFormatter(object):
     *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
         The final log message is computed as ``msg % args``.
 
+    Users can define their own ``LogFormatter`` class if they want to customise how
+    each action is logged or if they want to omit it entirely. In order to omit
+    logging an action the method must return ``None``.
+
     Here is an example on how to create a custom log formatter to lower the severity level of
     the log message when an item is dropped from the pipeline::
 


Test patch:
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,10 +1,18 @@
 import unittest
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase as TwistedTestCase
 import six
 
-from scrapy.spiders import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import DropItem
 from scrapy.http import Request, Response
 from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
+from scrapy.spiders import Spider
+from tests.mockserver import MockServer
+from tests.spiders import ItemSpider
 
 
 class CustomItem(Item):
@@ -89,5 +97,61 @@ def test_flags_in_request(self):
         pass
 
 
+class SkipMessagesLogFormatter(LogFormatter):
+    def crawled(self, *args, **kwargs):
+        return None
+
+    def scraped(self, *args, **kwargs):
+        return None
+
+    def dropped(self, *args, **kwargs):
+        return None
+
+
+class DropSomeItemsPipeline(object):
+    drop = True
+
+    def process_item(self, item, spider):
+        if self.drop:
+            self.drop = False
+            raise DropItem("Ignoring item")
+        else:
+            self.drop = True
+
+class ShowOrSkipMessagesTestCase(TwistedTestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.base_settings = {
+            'LOG_LEVEL': 'DEBUG',
+            'ITEM_PIPELINES': {
+                __name__ + '.DropSomeItemsPipeline': 300,
+            },
+        }
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_show_messages(self):
+        crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+        with LogCapture() as lc:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
+        self.assertIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+        self.assertIn("Dropped: Ignoring item", str(lc))
+
+    @defer.inlineCallbacks
+    def test_skip_messages(self):
+        settings = self.base_settings.copy()
+        settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
+        crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+        with LogCapture() as lc:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
+        self.assertNotIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+        self.assertNotIn("Dropped: Ignoring item", str(lc))
+
+
 if __name__ == "__main__":
     unittest.main()