Repository: scrapy/scrapy (python)

Issue:
Increase verbosity for ItemLoader, Compose and MapCompose errors
ItemLoader already provides error raising for the case of the output processor:

```
    def get_output_value(self, field_name):
        proc = self.get_output_processor(field_name)
        proc = wrap_loader_context(proc, self.context)
        try:
            return proc(self._values[field_name])
        except Exception as e:
            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
                (field_name, self._values[field_name], type(e)._name__, str(e)))
```

It could be helpful to extend this behaviour could to other ItemLoader methods:

- `_process_input_value` for input processors 

```
    def _process_input_value(self, field_name, value):
        proc = self.get_input_processor(field_name)
        proc = wrap_loader_context(proc, self.context)
        return proc(value)
```
- `get_value` for processors that are passed as an argument to `add_css`, `add_xpath` or `add_value`
```
    def get_value(self, value, *processors, **kw):
        regex = kw.get('re', None)
        if regex:
            value = arg_to_iter(value)
            value = flatten(extract_regex(regex, x) for x in value)

        for proc in processors:
            if value is None:
                break
            proc = wrap_loader_context(proc, self.context)
            value = proc(value)
        return value
```
Also, `Compose` and `MapCompose` could raise errors occurring while processing the values.

Pull request description:
[MRG+1] Itemloader errors #3836
Added error messages to ItemLoader (for input processor, and processor as argument) and also for MapCompose and Compose. 

For MapCompose and Compose, I'm not sure though if the format is correct, since these errors will be displayed when processing the input/out processors and they will be nested to these ones, example:
```
ValueError: Error with processor MapCompose value=['change,deep-thoughts,thinking,world'] 
error='ValueError: Error in MapCompose with function _strip 
value=['change,deep-thoughts,thinking,world'] error='TypeError: must be str, not int''
```

I would appreciate some suggestions here.

Fixes #3836

Patch:
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 20f0f90c3..844e3828c 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -109,8 +109,14 @@ class ItemLoader(object):
         for proc in processors:
             if value is None:
                 break
+            _proc = proc
             proc = wrap_loader_context(proc, self.context)
-            value = proc(value)
+            try:
+                value = proc(value)
+            except Exception as e:
+                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
+                                 (_proc.__class__.__name__, value,
+                                  type(e).__name__, str(e)))
         return value
 
     def load_item(self):
@@ -150,8 +156,15 @@ class ItemLoader(object):
 
     def _process_input_value(self, field_name, value):
         proc = self.get_input_processor(field_name)
+        _proc = proc
         proc = wrap_loader_context(proc, self.context)
-        return proc(value)
+        try:
+            return proc(value)
+        except Exception as e:
+            raise ValueError(
+                "Error with input processor %s: field=%r value=%r "
+                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
+                                    value, type(e).__name__, str(e)))
 
     def _get_item_field_attr(self, field_name, key, default=None):
         if isinstance(self.item, Item):
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index bf7c74bfe..468aec2cc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -25,7 +25,13 @@ class MapCompose(object):
         for func in wrapped_funcs:
             next_values = []
             for v in values:
-                next_values += arg_to_iter(func(v))
+                try:
+                    next_values += arg_to_iter(func(v))
+                except Exception as e:
+                    raise ValueError("Error in MapCompose with "
+                                     "%s value=%r error='%s: %s'" %
+                                     (str(func), value, type(e).__name__,
+                                      str(e)))
             values = next_values
         return values
 
@@ -46,7 +52,12 @@ class Compose(object):
         for func in wrapped_funcs:
             if value is None and self.stop_on_none:
                 break
-            value = func(value)
+            try:
+                value = func(value)
+            except Exception as e:
+                raise ValueError("Error in Compose with "
+                                 "%s value=%r error='%s: %s'" %
+                                 (str(func), value, type(e).__name__, str(e)))
         return value
 
 
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 95dca9a3f..404bbf5bf 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -3,7 +3,7 @@ from __future__ import print_function
 import functools
 import logging
 from collections import defaultdict
-from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
+from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
 from scrapy.settings import Settings
@@ -139,30 +139,6 @@ class MediaPipeline(object):
             result.cleanFailure()
             result.frames = []
             result.stack = None
-
-            # This code fixes a memory leak by avoiding to keep references to
-            # the Request and Response objects on the Media Pipeline cache.
-            #
-            # Twisted inline callbacks pass return values using the function
-            # twisted.internet.defer.returnValue, which encapsulates the return
-            # value inside a _DefGen_Return base exception.
-            #
-            # What happens when the media_downloaded callback raises another
-            # exception, for example a FileException('download-error') when
-            # the Response status code is not 200 OK, is that it stores the
-            # _DefGen_Return exception on the FileException context.
-            #
-            # To avoid keeping references to the Response and therefore Request
-            # objects on the Media Pipeline cache, we should wipe the context of
-            # the exception encapsulated by the Twisted Failure when its a
-            # _DefGen_Return instance.
-            #
-            # This problem does not occur in Python 2.7 since we don't have
-            # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
-            context = getattr(result.value, '__context__', None)
-            if isinstance(context, _DefGen_Return):
-                setattr(result.value, '__context__', None)
-
         info.downloading.remove(fp)
         info.downloaded[fp] = result  # cache result
         for wad in info.waiting.pop(fp):


Test patch:
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 5a8ee1b2e..ce0fa0701 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -456,6 +456,42 @@ class BasicItemLoaderTest(unittest.TestCase):
             'title': [u'Test item title 3', u'Test item 4'],
         })
 
+    def test_error_input_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_in = MapCompose(float)
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'])
+
+    def test_error_output_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = Compose(Join(), float)
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        with self.assertRaises(ValueError):
+            il.load_item()
+
+    def test_error_processor_as_argument(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'], Compose(float))
+
 
 class ProcessorsTest(unittest.TestCase):
 
@@ -482,13 +518,22 @@ class ProcessorsTest(unittest.TestCase):
         proc = Compose(str.upper)
         self.assertEqual(proc(None), None)
         proc = Compose(str.upper, stop_on_none=False)
-        self.assertRaises(TypeError, proc, None)
+        self.assertRaises(ValueError, proc, None)
+        proc = Compose(str.upper, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x
         proc = MapCompose(filter_world, six.text_type.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        proc = MapCompose(filter_world, six.text_type.upper)
+        self.assertEqual(proc(None), [])
+        proc = MapCompose(filter_world, six.text_type.upper)
+        self.assertRaises(ValueError, proc, [1])
+        proc = MapCompose(filter_world, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
 
 
 class SelectortemLoaderTest(unittest.TestCase):
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 28e39cefa..5f6a6d9e6 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,19 +1,15 @@
 from __future__ import print_function
-
-import sys
-
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
+from twisted.internet.defer import Deferred, inlineCallbacks
 
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.request import request_fingerprint
 from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy import signals
@@ -94,77 +90,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         self.pipe._modify_media_request(request)
         assert request.meta == {'handle_httpstatus_all': True}
 
-    def test_should_remove_req_res_references_before_caching_the_results(self):
-        """Regression test case to prevent a memory leak in the Media Pipeline.
-
-        The memory leak is triggered when an exception is raised when a Response
-        scheduled by the Media Pipeline is being returned. For example, when a
-        FileException('download-error') is raised because the Response status
-        code is not 200 OK.
-
-        It happens because we are keeping a reference to the Response object
-        inside the FileException context. This is caused by the way Twisted
-        return values from inline callbacks. It raises a custom exception
-        encapsulating the original return value.
-
-        The solution is to remove the exception context when this context is a
-        _DefGen_Return instance, the BaseException used by Twisted to pass the
-        returned value from those inline callbacks.
-
-        Maybe there's a better and more reliable way to test the case described
-        here, but it would be more complicated and involve running - or at least
-        mocking - some async steps from the Media Pipeline. The current test
-        case is simple and detects the problem very fast. On the other hand, it
-        would not detect another kind of leak happening due to old object
-        references being kept inside the Media Pipeline cache.
-
-        This problem does not occur in Python 2.7 since we don't have Exception
-        Chaining (https://www.python.org/dev/peps/pep-3134/).
-        """
-        # Create sample pair of Request and Response objects
-        request = Request('http://url')
-        response = Response('http://url', body=b'', request=request)
-
-        # Simulate the Media Pipeline behavior to produce a Twisted Failure
-        try:
-            # Simulate a Twisted inline callback returning a Response
-            # The returnValue method raises an exception encapsulating the value
-            returnValue(response)
-        except BaseException as exc:
-            def_gen_return_exc = exc
-            try:
-                # Simulate the media_downloaded callback raising a FileException
-                # This usually happens when the status code is not 200 OK
-                raise FileException('download-error')
-            except Exception as exc:
-                file_exc = exc
-                # Simulate Twisted capturing the FileException
-                # It encapsulates the exception inside a Twisted Failure
-                failure = Failure(file_exc)
-
-        # The Failure should encapsulate a FileException ...
-        self.assertEqual(failure.value, file_exc)
-        # ... and if we're running on Python 3 ...
-        if sys.version_info.major >= 3:
-            # ... it should have the returnValue exception set as its context
-            self.assertEqual(failure.value.__context__, def_gen_return_exc)
-
-        # Let's calculate the request fingerprint and fake some runtime data...
-        fp = request_fingerprint(request)
-        info = self.pipe.spiderinfo
-        info.downloading.add(fp)
-        info.waiting[fp] = []
-
-        # When calling the method that caches the Request's result ...
-        self.pipe._cache_result_and_execute_waiters(failure, fp, info)
-        # ... it should store the Twisted Failure ...
-        self.assertEqual(info.downloaded[fp], failure)
-        # ... encapsulating the original FileException ...
-        self.assertEqual(info.downloaded[fp].value, file_exc)
-        # ... but it should not store the returnValue exception on its context
-        context = getattr(info.downloaded[fp].value, '__context__', None)
-        self.assertIsNone(context)
-
 
 class MockedMediaPipeline(MediaPipeline):