Repository: scrapy/itemloaders

Issue:
Feature Request: Adding "add_jmes" and "replace_jmes" method to ItemLoader
So, currently the `ItemLoader` class has 6 methods for loading values:
`add_xpath()`
`replace_xpath()`
`add_css()`
`replace_css()`
`add_value()`
`replace_value()`

Could we add another 2 more methods for loading data through JmesPath selectors. Currently, I have to use the `SelectJmes` processor to do this. Eventually, it looks really ugly and ends up taking so much line real estate. 
I did a hack where I extended the ItemLoader to include those 2 extra methods that are desperately needed.

When there is json data to parse instead of html, JmesPath selectors are the best way to go for parsing, so there should be support for JmesPath selectors in the ItemLoader class as well.

Discussion:

Patch:
diff --git a/itemloaders/__init__.py b/itemloaders/__init__.py
index 09bffba..a84da84 100644
--- a/itemloaders/__init__.py
+++ b/itemloaders/__init__.py
@@ -19,7 +19,7 @@ def unbound_method(method):
     (no need to define an unused first 'self' argument)
     """
     with suppress(AttributeError):
-        if '.' not in method.__qualname__:
+        if "." not in method.__qualname__:
             return method.__func__
     return method
 
@@ -36,12 +36,12 @@ class ItemLoader:
 
     :param item: The item instance to populate using subsequent calls to
         :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
-        or :meth:`~ItemLoader.add_value`.
+        :meth:`~ItemLoader.add_jmes` or :meth:`~ItemLoader.add_value`.
     :type item: :class:`dict` object
 
     :param selector: The selector to extract data from, when using the
-        :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
-        (resp. :meth:`replace_css`) method.
+        :meth:`add_xpath` (resp. :meth:`add_css`, :meth:`add_jmes`) or :meth:`replace_xpath`
+        (resp. :meth:`replace_css`, :meth:`replace_jmes`) method.
     :type selector: :class:`~parsel.selector.Selector` object
 
     The item, selector and the remaining keyword arguments are
@@ -105,7 +105,7 @@ class ItemLoader:
         if item is None:
             item = self.default_item_class()
         self._local_item = item
-        context['item'] = item
+        context["item"] = item
         self.context = context
         self.parent = parent
         self._local_values = {}
@@ -138,9 +138,7 @@ class ItemLoader:
         """
         selector = self.selector.xpath(xpath)
         context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
+        subloader = self.__class__(item=self.item, parent=self, **context)
         return subloader
 
     def nested_css(self, css, **context):
@@ -153,9 +151,7 @@ class ItemLoader:
         """
         selector = self.selector.css(css)
         context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
+        subloader = self.__class__(item=self.item, parent=self, **context)
         return subloader
 
     def add_value(self, field_name, value, *processors, re=None, **kw):
@@ -246,9 +242,10 @@ class ItemLoader:
             try:
                 value = proc(value)
             except Exception as e:
-                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
-                                 (_proc.__class__.__name__, value,
-                                  type(e).__name__, str(e)))
+                raise ValueError(
+                    "Error with processor %s value=%r error='%s: %s'"
+                    % (_proc.__class__.__name__, value, type(e).__name__, str(e))
+                )
         return value
 
     def load_item(self):
@@ -276,30 +273,28 @@ class ItemLoader:
         try:
             return proc(value)
         except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
-                             (field_name, value, type(e).__name__, str(e)))
+            raise ValueError(
+                "Error with output processor: field=%r value=%r error='%s: %s'"
+                % (field_name, value, type(e).__name__, str(e))
+            )
 
     def get_collected_values(self, field_name):
         """Return the collected values for the given field."""
         return self._values.get(field_name, [])
 
     def get_input_processor(self, field_name):
-        proc = getattr(self, '%s_in' % field_name, None)
+        proc = getattr(self, "%s_in" % field_name, None)
         if not proc:
             proc = self._get_item_field_attr(
-                field_name,
-                'input_processor',
-                self.default_input_processor
+                field_name, "input_processor", self.default_input_processor
             )
         return unbound_method(proc)
 
     def get_output_processor(self, field_name):
-        proc = getattr(self, '%s_out' % field_name, None)
+        proc = getattr(self, "%s_out" % field_name, None)
         if not proc:
             proc = self._get_item_field_attr(
-                field_name,
-                'output_processor',
-                self.default_output_processor
+                field_name, "output_processor", self.default_output_processor
             )
         return unbound_method(proc)
 
@@ -316,8 +311,15 @@ class ItemLoader:
         except Exception as e:
             raise ValueError(
                 "Error with input processor %s: field=%r value=%r "
-                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
-                                    value, type(e).__name__, str(e)))
+                "error='%s: %s'"
+                % (
+                    _proc.__class__.__name__,
+                    field_name,
+                    value,
+                    type(e).__name__,
+                    str(e),
+                )
+            )
 
     def _check_selector_method(self):
         if self.selector is None:
@@ -439,3 +441,63 @@ class ItemLoader:
         self._check_selector_method()
         csss = arg_to_iter(csss)
         return flatten(self.selector.css(css).getall() for css in csss)
+
+    def add_jmes(self, field_name, jmes, *processors, re=None, **kw):
+        """
+        Similar to :meth:`ItemLoader.add_value` but receives a JMESPath selector
+        instead of a value, which is used to extract a list of unicode strings
+        from the selector associated with this :class:`ItemLoader`.
+
+        See :meth:`get_jmes` for ``kwargs``.
+
+        :param jmes: the JMESPath selector to extract data from
+        :type jmes: str
+
+        Examples::
+
+            # HTML snippet: {"name": "Color TV"}
+            loader.add_jmes('name')
+            # HTML snippet: {"price": the price is $1200"}
+            loader.add_jmes('price', TakeFirst(), re='the price is (.*)')
+        """
+        values = self._get_jmesvalues(jmes)
+        self.add_value(field_name, values, *processors, re=re, **kw)
+
+    def replace_jmes(self, field_name, jmes, *processors, re=None, **kw):
+        """
+        Similar to :meth:`add_jmes` but replaces collected data instead of adding it.
+        """
+        values = self._get_jmesvalues(jmes)
+        self.replace_value(field_name, values, *processors, re=re, **kw)
+
+    def get_jmes(self, jmes, *processors, re=None, **kw):
+        """
+        Similar to :meth:`ItemLoader.get_value` but receives a JMESPath selector
+        instead of a value, which is used to extract a list of unicode strings
+        from the selector associated with this :class:`ItemLoader`.
+
+        :param jmes: the JMESPath selector to extract data from
+        :type jmes: str
+
+        :param re: a regular expression to use for extracting data from the
+            selected JMESPath
+        :type re: str or typing.Pattern
+
+        Examples::
+
+            # HTML snippet: {"name": "Color TV"}
+            loader.get_jmes('name')
+            # HTML snippet: {"price": the price is $1200"}
+            loader.get_jmes('price', TakeFirst(), re='the price is (.*)')
+        """
+        values = self._get_jmesvalues(jmes)
+        return self.get_value(values, *processors, re=re, **kw)
+
+    def _get_jmesvalues(self, jmess):
+        self._check_selector_method()
+        jmess = arg_to_iter(jmess)
+        if not hasattr(self.selector, "jmespath"):
+            raise AttributeError(
+                "Please install parsel >= 1.8.1 to get jmespath support"
+            )
+        return flatten(self.selector.jmespath(jmes).getall() for jmes in jmess)


Test patch:
diff --git a/tests/test_selector_loader.py b/tests/test_selector_loader.py
index 170b56f..972de71 100644
--- a/tests/test_selector_loader.py
+++ b/tests/test_selector_loader.py
@@ -1,5 +1,6 @@
 import re
 import unittest
+from unittest.mock import MagicMock
 
 from parsel import Selector
 
@@ -23,6 +24,18 @@ class SelectortemLoaderTest(unittest.TestCase):
     </html>
     """)
 
+    jmes_selector = Selector(text="""
+    {
+      "name": "marta",
+      "description": "paragraph",
+      "website": {
+        "url": "http://www.scrapy.org",
+        "name": "homepage"
+      },
+      "logo": "/images/logo.png"
+    }
+    """)
+
     def test_init_method(self):
         loader = CustomItemLoader()
         self.assertEqual(loader.selector, None)
@@ -172,3 +185,77 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(loader.get_output_value('url'), ['http://www.scrapy.org'])
         loader.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
         self.assertEqual(loader.get_output_value('url'), ['scrapy.org'])
+
+    def test_jmes_not_installed(self):
+        selector = MagicMock(spec=Selector)
+        del selector.jmespath
+        loader = CustomItemLoader(selector=selector)
+        with self.assertRaises(AttributeError) as err:
+            loader.add_jmes("name", "name", re="ma")
+
+        self.assertEqual(str(err.exception), "Please install parsel >= 1.8.1 to get jmespath support")
+
+    def test_add_jmes_re(self):
+        loader = CustomItemLoader(selector=self.jmes_selector)
+        loader.add_jmes("name", "name", re="ma")
+        self.assertEqual(loader.get_output_value("name"), ["Ma"])
+
+        loader.add_jmes("url", "website.url", re="http://(.+)")
+        self.assertEqual(loader.get_output_value("url"), ["www.scrapy.org"])
+
+        loader = CustomItemLoader(selector=self.jmes_selector)
+        loader.add_jmes("name", "name", re=re.compile("ma"))
+        self.assertEqual(loader.get_output_value("name"), ["Ma"])
+
+        loader.add_jmes("url", "website.url", re=re.compile("http://(.+)"))
+        self.assertEqual(loader.get_output_value("url"), ["www.scrapy.org"])
+
+    def test_get_jmes(self):
+        loader = CustomItemLoader(selector=self.jmes_selector)
+        self.assertEqual(loader.get_jmes("description"), ["paragraph"])
+        self.assertEqual(loader.get_jmes("description", TakeFirst()), "paragraph")
+        self.assertEqual(loader.get_jmes("description", TakeFirst(), re="pa"), "pa")
+
+        self.assertEqual(
+            loader.get_jmes(["description", "name"]), ["paragraph", "marta"]
+        )
+        self.assertEqual(
+            loader.get_jmes(["website.url", "logo"]),
+            ["http://www.scrapy.org", "/images/logo.png"],
+        )
+
+    def test_replace_jmes(self):
+        loader = CustomItemLoader(selector=self.jmes_selector)
+        self.assertTrue(loader.selector)
+        loader.add_jmes("name", "name")
+        self.assertEqual(loader.get_output_value("name"), ["Marta"])
+        loader.replace_jmes("name", "description")
+        self.assertEqual(loader.get_output_value("name"), ["Paragraph"])
+
+        loader.replace_jmes("name", ["description", "name"])
+        self.assertEqual(loader.get_output_value("name"), ["Paragraph", "Marta"])
+
+        loader.add_jmes("url", "website.url", re="http://(.+)")
+        self.assertEqual(loader.get_output_value("url"), ["www.scrapy.org"])
+        loader.replace_jmes("url", "logo")
+        self.assertEqual(loader.get_output_value("url"), ["/images/logo.png"])
+
+    def test_replace_jmes_multi_fields(self):
+        loader = CustomItemLoader(selector=self.jmes_selector)
+        loader.add_jmes(None, 'name', TakeFirst(), lambda x: {'name': x})
+        self.assertEqual(loader.get_output_value('name'), ['Marta'])
+        loader.replace_jmes(None, 'description', TakeFirst(), lambda x: {'name': x})
+        self.assertEqual(loader.get_output_value('name'), ['Paragraph'])
+
+        loader.add_jmes(None, 'website.url', TakeFirst(), lambda x: {'url': x})
+        self.assertEqual(loader.get_output_value('url'), ['http://www.scrapy.org'])
+        loader.replace_jmes(None, 'logo', TakeFirst(), lambda x: {'url': x})
+        self.assertEqual(loader.get_output_value('url'), ['/images/logo.png'])
+
+    def test_replace_jmes_re(self):
+        loader = CustomItemLoader(selector=self.jmes_selector)
+        self.assertTrue(loader.selector)
+        loader.add_jmes('url', 'website.url')
+        self.assertEqual(loader.get_output_value('url'), ['http://www.scrapy.org'])
+        loader.replace_jmes('url', 'website.url', re=r'http://www\.(.+)')
+        self.assertEqual(loader.get_output_value('url'), ['scrapy.org'])