Repository: scrapy/scrapy (python)

Issue:
Add support for meta in Spider Contracts
<!--

Thanks for taking an interest in Scrapy!

If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.

Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md

The following is a suggested template to structure your pull request, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#writing-patches and https://doc.scrapy.org/en/latest/contributing.html#submitting-patches

-->

## Summary

Today we support `cb_kwargs` but we have scenarios where the data is passed using `meta`.

## Motivation

I have some spiders that use `meta` to store information, e.g: `cookiejar`

## Describe alternatives you've considered

Use callback args.


Pull request description:
Add support for meta in Spider Contracts
close #6468

Patch:
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 2d61026e9..61aef4bbb 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -20,13 +20,13 @@ following example:
         This function parses a sample response. Some contracts are mingled
         with this docstring.
 
-        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        @url http://www.example.com/s?field-keywords=selfish+gene
         @returns items 1 16
         @returns requests 0 0
         @scrapes Title Author Year Price
         """
 
-This callback is tested using three built-in contracts:
+You can use the following contracts:
 
 .. module:: scrapy.contracts.default
 
@@ -46,6 +46,14 @@ This callback is tested using three built-in contracts:
 
     @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
 
+.. class:: MetadataContract
+
+    This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`
+    attribute for the sample request. It must be a valid JSON dictionary.
+    ::
+
+    @meta {"arg1": "value1", "arg2": "value2", ...}
+
 .. class:: ReturnsContract
 
     This contract (``@returns``) sets lower and upper bounds for the items and
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 71ca4168a..e7b11d426 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract):
         return args
 
 
+class MetadataContract(Contract):
+    """Contract to set metadata arguments for the request.
+    The value should be JSON-encoded dictionary, e.g.:
+
+    @meta {"arg1": "some value"}
+    """
+
+    name = "meta"
+
+    def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
+        args["meta"] = json.loads(" ".join(self.args))
+        return args
+
+
 class ReturnsContract(Contract):
     """Contract to check the output of a callback
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 932475fb5..7ba0128a5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     "scrapy.contracts.default.UrlContract": 1,
     "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
+    "scrapy.contracts.default.MetadataContract": 1,
     "scrapy.contracts.default.ReturnsContract": 2,
     "scrapy.contracts.default.ScrapesContract": 3,
 }


Test patch:
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c9c12f0d8..d578b3af4 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -8,6 +8,7 @@ from scrapy import FormRequest
 from scrapy.contracts import Contract, ContractsManager
 from scrapy.contracts.default import (
     CallbackKeywordArgumentsContract,
+    MetadataContract,
     ReturnsContract,
     ScrapesContract,
     UrlContract,
@@ -29,6 +30,10 @@ class ResponseMock:
     url = "http://scrapy.org"
 
 
+class ResponseMetaMock(ResponseMock):
+    meta = None
+
+
 class CustomSuccessContract(Contract):
     name = "custom_success_contract"
 
@@ -195,6 +200,33 @@ class TestSpider(Spider):
         """
         pass
 
+    def returns_request_meta(self, response):
+        """method which returns request
+        @url https://example.org
+        @meta {"cookiejar": "session1"}
+        @returns requests 1
+        """
+        return Request(
+            "https://example.org", meta=response.meta, callback=self.returns_item_meta
+        )
+
+    def returns_item_meta(self, response):
+        """method which returns item
+        @url http://scrapy.org
+        @meta {"key": "example"}
+        @returns items 1 1
+        """
+        return TestItem(name="example", url=response.url)
+
+    def returns_error_missing_meta(self, response):
+        """method which depends of metadata be defined
+
+        @url http://scrapy.org
+        @returns items 1
+        """
+        key = response.meta["key"]
+        yield {key: "value"}
+
 
 class CustomContractSuccessSpider(Spider):
     name = "custom_contract_success_spider"
@@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase):
     contracts = [
         UrlContract,
         CallbackKeywordArgumentsContract,
+        MetadataContract,
         ReturnsContract,
         ScrapesContract,
         CustomFormContract,
@@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response, **request.cb_kwargs)
         self.should_error()
 
+    def test_meta(self):
+        spider = TestSpider()
+
+        # extract contracts correctly
+        contracts = self.conman.extract_contracts(spider.returns_request_meta)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, MetadataContract, ReturnsContract]),
+        )
+
+        contracts = self.conman.extract_contracts(spider.returns_item_meta)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, MetadataContract, ReturnsContract]),
+        )
+
+        response = ResponseMetaMock()
+
+        # returns_request
+        request = self.conman.from_method(spider.returns_request_meta, self.results)
+        assert request.meta["cookiejar"] == "session1"
+        response.meta = request.meta
+        request.callback(response)
+        assert response.meta["cookiejar"] == "session1"
+        self.should_succeed()
+
+        response = ResponseMetaMock()
+
+        # returns_item
+        request = self.conman.from_method(spider.returns_item_meta, self.results)
+        assert request.meta["key"] == "example"
+        response.meta = request.meta
+        request.callback(ResponseMetaMock)
+        assert response.meta["key"] == "example"
+        self.should_succeed()
+
+        response = ResponseMetaMock()
+
+        request = self.conman.from_method(
+            spider.returns_error_missing_meta, self.results
+        )
+        request.callback(response)
+        self.should_error()
+
     def test_returns(self):
         spider = TestSpider()
         response = ResponseMock()