Repository: scrapy/scrapy
Pull request #6470 (2024-08-27 11:42:00)

Refactor genspider slightly so template variables can be overridden


Patch:
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 2d61026e9..61aef4bbb 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -20,13 +20,13 @@ following example:
         This function parses a sample response. Some contracts are mingled
         with this docstring.
 
-        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        @url http://www.example.com/s?field-keywords=selfish+gene
         @returns items 1 16
         @returns requests 0 0
         @scrapes Title Author Year Price
         """
 
-This callback is tested using three built-in contracts:
+You can use the following contracts:
 
 .. module:: scrapy.contracts.default
 
@@ -46,6 +46,14 @@ This callback is tested using three built-in contracts:
 
     @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
 
+.. class:: MetadataContract
+
+    This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`
+    attribute for the sample request. It must be a valid JSON dictionary.
+    ::
+
+    @meta {"arg1": "value1", "arg2": "value2", ...}
+
 .. class:: ReturnsContract
 
     This contract (``@returns``) sets lower and upper bounds for the items and
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2649fb23d..6c3713f8f 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -116,26 +116,34 @@ class Command(ScrapyCommand):
             if opts.edit:
                 self.exitcode = os.system(f'scrapy edit "{name}"')  # nosec
 
-    def _genspider(
+    def _generate_template_variables(
         self,
         module: str,
         name: str,
         url: str,
         template_name: str,
-        template_file: Union[str, os.PathLike],
-    ) -> None:
-        """Generate the spider module, based on the given template"""
+    ):
         capitalized_module = "".join(s.capitalize() for s in module.split("_"))
-        domain = extract_domain(url)
-        tvars = {
+        return {
             "project_name": self.settings.get("BOT_NAME"),
             "ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
             "module": module,
             "name": name,
             "url": url,
-            "domain": domain,
+            "domain": extract_domain(url),
             "classname": f"{capitalized_module}Spider",
         }
+
+    def _genspider(
+        self,
+        module: str,
+        name: str,
+        url: str,
+        template_name: str,
+        template_file: Union[str, os.PathLike],
+    ) -> None:
+        """Generate the spider module, based on the given template"""
+        tvars = self._generate_template_variables(module, name, url, template_name)
         if self.settings.get("NEWSPIDER_MODULE"):
             spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
             assert spiders_module.__file__
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 71ca4168a..e7b11d426 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract):
         return args
 
 
+class MetadataContract(Contract):
+    """Contract to set metadata arguments for the request.
+    The value should be JSON-encoded dictionary, e.g.:
+
+    @meta {"arg1": "some value"}
+    """
+
+    name = "meta"
+
+    def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
+        args["meta"] = json.loads(" ".join(self.args))
+        return args
+
+
 class ReturnsContract(Contract):
     """Contract to check the output of a callback
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 932475fb5..7ba0128a5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     "scrapy.contracts.default.UrlContract": 1,
     "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
+    "scrapy.contracts.default.MetadataContract": 1,
     "scrapy.contracts.default.ReturnsContract": 2,
     "scrapy.contracts.default.ScrapesContract": 3,
 }


Test patch:
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c9c12f0d8..d578b3af4 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -8,6 +8,7 @@ from scrapy import FormRequest
 from scrapy.contracts import Contract, ContractsManager
 from scrapy.contracts.default import (
     CallbackKeywordArgumentsContract,
+    MetadataContract,
     ReturnsContract,
     ScrapesContract,
     UrlContract,
@@ -29,6 +30,10 @@ class ResponseMock:
     url = "http://scrapy.org"
 
 
+class ResponseMetaMock(ResponseMock):
+    meta = None
+
+
 class CustomSuccessContract(Contract):
     name = "custom_success_contract"
 
@@ -195,6 +200,33 @@ class TestSpider(Spider):
         """
         pass
 
+    def returns_request_meta(self, response):
+        """method which returns request
+        @url https://example.org
+        @meta {"cookiejar": "session1"}
+        @returns requests 1
+        """
+        return Request(
+            "https://example.org", meta=response.meta, callback=self.returns_item_meta
+        )
+
+    def returns_item_meta(self, response):
+        """method which returns item
+        @url http://scrapy.org
+        @meta {"key": "example"}
+        @returns items 1 1
+        """
+        return TestItem(name="example", url=response.url)
+
+    def returns_error_missing_meta(self, response):
+        """method which depends of metadata be defined
+
+        @url http://scrapy.org
+        @returns items 1
+        """
+        key = response.meta["key"]
+        yield {key: "value"}
+
 
 class CustomContractSuccessSpider(Spider):
     name = "custom_contract_success_spider"
@@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase):
     contracts = [
         UrlContract,
         CallbackKeywordArgumentsContract,
+        MetadataContract,
         ReturnsContract,
         ScrapesContract,
         CustomFormContract,
@@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response, **request.cb_kwargs)
         self.should_error()
 
+    def test_meta(self):
+        spider = TestSpider()
+
+        # extract contracts correctly
+        contracts = self.conman.extract_contracts(spider.returns_request_meta)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, MetadataContract, ReturnsContract]),
+        )
+
+        contracts = self.conman.extract_contracts(spider.returns_item_meta)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
+            frozenset([UrlContract, MetadataContract, ReturnsContract]),
+        )
+
+        response = ResponseMetaMock()
+
+        # returns_request
+        request = self.conman.from_method(spider.returns_request_meta, self.results)
+        assert request.meta["cookiejar"] == "session1"
+        response.meta = request.meta
+        request.callback(response)
+        assert response.meta["cookiejar"] == "session1"
+        self.should_succeed()
+
+        response = ResponseMetaMock()
+
+        # returns_item
+        request = self.conman.from_method(spider.returns_item_meta, self.results)
+        assert request.meta["key"] == "example"
+        response.meta = request.meta
+        request.callback(ResponseMetaMock)
+        assert response.meta["key"] == "example"
+        self.should_succeed()
+
+        response = ResponseMetaMock()
+
+        request = self.conman.from_method(
+            spider.returns_error_missing_meta, self.results
+        )
+        request.callback(response)
+        self.should_error()
+
     def test_returns(self):
         spider = TestSpider()
         response = ResponseMock()