Repository: scrapy/scrapy

Issue:
Contracts can't be written for callbacks that receive cb_kwargs
<!--

Thanks for taking an interest in Scrapy!

If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.

Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md

The following is a suggested template to structure your issue, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#reporting-bugs

-->

### Description

Contracts can't be written for callbacks that receive `cb_kwargs`

### Steps to Reproduce
1. Run `scrapy check` with the following spider:
```
class CbKwargsContract(scrapy.Spider):
    name = 'cb_kwargs_contract'

    def start_requests(self):
        yield scrapy.Request("https://httpbin.org/get", cb_kwargs={"arg1": "foo"})

    def parse(self, response, arg1):
        """
        @url https://httpbin.org/get
        """
        self.logger.info(arg1)
```

**Expected behavior:**
Ideally would be possible to add args to the contract (e.g., `@url https://httpbin.org/get foo`)

**Actual behavior:** Fails with a TypeError

**Reproduces how often:** 100%

### Versions
Scrapy 1.7.3


Discussion:
I can reproduce:
```
E
======================================================================
ERROR: [cb_kwargs_contract] parse (callback)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/.../lib/python3.6/site-packages/scrapy/contracts/__init__.py", line 95, in cb_wrapper
    output = cb(response)
TypeError: parse() missing 1 required positional argument: 'arg1'

----------------------------------------------------------------------
Ran 0 contracts in 0.984s

FAILED (errors=1)
```
My first thought was to replace `output = cb(response)` with `output = cb(response, **request.cb_kwargs)` in https://github.com/scrapy/scrapy/blob/1.7.3/scrapy/contracts/__init__.py#L95. However, with that change I still get `TypeError: parse() missing 1 required positional argument: 'arg1'`.

~I suspect this is due to the fact that the parsing method is not called as a Deferred callback, and the arguments are interpreted as positional instead of keyword ones. Refactoring the parsing method signature to add a default value for the expected argument (something like `arg1=None`) does work in that scenario, but I think we need a slightly more complex solution is in order to prevent that restriction.~

Scratch that, the request is generated from the contract definition, hence the `cb_kwargs` attribute is in fact empty.

Resolving patch:
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -90,9 +90,9 @@ def _clean_req(self, request, method, results):
         cb = request.callback
 
         @wraps(cb)
-        def cb_wrapper(response):
+        def cb_wrapper(response, **cb_kwargs):
             try:
-                output = cb(response)
+                output = cb(response, **cb_kwargs)
                 output = list(iterate_spider_output(output))
             except Exception:
                 case = _create_testcase(method, 'callback')
@@ -121,7 +121,7 @@ def add_pre_hook(self, request, results):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
+            def wrapper(response, **cb_kwargs):
                 try:
                     results.startTest(self.testcase_pre)
                     self.pre_process(response)
@@ -133,7 +133,7 @@ def wrapper(response):
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response)))
+                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
 
             request.callback = wrapper
 
@@ -144,8 +144,8 @@ def add_post_hook(self, request, results):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
-                output = list(iterate_spider_output(cb(response)))
+            def wrapper(response, **cb_kwargs):
+                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,3 +1,5 @@
+import json
+
 from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
@@ -18,6 +20,20 @@ def adjust_request_args(self, args):
         return args
 
 
+class CallbackKeywordArgumentsContract(Contract):
+    """ Contract to set the keyword arguments for the request.
+        The value should be a JSON-encoded dictionary, e.g.:
+
+        @cb_kwargs {"arg1": "some value"}
+    """
+
+    name = 'cb_kwargs'
+
+    def adjust_request_args(self, args):
+        args['cb_kwargs'] = json.loads(' '.join(self.args))
+        return args
+
+
 class ReturnsContract(Contract):
     """ Contract to check the output of a callback
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -291,6 +291,7 @@
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,
+    'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1,
     'scrapy.contracts.default.ReturnsContract': 2,
     'scrapy.contracts.default.ScrapesContract': 3,
 }


Test patch:
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -14,6 +14,7 @@
 from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
+    CallbackKeywordArgumentsContract,
     ReturnsContract,
     ScrapesContract,
 )
@@ -70,6 +71,37 @@ def returns_item(self, response):
         """
         return TestItem(url=response.url)
 
+    def returns_request_cb_kwargs(self, response, url):
+        """ method which returns request
+        @url https://example.org
+        @cb_kwargs {"url": "http://scrapy.org"}
+        @returns requests 1
+        """
+        return Request(url, callback=self.returns_item_cb_kwargs)
+
+    def returns_item_cb_kwargs(self, response, name):
+        """ method which returns item
+        @url http://scrapy.org
+        @cb_kwargs {"name": "Scrapy"}
+        @returns items 1 1
+        """
+        return TestItem(name=name, url=response.url)
+
+    def returns_item_cb_kwargs_error_unexpected_keyword(self, response):
+        """ method which returns item
+        @url http://scrapy.org
+        @cb_kwargs {"arg": "value"}
+        @returns items 1 1
+        """
+        return TestItem(url=response.url)
+
+    def returns_item_cb_kwargs_error_missing_argument(self, response, arg):
+        """ method which returns item
+        @url http://scrapy.org
+        @returns items 1 1
+        """
+        return TestItem(url=response.url)
+
     def returns_dict_item(self, response):
         """ method which returns item
         @url http://scrapy.org
@@ -172,6 +204,7 @@ class InheritsTestSpider(TestSpider):
 class ContractsManagerTest(unittest.TestCase):
     contracts = [
         UrlContract,
+        CallbackKeywordArgumentsContract,
         ReturnsContract,
         ScrapesContract,
         CustomFormContract,
@@ -211,6 +244,51 @@ def test_contracts(self):
         request = self.conman.from_method(spider.parse_no_url, self.results)
         self.assertEqual(request, None)
 
+    def test_cb_kwargs(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        # extract contracts correctly
+        contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+        
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument)
+        self.assertEqual(len(contracts), 2)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, ReturnsContract]))
+
+        # returns_request
+        request = self.conman.from_method(spider.returns_request_cb_kwargs, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_succeed()
+
+        # returns_item
+        request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_succeed()
+
+        # returns_item (error, callback doesn't take keyword arguments)
+        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_error()
+
+        # returns_item (error, contract doesn't provide keyword arguments)
+        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_missing_argument, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_error()
+
     def test_returns(self):
         spider = TestSpider()
         response = ResponseMock()