Repository: scrapy/scrapy

Issue:
Disk queues don't preserve Request class
When a Request subclass (e.g. FormRequest) is sent to a disk queue a bare Request is what you get back. 

This is inconvenient for scrapy-splash: Splash requests all have Splash URL as request.url, but for logging it is nice to display the requested URL, not only Splash URL. In scrapy-splash this is implemented by changing `__repr__` in a Request subclass, but it works only when request is kept in memory.


Discussion:
codecov-io: ## [Current coverage](https://codecov.io/gh/scrapy/scrapy/pull/2510?src=pr) is 83.46% (diff: 100%)
> Merging [#2510](https://codecov.io/gh/scrapy/scrapy/pull/2510?src=pr) into [master](https://codecov.io/gh/scrapy/scrapy/branch/master?src=pr) will decrease coverage by **<.01%**

```diff
@@             master      #2510   diff @@
==========================================
  Files           161        161          
  Lines          8780       8779     -1   
  Methods           0          0          
  Messages          0          0          
  Branches       1288       1288          
==========================================
- Hits           7328       7327     -1   
  Misses         1204       1204          
  Partials        248        248          
```

> Powered by [Codecov](https://codecov.io?src=pr). Last update [4620d2f...5034821](https://codecov.io/gh/scrapy/scrapy/compare/4620d2f4256093296c490b412a95e4eb9aa5bd41...5034821d028fefbcb35e043f481fa1edc849e578?src=pr)

Patch:
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 7e1e99e48..2fceb0d94 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -5,6 +5,7 @@ import six
 
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.misc import load_object
 
 
 def request_to_dict(request, spider=None):
@@ -32,6 +33,8 @@ def request_to_dict(request, spider=None):
         'priority': request.priority,
         'dont_filter': request.dont_filter,
     }
+    if type(request) is not Request:
+        d['_class'] = request.__module__ + '.' + request.__class__.__name__
     return d
 
 
@@ -47,7 +50,8 @@ def request_from_dict(d, spider=None):
     eb = d['errback']
     if eb and spider:
         eb = _get_method(spider, eb)
-    return Request(
+    request_cls = load_object(d['_class']) if '_class' in d else Request
+    return request_cls(
         url=to_native_str(d['url']),
         callback=cb,
         errback=eb,


Test patch:
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index a62f13e21..5b889ab5d 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-from scrapy.http import Request
+from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
@@ -42,6 +42,7 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_same_request(request, request2)
 
     def _assert_same_request(self, r1, r2):
+        self.assertEqual(r1.__class__, r2.__class__)
         self.assertEqual(r1.url, r2.url)
         self.assertEqual(r1.callback, r2.callback)
         self.assertEqual(r1.errback, r2.errback)
@@ -54,6 +55,12 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
 
+    def test_request_class(self):
+        r = FormRequest("http://www.example.com")
+        self._assert_serializes_ok(r, spider=self.spider)
+        r = CustomRequest("http://www.example.com")
+        self._assert_serializes_ok(r, spider=self.spider)
+
     def test_callback_serialization(self):
         r = Request("http://www.example.com", callback=self.spider.parse_item,
                     errback=self.spider.handle_error)
@@ -77,3 +84,7 @@ class TestSpider(Spider):
 
     def handle_error(self, failure):
         pass
+
+
+class CustomRequest(Request):
+    pass