Repository: scrapy/scrapy

Issue:
deprecate Spider.make_requests_from_url
Hey,

`Spider.make_requests_from_url` is a shortcut for a single use case: sending initial requests from a spider with start_urls attribute. It saves a single line of code (`for url in start_urls`) in one specific use case, but yet we have to document it, users have to read about it and understand what it is for. 

The name suggests it is more general - but we can't yield Link objects and get requests created by `make_requests_from_url`, it does nothing for CrawlSpider (I'd expect URLs to be passed through this method), and it is ignored for SitemapSpider. This is inconsistent.

What about deprecating `make_requests_from_url` and removing it from docs? IMHO it will make API simpler, this hook does nothing useful now, and if we ever want a general 'process URL and get a Request' method we'd have to use another name for backwards compatibility anyways.



Discussion:
I agree; this method isn't that useful. Also now you even have a shorter import for `Request` if you wanna be lazy (:

+1


Resolving patch:
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -12,6 +12,7 @@
 from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.deprecate import method_is_overridden
 
 
 class Spider(object_ref):
@@ -66,10 +67,23 @@ def _set_crawler(self, crawler):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        for url in self.start_urls:
-            yield self.make_requests_from_url(url)
+        cls = self.__class__
+        if method_is_overridden(cls, Spider, 'make_requests_from_url'):
+            warnings.warn(
+                "Spider.make_requests_from_url method is deprecated; it "
+                "won't be called in future Scrapy releases. Please "
+                "override Spider.start_requests method instead (see %s.%s)." % (
+                    cls.__module__, cls.__name__
+                ),
+            )
+            for url in self.start_urls:
+                yield self.make_requests_from_url(url)
+        else:
+            for url in self.start_urls:
+                yield Request(url, dont_filter=True)
 
     def make_requests_from_url(self, url):
+        """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
     def parse(self, response):
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -156,3 +156,35 @@ def update_classpath(path):
                           ScrapyDeprecationWarning)
             return new_path
     return path
+
+
+def method_is_overridden(subclass, base_class, method_name):
+    """ 
+    Return True if a method named ``method_name`` of a ``base_class`` 
+    is overridden in a ``subclass``. 
+    
+    >>> class Base(object):
+    ...     def foo(self):
+    ...         pass
+    >>> class Sub1(Base):
+    ...     pass
+    >>> class Sub2(Base):
+    ...     def foo(self):
+    ...         pass
+    >>> class Sub3(Sub1):
+    ...     def foo(self):
+    ...         pass
+    >>> class Sub4(Sub2):
+    ...     pass
+    >>> method_is_overridden(Sub1, Base, 'foo')
+    False
+    >>> method_is_overridden(Sub2, Base, 'foo')
+    True
+    >>> method_is_overridden(Sub3, Base, 'foo')
+    True
+    >>> method_is_overridden(Sub4, Base, 'foo')
+    True
+    """
+    base_method = getattr(base_class, method_name)
+    sub_method = getattr(subclass, method_name)
+    return base_method.__code__ is not sub_method.__code__


Test patch:
diff --git a/tests/spiders.py b/tests/spiders.py
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -170,10 +170,7 @@ def start_requests(self):
         for i in range(0, self.distinct_urls):
             for j in range(0, self.dupe_factor):
                 url = "http://localhost:8998/echo?headers=1&body=test%d" % i
-                yield self.make_requests_from_url(url)
-
-    def make_requests_from_url(self, url):
-        return Request(url, dont_filter=self.dont_filter)
+                yield Request(url, dont_filter=self.dont_filter)
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
         super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
diff --git a/tests/test_engine.py b/tests/test_engine.py
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -66,8 +66,8 @@ def parse_item(self, response):
 
 
 class TestDupeFilterSpider(TestSpider):
-    def make_requests_from_url(self, url):
-        return Request(url)  # dont_filter=False
+    def start_requests(self):
+        return (Request(url) for url in self.start_urls)  # no dont_filter=True
 
 
 class DictItemsSpider(TestSpider):
diff --git a/tests/test_spider.py b/tests/test_spider.py
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -345,7 +345,7 @@ def test_get_sitemap_urls_from_robotstxt(self):
                           'http://www.example.com/sitemap-relative-url.xml'])
 
 
-class BaseSpiderDeprecationTest(unittest.TestCase):
+class DeprecationTest(unittest.TestCase):
 
     def test_basespider_is_deprecated(self):
         with warnings.catch_warnings(record=True) as w:
@@ -399,6 +399,29 @@ def test_crawl_spider(self):
         assert isinstance(CrawlSpider(name='foo'), Spider)
         assert isinstance(CrawlSpider(name='foo'), BaseSpider)
 
+    def test_make_requests_from_url_deprecated(self):
+        class MySpider4(Spider):
+            name = 'spider1'
+            start_urls = ['http://example.com']
 
-if __name__ == '__main__':
-    unittest.main()
+        class MySpider5(Spider):
+            name = 'spider2'
+            start_urls = ['http://example.com']
+
+            def make_requests_from_url(self, url):
+                return Request(url + "/foo", dont_filter=True)
+
+        with warnings.catch_warnings(record=True) as w:
+            # spider without overridden make_requests_from_url method
+            # doesn't issue a warning
+            spider1 = MySpider4()
+            self.assertEqual(len(list(spider1.start_requests())), 1)
+            self.assertEqual(len(w), 0)
+
+            # spider with overridden make_requests_from_url issues a warning,
+            # but the method still works
+            spider2 = MySpider5()
+            requests = list(spider2.start_requests())
+            self.assertEqual(len(requests), 1)
+            self.assertEqual(requests[0].url, 'http://example.com/foo')
+            self.assertEqual(len(w), 1)