Repository: scrapy/scrapy
Pull request #1128 (2015-04-02 02:46:43)

[MRG+1] Move spider settings population from CrawlerRunner to Crawler.__init__
This change was discussed in #873

This relocation has some advantages:
* Removes functionality from CrawlerRunner: After this change and moving the logging initialization to Crawler.\__init\__ as well (made in the python logging PR #1060) we can run crawlers without CrawlerRunner intervention.

* Settings given to each Crawler will be frozen after the extensions initialization. This allows the extensions to update the settings. That's something we want to implement for the Simplified Addons Idea for this GSoC (issue #591).


Patch:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 52d4069ac..692a896be 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -18,13 +18,17 @@ class Crawler(object):
 
     def __init__(self, spidercls, settings):
         self.spidercls = spidercls
-        self.settings = settings
+        self.settings = settings.copy()
+
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
 
+        self.spidercls.update_settings(self.settings)
+        self.settings.freeze()
+
         self.crawling = False
         self.spider = None
         self.engine = None
@@ -95,11 +99,7 @@ class CrawlerRunner(object):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spiders.load(spidercls)
-
-        crawler_settings = self.settings.copy()
-        spidercls.update_settings(crawler_settings)
-        crawler_settings.freeze()
-        return Crawler(spidercls, crawler_settings)
+        return Crawler(spidercls, self.settings)
 
     def _setup_crawler_logging(self, crawler):
         log_observer = log.start_from_crawler(crawler)


Test patch:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 5128ea6e9..34fc1a367 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -25,16 +25,6 @@ class CrawlerTestCase(unittest.TestCase):
             self.crawler.spiders
             self.assertEqual(len(w), 1, "Warn deprecated access only once")
 
-
-class CrawlerRunnerTest(unittest.TestCase):
-
-    def setUp(self):
-        self.crawler_runner = CrawlerRunner(Settings())
-
-    def tearDown(self):
-        return self.crawler_runner.stop()
-
-    @defer.inlineCallbacks
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
         project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@@ -42,12 +32,13 @@ class CrawlerRunnerTest(unittest.TestCase):
         class CustomSettingsSpider(DefaultSpider):
             custom_settings = spider_settings
 
-        self.crawler_runner.settings.setdict(project_settings,
-                                             priority='project')
+        settings = Settings()
+        settings.setdict(project_settings, priority='project')
+        crawler = Crawler(CustomSettingsSpider, settings)
 
-        d = self.crawler_runner.crawl(CustomSettingsSpider)
-        crawler = list(self.crawler_runner.crawlers)[0]
-        yield d
         self.assertEqual(crawler.settings.get('TEST1'), 'spider')
         self.assertEqual(crawler.settings.get('TEST2'), 'spider')
         self.assertEqual(crawler.settings.get('TEST3'), 'project')
+
+        self.assertFalse(settings.frozen)
+        self.assertTrue(crawler.settings.frozen)