Repository: scrapy/scrapy

Issue:
Command parse unhandled error :AttributeError: 'NoneType' object has no attribute 'start_requests'
Scrapy version :1.5.0
When i run the command **scrapy parse http://www.baidu.com**, and the url www.baidu.com dosn't  have spider matched , then i got the error:

> 2018-03-11 16:23:35 [scrapy.utils.log] INFO: Scrapy 1.5.0 started (bot: DouTu)
> 2018-03-11 16:23:35 [scrapy.utils.log] INFO: Versions: lxml 4.2.1.0, libxml2 2.9.8, cssselect 1.0.3, parsel 1.4.0, w3lib 1.19.0, Twisted 17.9.0, Python 2.7.12 (default, Dec  4 2017, 14:50:18) - [GCC 5.4.0 20160609], pyOpenSSL 17.5.0 (OpenSSL 1.1.0h  27 Mar 2018), cryptography 2.2.2, Platform Linux-4.13.0-38-generic-x86_64-with-Ubuntu-16.04-xenial
> 2018-05-18 16:23:35 [scrapy.commands.parse] ERROR: Unable to find spider for: http://www.baidu.com
> Traceback (most recent call last):
>   File "/home/wangsir/code/sourceWorkSpace/scrapy/cmdline.py", line 239, in <module>
>     execute(['scrapy','parse','http://www.baidu.com'])
>   File "/home/wangsir/code/sourceWorkSpace/scrapy/cmdline.py", line 168, in execute
>     _run_print_help(parser, _run_command, cmd, args, opts)
>   File "/home/wangsir/code/sourceWorkSpace/scrapy/cmdline.py", line 98, in _run_print_help
>     func(*a, **kw)
>   File "/home/wangsir/code/sourceWorkSpace/scrapy/cmdline.py", line 176, in _run_command
>     cmd.run(args, opts)
>   File "/home/wangsir/code/sourceWorkSpace/scrapy/commands/parse.py", line 250, in run
>     self.set_spidercls(url, opts)
>   File "/home/wangsir/code/sourceWorkSpace/scrapy/commands/parse.py", line 151, in set_spidercls
>     self.spidercls.start_requests = _start_requests
> AttributeError: 'NoneType' object has no attribute 'start_requests'.

The failed reason should be follwing code(scrapy/commands/parse.py line 151):
         **`self.spidercls.start_requests = _start_requests`**
because the url www.baidu.com dosn't  have spider matched,so self.spidercls is none,so self.spidercls.start_requests throw the error.

Discussion:

Patch:
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a3f6b96f4..99fc8f955 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
 
         def _start_requests(spider):
             yield self.prepare_request(spider, Request(url), opts)
-        self.spidercls.start_requests = _start_requests
+        if self.spidercls:
+            self.spidercls.start_requests = _start_requests
 
     def start_parsing(self, url, opts):
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index f5081a7d7..9de119a10 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -33,8 +33,8 @@ class MemoryUsage:
         self.crawler = crawler
         self.warned = False
         self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
-        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
-        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
+        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
+        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
         self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@@ -77,7 +77,7 @@ class MemoryUsage:
     def _check_limit(self):
         if self.get_virtual_size() > self.limit:
             self.crawler.stats.set_value('memusage/limit_reached', 1)
-            mem = self.limit / 1024 / 1024
+            mem = self.limit/1024/1024
             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
@@ -94,11 +94,11 @@ class MemoryUsage:
                 self.crawler.stop()
 
     def _check_warning(self):
-        if self.warned:  # warn only once
+        if self.warned: # warn only once
             return
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value('memusage/warning_reached', 1)
-            mem = self.warning / 1024 / 1024
+            mem = self.warning/1024/1024
             logger.warning("Memory usage reached %(memusage)dM",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 739e6b958..bcdbaff24 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,7 +8,6 @@ from scrapy import signals
 from scrapy.mail import MailSender
 from scrapy.exceptions import NotConfigured
 
-
 class StatsMailer:
 
     def __init__(self, stats, recipients, mail):


Test patch:
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
index 72bb986bc..1f5a44010 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -8,7 +8,6 @@ class AsyncioReactorSpider1(scrapy.Spider):
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
 
-
 class AsyncioReactorSpider2(scrapy.Spider):
     name = 'asyncio_reactor2'
     custom_settings = {
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index f21ee971d..0d992be56 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,7 @@
 import os
 import argparse
 from os.path import join, abspath, isfile, exists
+
 from twisted.internet import defer
 from scrapy.commands import parse
 from scrapy.settings import Settings
@@ -222,6 +223,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_not_exists_with_not_matched_url(self):
+        status, out, stderr = yield self.execute([self.url('/invalid_url')])
+        self.assertEqual(status, 0)
+
     @defer.inlineCallbacks
     def test_output_flag(self):
         """Checks if a file was created successfully having