Repository: scrapy/scrapy

Issue:
Tried to stop a LoopingCall that was not running
I'm using mysql to store my spider data, but when I set the piplines to store data into local mysql server, it raise an error.
pipelines.py

```
import json
import requests
from mysql.connector import connection

MYSQL_SERVER = '192.168.1.90'   # using this, the spider can run
MYSQL_SERVER = 'localhost'   # using this, the spider raise an error
MYSQL_DB = 'scrapy'
MYSQL_USER = 'crawler'
MYSQL_PASS = 'crawl'
MYSQL_TABLE = 'pm25in'

class Pm25InPipeline(object):
    def __init__(self):                                                                
        pass

    def process_item(self, item, spider):                                              
        command = '''insert into {table} (monitortime, monitorcity, monitorpoint,      
            AQIindex, airsituation, primarypullutant, PM25content, PM10content,        
            CO, NO2, O3_1h, O3_8h, SO2)                                                
            values ( "{monitortime}", "{monitorcity}","{monitorpoint}", "{AQIindex}",  
            "{airsituation}", "{primarypollutant}", {PM25content}, {PM10content},      
            {CO}, {NO2}, {O3_1h}, {O3_8h}, {SO2} );                                    
            '''.format(table=MYSQL_TABLE, **dict(item))                                
        self.cursor.execute(command)                                                   
        return item                                                                    

    def open_spider(self, spider):                                                     
        self.cnx = connection.MySQLConnection(                                         
            host=MYSQL_SERVER,                                                         
            user=MYSQL_USER,                                                           
            password=MYSQL_PASS,                                                       
            database=MYSQL_DB,                                                         
            charset='utf8')                                                            
        self.cursor = self.cnx.cursor()                                                

    def close_spider(self,spider):                                                     
        self.cnx.commit()                                                              
        self.cnx.close()                                                               
```

the Traceback:

```
wangx@wangx-PC:~/github/pm25in$ scrapy crawl pm25spider
Unhandled error in Deferred:


Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/scrapy/commands/crawl.py", line 57, in run
    self.crawler_process.crawl(spname, **opts.spargs)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 163, in crawl
    return self._crawl(crawler, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 167, in _crawl
    d = crawler.crawl(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 1274, in unwindGenerator
    return _inlineCallbacks(None, gen, Deferred())
--- <exception caught here> ---
  File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 87, in crawl
    yield self.engine.close()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 100, in close
    return self._close_all_spiders()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 340, in _close_all_spiders
    dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 298, in close_spider
    dfd = slot.close()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 44, in close
    self._maybe_fire_closing()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 51, in _maybe_fire_closing
    self.heartbeat.stop()
  File "/usr/local/lib/python2.7/dist-packages/twisted/internet/task.py", line 202, in stop
    assert self.running, ("Tried to stop a LoopingCall that was "
exceptions.AssertionError: Tried to stop a LoopingCall that was not running.
```



Discussion:
Hey, I has a same error log like yours, and my problem is **"connection.MySQLConnection()"** not connect successfully.

When I connect to my MySQL, Scrapy Crawl is work!

Hope this help for you.

mysql的连接，执行和关闭.   你为啥不直接放到 process_item 一次性完成呢？

process_item 內若要一直使用到 MySQL 的話，只在 open_spider 與 close_spider 做一次 開啟/關閉 應該會比較好吧

我都是用django的...  感觉scrapy+django 配合起来很好用

i got same error,and found  this occur because of file "setting" configuration not right,you may check it.

@shartoo can you give some hint about the "setting" configuration?

like this

```
ITEM_PIPELINES = {
    'bdtb.pipelines.bdtb_pipeline': 300,
    'bdtb.item2mysql_pipeline.MySQLStorePipeline':300
}
```

you should check if  the path is right.

I have a working Scrapy project with two named spiders that are running successfully with Scrapy v1.0.1 and Python v2.7.9.
One of the two spiders uses `MySQL-python` library to connect to a MySQL database.

Today, I updated Python to 2.7.11+ as well as Scrapy to 1.1.0 and received a similar error

``` python
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/scrapy/commands/crawl.py", line 57, in run
    self.crawler_process.crawl(spname, **opts.spargs)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 163, in crawl
    return self._crawl(crawler, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 167, in _crawl
    d = crawler.crawl(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 1274, in unwindGenerator
    return _inlineCallbacks(None, gen, Deferred())
--- <exception caught here> ---
  File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/lib/python2.7/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 87, in crawl
    yield self.engine.close()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 100, in close
    return self._close_all_spiders()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 340, in _close_all_spiders
    dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 298, in close_spider
    dfd = slot.close()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 44, in close
    self._maybe_fire_closing()
  File "/usr/local/lib/python2.7/dist-packages/scrapy/core/engine.py", line 51, in _maybe_fire_closing
    self.heartbeat.stop()
  File "/usr/lib/python2.7/dist-packages/twisted/internet/task.py", line 202, in stop
    assert self.running, ("Tried to stop a LoopingCall that was "
exceptions.AssertionError: Tried to stop a LoopingCall that was not running.
```

I have also rectified my `settings.py` file as recommended by @shartoo with no luck: 

``` python
ITEM_PIPELINES = {
    'tttscraper.pipelines.MySQLStorePipeline':300,
    'tttscraper.pipelines.CustomFilePipeline':300,
}
```

For now, I've reverted Scrapy to version 1.0.1 and the code works again. (Python is still at v2.7.11+)

I met this error, when my pipeline.py had some bug.

updating my custom scheduler fix this issue. 

my case was `pqclass` is missing from `from_crawler` classmethod. for futher info check and compare your custom scheduler with scrapy's original scheduler

got the hint after trying `scrapy shell <URL GOES HERE>` and found some error trace leading to my buggy  customscheduler.

AssertionError: Tried to stop a LoopingCall that was not running.

I was having this issue even it works the day before with `pymongo`.

I was able to fix reinstalling pymongo with `pip install pymongo --upgrade`.

No version bumbed but problem fixed.


Resolving patch:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -48,7 +48,8 @@ def _maybe_fire_closing(self):
         if self.closing and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
-                self.heartbeat.stop()
+                if self.heartbeat.running:
+                    self.heartbeat.stop()
             self.closing.callback(None)
 
 
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -15,6 +15,7 @@ def __init__(self, stats, interval=60.0):
         self.stats = stats
         self.interval = interval
         self.multiplier = 60.0 / self.interval
+        self.task = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -47,5 +48,5 @@ def log(self, spider):
         logger.info(msg, log_args, extra={'spider': spider})
 
     def spider_closed(self, spider, reason):
-        if self.task.running:
+        if self.task and self.task.running:
             self.task.stop()


Test patch:
diff --git a/tests/pipelines.py b/tests/pipelines.py
new file mode 100644
--- /dev/null
+++ b/tests/pipelines.py
@@ -0,0 +1,11 @@
+"""
+Some pipelines used for testing
+"""
+
+class ZeroDivisionErrorPipeline(object):
+
+    def open_spider(self, spider):
+        a = 1/0
+
+    def process_item(self, item, spider):
+        return item
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -250,6 +250,19 @@ def start_requests(self):
         yield self.assertFailure(crawler.crawl(), TestError)
         self.assertFalse(crawler.crawling)
 
+    @defer.inlineCallbacks
+    def test_open_spider_error_on_faulty_pipeline(self):
+        settings = {
+            "ITEM_PIPELINES": {
+                "tests.pipelines.ZeroDivisionErrorPipeline": 300,
+            }
+        }
+        crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+        yield self.assertFailure(
+            self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
+            ZeroDivisionError)
+        self.assertFalse(crawler.crawling)
+
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):
         crawler = self.runner.create_crawler(SimpleSpider)