Repository: scrapy/scrapy

Issue:
engine status util references removed engine.slots attribute
```
Traceback (most recent call last): Less
      File "/usr/lib/pymodules/python2.7/scrapy/xlib/pydispatch/robustapply.py", line 54, in robustApply
        return receiver(*arguments, **named)
      File "/usr/lib/pymodules/python2.7/scrapy/contrib/memusage.py", line 63, in engine_started
        tsk.start(60.0, now=True)
      File "/usr/lib/python2.7/dist-packages/twisted/internet/task.py", line 163, in start
        self()
      File "/usr/lib/python2.7/dist-packages/twisted/internet/task.py", line 208, in __call__
        d = defer.maybeDeferred(self.f, *self.a, **self.kw)
    --- <exception caught here> ---
      File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 134, in maybeDeferred
        result = f(*args, **kw)
      File "/usr/lib/pymodules/python2.7/scrapy/contrib/memusage.py", line 103, in _check_warning
        self._send_report(self.notify_mails, subj)
      File "/usr/lib/pymodules/python2.7/scrapy/contrib/memusage.py", line 116, in _send_report
        s += pformat(get_engine_status(self.crawler.engine))
      File "/usr/lib/pymodules/python2.7/scrapy/utils/engine.py", line 33, in get_engine_status
        for spider in engine.slots.keys():
    exceptions.AttributeError: 'ExecutionEngine' object has no attribute 'slots'
```



Discussion:

Resolving patch:
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -5,14 +5,13 @@
 
 def get_engine_status(engine):
     """Return a report of the current engine status"""
-    global_tests = [
+    tests = [
         "time()-engine.start_time",
         "engine.has_capacity()",
         "len(engine.downloader.active)",
         "engine.scraper.is_idle()",
-    ]
-    spider_tests = [
-        "engine.spider_is_idle(spider)",
+        "engine.spider.name",
+        "engine.spider_is_idle(engine.spider)",
         "engine.slot.closing",
         "len(engine.slot.inprogress)",
         "len(engine.slot.scheduler.dqs or [])",
@@ -24,34 +23,23 @@ def get_engine_status(engine):
         "engine.scraper.slot.needs_backout()",
     ]
 
-    status = {'global': [], 'spiders': {}}
-    for test in global_tests:
+    checks = []
+    for test in tests:
         try:
-            status['global'] += [(test, eval(test))]
+            checks += [(test, eval(test))]
         except Exception as e:
-            status['global'] += [(test, "%s (exception)" % type(e).__name__)]
-    for spider in engine.slots.keys():
-        x = []
-        for test in spider_tests:
-            try:
-                x += [(test, eval(test))]
-            except Exception as e:
-                x += [(test, "%s (exception)" % type(e).__name__)]
-            status['spiders'][spider] = x
-    return status
+            checks += [(test, "%s (exception)" % type(e).__name__)]
+
+    return checks
 
 def format_engine_status(engine=None):
-    status = get_engine_status(engine)
+    checks = get_engine_status(engine)
     s = "Execution engine status\n\n"
-    for test, result in status['global']:
+    for test, result in checks:
         s += "%-47s : %s\n" % (test, result)
     s += "\n"
-    for spider, tests in status['spiders'].items():
-        s += "Spider: %s\n" % spider
-        for test, result in tests:
-            s += "  %-50s : %s\n" % (test, result)
+
     return s
 
 def print_engine_status(engine):
     print(format_engine_status(engine))
-


Test patch:
diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py
--- a/scrapy/tests/spiders.py
+++ b/scrapy/tests/spiders.py
@@ -137,6 +137,8 @@ def parse(self, response):
 class SingleRequestSpider(MetaSpider):
 
     seed = None
+    callback_func = None
+    errback_func = None
 
     def start_requests(self):
         if isinstance(self.seed, Request):
@@ -146,8 +148,12 @@ def start_requests(self):
 
     def parse(self, response):
         self.meta.setdefault('responses', []).append(response)
+        if callable(self.callback_func):
+            return self.callback_func(response)
         if 'next' in response.meta:
             return response.meta['next']
 
     def on_error(self, failure):
         self.meta['failure'] = failure
+        if callable(self.errback_func):
+            return self.errback_func(failure)
diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py
--- a/scrapy/tests/test_crawl.py
+++ b/scrapy/tests/test_crawl.py
@@ -192,3 +192,18 @@ def test_referer_header(self):
         # last request explicitly sets a Referer header
         echo3 = json.loads(spider.meta['responses'][3].body)
         self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
+
+    @defer.inlineCallbacks
+    def test_engine_status(self):
+        from scrapy.utils.engine import get_engine_status
+        est = []
+
+        def cb(response):
+            est.append(get_engine_status(spider.crawler.engine))
+
+        spider = SingleRequestSpider(seed='http://localhost:8998/', callback_func=cb)
+        yield docrawl(spider)
+        self.assertEqual(len(est), 1, est)
+        s = dict(est[0])
+        self.assertEqual(s['engine.spider.name'], spider.name)
+        self.assertEqual(s['len(engine.scraper.slot.active)'], 1)