Repository: scrapy/scrapy

Issue:
DNSCACHE_ENABLED=False not working
Originally reported by @softwarevamp on [StackOverflow](https://stackoverflow.com/questions/44877296/scrapy-with-dnscache-enabled-false-not-working):

> When i run scrapy shell with `DNSCACHE_ENABLED=False` got
```
KeyError: 'dictionary is empty'
twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: www.mydomain.com.
```

```
    2017-07-03 03:09:12 [twisted] CRITICAL: while looking up www.mydomain.com with <scrapy.resolver.CachingThreadedResolver object at 0x3fd0050>
    Traceback (most recent call last):
      File "/usr/lib64/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/lib64/python2.7/site-packages/scrapy/resolver.py", line 29, in _cache_result
        dnscache[name] = result
      File "/usr/lib64/python2.7/site-packages/scrapy/utils/datatypes.py", line 305, in __setitem__
        self.popitem(last=False)
      File "/usr/lib64/python2.7/collections.py", line 159, in popitem
        raise KeyError('dictionary is empty')
    KeyError: 'dictionary is empty'
    2017-07-03 03:09:12 [scrapy.downloadermiddlewares.retry] DEBUG: Gave up retrying <GET //www.mydomain.com/> (failed 3 times): DNS lookup failed: no results for hostname lookup: www.mydomain.com.
    Traceback (most recent call last):
      File "/usr/bin/scrapy", line 11, in <module>
        sys.exit(execute())
      File "/usr/lib64/python2.7/site-packages/scrapy/cmdline.py", line 149, in execute
        _run_print_help(parser, _run_command, cmd, args, opts)
      File "/usr/lib64/python2.7/site-packages/scrapy/cmdline.py", line 89, in _run_print_help
        func(*a, **kw)
      File "/usr/lib64/python2.7/site-packages/scrapy/cmdline.py", line 156, in _run_command
        cmd.run(args, opts)
      File "/usr/lib64/python2.7/site-packages/scrapy/commands/shell.py", line 73, in run
        shell.start(url=url, redirect=not opts.no_redirect)
      File "/usr/lib64/python2.7/site-packages/scrapy/shell.py", line 48, in start
        self.fetch(url, spider, redirect=redirect)
      File "/usr/lib64/python2.7/site-packages/scrapy/shell.py", line 115, in fetch
        reactor, self._schedule, request, spider)
      File "/usr/lib64/python2.7/site-packages/twisted/internet/threads.py", line 122, in blockingCallFromThread
        result.raiseException()
      File "<string>", line 2, in raiseException
    twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: www.mydomain.com.
```

> Any thoughts welcome


Discussion:
This patch seems to do the trick:

```
$ git diff
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 4f4f0b0..0aaced7 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -22,7 +22,8 @@ class CachingThreadedResolver(ThreadedResolver):
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
-        d.addCallback(self._cache_result, name)
+        if dnscache.limit:
+            d.addCallback(self._cache_result, name)
         return d
 
     def _cache_result(self, result, name):
```

Resolving patch:
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -22,7 +22,8 @@ def getHostByName(self, name, timeout=None):
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
-        d.addCallback(self._cache_result, name)
+        if dnscache.limit:
+            d.addCallback(self._cache_result, name)
         return d
 
     def _cache_result(self, result, name):


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -226,6 +226,27 @@ def test_runspider_log_level(self):
         self.assertNotIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
 
+    def test_runspider_dnscache_disabled(self):
+        # see https://github.com/scrapy/scrapy/issues/2811
+        # The spider below should not be able to connect to localhost:12345,
+        # which is intended,
+        # but this should not be because of DNS lookup error
+        # assumption: localhost will resolve in all cases (true?)
+        log = self.get_log("""
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+    start_urls = ['http://localhost:12345']
+
+    def parse(self, response):
+        return {'test': 'value'}
+""",
+                           args=('-s', 'DNSCACHE_ENABLED=False'))
+        print(log)
+        self.assertNotIn("DNSLookupError", log)
+        self.assertIn("INFO: Spider opened", log)
+
     def test_runspider_log_short_names(self):
         log1 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=1'))