Repository: scrapy/scrapy

Issue:
Anonymous FTP download fails with KeyError: ftp_user
Tested with Scrapy 1.2.0

```
$ scrapy version -v
Scrapy    : 1.2.0
lxml      : 3.6.4.0
libxml2   : 2.9.4
Twisted   : 16.4.1
Python    : 2.7.12 (default, Jul  1 2016, 15:12:24) - [GCC 5.4.0 20160609]
pyOpenSSL : 16.1.0 (OpenSSL 1.0.2g  1 Mar 2016)
Platform  : Linux-4.4.0-43-generic-x86_64-with-Ubuntu-16.04-xenial


$ scrapy shell ftp://ftp.eu.metabrainz.org/pub/musicbrainz/data/fullexport/20161019-001816/MD5SUMS
2016-10-20 16:01:27 [scrapy] INFO: Scrapy 1.2.0 started (bot: scrapybot)
(...)
2016-10-20 16:01:27 [scrapy] INFO: Spider opened
Traceback (most recent call last):
  File "/home/paul/.virtualenvs/scrapy12/bin/scrapy", line 11, in <module>
    sys.exit(execute())
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/cmdline.py", line 142, in execute
    _run_print_help(parser, _run_command, cmd, args, opts)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/cmdline.py", line 88, in _run_print_help
    func(*a, **kw)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/cmdline.py", line 149, in _run_command
    cmd.run(args, opts)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/commands/shell.py", line 71, in run
    shell.start(url=url)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/shell.py", line 47, in start
    self.fetch(url, spider)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/shell.py", line 112, in fetch
    reactor, self._schedule, request, spider)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/twisted/internet/threads.py", line 122, in blockingCallFromThread
    result.raiseException()
  File "<string>", line 2, in raiseException
KeyError: 'ftp_user'
```


Discussion:

Resolving patch:
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import urlparse, unquote
+from six.moves.urllib.parse import unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed
@@ -38,6 +38,8 @@
 
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.httpobj import urlparse_cached
+
 
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename=None):
@@ -64,14 +66,19 @@ class FTPDownloadHandler(object):
         "default": 503,
     }
 
-    def __init__(self, setting):
-        pass
+    def __init__(self, settings):
+        self.default_user = settings['FTP_USER']
+        self.default_password = settings['FTP_PASSWORD']
+        self.passive_mode = settings['FTP_PASSIVE_MODE']
 
     def download_request(self, request, spider):
-        parsed_url = urlparse(request.url)
-        creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],
-                                    request.meta["ftp_password"],
-                                    passive=request.meta.get("ftp_passive", 1))
+        parsed_url = urlparse_cached(request)
+        user = request.meta.get("ftp_user", self.default_user)
+        password = request.meta.get("ftp_password", self.default_password)
+        passive_mode = 1 if bool(request.meta.get("ftp_passive",
+                                                  self.passive_mode)) else 0
+        creator = ClientCreator(reactor, FTPClient, user, password,
+            passive=passive_mode)
         return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
                                 request, unquote(parsed_url.path))
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,6 +161,10 @@
 
 FILES_STORE_S3_ACL = 'private'
 
+FTP_USER = 'anonymous'
+FTP_PASSWORD = 'guest'
+FTP_PASSIVE_MODE = True
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False


Test patch:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -5,6 +5,7 @@
     from unittest import mock
 except ImportError:
     import mock
+import shutil
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -676,10 +677,11 @@ def test_request_signing7(self):
             b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
 
 
-class FTPTestCase(unittest.TestCase):
+class BaseFTPTestCase(unittest.TestCase):
 
     username = "scrapy"
     password = "passwd"
+    req_meta = {"ftp_user": username, "ftp_password": password}
 
     if six.PY3:
         skip = "Twisted missing ftp support for PY3"
@@ -709,6 +711,9 @@ def setUp(self):
         self.download_handler = FTPDownloadHandler(Settings())
         self.addCleanup(self.port.stopListening)
 
+    def tearDown(self):
+        shutil.rmtree(self.directory)
+
     def _add_test_callbacks(self, deferred, callback=None, errback=None):
         def _clean(data):
             self.download_handler.client.transport.loseConnection()
@@ -722,7 +727,7 @@ def _clean(data):
 
     def test_ftp_download_success(self):
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": self.password})
+                          meta=self.req_meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -734,7 +739,7 @@ def _test(r):
     def test_ftp_download_path_with_spaces(self):
         request = Request(
             url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,
-            meta={"ftp_user": self.username, "ftp_password": self.password}
+            meta=self.req_meta
         )
         d = self.download_handler.download_request(request, None)
 
@@ -746,7 +751,7 @@ def _test(r):
 
     def test_ftp_download_notexist(self):
         request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": self.password})
+                          meta=self.req_meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -755,8 +760,10 @@ def _test(r):
 
     def test_ftp_local_filename(self):
         local_fname = "/tmp/file.txt"
+        meta = {"ftp_local_filename": local_fname}
+        meta.update(self.req_meta)
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})
+                          meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -768,13 +775,52 @@ def _test(r):
             os.remove(local_fname)
         return self._add_test_callbacks(d, _test)
 
+
+class FTPTestCase(BaseFTPTestCase):
+
     def test_invalid_credentials(self):
         from twisted.protocols.ftp import ConnectionLost
 
+        meta = dict(self.req_meta)
+        meta.update({"ftp_password": 'invalid'})
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": 'invalid'})
+                          meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
             self.assertEqual(r.type, ConnectionLost)
         return self._add_test_callbacks(d, errback=_test)
+
+
+class AnonymousFTPTestCase(BaseFTPTestCase):
+
+    username = "anonymous"
+    req_meta = {}
+
+    def setUp(self):
+        from twisted.protocols.ftp import FTPRealm, FTPFactory
+        from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+
+        # setup dir and test file
+        self.directory = self.mktemp()
+        os.mkdir(self.directory)
+
+        fp = FilePath(self.directory)
+        fp.child('file.txt').setContent("I have the power!")
+        fp.child('file with spaces.txt').setContent("Moooooooooo power!")
+
+        # setup server for anonymous access
+        realm = FTPRealm(anonymousRoot=self.directory)
+        p = portal.Portal(realm)
+        p.registerChecker(checkers.AllowAnonymousAccess(),
+                          credentials.IAnonymous)
+
+        self.factory = FTPFactory(portal=p,
+                                  userAnonymous=self.username)
+        self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
+        self.portNum = self.port.getHost().port
+        self.download_handler = FTPDownloadHandler(Settings())
+        self.addCleanup(self.port.stopListening)
+
+    def tearDown(self):
+        shutil.rmtree(self.directory)