Repository: spotify/luigi

Issue:
FTP RemoteTarget tries to create local temp file in root
luigi.contrib.ftp.RemoteTarget open('r') tries to create a local temporary file based on the remote file path with a random suffix but using the entire path of the remote file.

In my case, the remote file is in the root folder of the FTP server so it tries to create the same file locally with '-luigi-tmp-%09d' as a suffix.  Because I don't have write permission to my root folder it fails.



Discussion:
+1
https://github.com/spotify/luigi/blob/7a63da8b7c6fc48fde6479ae969581c160d6516c/luigi/contrib/ftp.py#L261

I cannot think of any reason that this needs to be at self.path instead of something like '/tmp/' in unix.

yeah that sounds messed up. would love it if you can send a PR


Resolving patch:
diff --git a/luigi/contrib/ftp.py b/luigi/contrib/ftp.py
--- a/luigi/contrib/ftp.py
+++ b/luigi/contrib/ftp.py
@@ -29,6 +29,7 @@
 import ftplib
 import os
 import random
+import tempfile
 import io
 
 import luigi
@@ -112,13 +113,15 @@ def _rm_recursive(self, ftp, path):
                 continue
 
             try:
-                ftp.cwd(name)  # if we can cwd to it, it's a folder
-                ftp.cwd(wd)  # don't try a nuke a folder we're in
+                ftp.cwd(name)   # if we can cwd to it, it's a folder
+                ftp.cwd(wd)   # don't try a nuke a folder we're in
+                ftp.cwd(path)  # then go back to where we were
                 self._rm_recursive(ftp, name)
-            except ftplib.all_errors:
+            except ftplib.all_errors as e:
                 ftp.delete(name)
 
         try:
+            ftp.cwd(wd)  # do not delete the folder that we are in
             ftp.rmd(path)
         except ftplib.all_errors as e:
             print('_rm_recursive: Could not remove {0}: {1}'.format(path, e))
@@ -258,7 +261,8 @@ def open(self, mode):
             return self.format.pipe_writer(AtomicFtpFile(self._fs, self.path))
 
         elif mode == 'r':
-            self.__tmp_path = self.path + '-luigi-tmp-%09d' % random.randrange(0, 1e10)
+            temp_dir = os.path.join(tempfile.gettempdir(), 'luigi-contrib-ftp')
+            self.__tmp_path = temp_dir + '/' + self.path.lstrip('/') + '-luigi-tmp-%09d' % random.randrange(0, 1e10)
             # download file to local
             self._fs.get(self.path, self.__tmp_path)
 


Test patch:
diff --git a/test/_test_ftp.py b/test/_test_ftp.py
--- a/test/_test_ftp.py
+++ b/test/_test_ftp.py
@@ -15,9 +15,17 @@
 # limitations under the License.
 #
 
+# this is an integration test. to run this test requires that an actuall FTP server
+# is running somewhere. to run a local ftp server do the following
+# pip install pyftpdlib==1.5.0
+# mkdir /tmp/luigi-test-ftp/
+# sudo python -m _test_ftp
+
+
 import datetime
 import ftplib
 import os
+import shutil
 from helpers import unittest
 try:
     from cStringIO import StringIO
@@ -31,7 +39,7 @@
 FILE2 = """this is file2"""
 FILE3 = """this is file3"""
 
-HOST = "some_host"
+HOST = "localhost"
 USER = "luigi"
 PWD = "some_password"
 
@@ -168,6 +176,10 @@ def test_get(self):
         remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD)
         remotetarget.get(local_filepath)
 
+        # make sure that it can open file
+        with remotetarget.open('r') as fin:
+            self.assertEqual(fin.read(), "something to fill")
+
         # file is successfuly created
         self.assertTrue(os.path.exists(local_filepath))
 
@@ -176,7 +188,7 @@ def test_get(self):
         delayed_remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD, mtime=ts)
         self.assertTrue(delayed_remotetarget.exists())
 
-        ts = datetime.datetime.now() + datetime.timedelta(minutes=2)
+        ts = datetime.datetime.now() + datetime.timedelta(days=2)  # who knows what timezone it is in
         delayed_remotetarget = RemoteTarget(remote_file, HOST, username=USER, password=PWD, mtime=ts)
         self.assertFalse(delayed_remotetarget.exists())
 
@@ -188,3 +200,27 @@ def test_get(self):
         ftp.cwd("/")
         ftp.rmd("test")
         ftp.close()
+
+
+def _run_ftp_server():
+    from pyftpdlib.authorizers import DummyAuthorizer
+    from pyftpdlib.handlers import FTPHandler
+    from pyftpdlib.servers import FTPServer
+
+    # Instantiate a dummy authorizer for managing 'virtual' users
+    authorizer = DummyAuthorizer()
+
+    tmp_folder = '/tmp/luigi-test-ftp-server/'
+    if os.path.exists(tmp_folder):
+        shutil.rmtree(tmp_folder)
+    os.mkdir(tmp_folder)
+
+    authorizer.add_user(USER, PWD, tmp_folder, perm='elradfmwM')
+    handler = FTPHandler
+    handler.authorizer = authorizer
+    address = ('localhost', 21)
+    server = FTPServer(address, handler)
+    server.serve_forever()
+
+if __name__ == '__main__':
+    _run_ftp_server()