Repository: spotify/luigi

Issue:
`from __future__ import absolute_import` throws Luigi off
If a taks file is part of a python package, and it uses `from __future__ import absolute_import`, the Luigi packer gets confused and only adds the file as a root module to the `jar` file sent to nodes.

The fix for the task file is very easy (remove the `from __future__ import`) but it took us a while to figure out what was going on.



Discussion:
is it possible to fix in luigi?

I think so, we haven't had the time to dig into the root cause yet.

I thought that creating the issue was a good way to at least point people in the right direction if they run into the same problem.


Resolving patch:
diff --git a/luigi/hadoop.py b/luigi/hadoop.py
--- a/luigi/hadoop.py
+++ b/luigi/hadoop.py
@@ -86,13 +86,18 @@ def add(src, dst):
         logger.debug('adding to tar: %s -> %s', src, dst)
         tar.add(src, dst)
     for package in packages:
-        if package.__package__ and package.__package__ != package.__name__:
-            # Replace with package instead of submodule
-            package = __import__(package.__package__, None, None, 'non_empty')
+        # Put a submodule's entire package in the archive. This is the
+        # magic that usually packages everything you need without
+        # having to attach packages/modules explicitly
+        if not hasattr(package, "__path__") and '.' in package.__name__:
+            package = __import__(package.__name__.rpartition('.')[0], None, None, 'non_empty')
 
         n = package.__name__.replace(".", "/")
 
         if hasattr(package, "__path__"):
+            # TODO: (BUG) picking only the first path does not
+            # properly deal with namespaced packages in different
+            # directories
             p = package.__path__[0]
 
             if p.endswith('.egg') and os.path.isfile(p):


Test patch:
diff --git a/test/create_packages_archive_root/module.py b/test/create_packages_archive_root/module.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/module.py
@@ -0,0 +1 @@
+# Copyright (c) 2014 Spotify AB
diff --git a/test/create_packages_archive_root/package/__init__.py b/test/create_packages_archive_root/package/__init__.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/package/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) 2014 Spotify AB
diff --git a/test/create_packages_archive_root/package/submodule.py b/test/create_packages_archive_root/package/submodule.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/package/submodule.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2014 Spotify AB
+import os
diff --git a/test/create_packages_archive_root/package/submodule_with_absolute_import.py b/test/create_packages_archive_root/package/submodule_with_absolute_import.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/package/submodule_with_absolute_import.py
@@ -0,0 +1,3 @@
+# Copyright (c) 2014 Spotify AB
+from __future__ import absolute_import
+import os
diff --git a/test/create_packages_archive_root/package/submodule_without_imports.py b/test/create_packages_archive_root/package/submodule_without_imports.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/package/submodule_without_imports.py
@@ -0,0 +1 @@
+# Copyright (c) 2014 Spotify AB
diff --git a/test/create_packages_archive_root/package/subpackage/__init__.py b/test/create_packages_archive_root/package/subpackage/__init__.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/package/subpackage/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) 2014 Spotify AB
diff --git a/test/create_packages_archive_root/package/subpackage/submodule.py b/test/create_packages_archive_root/package/subpackage/submodule.py
new file mode 100644
--- /dev/null
+++ b/test/create_packages_archive_root/package/subpackage/submodule.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2014 Spotify AB
+import os
diff --git a/test/hadoop_test.py b/test/hadoop_test.py
--- a/test/hadoop_test.py
+++ b/test/hadoop_test.py
@@ -12,6 +12,9 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
+import mock
+import os
+import sys
 import unittest
 import subprocess
 import luigi
@@ -234,5 +237,73 @@ def run():
         self.assertEquals(len(excs), 1) # should have been set
         self.assertTrue(type(excs[0]), FailingJobException)
 
+class CreatePackagesArchive(unittest.TestCase):
+    def setUp(self):
+        sys.path.append(os.path.join('test', 'create_packages_archive_root'))
+
+    def tearDown(self):
+        sys.path.remove(os.path.join('test', 'create_packages_archive_root'))
+
+    def _assert_module(self, add):
+        add.assert_called_once_with('test/create_packages_archive_root/module.py',
+                                    'module.py')
+
+    def _assert_package(self, add):
+        add.assert_any_call('test/create_packages_archive_root/package/__init__.py', 'package/__init__.py')
+        add.assert_any_call('test/create_packages_archive_root/package/submodule.py', 'package/submodule.py')
+        add.assert_any_call('test/create_packages_archive_root/package/submodule_with_absolute_import.py', 'package/submodule_with_absolute_import.py')
+        add.assert_any_call('test/create_packages_archive_root/package/submodule_without_imports.py', 'package/submodule_without_imports.py')
+        add.assert_any_call('test/create_packages_archive_root/package/subpackage/__init__.py', 'package/subpackage/__init__.py')
+        add.assert_any_call('test/create_packages_archive_root/package/subpackage/submodule.py', 'package/subpackage/submodule.py')
+        assert add.call_count == 6
+
+    def _assert_package_subpackage(self, add):
+        add.assert_any_call('test/create_packages_archive_root/package/__init__.py', 'package/__init__.py')
+        add.assert_any_call('test/create_packages_archive_root/package/subpackage/__init__.py', 'package/subpackage/__init__.py')
+        add.assert_any_call('test/create_packages_archive_root/package/subpackage/submodule.py', 'package/subpackage/submodule.py')
+        assert add.call_count == 3
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_module(self, tar):
+        module = __import__("module", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([module], '/dev/null')
+        self._assert_module(tar.return_value.add)
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_package(self, tar):
+        package = __import__("package", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([package], '/dev/null')
+        self._assert_package(tar.return_value.add)
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_package_submodule(self, tar):
+        package_submodule = __import__("package.submodule", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([package_submodule], '/dev/null')
+        self._assert_package(tar.return_value.add)
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_package_submodule_with_absolute_import(self, tar):
+        package_submodule_with_absolute_import = __import__("package.submodule_with_absolute_import", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([package_submodule_with_absolute_import], '/dev/null')
+        self._assert_package(tar.return_value.add)
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_package_submodule_without_imports(self, tar):
+        package_submodule_without_imports = __import__("package.submodule_without_imports", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([package_submodule_without_imports], '/dev/null')
+        self._assert_package(tar.return_value.add)
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_package_subpackage(self, tar):
+        package_subpackage = __import__("package.subpackage", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([package_subpackage], '/dev/null')
+        self._assert_package_subpackage(tar.return_value.add)
+
+    @mock.patch('tarfile.open')
+    def test_create_packages_archive_package_subpackage_submodule(self, tar):
+        package_subpackage_submodule = __import__("package.subpackage.submodule", None, None, 'dummy')
+        luigi.hadoop.create_packages_archive([package_subpackage_submodule], '/dev/null')
+        self._assert_package_subpackage(tar.return_value.add)
+
 if __name__ == '__main__':
     HadoopJobTest.test_run_real()