Repository: conda/conda-build

Issue:
conda-build crashes when building PEP420 namespace packages
There is a very trivial bug in `conda_build.utils.copy_into()`, causing it to crash when building PEP420 namespace packages. To reproduce the issue, all of the following conditions must be met:
- your package must be a native PEP420 namespace package (`python >=3.3`)
- your package must depend on (and therefore install) other packages mapping to the same namespace
- you must build all of these packages with `recipe['build']['preserve_egg_dir'] = False`
- your namespace must be a multi-level directory 

This is easier to explain by example:

```
found egg dir: C:\ANACONDA\envs\_build\Lib\site-packages\example-4.3.0-py3.5-win-amd64.egg
...
Traceback (most recent call last):
  File "C:\ANACONDA\Scripts\conda-build-script.py", line 4, in <module>
    sys.exit(main())
  File "C:\ANACONDA\lib\site-packages\conda_build\main_build.py", line 144, in main
    args_func(args, p)
  File "C:\ANACONDA\lib\site-packages\conda_build\main_build.py", line 389, in args_func
    args.func(args, p)
  File "C:\ANACONDA\lib\site-packages\conda_build\main_build.py", line 332, in execute
    dirty=args.dirty, activate=args.activate)
  File "C:\ANACONDA\lib\site-packages\conda_build\build.py", line 606, in build
    preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))
  File "C:\ANACONDA\lib\site-packages\conda_build\post.py", line 162, in post_process
    remove_easy_install_pth(files, preserve_egg_dir=preserve_egg_dir)
  File "C:\ANACONDA\lib\site-packages\conda_build\post.py", line 119, in remove_easy_install_pth
    utils.copy_into(join(egg_path, fn), join(sp_dir, fn))
  File "C:\ANACONDA\lib\site-packages\conda_build\utils.py", line 54, in copy_into
    shutil.copytree(srcname, dstname)
  File "C:\ANACONDA\lib\shutil.py", line 177, in copytree
    os.makedirs(dst)
  File "C:\ANACONDA\lib\os.py", line 157, in makedirs
    mkdir(name, mode)
WindowsError: [Error 183] Cannot create a file when that file already exists: 'C:\ANACONDA\envs\_build\Lib\site-packages\ns\sub'
```

Here `ns` is the top level namespace directory and `sub` is a sub-package, but also a namespace package:

```
+ ns
   |
   + sub
       |
       module.py
```

The problem is that conda is trying to copy `ns\sub` to site-packages at build time. However, this directory already exists because another package -- a runtime dependency -- is already installed in site packages providing the same namespace directory layout:

```
+ ns
   |
   + sub
       |
       dependency.py
```

`conda-build` needs to be able to "merge" the `sub` directories coming from all namespace packages. Deleting or skipping directories if they exist will corrupt the namespace and produce incorrect results. Here is the quickest way to fix this:

``` python
from distutils.dir_util import copy_tree

def copy_into(src, dst):
    "Copy all the files and directories in src to the directory dst"

    if not isdir(src):
        tocopy = [src]
    else:
        tocopy = os.listdir(src)
    for afile in tocopy:
        srcname = os.path.join(src, afile)
        dstname = os.path.join(dst, afile)

        if os.path.isdir(srcname):
            copy_tree(srcname, dstname)
        else:
            shutil.copy2(srcname, dstname)
```

i.e. just replace `shutil.copytree()` with `distutils.dir_util.copy_tree()`.



Discussion:

Resolving patch:
diff --git a/conda_build/utils.py b/conda_build/utils.py
--- a/conda_build/utils.py
+++ b/conda_build/utils.py
@@ -8,8 +8,9 @@
 import zipfile
 import subprocess
 import operator
-from os.path import dirname, getmtime, getsize, isdir, join
+from os.path import dirname, getmtime, getsize, isdir, isfile, join
 from collections import defaultdict
+from distutils.dir_util import copy_tree
 
 from conda.utils import md5_file, unix_path_to_win
 from conda.compat import PY3, iteritems
@@ -51,11 +52,28 @@ def copy_into(src, dst):
         dstname = os.path.join(dst, afile)
 
         if os.path.isdir(srcname):
-            shutil.copytree(srcname, dstname)
+            merge_tree(srcname, dstname)
         else:
             shutil.copy2(srcname, dstname)
 
 
+def merge_tree(src, dst):
+    """
+    Merge src into dst recursively by copying all files from src into dst.
+    Return a list of all files copied.
+    
+    Like copy_tree(src, dst), but raises an error if merging the two trees
+    would overwrite any files.
+    """
+    new_files = copy_tree(src, dst, dry_run=True)
+    existing = [f for f in new_files if isfile(f)]
+
+    if existing:
+        raise IOError("Can't merge {0} into {1}: file exists: {2}".format(src, dst, existing[0]))
+
+    return copy_tree(src, dst)
+
+
 def relative(f, d='lib'):
     assert not f.startswith('/'), f
     assert not d.startswith('/'), d


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,8 +1,60 @@
 import unittest
+import tempfile
+import shutil
+import os
 
 import conda_build.utils as utils
 
 
+class TestCopyInto(unittest.TestCase):
+
+    def setUp(self):
+        self.src = tempfile.mkdtemp()
+        self.dst = tempfile.mkdtemp()
+
+        self.namespace = os.path.join(self.src, 'namespace')
+        self.package = os.path.join(self.namespace, 'package')
+        self.module = os.path.join(self.package, 'module.py')
+
+        os.makedirs(self.namespace)
+        os.makedirs(self.package)
+        self.makefile(self.module)
+
+    def makefile(self, name, contents=''):
+
+        name = os.path.abspath(name)
+        path = os.path.dirname(name)
+
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        with open(name, 'w') as f:
+            f.write(contents)
+
+    def test_copy_source_tree(self):
+        utils.copy_into(self.src, self.dst)
+        self.assertTrue(os.path.isfile(os.path.join(self.dst, 'namespace', 'package', 'module.py')))
+
+    def test_merge_namespace_trees(self):
+    
+        dep = os.path.join(self.dst, 'namespace', 'package', 'dependency.py')
+        self.makefile(dep)
+
+        utils.copy_into(self.src, self.dst)
+        self.assertTrue(os.path.isfile(os.path.join(self.dst, 'namespace', 'package', 'module.py')))
+        self.assertTrue(os.path.isfile(dep))
+
+    def test_disallow_merge_conflicts(self):
+
+        duplicate = os.path.join(self.dst, 'namespace', 'package', 'module.py')
+        self.makefile(duplicate)
+        self.assertRaises(IOError, utils.copy_into, self.src, self.dst)
+
+    def tearDown(self):
+        shutil.rmtree(self.dst)
+        shutil.rmtree(self.src)
+
+
 class TestUtils(unittest.TestCase):
 
     def test_relative_default(self):