Repository: conda/conda-build

Issue:
paths.json not updated during causing installation failure in conda 4.3.4
When converting from a win-64 package to a linux-64 package the `Lib/site-packages` directory should become `lib/python2.7/site-packages`. This is correctly updated in 'files' but should be updated in paths.json also. Otherwise this error is given when installing under conda 4.3.4 on Linux:

`
CondaError: Cannot link a source that does not exist. /home/NETWORK/e.hughes/miniconda3/pkgs/package-v1.0.0/Lib/site-packages/package-v1.0.0-py2.7.egg-info
`


Discussion:

Resolving patch:
diff --git a/conda_build/convert.py b/conda_build/convert.py
--- a/conda_build/convert.py
+++ b/conda_build/convert.py
@@ -162,6 +162,20 @@ def tar_update(source, dest, file_map, verbose=True, quiet=False):
     finally:
         t.close()
 
+def _check_paths_version(paths):
+    """Verify that we can handle this version of a paths file"""
+    # For now we only accept 1, but its possible v2 will still have the structure we need
+    # If so just update this if statement.
+    if paths['paths_version'] != 1:
+        raise RuntimeError("Cannot handle info/paths.json paths_version other than 1")
+
+def _update_paths(paths, mapping_dict):
+    """Given a paths file, update it such that old paths are replaced with new"""
+    updated_paths = deepcopy(paths)
+    for path in updated_paths['paths']:
+        if path['_path'] in mapping_dict:
+            path['_path'] = mapping_dict[path['_path']]
+    return updated_paths
 
 path_mapping_bat_proxy = [
     (re.compile(r'bin/(.*)(\.py)'), r'Scripts/\1.bat'),
@@ -194,6 +208,11 @@ def tar_update(source, dest, file_map, verbose=True, quiet=False):
 
 def get_pure_py_file_map(t, platform):
     info = json.loads(t.extractfile('info/index.json').read().decode('utf-8'))
+    try:
+        paths = json.loads(t.extractfile('info/paths.json').read().decode('utf-8'))
+        _check_paths_version(paths)
+    except KeyError:
+        paths = None
     source_plat = info['platform']
     source_type = 'unix' if source_plat in {'osx', 'linux'} else 'win'
     dest_plat, dest_arch = platform.split('-')
@@ -235,6 +254,8 @@ def get_pure_py_file_map(t, platform):
 
     members = t.getmembers()
     file_map = {}
+    paths_mapping_dict = {} # keep track of what we change in files
+    pathmember = None
     for member in members:
         # Update metadata
         if member.path == 'info/index.json':
@@ -250,6 +271,9 @@ def get_pure_py_file_map(t, platform):
             # We have to do this at the end when we have all the files
             filemember = deepcopy(member)
             continue
+        elif member.path == 'info/paths.json':
+            pathmember = deepcopy(member)
+            continue
         elif member.path == 'info/has_prefix':
             if source_type == 'unix' and dest_type == 'win':
                 # has_prefix is not needed on Windows
@@ -270,6 +294,7 @@ def get_pure_py_file_map(t, platform):
                 file_map[newpath] = newmember
                 loc = files.index(oldpath)
                 files[loc] = newpath
+                paths_mapping_dict[oldpath] = newpath
                 break
         else:
             file_map[oldpath] = member
@@ -292,11 +317,21 @@ def get_pure_py_file_map(t, platform):
                     batseen.add(oldpath)
                     files.append(newpath)
 
+    # Change paths.json the same way that we changed files
+    if paths:
+        updated_paths = _update_paths(paths, paths_mapping_dict)
+        paths = json.dumps(updated_paths, sort_keys=True,
+                           indent=4, separators=(',', ': '))
     files = '\n'.join(sorted(files)) + '\n'
     if PY3:
         files = bytes(files, 'utf-8')
+        if paths:
+            paths = bytes(paths, 'utf-8')
     filemember.size = len(files)
     file_map['info/files'] = filemember, bytes_io(files)
+    if pathmember:
+        pathmember.size = len(paths)
+        file_map['info/paths.json'] = pathmember, bytes_io(paths)
 
     return file_map
 


Test patch:
diff --git a/tests/test_api_convert.py b/tests/test_api_convert.py
--- a/tests/test_api_convert.py
+++ b/tests/test_api_convert.py
@@ -1,4 +1,6 @@
 import os
+import json
+import tarfile
 
 import pytest
 
@@ -19,19 +21,39 @@ def test_convert_exe_raises():
         api.convert("some_wheel.exe")
         assert "cannot convert:" in str(exc)
 
+def assert_package_paths_matches_files(package_path):
+    """Ensure that info/paths.json matches info/files"""
+    with tarfile.open(package_path) as t:
+        files_content = t.extractfile('info/files').read().decode('utf-8')
+        files_set = set(line for line in files_content.split('\n') if line)
+        paths_content = json.loads(t.extractfile('info/paths.json').read().decode('utf-8'))
+
+    for path_entry in paths_content['paths']:
+        assert path_entry['_path'] in files_set
+        files_set.remove(path_entry['_path'])
+
+    assert not files_set # Check that we've seen all the entries in files
 
 @pytest.mark.serial
 @pytest.mark.parametrize('base_platform', ['linux', 'win', 'osx'])
-def test_convert_platform_to_others(testing_workdir, base_platform):
-    f = 'http://repo.continuum.io/pkgs/free/{}-64/itsdangerous-0.24-py27_0.tar.bz2'.format(base_platform)
-    fn = "itsdangerous-0.24-py27_0.tar.bz2"
+@pytest.mark.parametrize('package', [('itsdangerous-0.24', 'itsdangerous.py'),
+                                     ('py-1.4.32', 'py/__init__.py')])
+def test_convert_platform_to_others(testing_workdir, base_platform, package):
+    package_name, example_file = package
+    f = 'http://repo.continuum.io/pkgs/free/{}-64/{}-py27_0.tar.bz2'.format(base_platform, package_name)
+    fn = "{}-py27_0.tar.bz2".format(package_name)
     download(f, fn)
-    api.convert(fn, platforms='all', quiet=False, verbose=True)
+    expected_paths_json = package_has_file(fn, 'info/paths.json')
+    api.convert(fn, platforms='all', quiet=False, verbose=False)
     for platform in ['osx-64', 'win-64', 'win-32', 'linux-64', 'linux-32']:
         python_folder = 'lib/python2.7' if not platform.startswith('win') else 'Lib'
-        assert package_has_file(os.path.join(platform, fn),
-                                '{}/site-packages/itsdangerous.py'.format(python_folder))
+        package = os.path.join(platform, fn)
+        assert package_has_file(package,
+                                '{}/site-packages/{}'.format(python_folder, example_file))
 
+        if expected_paths_json:
+            assert package_has_file(package, 'info/paths.json')
+            assert_package_paths_matches_files(package)
 
 @pytest.mark.serial
 @pytest.mark.skipif(on_win, reason="we create the package to be converted in *nix, so don't run on win.")