Repository: conda/conda-build

Issue:
Python 3 packages missing __pycache__ have .pyc
Seeing a number of occurrences of packages built on Python 3 with `conda-build` that have `.pyc` files instead of `__pycache__` directories. This is actually causing a world of hurt in some cases as it results in other packages potentially containing `__pycache__` directories for their dependencies due to our compilation of all Python files in the build directory. In turn this results in challenging to debug problems like this one ( https://github.com/conda-forge/conda-smithy/issues/289 ).

xref: https://github.com/conda-forge/setuptools-feedstock/issues/18



Discussion:
Should add this is most acute on Python 3.5 in particular. Not really seeing this with Python 3.4.

I think I'm just going to gut the bit that tries to get rid of the pycache folder.


Resolving patch:
diff --git a/conda_build/post.py b/conda_build/post.py
--- a/conda_build/post.py
+++ b/conda_build/post.py
@@ -22,6 +22,7 @@
 from .conda_interface import lchmod
 from .conda_interface import walk_prefix
 from .conda_interface import md5_file
+from .conda_interface import PY3
 
 from conda_build import environ
 from conda_build import utils
@@ -142,32 +143,6 @@ def rm_py_along_so(prefix):
                         os.unlink(join(root, name + ext))
 
 
-def coerce_pycache_to_old_style(files, cwd):
-    """
-    For the sake of simplicity, remove the filename additions, like .cpython-35.pyc
-
-    Since Conda allows only one Python install in a given prefix, there is no reason
-    for these additional suffixes.  Newer Python will find these pyc files without issue.
-    """
-    for f in files:
-        if not os.path.exists(f):
-            f = os.path.join(cwd, f)
-        if not os.path.isfile(f) or not f.endswith('.py'):
-            continue
-        if '/' in f or '\\' in f:
-            folder = os.path.join(cwd, os.path.dirname(f), '__pycache__')
-        else:
-            folder = os.path.join(cwd, '__pycache__')
-        fname = os.path.join(folder, os.path.splitext(os.path.basename(f))[0] +
-                 '.cpython-{0}{1}.pyc'.format(sys.version_info.major,
-                                              sys.version_info.minor))
-        if os.path.isfile(fname):
-            os.rename(fname, f + 'c')
-    for root, _, files in os.walk(cwd):
-        if root.endswith("__pycache__") and not files:
-            os.rmdir(root)
-
-
 def compile_missing_pyc(files, cwd, python_exe):
     compile_files = []
     for fn in files:
@@ -179,14 +154,15 @@ def compile_missing_pyc(files, cwd, python_exe):
         else:
             if fn.startswith('bin'):
                 continue
-        if fn.endswith(".py") and fn + 'c' not in files:
+        cache_prefix = "__pycache__/" if PY3 else ""
+        if (fn.endswith(".py") and
+                os.path.dirname(fn) + cache_prefix + os.path.basename(fn) + 'c' not in files):
             compile_files.append(fn)
 
     if compile_files and os.path.isfile(python_exe):
         print('compiling .pyc files...')
         for f in compile_files:
             call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd)
-        coerce_pycache_to_old_style(compile_files, cwd=cwd)
 
 
 def post_process(files, prefix, config, preserve_egg_dir=False):


Test patch:
diff --git a/tests/test_api_build.py b/tests/test_api_build.py
--- a/tests/test_api_build.py
+++ b/tests/test_api_build.py
@@ -21,7 +21,7 @@
 from conda_build.os_utils.external import find_executable
 
 from .utils import (metadata_dir, fail_dir, is_valid_dir, testing_workdir, test_config,
-                    test_metadata)
+                    test_metadata, add_mangling)
 
 # define a few commonly used recipes - use os.path.join(metadata_dir, recipe) elsewhere
 empty_sections = os.path.join(metadata_dir, "empty_sections")
@@ -388,9 +388,9 @@ def test_compileall_compiles_all_good_files(testing_workdir, test_config):
     for f in good_files:
         assert package_has_file(output_file, f)
         # look for the compiled file also
-        assert package_has_file(output_file, f + 'c')
+        assert package_has_file(output_file, add_mangling(f))
     assert package_has_file(output_file, bad_file)
-    assert not package_has_file(output_file, bad_file + 'c')
+    assert not package_has_file(output_file, add_mangling(bad_file))
 
 
 def test_render_setup_py_old_funcname(testing_workdir, test_config, caplog):
diff --git a/tests/test_post.py b/tests/test_post.py
--- a/tests/test_post.py
+++ b/tests/test_post.py
@@ -8,7 +8,7 @@
 from conda_build import post
 from conda_build.utils import on_win
 
-from .utils import test_config, testing_workdir
+from .utils import test_config, testing_workdir, add_mangling
 
 
 def test_compile_missing_pyc(testing_workdir):
@@ -19,36 +19,9 @@ def test_compile_missing_pyc(testing_workdir):
                                  'metadata', '_compile-test'), tmp)
     post.compile_missing_pyc(os.listdir(tmp), cwd=tmp,
                                 python_exe=sys.executable)
-    files = os.listdir(tmp)
     for f in good_files:
-        assert f + 'c' in files
-    assert bad_file + 'c' not in files
-
-
-@pytest.mark.skipif(not PY3, reason="test applies to py3 only")
-def test_coerce_pycache_to_old_style(testing_workdir):
-    os.makedirs(os.path.join(testing_workdir, '__pycache__'))
-    os.makedirs(os.path.join(testing_workdir, 'testdir', '__pycache__'))
-    with open(os.path.join(testing_workdir, 'test.py'), 'w') as f:
-        f.write("\n")
-    with open(os.path.join(testing_workdir, '__pycache__', 'test.cpython-{0}{1}.pyc'.format(
-            sys.version_info.major, sys.version_info.minor)), 'w') as f:
-        f.write("\n")
-    with open(os.path.join(testing_workdir, 'testdir', 'test.py'), 'w') as f:
-        f.write("\n")
-    with open(os.path.join(testing_workdir, 'testdir', '__pycache__',
-                           'test.cpython-{0}{1}.pyc'.format(
-                               sys.version_info.major, sys.version_info.minor)), 'w') as f:
-        f.write("\n")
-
-    for root, dirs, files in os.walk(testing_workdir):
-        fs = [os.path.join(root, _) for _ in files]
-        post.coerce_pycache_to_old_style(fs, cwd=testing_workdir)
-    assert os.path.isfile(os.path.join(testing_workdir, 'test.pyc')), os.listdir(testing_workdir)
-    assert os.path.isfile(os.path.join(testing_workdir, 'testdir', 'test.pyc')), \
-        os.listdir(os.path.join(testing_workdir, 'testdir'))
-    for root, dirs, files in os.walk(testing_workdir):
-        assert '__pycache__' not in dirs
+        assert os.path.isfile(os.path.join(tmp, add_mangling(f)))
+    assert not os.path.isfile(os.path.join(tmp, add_mangling(bad_file)))
 
 
 @pytest.mark.skipif(on_win, reason="no linking on win")
diff --git a/tests/utils.py b/tests/utils.py
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -6,6 +6,7 @@
 
 import pytest
 
+from conda_build.conda_interface import PY3
 from conda_build.config import Config
 from conda_build.metadata import MetaData
 from conda_build.utils import on_win, prepend_bin_path
@@ -101,3 +102,12 @@ def package_has_file(package_path, file_path):
     except tarfile.ReadError:
         raise RuntimeError("Could not extract metadata from %s. "
                            "File probably corrupt." % package_path)
+
+
+def add_mangling(filename):
+    if PY3:
+        filename = os.path.splitext(filename)[0] + '.cpython-{0}{1}.py'.format(
+            sys.version_info.major, sys.version_info.minor)
+        filename = os.path.join(os.path.dirname(filename), '__pycache__',
+                                os.path.basename(filename))
+    return filename + 'c'