Repository: benfogle/crossenv (language: python)

Issue:
Shebang in 1.1.0 changed; activating venv fails
I tried upgrading to 1.1.0, but activating venv fails now:
```
/bin/sh: /home/geulig/linux.git/user/python_modules/venv/cross/bin/pip3: /home/geulig/linux.git/user/python/build/bin/python: bad interpreter: No such file or directory
make[2]: *** [Makefile:40: all] Error 1
```
The shebang of venv/cross/bin/pip3 is
`#!/home/geulig/linux.git/user/python/build/bin/python`
and yes, this file does not exist.

Version 1.0 still works. Here the shebang is
`#!/home/geulig/linux.git/user/python_modules/venv/cross/bin/python`

Setup is done like this:
```
PYTHONHOME="/opt/python-3.6.13" \
PYTHONPATH="/opt/python-3.6.13/lib/python3.6/site-packages:/opt/python-3.6.13/lib64/python3.6/lib-dynload" \
        /opt/python-3.6.13/bin/python3 -m crossenv /home/geulig/linux.git/user/python/build/python venv
. venv/bin/activate
```

Maybe this has something to do with the changes for #44/#46?


Discussion / hints:
I don't think it's #44/#46 because when the shebang line is changed, it get's changed to `#!/bin/sh`.

Is `/home/geulig/linux.git/user/python/build/python` a build directory? If so, does the same issue occur if you install it somewhere and run crossenv with the installed version?
I can reproduce this when `host-python` is in a build directory. I am investigating the cause.

Fix patch:
diff --git a/crossenv/scripts/site.py.tmpl b/crossenv/scripts/site.py.tmpl
index 4a061a3..fb501ce 100644
--- a/crossenv/scripts/site.py.tmpl
+++ b/crossenv/scripts/site.py.tmpl
@@ -81,6 +81,9 @@ class CrossenvFinder(importlib.abc.MetaPathFinder):
     """
 
     PATCHES = {
+        'importlib.machinery': '{{context.lib_path}}/importlib-machinery-patch.py',
+        'sys': '{{context.lib_path}}/sys-patch.py',
+        'os': '{{context.lib_path}}/os-patch.py',
         'sysconfig': '{{context.lib_path}}/sysconfig-patch.py',
         'distutils.sysconfig': '{{context.lib_path}}/distutils-sysconfig-patch.py',
         'platform': '{{context.lib_path}}/platform-patch.py',
@@ -88,6 +91,12 @@ class CrossenvFinder(importlib.abc.MetaPathFinder):
         'pip._vendor.pkg_resources': '{{context.lib_path}}/pkg_resources-patch.py',
     }
 
+    def __init__(self):
+        # At startup, manually patch things that have already been loaded.  We
+        # can't re-load them because they might be used in many places already.
+        # This will be importlib.machinery, sys, and os at the very least.
+        self.manually_patch_loaded()
+
     def find_spec(self, fullname, path, target=None):
         spec = self._find_sysconfigdata(fullname, path, target)
         if spec:
@@ -124,6 +133,18 @@ class CrossenvFinder(importlib.abc.MetaPathFinder):
         spec.loader = CrossenvPatchLoader(spec.loader, patch)
         return spec
 
+    def manually_patch_loaded(self):
+        # do this one first so import machinery works for any subsequent
+        # patches
+        import importlib.machinery
+        _patch_module(importlib.machinery, self.PATCHES['importlib.machinery'])
+
+        for name, module in sys.modules.items():
+            if name == 'importlib.machinery':
+                continue
+            if name in self.PATCHES:
+                _patch_module(module, self.PATCHES[name])
+
 # add just before the real path finder
 try:
     _index = sys.meta_path.index(importlib.machinery.PathFinder)
@@ -131,14 +152,9 @@ except ValueError:
     _index = 0
 sys.meta_path.insert(_index, CrossenvFinder())
 
-# Fixup sys, os, etc. We can't safely re-import them, because they're already
-# being used in many places, so we'll manually apply the patch. Do importlib.machinery
-# first so that our imports work for the others.
-_patch_module(importlib.machinery, '{{context.lib_path}}/importlib-machinery-patch.py')
-_patch_module(sys, '{{context.lib_path}}/sys-patch.py')
-_patch_module(os, '{{context.lib_path}}/os-patch.py')
-
 # Re-import the real site module, so Python can continue booting. Crossenv is
-# ready!
+# ready! We do want to remove sysconfig or any other module that relies on
+# patching after site has messed with sys.
 del sys.modules['site']
+sys.modules.pop('sysconfig', None)
 import site
diff --git a/crossenv/scripts/sysconfig-patch.py.tmpl b/crossenv/scripts/sysconfig-patch.py.tmpl
index fc2a71f..d64be1d 100644
--- a/crossenv/scripts/sysconfig-patch.py.tmpl
+++ b/crossenv/scripts/sysconfig-patch.py.tmpl
@@ -1,7 +1,12 @@
 # Patch the things that depend on os.environ
 _PROJECT_BASE = {{repr(self.host_project_base)}}
 
-_PYTHON_BUILD = is_python_build(True) # re-eval after patching
+# Things that need to be re-evaluated after patching
+_PYTHON_BUILD = is_python_build(True)
+_PREFIX = os.path.normpath(sys.prefix)
+_BASE_PREFIX = os.path.normpath(sys.base_prefix)
+_EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
 
 def get_makefile_filename():
     return {{repr(self.host_makefile)}}


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index d1265eb..0b835e3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,7 +11,7 @@
 # Fixtures for everyone. Make sure to include fixture-of-fixture
 # dependencies...
 from .resources import host_python, build_python, architecture, \
-        python_version, get_resource
+        python_version, get_resource, crossenv_setup
 
 def pytest_addoption(parser):
     parser.addoption('--coverage', action='store_true',
diff --git a/tests/prebuilt/src/CMakeLists.txt b/tests/prebuilt/src/CMakeLists.txt
index 313a782..82cb1c9 100644
--- a/tests/prebuilt/src/CMakeLists.txt
+++ b/tests/prebuilt/src/CMakeLists.txt
@@ -379,6 +379,7 @@ foreach(index RANGE ${num_versions})
         INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/${pyversion}/build
         DOWNLOAD_DIR        python-download
         SOURCE_DIR          python-source-${pyversion}
+        BINARY_DIR          ${CMAKE_INSTALL_PREFIX}/python-obj/${pyversion}/build
         ${download}
         CONFIGURE_COMMAND   <SOURCE_DIR>/configure
                                 --prefix=<INSTALL_DIR>
@@ -408,6 +409,7 @@ foreach(index RANGE ${num_versions})
         INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/${pyversion}/aarch64
         DOWNLOAD_COMMAND    ""
         SOURCE_DIR          python-source-${pyversion}
+        BINARY_DIR          ${CMAKE_INSTALL_PREFIX}/python-obj/${pyversion}/aarch64
         CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
                             <SOURCE_DIR>/configure
                                 --prefix=<INSTALL_DIR>
@@ -443,6 +445,7 @@ foreach(index RANGE ${num_versions})
         INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/${pyversion}/armhf
         DOWNLOAD_COMMAND    ""
         SOURCE_DIR          python-source-${pyversion}
+        BINARY_DIR          ${CMAKE_INSTALL_PREFIX}/python-obj/${pyversion}/armhf
         CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
                             <SOURCE_DIR>/configure
                                 --prefix=<INSTALL_DIR>
diff --git a/tests/resources.py b/tests/resources.py
index 019e171..618102d 100644
--- a/tests/resources.py
+++ b/tests/resources.py
@@ -14,6 +14,8 @@
 from .testutils import ExecEnvironment, hash_file, open_lock_file
 
 Architecture = namedtuple('Architecture', 'name system machine')
+TestSetup = namedtuple('TestSetup',
+        'name architecture version host_python_tag build_python_tag')
 
 ARCHITECTURES = [
     Architecture(
@@ -130,6 +132,39 @@
             'QEMU_LD_PREFIX': '$EXTRACTED/prebuilt_musl_arm_aarch64/musl-toolchain/arm-linux-musleabihf',
         }
     },
+    # All the same, but from a build directory.
+    'build-python:master:obj': {
+        'source': 'python-obj/master/build',
+        'binary': 'python',
+        'env': {
+            'PATH': '$EXTRACTED/prebuilt_musl_arm_aarch64/musl-toolchain/bin:$PATH',
+        },
+    },
+    'host-python:master:x86_64-linux-gnu:obj': {
+        'source': 'python-obj/master/build',
+        'binary': 'python',
+        'env': {
+            'PATH': '$EXTRACTED/prebuilt_musl_arm_aarch64/musl-toolchain/bin:$PATH',
+        },
+    },
+    'host-python:master:aarch64-linux-musl:obj': {
+        'source': 'python-obj/master/aarch64',
+        'binary': 'python',
+        'env': {
+            'QEMU_LD_PREFIX': '$EXTRACTED/prebuilt_musl_arm_aarch64/musl-toolchain/aarch64-linux-musl',
+            'LD_LIBRARY_PATH': '$SOURCE',
+        },
+    },
+    'host-python:master:arm-linux-musleabihf:obj': {
+        'source': 'python-obj/master/armhf',
+        'binary': 'python',
+        'env': {
+            'QEMU_LD_PREFIX': '$EXTRACTED/prebuilt_musl_arm_aarch64/musl-toolchain/arm-linux-musleabihf',
+            'LD_LIBRARY_PATH': '$SOURCE',
+        }
+    },
+
+    # finally, source files
     'hello-module:source': {
         'source': 'hello',
     },
@@ -271,14 +306,57 @@ def derive(self):
     def __repr__(self):
         return '<Resource {}>'.format(self.binary or self.path)
 
-
-@pytest.fixture(params=ARCHITECTURES, scope='session', ids=lambda a: a.name)
-def architecture(request):
+def collect_test_setups():
+    # build_installed and host_installed, below, are causing us to hit some
+    # issues with CI runners as the number of variants explodes. We'll leave
+    # them disabled except for local tests, pending a later fix.
+    if os.environ.get('CROSSENV_TEST_INPLACE'):
+        installed = (True, False)
+    else:
+        installed = (True,)
+
+    setups = []
+    for version in PY_VERSIONS:
+        for arch in ARCHITECTURES:
+            for build_installed in installed:
+                parts = ['build-python', version]
+                if not build_installed:
+                    parts.append('obj')
+                build_python_tag = ':'.join(parts)
+                if not Resource.exists(build_python_tag):
+                    continue
+
+                #for host_installed in (True, False):
+                for host_installed in installed:
+                    parts = ['host-python', version, arch.name]
+
+                    if not host_installed:
+                        parts.append('obj')
+                    host_python_tag = ':'.join(parts)
+
+                    if not Resource.exists(host_python_tag):
+                        continue
+
+                    build_installed = 'installed' if build_installed else 'inplace'
+                    host_installed = 'installed' if host_installed else 'inplace'
+
+                    name = ':'.join([arch.name, version, build_installed,
+                        host_installed])
+                    setups.append(TestSetup(name, arch, version,
+                        host_python_tag, build_python_tag))
+    return setups
+
+@pytest.fixture(params=collect_test_setups(), scope='session', ids=lambda s: s.name)
+def crossenv_setup(request):
     return request.param
 
-@pytest.fixture(params=PY_VERSIONS, scope='session')
-def python_version(request):
-    return request.param
+@pytest.fixture(scope='session')
+def architecture(crossenv_setup):
+    return crossenv_setup.architecture
+
+@pytest.fixture(scope='session')
+def python_version(crossenv_setup):
+    return crossenv_setup.version
 
 def setup_coverage(venv_python):
     """Get code coverage, if requested. This is tricker than normal, because
@@ -313,16 +391,16 @@ def setup_coverage(venv_python):
     venv_python.setenv('COVERAGE_PROCESS_START', coverage_config.resolve())
 
 @pytest.fixture(scope='session')
-def build_python(request, python_version, tmp_path_factory):
-    build_python_tag = 'build-python:{}'.format(python_version)
+def build_python(request, crossenv_setup, tmp_path_factory):
+    build_python_tag = crossenv_setup.build_python_tag
     try:
         base = Resource(build_python_tag)
     except KeyError:
         pytest.skip('No build-python version {} available'.format(
-            python_version))
+            crossenv_setup.version))
     except FileNotFoundError:
         pytest.skip('Build-python version {} registerd, but not found on '
-                'disk'.format(python_version))
+                'disk'.format(crossenv_setup.version))
 
     # Make a virtualenv for build-python. This is not necessary for the tests,
     # but it lets us install coverage and other support packages so we can
@@ -339,16 +417,16 @@ def build_python(request, python_version, tmp_path_factory):
     return venv_python
 
 @pytest.fixture(scope='session')
-def host_python(architecture, python_version):
-    host_python_tag = 'host-python:{}:{}'.format(python_version, architecture.name)
+def host_python(crossenv_setup):
+    host_python_tag = crossenv_setup.host_python_tag
     try:
         return Resource(host_python_tag)
     except KeyError:
         pytest.skip('No Python version {} available for {}'.format(
-            python_version, architecture.name))
+            crossenv_setup.version, crossenv_setup.architecture.name))
     except FileNotFoundError:
         pytest.skip('Python version {} for {} registerd, but not found on '
-                'disk'.format(python_version, architecture.name))
+            'disk'.format(crossenv_setup.version, crossenv_setup.architecture.name))
 
 
 @pytest.fixture(scope='session')