Repository: benfogle/crossenv (language: python)

Issue:
Update for Python 3.9
There don't appear to be major changes, but some small fixes are needed.
support python 3.9



Discussion / hints:

Fix patch:
diff --git a/crossenv/scripts/site.py.tmpl b/crossenv/scripts/site.py.tmpl
index 47f7e5e..bae8146 100644
--- a/crossenv/scripts/site.py.tmpl
+++ b/crossenv/scripts/site.py.tmpl
@@ -31,13 +31,12 @@ os.environ['PYTHON_CROSSENV'] = 'x'
 # We'll try not to do that unless we really have to.
 for name in ['_PYTHON_PROJECT_BASE', '_PYTHON_HOST_PLATFORM',
              '_PYTHON_SYSCONFIGDATA_NAME', 'PYTHONHOME', 'PYTHONPATH']:
-    # This is the same as os.putenv/os.unsetenv, but I want to be clear that
-    # I'm trying to break something.
     try:
-        os.environ.putenv(name, os.environ['_OLD_' + name])
+        # Using os.putenv causes the environments to diverge
+        os.putenv(name, os.environ['_OLD_' + name])
         del os.environ['_OLD_' + name]
     except KeyError:
-        os.environ.unsetenv(name)
+        os.unsetenv(name)
 
 
 # sysconfig isn't _quite_ set up right, because it queries from a not-yet fixed
@@ -135,8 +134,10 @@ os.uname = uname
 import platform
 processor = config.get('uname', 'processor', fallback=None)
 if processor is None:
-    processor = config.get('uname', 'machine', fallback=None)
-uname_result2 = platform.uname_result(
+    processor = config.get('uname', 'machine', fallback='')
+platform_uname_result_type = namedtuple('uname_result',
+                    'system node release version machine processor')
+uname_result2 = platform_uname_result_type(
         config.get('uname', 'sysname', fallback=''),
         config.get('uname', 'nodename', fallback=''),
         config.get('uname', 'release', fallback=''),


Test patch:
diff --git a/tests/prebuilt/prebuilt_python3.9.0.tar.xz b/tests/prebuilt/prebuilt_python3.9.0.tar.xz
new file mode 100644
index 0000000..989ee78
--- /dev/null
+++ b/tests/prebuilt/prebuilt_python3.9.0.tar.xz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:280fbf0ccaf5e015b4af48160dfd97eb5d6fef3650a62fe91b278bacce019a27
+size 92312252
diff --git a/tests/prebuilt/src/CMakeLists.txt b/tests/prebuilt/src/CMakeLists.txt
index 9a12352..cb89bf1 100644
--- a/tests/prebuilt/src/CMakeLists.txt
+++ b/tests/prebuilt/src/CMakeLists.txt
@@ -15,6 +15,18 @@ cmake_minimum_required(VERSION 3.6)
 project(crossenv-artifacts NONE)
 include(ExternalProject)
 
+set(PYTHON_VERSIONS
+    3.8.1
+    3.9.0
+    )
+set(PYTHON_URLS
+    https://www.python.org/ftp/python/3.8.1/Python-3.8.1.tar.xz
+    https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tar.xz
+    )
+set(PYTHON_HASHES
+    MD5=b3fb85fd479c0bf950c626ef80cacb57
+    MD5=6ebfe157f6e88d9eabfbaf3fa92129f6
+    )
 
 ##############################################################################
 # MUSL-gcc toolchains
@@ -306,85 +318,94 @@ add_custom_target(prereqs
 # as part of build-python
 ##############################################################################
 
-ExternalProject_Add(build-python-3.8.1
-    INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/3.8.1/build
-    DOWNLOAD_DIR        python-download
-    SOURCE_DIR          python-source-3.8.1
-    URL                 https://www.python.org/ftp/python/3.8.1/Python-3.8.1.tar.xz
-    URL_HASH            MD5=b3fb85fd479c0bf950c626ef80cacb57
-    CONFIGURE_COMMAND   <SOURCE_DIR>/configure
-                            --prefix=<INSTALL_DIR>
-    BUILD_COMMAND       make -j8
-    INSTALL_COMMAND     make install
-)
-ExternalProject_Get_Property(build-python-3.8.1 INSTALL_DIR)
-set(BUILD_PYTHON_BIN ${INSTALL_DIR}/bin)
-set(BUILD_PYTHON_LIB ${INSTALL_DIR}/lib)
-set(BUILD_PYTHON_PATH ${MUSL_TOOLCHAIN_BIN}:${BUILD_PYTHON_BIN}:$ENV{PATH})
-
-
-# set rpath to run from install directory. Prevents a mess of LD_LIBRARY_PATH when
-# we're running under emulation.
-set(rpaths
-    -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/aarch64-linux-musl/lib'
-    -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/aarch64-linux-musl/usr/lib'
-    -Wl,-rpath='\$\${ORIGIN}/../lib'
-)
-string(REPLACE ";" " " rpaths "${rpaths}") # join ;-list with " "
-
-
-ExternalProject_Add(host-python-3.8.1-aarch64
-    INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/3.8.1/aarch64
-    DOWNLOAD_COMMAND    ""
-    SOURCE_DIR          python-source-3.8.1
-    CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
-                        <SOURCE_DIR>/configure
-                            --prefix=<INSTALL_DIR>
-                            --enable-shared
-                            --host=aarch64-linux-musl
-                            --build=x86_64-linux-gnu
-                            --without-ensurepip
-                            ac_cv_buggy_getaddrinfo=no
-                            ac_cv_file__dev_ptmx=yes
-                            ac_cv_file__dev_ptc=no
-                            "LDFLAGS=${rpaths}"
-    BUILD_COMMAND       ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
-                        make -j8
-    INSTALL_COMMAND     ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
-                        make install
-    DEPENDS             prereqs
-                        build-python-3.8.1
-)
-
-
-# set rpath to run from install directory. Prevents a mess of LD_LIBRARY_PATH when
-# we're running under emulation.
-set(rpaths
-    -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/arm-linux-musleabi/lib'
-    -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/arm-linux-musleabi/usr/lib'
-    -Wl,-rpath='\$\${ORIGIN}/../lib'
-)
-string(REPLACE ";" " " rpaths "${rpaths}") # join ;-list with " "
-
-ExternalProject_Add(host-python-3.8.1-armhf
-    INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/3.8.1/armhf
-    DOWNLOAD_COMMAND    ""
-    SOURCE_DIR          python-source-3.8.1
-    CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
-                        <SOURCE_DIR>/configure
-                            --prefix=<INSTALL_DIR>
-                            --enable-shared
-                            --host=arm-linux-musleabihf
-                            --build=x86_64-linux-gnu
-                            --without-ensurepip
-                            ac_cv_buggy_getaddrinfo=no
-                            ac_cv_file__dev_ptmx=yes
-                            ac_cv_file__dev_ptc=no
-                            "LDFLAGS=${rpaths}"
-    BUILD_COMMAND       ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
-                        make -j8
-    INSTALL_COMMAND     ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
-                        make install
-    DEPENDS             prereqs
-                        build-python-3.8.1
-)
+list(LENGTH PYTHON_VERSIONS num_versions)
+math(EXPR num_versions "${num_versions} - 1")
+
+foreach(index RANGE ${num_versions})
+    list(GET PYTHON_VERSIONS ${index} pyversion)
+    list(GET PYTHON_URLS ${index} pyurl)
+    list(GET PYTHON_HASHES ${index} pyhash)
+
+    ExternalProject_Add(build-python-${pyversion}
+        INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/${pyversion}/build
+        DOWNLOAD_DIR        python-download
+        SOURCE_DIR          python-source-${pyversion}
+        URL                 ${pyurl}
+        URL_HASH            ${pyhash}
+        CONFIGURE_COMMAND   <SOURCE_DIR>/configure
+                                --prefix=<INSTALL_DIR>
+        BUILD_COMMAND       make -j8
+        INSTALL_COMMAND     make install
+    )
+    ExternalProject_Get_Property(build-python-${pyversion} INSTALL_DIR)
+    set(BUILD_PYTHON_BIN ${INSTALL_DIR}/bin)
+    set(BUILD_PYTHON_LIB ${INSTALL_DIR}/lib)
+    set(BUILD_PYTHON_PATH ${MUSL_TOOLCHAIN_BIN}:${BUILD_PYTHON_BIN}:$ENV{PATH})
+
+
+    # set rpath to run from install directory. Prevents a mess of LD_LIBRARY_PATH when
+    # we're running under emulation. We might not be able to completely avoid that.
+    set(rpaths
+        -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/aarch64-linux-musl/lib'
+        -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/aarch64-linux-musl/usr/lib'
+        -Wl,-rpath='\$\${ORIGIN}/../lib'
+    )
+    string(REPLACE ";" " " rpaths "${rpaths}") # join ;-list with " "
+
+
+    ExternalProject_Add(host-python-${pyversion}-aarch64
+        INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/${pyversion}/aarch64
+        DOWNLOAD_COMMAND    ""
+        SOURCE_DIR          python-source-${pyversion}
+        CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
+                            <SOURCE_DIR>/configure
+                                --prefix=<INSTALL_DIR>
+                                --enable-shared
+                                --host=aarch64-linux-musl
+                                --build=x86_64-linux-gnu
+                                --without-ensurepip
+                                ac_cv_buggy_getaddrinfo=no
+                                ac_cv_file__dev_ptmx=yes
+                                ac_cv_file__dev_ptc=no
+                                "LDFLAGS=${rpaths}"
+        BUILD_COMMAND       ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
+                            make -j8
+        INSTALL_COMMAND     ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
+                            make install
+        DEPENDS             prereqs
+                            build-python-${pyversion}
+    )
+
+
+    # set rpath to run from install directory. Prevents a mess of LD_LIBRARY_PATH when
+    # we're running under emulation.
+    set(rpaths
+        -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/arm-linux-musleabi/lib'
+        -Wl,-rpath='\$\${ORIGIN}/../../../../musl-toolchain/arm-linux-musleabi/usr/lib'
+        -Wl,-rpath='\$\${ORIGIN}/../lib'
+    )
+    string(REPLACE ";" " " rpaths "${rpaths}") # join ;-list with " "
+
+    ExternalProject_Add(host-python-${pyversion}-armhf
+        INSTALL_DIR         ${CMAKE_INSTALL_PREFIX}/python/${pyversion}/armhf
+        DOWNLOAD_COMMAND    ""
+        SOURCE_DIR          python-source-${pyversion}
+        CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
+                            <SOURCE_DIR>/configure
+                                --prefix=<INSTALL_DIR>
+                                --enable-shared
+                                --host=arm-linux-musleabihf
+                                --build=x86_64-linux-gnu
+                                --without-ensurepip
+                                ac_cv_buggy_getaddrinfo=no
+                                ac_cv_file__dev_ptmx=yes
+                                ac_cv_file__dev_ptc=no
+                                "LDFLAGS=${rpaths}"
+        BUILD_COMMAND       ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
+                            make -j8
+        INSTALL_COMMAND     ${CMAKE_COMMAND} -E env PATH=${BUILD_PYTHON_PATH}
+                            make install
+        DEPENDS             prereqs
+                            build-python-${pyversion}
+    )
+endforeach()
diff --git a/tests/resources.py b/tests/resources.py
index 8b31fe9..5960064 100644
--- a/tests/resources.py
+++ b/tests/resources.py
@@ -29,9 +29,17 @@
 
 PY_VERSIONS = [
     '3.8.1',
+    '3.9.0',
 ]
 
-
+# This structure declares all the prebuilt resources we need. Each tag refers
+# to a file or directory, that might need to be extracted. Really, we just
+# unconditionally extract all archive sources defined below to the same $SOURCE
+# directory before we begin. (We leave directory sources as-is.) Multiple
+# resources can share a source archive: only one copy will be extracted.
+#
+# The binary attribute would be e.g., an executable to run. The env field is a
+# dictionary specifying the environment needed to run it.
 PREBUILT_RESOURCES = {
     'build-python:3.8.1': {
         'source': 'prebuilt_musl_arm_aarch64.tar.xz',
@@ -54,6 +62,27 @@
             'QEMU_LD_PREFIX': '$SOURCE/prebuilt_musl_arm_aarch64/musl-toolchain/arm-linux-musleabihf',
         }
     },
+    'build-python:3.9.0': {
+        'source': 'prebuilt_python3.9.0.tar.xz',
+        'binary': 'prebuilt_python3.9.0/build/bin/python3',
+        'env': {
+            'PATH': '$SOURCE/prebuilt_musl_arm_aarch64/musl-toolchain/bin:$PATH',
+        },
+    },
+    'host-python:3.9.0:aarch64-linux-musl': {
+        'source': 'prebuilt_python3.9.0.tar.xz',
+        'binary': 'prebuilt_python3.9.0/aarch64/bin/python3',
+        'env': {
+            'QEMU_LD_PREFIX': '$SOURCE/prebuilt_musl_arm_aarch64/musl-toolchain/aarch64-linux-musl',
+        }
+    },
+    'host-python:3.9.0:arm-linux-musleabihf': {
+        'source': 'prebuilt_python3.9.0.tar.xz',
+        'binary': 'prebuilt_python3.9.0/armhf/bin/python3',
+        'env': {
+            'QEMU_LD_PREFIX': '$SOURCE/prebuilt_musl_arm_aarch64/musl-toolchain/arm-linux-musleabihf',
+        }
+    },
     'hello-module:source': {
         'source': 'hello',
     },
@@ -61,21 +90,28 @@
 
 
 class PrebuiltBlobs:
-    def __init__(self):
-        this_dir = Path(__file__).parent
+    """We assume that all pre-built blobs defined in RESOURCES are needed,
+    becuase we don't want to do dependency management. Everything that
+    needs to be unpacked will be unpacked in the same directory. Everything
+    that's already unpacked remains in place"""
 
+    def __init__(self, resources):
+        this_dir = Path(__file__).parent
+        self.resources = resources
         self.source_paths = [ this_dir/'prebuilt', this_dir/'sources' ]
+        self.extract_base = None
         self.cache_dir = None
-        self._existing_blobs = {}
 
-    def get(self, source):
-        path = self._existing_blobs.get(source)
-        if path is not None:
-            return path
+        for tag, info in resources.items():
+            if 'source' not in info:
+                raise KeyError(
+                    "Resource {} missing 'source' field".format(tag))
+
 
+    def get(self, source):
         source = self.find_source(source)
         if source.is_file():
-            return self.unpack_if_needed(source)
+            return self.extract_dir
         elif source.is_dir():
             return source # Just use it in place.
         else:
@@ -88,11 +124,28 @@ def find_source(self, source):
                 return result
         raise FileNotFoundError("No such file: {}".format(source))
 
-    def unpack_if_needed(self, archive):
+    @property
+    def extract_dir(self):
         if not self.cache_dir:
             raise ValueError("Need to configure cache_dir for prebuilt blobs!")
-        digest = hash_file(archive)
-        path = self.cache_dir / digest
+
+        if not self.extract_base:
+            # Find a hash-of-hash that will be our extraction directory. If
+            # things change, we won't have a conflict.
+            archives = set()
+            for tag, info in self.resources.items():
+                src = self.find_source(info['source'])
+                if src.is_file():
+                    archives.add(src)
+
+            digest = hashlib.sha256()
+            for arc in sorted(archives):
+                if arc.is_file():
+                    digest.update(hash_file(arc).encode('ascii'))
+
+            self.extract_base = digest.hexdigest()
+
+        path = self.cache_dir / self.extract_base
 
         # Our caching check isn't real smart: if the directory exists,
         # then we'll assume everything is okay.
@@ -101,29 +154,27 @@ def unpack_if_needed(self, archive):
 
         path.mkdir(parents=True, exist_ok=False)
         try:
-            shutil.unpack_archive(str(archive), str(path))
+            for arc in archives:
+                shutil.unpack_archive(str(arc), str(path))
         except:
             shutil.rmtree(path)
             raise
         return path
 
-prebuilt_blobs = PrebuiltBlobs()
+prebuilt_blobs = PrebuiltBlobs(PREBUILT_RESOURCES)
 
 class Resource(ExecEnvironment):
+    """A container for running or accessing a particular resource"""
+
     def __init__(self, tag):
         super().__init__()
 
         try:
-            info = PREBUILT_RESOURCES[tag]
+            info = prebuilt_blobs.resources[tag]
         except KeyError:
             raise KeyError("No such resource {}".format(tag))
 
-        try:
-            self.source = info['source']
-        except KeyError:
-            raise KeyError(
-                "Resource {} missing 'source' field".format(tag))
-
+        self.source = info['source']
         self.path = prebuilt_blobs.get(self.source)
         self.binary = info.get('binary')
         if self.binary:
@@ -137,9 +188,11 @@ def __init__(self, tag):
 
     @classmethod
     def exists(cls, tag):
-        return tag in PREBUILT_RESOURCES
+        """Does this resource tag even exist?"""
+        return tag in prebuilt_blobs.resources
 
     def make_copy(self, destdir = None, symlinks=True):
+        """Make a copy of the file or directory"""
         if destdir is None:
             if self._get_temp is None:
                 raise ValueError("Must explcitly set destdir")