Repository: ReproNim/reproman

Issue:
env -0 not supported on macOS
reproman/resource/session.py uses `env -0` to get the environment, but `-0` is not a supported option to `env` on macOS.

Discussion:

Patch:
diff --git a/reproman/distributions/conda.py b/reproman/distributions/conda.py
index 4cea4e9..21f1599 100644
--- a/reproman/distributions/conda.py
+++ b/reproman/distributions/conda.py
@@ -85,7 +85,9 @@ def get_miniconda_url(conda_platform, python_version):
         raise ValueError("Unsupported platform %s for conda installation" %
                          conda_platform)
     platform += "-x86_64" if ("64" in conda_platform) else "-x86"
-    return "https://repo.anaconda.com/miniconda/Miniconda%s-latest-%s.sh" \
+    # FIXME: We need to update this our conda tracer to work with conda's newer
+    # than 4.6.14. See gh-443.
+    return "https://repo.continuum.io/miniconda/Miniconda%s-4.6.14-%s.sh" \
                     % (python_version[0], platform)
 
 
@@ -182,22 +184,22 @@ class CondaDistribution(Distribution):
                 # TODO: Determine if we can detect miniconda vs anaconad
                 miniconda_url = get_miniconda_url(self.platform,
                                                   self.python_version)
-                session.execute_command(
-                    "curl --fail --silent --show-error --location "
-                    "--output {}/miniconda.sh {}"
-                    .format(tmp_dir, miniconda_url))
+                session.execute_command("curl %s -o %s/miniconda.sh" %
+                                        (miniconda_url, tmp_dir))
                 # NOTE: miniconda.sh makes parent directories automatically
                 session.execute_command("bash -b %s/miniconda.sh -b -p %s" %
                                         (tmp_dir, self.path))
-            envs = sorted(
-                self.environments,
-                # Create/update the root environment before handling anything
-                # else.
-                key=lambda x: "_" if x.name == "root" else "_" + x.name)
-            for env in envs:
+            ## Update root version of conda
+            session.execute_command(
+               "%s/bin/conda install -y conda=%s python=%s" %
+               (self.path, self.conda_version,
+                self.get_simple_python_version(self.python_version)))
+
+            # Loop through non-root packages, creating the conda-env config
+            for env in self.environments:
                 export_contents = self.create_conda_export(env)
                 with make_tempfile(export_contents) as local_config:
-                    remote_config = os.path.join(tmp_dir, env.name + ".yaml")
+                    remote_config = os.path.join(tmp_dir, env.name)
                     session.put(local_config, remote_config)
                     if not session.isdir(env.path):
                         try:
@@ -278,7 +280,6 @@ class CondaTracer(DistributionTracer):
 
     def _init(self):
         self._get_conda_env_path = PathRoot(self._is_conda_env_path)
-        self._get_conda_dist_path = PathRoot(self._is_conda_dist_path)
 
     def _get_packagefields_for_files(self, files):
         raise NotImplementedError("TODO")
@@ -334,27 +335,20 @@ class CondaTracer(DistributionTracer):
 
         # If there are pip dependencies, they'll be listed under a
         # {"pip": [...]} entry.
-        pip_pkgs = set()
+        pip_pkgs = []
         for dep in dependencies:
             if isinstance(dep, dict) and "pip" in dep:
                 # Pip packages are recorded in conda exports as "name (loc)",
                 # "name==version" or "name (loc)==version".
-                pip_pkgs = {p.split("=")[0].split(" ")[0] for p in dep["pip"]}
+                pip_pkgs = [p.split("=")[0].split(" ")[0] for p in dep["pip"]]
                 break
 
-        pip = conda_path + "/bin/pip"
-        if not self._session.exists(pip):
-            return {}, {}
-
-        pkgs_editable = set(piputils.get_pip_packages(
-            self._session, pip, restriction="editable"))
-        pip_pkgs.update(pkgs_editable)
-
         if not pip_pkgs:
             return {}, {}
 
+        pip = conda_path + "/bin/pip"
         packages, file_to_package_map = piputils.get_package_details(
-            self._session, pip, pip_pkgs, editable_packages=pkgs_editable)
+            self._session, pip, pip_pkgs)
         for entry in packages.values():
             entry["installer"] = "pip"
         return packages, file_to_package_map
@@ -397,10 +391,6 @@ class CondaTracer(DistributionTracer):
     def _is_conda_env_path(self, path):
         return self._session.exists('%s/conda-meta' % path)
 
-    def _is_conda_dist_path(self, path):
-        return (self._session.exists(path + "/envs")
-                and self._is_conda_env_path(path))
-
     def identify_distributions(self, paths):
         conda_paths = set()
         root_to_envs = defaultdict(list)
@@ -427,7 +417,8 @@ class CondaTracer(DistributionTracer):
             # Find the root path for the environment
             # TODO: cache/memoize for those paths which have been considered
             # since will be asked again below
-            root_path = self._get_conda_dist_path(conda_path)
+            conda_info = self._get_conda_info(conda_path)
+            root_path = conda_info.get('root_prefix')
             if not root_path:
                 lgr.warning("Could not find root path for conda environment %s"
                             % conda_path)
diff --git a/reproman/distributions/piputils.py b/reproman/distributions/piputils.py
index 1bbc872..4e5709a 100644
--- a/reproman/distributions/piputils.py
+++ b/reproman/distributions/piputils.py
@@ -117,8 +117,7 @@ def get_pip_packages(session, which_pip, restriction=None):
     return (p["name"] for p in json.loads(out))
 
 
-def get_package_details(session, which_pip, packages=None,
-                        editable_packages=None):
+def get_package_details(session, which_pip, packages=None):
     """Get package details from `pip show` and `pip list`.
 
     This is similar to `pip_show`, but it uses `pip list` to get information
@@ -133,9 +132,6 @@ def get_package_details(session, which_pip, packages=None,
     packages : list of str, optional
         Package names.  If not given, all packages returned by `pip list` are
         used.
-    editable_packages : collection of str
-        If a package name is in this collection, mark it as editable. Passing
-        this saves a call to `which_pip`.
 
     Returns
     -------
@@ -144,9 +140,8 @@ def get_package_details(session, which_pip, packages=None,
     """
     if packages is None:
         packages = list(get_pip_packages(session, which_pip))
-    if editable_packages is None:
-        editable_packages = set(
-            get_pip_packages(session, which_pip, restriction="editable"))
+    editable_packages = set(
+        get_pip_packages(session, which_pip, restriction="editable"))
     details, file_to_pkg = pip_show(session, which_pip, packages)
 
     for pkg in details:
diff --git a/reproman/resource/session.py b/reproman/resource/session.py
index a4d17d5..19946be 100644
--- a/reproman/resource/session.py
+++ b/reproman/resource/session.py
@@ -17,6 +17,8 @@ import os
 import os.path as op
 import re
 from shlex import quote as shlex_quote
+import subprocess
+from ..support.exceptions import CommandError
 
 from reproman.support.exceptions import SessionRuntimeError
 from reproman.cmd import Runner
@@ -492,10 +494,20 @@ class POSIXSession(Session):
     # -0 is not provided by busybox's env command.  So if we decide to make it
     # even more portable - something to be done
     _GET_ENVIRON_CMD = ['env', '-0']
+    _ALT_GET_ENVIRON_CMD = ['perl', '-e', r'foreach (keys %ENV) {print "$_=$ENV{$_}\0";}']
 
     @borrowdoc(Session)
     def query_envvars(self):
-        out, err = self.execute_command(self._GET_ENVIRON_CMD)
+        try:
+            out, err = self.execute_command(self._GET_ENVIRON_CMD)
+        except CommandError:
+            # if this fails, we might need the altenative command...
+            if self._GET_ENVIRON_CMD == self.__class__._ALT_GET_ENVIRON_CMD:
+                # ...if it's already installed, we fail...
+                raise
+            # ...otherwise we install it and try again
+            self._GET_ENVIRON_CMD = self._ALT_GET_ENVIRON_CMD
+            return self.query_envvars()
         env = self._parse_envvars_output(out)
         # TODO:  should we update with our .env or .env_permament????
         return env
@@ -577,7 +589,7 @@ class POSIXSession(Session):
         # just run it and be done
         marker = "== =ReproMan == ="  # unique marker to be able to split away
         # possible output from the sourced script
-        get_env_command = " ".join('"%s"' % s for s in self._GET_ENVIRON_CMD)
+        get_env_command = " ".join("'%s'" % s for s in self._GET_ENVIRON_CMD)
         shell = shell or self.query_envvars().get('SHELL', None)
         if not isinstance(command, list):
             command = [command]


Test patch:
diff --git a/reproman/distributions/tests/test_conda.py b/reproman/distributions/tests/test_conda.py
index ec662bd..9ac27ec 100644
--- a/reproman/distributions/tests/test_conda.py
+++ b/reproman/distributions/tests/test_conda.py
@@ -33,11 +33,11 @@ def test_get_conda_platform_from_python():
 
 def test_get_miniconda_url():
     assert get_miniconda_url("linux-64", "2.7") == \
-           "https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh"
+           "https://repo.continuum.io/miniconda/Miniconda2-4.6.14-Linux-x86_64.sh"
     assert get_miniconda_url("linux-32", "3.4") == \
-           "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86.sh"
+           "https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86.sh"
     assert get_miniconda_url("osx-64", "3.5.1") == \
-           "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+           "https://repo.continuum.io/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh"
 
 
 def test_get_simple_python_version():
@@ -83,7 +83,7 @@ def test_create_conda_export():
                 md5=None,
                 size=None,
                 url=None,
-                files=["lib/python3.7/site-packages/rpaths.py"])],
+                files=["lib/python2.7/site-packages/rpaths.py"])],
         channels=[
             CondaChannel(
                 name="conda-forge",
@@ -107,8 +107,8 @@ def test_conda_init_install_and_detect(tmpdir):
     dist = CondaDistribution(
         name="conda",
         path=test_dir,
-        conda_version="4.8.2",
-        python_version="3.8.2",
+        conda_version="4.3.31",
+        python_version="2.7.14.final.0",
         platform=get_conda_platform_from_python(sys.platform) + "-64",
         environments=[
             CondaEnvironment(
@@ -118,7 +118,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="conda",
                         installer=None,
-                        version="4.8.2",
+                        version="4.3.31",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -128,7 +128,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="pip",
                         installer=None,
-                        version="20.0.2",
+                        version="9.0.1",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -159,7 +159,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="pip",
                         installer=None,
-                        version="20.0.2",
+                        version="9.0.1",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -185,7 +185,7 @@ def test_conda_init_install_and_detect(tmpdir):
                         md5=None,
                         size=None,
                         url=None,
-                        files=["lib/python3.8/site-packages/rpaths.py"])],
+                        files=["lib/python2.7/site-packages/rpaths.py"])],
                 channels=[
                     CondaChannel(
                         name="conda-forge",