Repository: ReproNim/reproman

Issue:
Our tracer is incompatible with latest conda
As described in  gh-445:

> [...] our distributions.conda isn't compatible
> with the latest conda in at least two ways. We use an environment
> file that doesn't have an extension that conda now expects. That's an
> easy fix, and there's a patch in gh-443. The bigger issue is that
> environments seem to no longer have their own bin/conda unless conda
> is explicitly installed in the environment. (This mirrors the then
> seemingly hard-to-trigger problem initially described in gh-173.) As
> a result, our method for identifying an environment's root prefix is
> broken. Based on a quick scan of envs/NAME/, there may no longer be
> an easy way to map from an environment to its root prefix.

Re: #443 

Discussion:

Patch:
diff --git a/reproman/distributions/conda.py b/reproman/distributions/conda.py
index db568e5..4cea4e9 100644
--- a/reproman/distributions/conda.py
+++ b/reproman/distributions/conda.py
@@ -85,9 +85,7 @@ def get_miniconda_url(conda_platform, python_version):
         raise ValueError("Unsupported platform %s for conda installation" %
                          conda_platform)
     platform += "-x86_64" if ("64" in conda_platform) else "-x86"
-    # FIXME: We need to update this our conda tracer to work with conda's newer
-    # than 4.6.14. See gh-443.
-    return "https://repo.anaconda.com/miniconda/Miniconda%s-4.6.14-%s.sh" \
+    return "https://repo.anaconda.com/miniconda/Miniconda%s-latest-%s.sh" \
                     % (python_version[0], platform)
 
 
@@ -191,17 +189,15 @@ class CondaDistribution(Distribution):
                 # NOTE: miniconda.sh makes parent directories automatically
                 session.execute_command("bash -b %s/miniconda.sh -b -p %s" %
                                         (tmp_dir, self.path))
-            ## Update root version of conda
-            session.execute_command(
-               "%s/bin/conda install -y conda=%s python=%s" %
-               (self.path, self.conda_version,
-                self.get_simple_python_version(self.python_version)))
-
-            # Loop through non-root packages, creating the conda-env config
-            for env in self.environments:
+            envs = sorted(
+                self.environments,
+                # Create/update the root environment before handling anything
+                # else.
+                key=lambda x: "_" if x.name == "root" else "_" + x.name)
+            for env in envs:
                 export_contents = self.create_conda_export(env)
                 with make_tempfile(export_contents) as local_config:
-                    remote_config = os.path.join(tmp_dir, env.name)
+                    remote_config = os.path.join(tmp_dir, env.name + ".yaml")
                     session.put(local_config, remote_config)
                     if not session.isdir(env.path):
                         try:
@@ -282,6 +278,7 @@ class CondaTracer(DistributionTracer):
 
     def _init(self):
         self._get_conda_env_path = PathRoot(self._is_conda_env_path)
+        self._get_conda_dist_path = PathRoot(self._is_conda_dist_path)
 
     def _get_packagefields_for_files(self, files):
         raise NotImplementedError("TODO")
@@ -337,20 +334,27 @@ class CondaTracer(DistributionTracer):
 
         # If there are pip dependencies, they'll be listed under a
         # {"pip": [...]} entry.
-        pip_pkgs = []
+        pip_pkgs = set()
         for dep in dependencies:
             if isinstance(dep, dict) and "pip" in dep:
                 # Pip packages are recorded in conda exports as "name (loc)",
                 # "name==version" or "name (loc)==version".
-                pip_pkgs = [p.split("=")[0].split(" ")[0] for p in dep["pip"]]
+                pip_pkgs = {p.split("=")[0].split(" ")[0] for p in dep["pip"]}
                 break
 
+        pip = conda_path + "/bin/pip"
+        if not self._session.exists(pip):
+            return {}, {}
+
+        pkgs_editable = set(piputils.get_pip_packages(
+            self._session, pip, restriction="editable"))
+        pip_pkgs.update(pkgs_editable)
+
         if not pip_pkgs:
             return {}, {}
 
-        pip = conda_path + "/bin/pip"
         packages, file_to_package_map = piputils.get_package_details(
-            self._session, pip, pip_pkgs)
+            self._session, pip, pip_pkgs, editable_packages=pkgs_editable)
         for entry in packages.values():
             entry["installer"] = "pip"
         return packages, file_to_package_map
@@ -393,6 +397,10 @@ class CondaTracer(DistributionTracer):
     def _is_conda_env_path(self, path):
         return self._session.exists('%s/conda-meta' % path)
 
+    def _is_conda_dist_path(self, path):
+        return (self._session.exists(path + "/envs")
+                and self._is_conda_env_path(path))
+
     def identify_distributions(self, paths):
         conda_paths = set()
         root_to_envs = defaultdict(list)
@@ -419,8 +427,7 @@ class CondaTracer(DistributionTracer):
             # Find the root path for the environment
             # TODO: cache/memoize for those paths which have been considered
             # since will be asked again below
-            conda_info = self._get_conda_info(conda_path)
-            root_path = conda_info.get('root_prefix')
+            root_path = self._get_conda_dist_path(conda_path)
             if not root_path:
                 lgr.warning("Could not find root path for conda environment %s"
                             % conda_path)
diff --git a/reproman/distributions/piputils.py b/reproman/distributions/piputils.py
index 4e5709a..1bbc872 100644
--- a/reproman/distributions/piputils.py
+++ b/reproman/distributions/piputils.py
@@ -117,7 +117,8 @@ def get_pip_packages(session, which_pip, restriction=None):
     return (p["name"] for p in json.loads(out))
 
 
-def get_package_details(session, which_pip, packages=None):
+def get_package_details(session, which_pip, packages=None,
+                        editable_packages=None):
     """Get package details from `pip show` and `pip list`.
 
     This is similar to `pip_show`, but it uses `pip list` to get information
@@ -132,6 +133,9 @@ def get_package_details(session, which_pip, packages=None):
     packages : list of str, optional
         Package names.  If not given, all packages returned by `pip list` are
         used.
+    editable_packages : collection of str
+        If a package name is in this collection, mark it as editable. Passing
+        this saves a call to `which_pip`.
 
     Returns
     -------
@@ -140,8 +144,9 @@ def get_package_details(session, which_pip, packages=None):
     """
     if packages is None:
         packages = list(get_pip_packages(session, which_pip))
-    editable_packages = set(
-        get_pip_packages(session, which_pip, restriction="editable"))
+    if editable_packages is None:
+        editable_packages = set(
+            get_pip_packages(session, which_pip, restriction="editable"))
     details, file_to_pkg = pip_show(session, which_pip, packages)
 
     for pkg in details:


Test patch:
diff --git a/reproman/distributions/tests/test_conda.py b/reproman/distributions/tests/test_conda.py
index 7ccd404..ec662bd 100644
--- a/reproman/distributions/tests/test_conda.py
+++ b/reproman/distributions/tests/test_conda.py
@@ -33,11 +33,11 @@ def test_get_conda_platform_from_python():
 
 def test_get_miniconda_url():
     assert get_miniconda_url("linux-64", "2.7") == \
-           "https://repo.anaconda.com/miniconda/Miniconda2-4.6.14-Linux-x86_64.sh"
+           "https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh"
     assert get_miniconda_url("linux-32", "3.4") == \
-           "https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86.sh"
+           "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86.sh"
     assert get_miniconda_url("osx-64", "3.5.1") == \
-           "https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh"
+           "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
 
 
 def test_get_simple_python_version():
@@ -83,7 +83,7 @@ def test_create_conda_export():
                 md5=None,
                 size=None,
                 url=None,
-                files=["lib/python2.7/site-packages/rpaths.py"])],
+                files=["lib/python3.7/site-packages/rpaths.py"])],
         channels=[
             CondaChannel(
                 name="conda-forge",
@@ -107,8 +107,8 @@ def test_conda_init_install_and_detect(tmpdir):
     dist = CondaDistribution(
         name="conda",
         path=test_dir,
-        conda_version="4.3.31",
-        python_version="2.7.14.final.0",
+        conda_version="4.8.2",
+        python_version="3.8.2",
         platform=get_conda_platform_from_python(sys.platform) + "-64",
         environments=[
             CondaEnvironment(
@@ -118,7 +118,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="conda",
                         installer=None,
-                        version="4.3.31",
+                        version="4.8.2",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -128,7 +128,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="pip",
                         installer=None,
-                        version="9.0.1",
+                        version="20.0.2",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -159,7 +159,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="pip",
                         installer=None,
-                        version="9.0.1",
+                        version="20.0.2",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -185,7 +185,7 @@ def test_conda_init_install_and_detect(tmpdir):
                         md5=None,
                         size=None,
                         url=None,
-                        files=["lib/python2.7/site-packages/rpaths.py"])],
+                        files=["lib/python3.8/site-packages/rpaths.py"])],
                 channels=[
                     CondaChannel(
                         name="conda-forge",