Repository: conda/conda

Issue:
Add pip shadow packages into PrefixData
Epic at #7053 

Teach PrefixData to recognize when python is installed in an environment, and then scan site-packages to find packages that don't have metadata in conda-meta.  Also replace `conda/egg_info.py` and how it influences `conda list`.


Discussion:

Resolving patch:
diff --git a/conda/cli/main_list.py b/conda/cli/main_list.py
--- a/conda/cli/main_list.py
+++ b/conda/cli/main_list.py
@@ -27,12 +27,10 @@ def print_export_header(subdir):
 
 def get_packages(installed, regex):
     pat = re.compile(regex, re.I) if regex else None
-    for dist in sorted(installed, key=lambda x: x.quad[0].lower()):
-        name = dist.quad[0]
-        if pat and pat.search(name) is None:
+    for prefix_rec in sorted(installed, key=lambda x: x.name.lower()):
+        if pat and pat.search(str(prefix_rec)) is None:
             continue
-
-        yield dist
+        yield prefix_rec
 
 
 def list_packages(prefix, installed, regex=None, format='human',
@@ -85,12 +83,14 @@ def print_packages(prefix, regex=None, format='human', piplist=False,
         if format == 'export':
             print_export_header(context.subdir)
 
-    installed = linked(prefix)
-    log.debug("installed conda packages:\n%s", installed)
-    if piplist and context.use_pip and format == 'human':
-        other_python = get_egg_info(prefix)
-        log.debug("other installed python packages:\n%s", other_python)
-        installed.update(other_python)
+    # installed = linked(prefix)
+    # log.debug("installed conda packages:\n%s", installed)
+    # if piplist and context.use_pip and format == 'human':
+    #     other_python = get_egg_info(prefix)
+    #     log.debug("other installed python packages:\n%s", other_python)
+    #     installed.update(other_python)
+
+    installed = tuple(PrefixData(prefix).iter_records())
 
     exitcode, output = list_packages(prefix, installed, regex, format=format,
                                      show_channel_urls=show_channel_urls)
diff --git a/conda/core/prefix_data.py b/conda/core/prefix_data.py
--- a/conda/core/prefix_data.py
+++ b/conda/core/prefix_data.py
@@ -3,21 +3,33 @@
 
 from glob import glob
 from logging import getLogger
-from os.path import isfile, join, lexists
+from os import listdir
+from os.path import basename, isfile, join, lexists
 
 from ..base.constants import CONDA_TARBALL_EXTENSION, PREFIX_MAGIC_FILE
 from ..base.context import context
 from ..common.compat import JSONDecodeError, itervalues, string_types, with_metaclass
 from ..common.constants import NULL
+from ..common.path import get_python_site_packages_short_path, win_path_ok
 from ..common.serialize import json_load
 from ..exceptions import (BasicClobberError, CondaDependencyError, CorruptedEnvironmentError,
                           maybe_raise)
 from ..gateways.disk.create import write_as_json_to_file
 from ..gateways.disk.delete import rm_rf
 from ..gateways.disk.test import file_path_is_writable
+from ..models.channel import Channel
 from ..models.dist import Dist
+from ..models.enums import PackageType, PathType
 from ..models.match_spec import MatchSpec
-from ..models.records import PackageRef, PrefixRecord
+from ..models.prefix_graph import PrefixGraph
+from ..models.records import (ComposableField, EnumField, ListField, PackageRef, PathDataV1,
+                              PathsData, PrefixRecord)
+
+try:
+    from cytoolz.itertoolz import concat, concatv
+except ImportError:  # pragma: no cover
+    from .._vendor.toolz.itertoolz import concat, concatv  # NOQA
+
 
 log = getLogger(__name__)
 
@@ -48,6 +60,7 @@ def load(self):
         self.__prefix_records = {}
         for meta_file in glob(join(self.prefix_path, 'conda-meta', '*.json')):
             self._load_single_record(meta_file)
+        self._load_site_packages()
 
     def reload(self):
         self.load()
@@ -135,8 +148,8 @@ def _load_single_record(self, prefix_record_json_path):
             except JSONDecodeError:
                 raise CorruptedEnvironmentError(self.prefix_path, prefix_record_json_path)
 
-        prefix_record = PrefixRecord(**json_data)
-        self.__prefix_records[prefix_record.name] = prefix_record
+            prefix_record = PrefixRecord(**json_data)
+            self.__prefix_records[prefix_record.name] = prefix_record
 
     @property
     def is_writable(self):
@@ -145,6 +158,142 @@ def is_writable(self):
             return None
         return file_path_is_writable(test_path)
 
+    def _has_python(self):
+        return 'python' in self._prefix_records
+
+    def _load_site_packages(self):
+        # 1. identify all package names in site-packages
+        # 2. determine if they are conda packages
+        # 3. load as pip packages if not
+        python_record = next(
+            (prefix_rec for prefix_rec in itervalues(self.__prefix_records) if prefix_rec.name == 'python'),
+            None
+        )
+        if not python_record:
+            return
+        prefix_graph = PrefixGraph(self.iter_records())
+        known_python_records = prefix_graph.all_descendants(python_record)
+
+        def norm_package_name(name):
+            return name.replace('.', '-').replace('_', '-').lower()
+
+        marker_file_endings = ('.egg-info/PKG-INFO', '.dist-info/RECORD', '.egg-info')
+        conda_python_packages = dict(
+            ((mf, prefix_rec)
+             for prefix_rec in known_python_records
+             for mf in prefix_rec.files
+             if mf.endswith(marker_file_endings))
+        )
+
+        non_conda_python_package_markers = []
+        site_packages_dir = get_python_site_packages_short_path(python_record.version)
+        sp_dir_full_path = join(self.prefix_path, win_path_ok(site_packages_dir))
+        sp_marker_endings = ('.dist-info', '.egg-info')
+        for fn in listdir(sp_dir_full_path):
+            if fn.endswith(sp_marker_endings):
+                if fn.endswith('.dist-info'):
+                    marker_file = "%s/%s/%s" % (site_packages_dir, fn, 'RECORD')
+                elif fn.endswith(".egg-info"):
+                    if isfile(join(sp_dir_full_path, fn)):
+                        marker_file = "%s/%s" % (site_packages_dir, fn)
+                    else:
+                        marker_file = "%s/%s/%s" % (site_packages_dir, fn, "PKG-INFO")
+                elif fn.endswith('.egg-link'):
+                    continue
+                elif fn.endswith('.pth'):
+                    continue
+                else:
+                    continue
+
+                if marker_file not in conda_python_packages:
+                    non_conda_python_package_markers.append(marker_file)
+
+        from pip._vendor.distlib.database import EggInfoDistribution, InstalledDistribution  # TODO: only compatible with pip 9.0
+        from pip._vendor.distlib.metadata import MetadataConflictError
+        from pip._vendor.distlib.util import parse_requirement
+
+        def get_pydist(marker_file):
+            if ".dist-info" in marker_file:
+                dist_file = join(self.prefix_path, win_path_ok(marker_file.rsplit('/', 1)[0]))
+                dist_cls = InstalledDistribution
+            elif marker_file.endswith(".egg-info"):
+                dist_file = join(self.prefix_path, win_path_ok(marker_file))
+                dist_cls = EggInfoDistribution
+            elif ".egg-info" in marker_file:
+                dist_file = join(self.prefix_path, win_path_ok(marker_file.rsplit('/', 1)[0]))
+                dist_cls = EggInfoDistribution
+            else:
+                raise NotImplementedError()
+            try:
+                pydist = dist_cls(dist_file)
+            except MetadataConflictError:
+                print("MetadataConflictError:", marker_file)
+                pydist = None
+            return pydist
+
+        python_recs = []
+        for marker_file in non_conda_python_package_markers:
+            pydist = get_pydist(marker_file)
+            if pydist is None:
+                continue
+            # x.provides  =>  [u'skdata (0.0.4)']
+            # x.run_requires  =>  set([u'joblib', u'scikit-learn', u'lockfile', u'numpy', u'nose (>=1.0)'])
+            # >>> list(x.list_installed_files())  =>  [(u'skdata/__init__.py', u'sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU', u'0'), (u'skdata/base.py', u'sha256=04MW02dky5T4nZb6Q0M351aRbAwLxd8voCK3nrAU-g0', u'5019'), (u'skdata/brodatz.py', u'sha256=NIPWLawJ59Fr037r0oT_gHe46WCo3UivuQ-cwxRU3ow', u'8492'), (u'skdata/caltech.py', u'sha256=cIfyMMRYggZ3Jkgc15tsYi_ZsZ7NpRqWh7mZ8bl6Fo0', u'8047'), (u'skdata/data_home.py', u'sha256=o5ChOI4v3Jd16JM3qWZlhrs5q-g_0yKa5-Oq44HC_K4', u'1297'), (u'skdata/diabetes.py', u'sha256=ny5Ihpc_eiIRYgzFn3Lm81fV0SZ1nyZQnqEmwb2PrS0', u'995'), (u'skdata/digits.py', u'sha256=DipeWAb3APpjXfmKmSumkfEFzuBW8XJ0
+
+            # TODO: normalize names against '.', '-', '_'
+            # TODO: ensure that this dist is *actually* the dist that matches conda-meta
+
+            if marker_file.endswith(".egg-info"):
+                paths_data = None
+            else:
+                _paths_data = []
+                for _path, _hash, _size in pydist.list_installed_files():
+                    if _hash:
+                        assert _hash.startswith('sha256='), (marker_file, _hash)
+                        sha256 = _hash[7:]
+                    else:
+                        sha256 = None
+                    _size = int(_size) if _size else None
+                    _paths_data.append(PathDataV1(
+                        _path=_path,
+                        path_type=PathType.hardlink,  # TODO
+                        sha256=sha256,
+                        size_in_bytes=_size
+                    ))
+                paths_data = PathsData(paths_version=1, paths=_paths_data)
+            # TODO: need to add entry points, "exports," and other files that might not be in RECORD
+
+            depends = tuple(
+                req.name for req in
+                # vars(req) => {'source': u'nose (>=1.0)', 'requirement': u'nose (>= 1.0)', 'extras': None, 'name': u'nose', 'url': None, 'constraints': [(u'>=', u'1.0')]}
+                (parse_requirement(r) for r in pydist.run_requires)
+            )
+            # TODO: need to add python (with version?) to deps
+
+            package_type = PackageType.SHADOW_PIP_UPDATEABLE if depends else PackageType.SHADOW_PIP_FROZEN
+
+            python_rec = PythonRecord(
+                package_type=PackageType.SHADOW_PIP_FROZEN,
+                name=pydist.name.lower(),
+                version=pydist.version,
+                channel=Channel('pypi'),
+                subdir='pypi',
+                fn=basename(marker_file),
+                build='pypi_0',
+                build_number=0,
+                paths_data=paths_data,
+                depends=depends
+            )
+            python_recs.append(python_rec)
+            self.__prefix_records[python_rec.name] = python_rec
+
+
+class PythonRecord(PackageRef):
+
+    depends = ListField(string_types, default=())
+    paths_data = ComposableField(PathsData, required=False, nullable=True, default_in_dump=False)
+    package_type = EnumField(PackageType, required=False, nullable=True)
+
 
 def get_python_version_for_prefix(prefix):
     # returns a string e.g. "2.7", "3.4", "3.5" or None
@@ -202,3 +351,8 @@ def is_linked(prefix, dist):
         return prefix_record
     else:
         return None
+
+
+if __name__ == '__main__':
+    pd = PrefixData('/Users/kfranz/anaconda')
+    pd._load_site_packages()
diff --git a/conda/models/enums.py b/conda/models/enums.py
--- a/conda/models/enums.py
+++ b/conda/models/enums.py
@@ -124,7 +124,8 @@ class PackageType(Enum):
     NOARCH_PYTHON = 'noarch_python'
     NOARCH_PRELINK_PYTHON = 'noarch_prelink_python'
     SHADOW_PRIVATE_ENV = 'shadow_private_env'
-    SHADOW_PIP = 'shadow_pip'
+    SHADOW_PIP_FROZEN = 'shadow_pip_frozen'  # insufficient information to modify the package
+    SHADOW_PIP_UPDATEABLE = 'shadow_pip_updateable'  # sufficient information to fully remove the package on disk (e.g. ALL paths owned by package are known), so that it can be replaced with a conda package
 
 
 class NoarchType(Enum):


Test patch:
diff --git a/tests/test_create.py b/tests/test_create.py
--- a/tests/test_create.py
+++ b/tests/test_create.py
@@ -1237,6 +1237,23 @@ def test_packages_not_found(self):
                                          use_exception_handler=True)
             assert "not-a-real-package" in stderr
 
+    def test_conda_pip_interop_dependency_satisfied_by_pip(self):
+        with make_temp_env("python") as prefix:
+            check_call(PYTHON_BINARY + " -m pip install itsdangerous",
+                       cwd=prefix, shell=True)
+
+            PrefixData._cache_.clear()
+            stdout, stderr = run_command(Commands.LIST, prefix)
+            assert 'itsdangerous' in stdout
+            assert not stderr
+
+            stdout, stderr = run_command(Commands.INSTALL, prefix, 'flask --dry-run --json', use_exception_handler=True)
+            json_obj = json.loads(stdout)
+            print(json_obj)
+            assert any(rec["name"] == "flask" for rec in json_obj["actions"]["LINK"])
+            assert not any(rec["name"] == "itsdangerous" for rec in json_obj["actions"]["LINK"])
+            assert not stderr
+
     @pytest.mark.skipif(on_win, reason="gawk is a windows only package")
     def test_search_gawk_not_win_1(self):
         with make_temp_env() as prefix: