Repository: jazzband/pip-tools

Issue:
Pip 18 support
Pip 18 has been released and there is some internal API changes which have to addressed to make the resolver work again.

Already did this for Prequ, and it shouldn't be very hard to port the changes to pip-tools. Here's the commit that adds Pip 18 support to Prequ: https://github.com/suutari/prequ/commit/fdc6f1d24535c05380454d8308de48a34a53d00b#diff-395e820748909fbb1b029281c5bbdf8c


Discussion:
For anyone else surprised by the pip version, note that pip 18 is now using a calendar based versioning and release scheme. 
Thanks for the report @suutari, and thanks for the info @rpkilby .

As you might have noted, I don't have much time to give to `pip-tools` right now. If someone wants to submit the PR, I'll do my best to join in on the review, as well as take care of the release in due time.

Resolving patch:
diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py
--- a/piptools/_compat/pip_compat.py
+++ b/piptools/_compat/pip_compat.py
@@ -2,17 +2,21 @@
 import importlib
 
 def do_import(module_path, subimport=None, old_path=None):
-    internal = 'pip._internal.{0}'.format(module_path)
     old_path = old_path or module_path
-    pip9 = 'pip.{0}'.format(old_path)
-    try:
-        _tmp = importlib.import_module(internal)
-    except ImportError:
-        _tmp = importlib.import_module(pip9)
-    if subimport:
-        return getattr(_tmp, subimport, _tmp)
-    return _tmp
-    
+    prefixes = ["pip._internal", "pip"]
+    paths = [module_path, old_path]
+    search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None]
+    package = subimport if subimport else None
+    for to_import in search_order:
+        if not subimport:
+            to_import, _, package = to_import.rpartition(".")
+        try:
+            imported = importlib.import_module(to_import)
+        except ImportError:
+            continue
+        else:
+            return getattr(imported, package)
+
 
 InstallRequirement = do_import('req.req_install', 'InstallRequirement')
 parse_requirements = do_import('req.req_file', 'parse_requirements')
@@ -24,7 +28,7 @@ def do_import(module_path, subimport=None, old_path=None):
 PackageFinder = do_import('index', 'PackageFinder')
 FormatControl = do_import('index', 'FormatControl')
 Wheel = do_import('wheel', 'Wheel')
-Command = do_import('basecommand', 'Command')
-cmdoptions = do_import('cmdoptions')
+Command = do_import('cli.base_command', 'Command', old_path='basecommand')
+cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions')
 get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils')
 PyPI = do_import('models.index', 'PyPI')
diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py
--- a/piptools/repositories/pypi.py
+++ b/piptools/repositories/pypi.py
@@ -24,12 +24,12 @@
                      make_install_requirement)
 from .base import BaseRepository
 
-
 try:
-    from pip._internal.operations.prepare import RequirementPreparer
-    from pip._internal.resolve import Resolver as PipResolver
+    from pip._internal.req.req_tracker import RequirementTracker
 except ImportError:
-    pass
+    @contextmanager
+    def RequirementTracker():
+        yield
 
 try:
     from pip._internal.cache import WheelCache
@@ -49,7 +49,6 @@ class PyPIRepository(BaseRepository):
     def __init__(self, pip_options, session):
         self.session = session
         self.pip_options = pip_options
-        self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control)
 
         index_urls = [pip_options.index_url] + pip_options.extra_index_urls
         if pip_options.no_index:
@@ -130,6 +129,62 @@ def find_best_match(self, ireq, prereleases=None):
             best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint
         )
 
+    def resolve_reqs(self, download_dir, ireq, wheel_cache):
+        results = None
+        try:
+            from pip._internal.operations.prepare import RequirementPreparer
+            from pip._internal.resolve import Resolver as PipResolver
+        except ImportError:
+            # Pip 9 and below
+            reqset = RequirementSet(
+                self.build_dir,
+                self.source_dir,
+                download_dir=download_dir,
+                wheel_download_dir=self._wheel_download_dir,
+                session=self.session,
+                wheel_cache=wheel_cache
+            )
+            results = reqset._prepare_file(self.finder, ireq)
+        else:
+            # pip >= 10
+            preparer_kwargs = {
+                'build_dir': self.build_dir,
+                'src_dir': self.source_dir,
+                'download_dir': download_dir,
+                'wheel_download_dir': self._wheel_download_dir,
+                'progress_bar': 'off',
+                'build_isolation': False
+            }
+            resolver_kwargs = {
+                'finder': self.finder,
+                'session': self.session,
+                'upgrade_strategy': "to-satisfy-only",
+                'force_reinstall': False,
+                'ignore_dependencies': False,
+                'ignore_requires_python': False,
+                'ignore_installed': True,
+                'isolated': False,
+                'wheel_cache': wheel_cache,
+                'use_user_site': False
+            }
+            resolver = None
+            preparer = None
+            with RequirementTracker() as req_tracker:
+                # Pip 18 uses a requirement tracker to prevent fork bombs
+                if req_tracker:
+                    preparer_kwargs['req_tracker'] = req_tracker
+                preparer = RequirementPreparer(**preparer_kwargs)
+                resolver_kwargs['preparer'] = preparer
+                reqset = RequirementSet()
+                ireq.is_direct = True
+                reqset.add_requirement(ireq)
+                resolver = PipResolver(**resolver_kwargs)
+                resolver.require_hashes = False
+                results = resolver._resolve_one(reqset, ireq)
+                reqset.cleanup_files()
+
+        return set(results)
+
     def get_dependencies(self, ireq):
         """
         Given a pinned or an editable InstallRequirement, returns a set of
@@ -156,50 +211,21 @@ def get_dependencies(self, ireq):
             if not os.path.isdir(self._wheel_download_dir):
                 os.makedirs(self._wheel_download_dir)
 
+            wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control)
+            prev_tracker = os.environ.get('PIP_REQ_TRACKER')
             try:
-                # Pip < 9 and below
-                reqset = RequirementSet(
-                    self.build_dir,
-                    self.source_dir,
-                    download_dir=download_dir,
-                    wheel_download_dir=self._wheel_download_dir,
-                    session=self.session,
-                    wheel_cache=self.wheel_cache,
-                )
-                self._dependencies_cache[ireq] = reqset._prepare_file(
-                    self.finder,
-                    ireq
-                )
-            except TypeError:
-                # Pip >= 10 (new resolver!)
-                preparer = RequirementPreparer(
-                    build_dir=self.build_dir,
-                    src_dir=self.source_dir,
-                    download_dir=download_dir,
-                    wheel_download_dir=self._wheel_download_dir,
-                    progress_bar='off',
-                    build_isolation=False
-                )
-                reqset = RequirementSet()
-                ireq.is_direct = True
-                reqset.add_requirement(ireq)
-                self.resolver = PipResolver(
-                    preparer=preparer,
-                    finder=self.finder,
-                    session=self.session,
-                    upgrade_strategy="to-satisfy-only",
-                    force_reinstall=False,
-                    ignore_dependencies=False,
-                    ignore_requires_python=False,
-                    ignore_installed=True,
-                    isolated=False,
-                    wheel_cache=self.wheel_cache,
-                    use_user_site=False,
-                )
-                self.resolver.resolve(reqset)
-                self._dependencies_cache[ireq] = reqset.requirements.values()
-            reqset.cleanup_files()
-        return set(self._dependencies_cache[ireq])
+                self._dependencies_cache[ireq] = self.resolve_reqs(download_dir, ireq, wheel_cache)
+            finally:
+                if 'PIP_REQ_TRACKER' in os.environ:
+                    if prev_tracker:
+                        os.environ['PIP_REQ_TRACKER'] = prev_tracker
+                    else:
+                        del os.environ['PIP_REQ_TRACKER']
+                try:
+                    self.wheel_cache.cleanup()
+                except AttributeError:
+                    pass
+        return self._dependencies_cache[ireq]
 
     def get_hashes(self, ireq):
         """


Test patch:
diff --git a/tests/test_repositories.py b/tests/test_repositories.py
--- a/tests/test_repositories.py
+++ b/tests/test_repositories.py
@@ -1,8 +1,11 @@
 from mock import MagicMock, patch
+from pip._vendor.packaging.version import parse as parse_version
+from pip import __version__ as pip_version
 from piptools._compat import PackageFinder, InstallRequirement
 
 from piptools.repositories.pypi import PyPIRepository
 from piptools.scripts.compile import get_pip_command
+import pytest
 
 
 def test_pypirepo_build_dir_is_str():
@@ -13,6 +16,10 @@ def test_pypirepo_source_dir_is_str():
     assert isinstance(get_pypi_repository().source_dir, str)
 
 
+@pytest.mark.skipif(
+    parse_version(pip_version) >= parse_version('10.0.0'),
+    reason="RequirementSet objects don't take arguments after pip 10."
+)
 def test_pypirepo_calls_reqset_with_str_paths():
     """
     Make sure that paths passed to RequirementSet init are str.