Repository: pypa/pipenv
Issue: Git dependency with sepcified branch gets overriden by dependency of other package.

This issue is simillar to #1779, but the solution stated there is not working for me.

I need to install specific version of a `PyInstaller` package, but `PyInstaller` is also a dependency of `PyUpdater` which is also required in my project and overrides other my specific `PyInstaller` version.

<details><summary>`$ python -m pipenv.help` output:</summary>

`python -m pipenv.help`

`C:\\Program Files (x86)\\Python36-32\\python.exe: No module named pipenv.help`


I am aware this is not how this output should look like, but I can not make it work. Any help here? I'm on pipenv version 11.0.2.
</details>

------------

##### Expected result

I need latest `develop` branch of PyInstaller installed and I don't want PyUpdater to override this with the `PyInstaller-3.3.1` version. As suggested in #1779 I placed PyUpdater above the PyInstaller in `[packages]` in Pipfile.

##### Actual result

Regardless whether the important part of Pipfile looks like this:
```
PyUpdater = "*"
PyInstaller = {git = "https://github.com/pyinstaller/pyinstaller", ref = "develop"}
```
or like this:
```
PyInstaller = {git = "https://github.com/pyinstaller/pyinstaller", ref = "develop"}
PyUpdater = "*"
```
the PyInstaller version in Pipfile.lock will be `3.3.1` and I end up with this version installed.

What is interesting is that if I uninstall PyInstaller with `pipenv uninstall pyinstaller` and then install it again with `pipenv install https://github.com/pyinstaller/pyinstaller/archive/develop.zip` I get the `Successfully installed PyInstaller-3.4.dev0+b31794cf1` but in fact the `3.3.1` version in installed instead.

##### Steps to replicate
```
# Pipfile
[[source]]

url = "https://pypi.python.org/simple"
verify_ssl = true
name = "pypi"

[packages]
PyUpdater = "*"
PyInstaller = {git = "https://github.com/pyinstaller/pyinstaller", ref = "develop"}
```
`pipenv install`
Check version: `pipenv run pyinstaller --version`

or

Create new clean environment
`pipenv install pyupdater`
`pipenv uninstall pyinstaller`
`pipenv install https://github.com/pyinstaller/pyinstaller/archive/develop.zip`
Check version: `pipenv run pyinstaller --version`

Changed files:
pipenv/utils.py, tests/integration/test_install_uri.py, pipenv/vendor/requirementslib/__init__.py, pipenv/core.py, tests/integration/test_install_markers.py, pipenv/vendor/requirementslib/utils.py, pipenv/vendor/requirementslib/models/requirements.py, pipenv/vendor/requirementslib/models/utils.py

diff --git a/pipenv/core.py b/pipenv/core.py
index 36c745ca..3758dfb4 100644
--- a/pipenv/core.py
+++ b/pipenv/core.py
@@ -1090,7 +1090,7 @@ def do_lock(
             lockfile[settings['lockfile_key']].update(dep_lockfile)
         # Add refs for VCS installs.
         # TODO: be smarter about this.
-        vcs_lines, vcs_lockfile = get_vcs_deps(
+        vcs_reqs, vcs_lockfile = get_vcs_deps(
             project,
             pip_freeze,
             which=which,
@@ -1100,6 +1100,7 @@ def do_lock(
             allow_global=system,
             dev=settings['dev']
         )
+        vcs_lines = [req.as_line() for req in vcs_reqs if req.editable]
         vcs_results = venv_resolve_deps(
             vcs_lines,
             which=which,
@@ -1111,13 +1112,26 @@ def do_lock(
             pypi_mirror=pypi_mirror,
         )
         for dep in vcs_results:
+            normalized = pep423_name(dep['name'])
             if not hasattr(dep, 'keys') or not hasattr(dep['name'], 'keys'):
                 continue
-            is_top_level = dep['name'] in vcs_lockfile
-            pipfile_entry = vcs_lockfile[dep['name']] if is_top_level else None
-            dep_lockfile = clean_resolved_dep(dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry)
+            is_top_level = (
+                dep['name'] in vcs_lockfile or
+                normalized in vcs_lockfile
+            )
+            if is_top_level:
+                try:
+                    pipfile_entry = vcs_lockfile[dep['name']]
+                except KeyError:
+                    pipfile_entry = vcs_lockfile[normalized]
+            else:
+                pipfile_entry = None
+            dep_lockfile = clean_resolved_dep(
+                dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry,
+            )
             vcs_lockfile.update(dep_lockfile)
         lockfile[settings['lockfile_key']].update(vcs_lockfile)
+
     # Support for --keep-outdated…
     if keep_outdated:
         for section_name, section in (
diff --git a/pipenv/utils.py b/pipenv/utils.py
index 9159ffe1..fa3d8b23 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -1135,7 +1135,11 @@ def extract_uri_from_vcs_dep(dep):
     return None
 
 
-def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):
+def resolve_ref(vcs_obj, target_dir, ref):
+    return vcs_obj.get_revision_sha(target_dir, ref)
+
+
+def obtain_vcs_req(vcs_obj, src_dir, name, rev=None):
     target_dir = os.path.join(src_dir, name)
     target_rev = vcs_obj.make_rev_options(rev)
     if not os.path.exists(target_dir):
@@ -1160,7 +1164,7 @@ def get_vcs_deps(
     from ._compat import TemporaryDirectory
 
     section = "vcs_dev_packages" if dev else "vcs_packages"
-    lines = []
+    reqs = []
     lockfile = {}
     try:
         packages = getattr(project, section)
@@ -1175,45 +1179,26 @@ def get_vcs_deps(
         )
         src_dir.mkdir(mode=0o775, exist_ok=True)
     vcs_registry = VcsSupport
-    vcs_uri_map = {
-        extract_uri_from_vcs_dep(v): {"name": k, "ref": v.get("ref")}
-        for k, v in packages.items()
-    }
-    for line in pip_freeze.strip().split("\\n"):
-        # if the line doesn't match a vcs dependency in the Pipfile,
-        # ignore it
-        _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
-        if not _vcs_match:
-            continue
-
-        pipfile_name = vcs_uri_map[_vcs_match]["name"]
-        pipfile_rev = vcs_uri_map[_vcs_match]["ref"]
-        pipfile_req = Requirement.from_pipfile(pipfile_name, packages[pipfile_name])
-        names = {pipfile_name}
-        backend = vcs_registry()._registry.get(pipfile_req.vcs)
-        # TODO: Why doesn't pip freeze list 'git+git://' formatted urls?
-        if line.startswith("-e ") and not "{0}+".format(pipfile_req.vcs) in line:
-            line = line.replace("-e ", "-e {0}+".format(pipfile_req.vcs))
-        installed = Requirement.from_line(line)
-        __vcs = backend(url=installed.req.uri)
-
-        names.add(installed.normalized_name)
+    for pkg_name, pkg_pipfile in packages.items():
+        requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile)
+        backend = vcs_registry()._registry.get(requirement.vcs)
+        __vcs = backend(url=requirement.req.vcs_uri)
         locked_rev = None
-        for _name in names:
-            locked_rev = install_or_update_vcs(
-                __vcs, src_dir.as_posix(), _name, rev=pipfile_rev
-            )
-        if installed.is_vcs:
-            installed.req.ref = locked_rev
-            lockfile[pipfile_name] = installed.pipfile_entry[1]
-        lines.append(line)
-    return lines, lockfile
+        name = requirement.normalized_name
+        locked_rev = obtain_vcs_req(
+            __vcs, src_dir.as_posix(), name, rev=pkg_pipfile.get("ref")
+        )
+        if requirement.is_vcs:
+            requirement.req.ref = locked_rev
+            lockfile[name] = requirement.pipfile_entry[1]
+        reqs.append(requirement)
+    return reqs, lockfile
 
 
 def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
     from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context
     allowed_marker_keys = ['markers'] + [k for k in marker_context.keys()]
-    name = dep['name']
+    name = pep423_name(dep['name'])
     # We use this to determine if there are any markers on top level packages
     # So we can make sure those win out during resolution if the packages reoccur
     dep_keys = [k for k in getattr(pipfile_entry, 'keys', list)()] if is_top_level else []
@@ -1224,6 +1209,7 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
         if key in dep:
             lockfile[key] = dep[key]
     # In case we lock a uri or a file when the user supplied a path
+    # remove the uri or file keys from the entry and keep the path
     if pipfile_entry and any(k in pipfile_entry for k in ['file', 'path']):
         fs_key = next((k for k in ['path', 'file'] if k in pipfile_entry), None)
         lockfile_key = next((k for k in ['uri', 'file', 'path'] if k in lockfile), None)
diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py
index 52550ba2..72ce5b71 100644
--- a/pipenv/vendor/requirementslib/__init__.py
+++ b/pipenv/vendor/requirementslib/__init__.py
@@ -1,5 +1,5 @@
 # -*- coding=utf-8 -*-
-__version__ = "0.2.0"
+__version__ = "1.0.1"
 
 
 from .exceptions import RequirementError
diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py
index 4fed0c6e..f0324be5 100644
--- a/pipenv/vendor/requirementslib/models/requirements.py
+++ b/pipenv/vendor/requirementslib/models/requirements.py
@@ -100,9 +100,9 @@ class NamedRequirement(BaseRequirement):
         return {name: pipfile_dict}
 
 
-LinkInfo = collections.namedtuple('LinkInfo', [
-    'vcs_type', 'prefer', 'relpath', 'path', 'uri', 'link',
-])
+LinkInfo = collections.namedtuple(
+    "LinkInfo", ["vcs_type", "prefer", "relpath", "path", "uri", "link"]
+)
 
 
 @attr.s
@@ -156,7 +156,7 @@ class FileRequirement(BaseRequirement):
         # Git allows `git@github.com...` lines that are not really URIs.
         # Add "ssh://" so we can parse correctly, and restore afterwards.
         fixed_line = add_ssh_scheme_to_git_uri(line)
-        added_ssh_scheme = (fixed_line != line)
+        added_ssh_scheme = fixed_line != line
 
         # We can assume a lot of things if this is a local filesystem path.
         if "://" not in fixed_line:
@@ -168,7 +168,7 @@ class FileRequirement(BaseRequirement):
                 relpath = get_converted_relative_path(path)
             except ValueError:
                 relpath = None
-            return LinkInfo(None, 'path', relpath, path, uri, link)
+            return LinkInfo(None, "path", relpath, path, uri, link)
 
         # This is an URI. We'll need to perform some elaborated parsing.
 
@@ -179,17 +179,15 @@ class FileRequirement(BaseRequirement):
         if "+" in original_scheme:
             vcs_type, scheme = original_scheme.split("+", 1)
             parsed_url = parsed_url._replace(scheme=scheme)
-            prefer = 'uri'
+            prefer = "uri"
         else:
             vcs_type = None
-            prefer = 'file'
+            prefer = "file"
 
-        if parsed_url.scheme == 'file' and parsed_url.path:
+        if parsed_url.scheme == "file" and parsed_url.path:
             # This is a "file://" URI. Use url_to_path and path_to_url to
             # ensure the path is absolute. Also we need to build relpath.
-            path = Path(url_to_path(
-                urllib_parse.urlunsplit(parsed_url),
-            )).as_posix()
+            path = Path(url_to_path(unquote(urllib_parse.urlunsplit(parsed_url)))).as_posix()
             try:
                 relpath = get_converted_relative_path(path)
             except ValueError:
@@ -200,21 +198,20 @@ class FileRequirement(BaseRequirement):
             path = None
             relpath = None
             # Cut the fragment, but otherwise this is fixed_line.
-            uri = urllib_parse.urlunsplit(
-                parsed_url._replace(scheme=original_scheme, fragment=''),
-            )
+            uri = unquote(urllib_parse.urlunsplit(
+                parsed_url._replace(scheme=original_scheme, fragment="")
+            ))
 
         if added_ssh_scheme:
             uri = strip_ssh_from_git_uri(uri)
 
         # Re-attach VCS prefix to build a Link.
-        link = Link(urllib_parse.urlunsplit(
-            parsed_url._replace(scheme=original_scheme),
+        link = Link(unquote(
+            urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme))
         ))
 
         return LinkInfo(vcs_type, prefer, relpath, path, uri, link)
 
-
     @uri.default
     def get_uri(self):
         if self.path and not self.uri:
@@ -342,9 +339,9 @@ class FileRequirement(BaseRequirement):
         # path - Local filesystem path.
         # uri - Absolute URI that is parsable with urlsplit.
         # One of these will be a string; the other would be None.
-        uri = pipfile.get('uri')
-        fil = pipfile.get('file')
-        path = pipfile.get('path')
+        uri = pipfile.get("uri")
+        fil = pipfile.get("file")
+        path = pipfile.get("path")
         if path and uri:
             raise ValueError("do not specify both 'path' and 'uri'")
         if path and fil:
@@ -356,7 +353,7 @@ class FileRequirement(BaseRequirement):
         # 'file' - A file:// URI (possibly with VCS prefix).
         # 'uri' - Any other URI.
         if path:
-            uri_scheme = 'path'
+            uri_scheme = "path"
         else:
             # URI is not currently a valid key in pipfile entries
             # see https://github.com/pypa/pipfile/issues/110
@@ -378,7 +375,11 @@ class FileRequirement(BaseRequirement):
 
     @property
     def line_part(self):
-        if (self._uri_scheme and self._uri_scheme == 'file') or (self.link.is_artifact or self.link.is_wheel) and self.link.url:
+        if (
+            (self._uri_scheme and self._uri_scheme == "file")
+            or (self.link.is_artifact or self.link.is_wheel)
+            and self.link.url
+        ):
             seed = unquote(self.link.url_without_fragment) or self.uri
         else:
             seed = self.formatted_path or self.link.url or self.uri
@@ -390,35 +391,47 @@ class FileRequirement(BaseRequirement):
 
     @property
     def pipfile_part(self):
-        pipfile_dict = {k: v for k, v in attr.asdict(self, filter=filter_none).items()}
+        pipfile_dict = attr.asdict(self, filter=filter_none).copy()
         name = pipfile_dict.pop("name")
-        if '_uri_scheme' in pipfile_dict:
-            pipfile_dict.pop('_uri_scheme')
+        if "_uri_scheme" in pipfile_dict:
+            pipfile_dict.pop("_uri_scheme")
         if "setup_path" in pipfile_dict:
             pipfile_dict.pop("setup_path")
         # For local paths and remote installable artifacts (zipfiles, etc)
-        collision_keys = {'file', 'uri', 'path'}
+        collision_keys = {"file", "uri", "path"}
         if self._uri_scheme:
             dict_key = self._uri_scheme
-            target_key = dict_key if dict_key in pipfile_dict else next((k for k in ('file', 'uri', 'path') if k in pipfile_dict), None)
+            target_key = (
+                dict_key
+                if dict_key in pipfile_dict
+                else next(
+                    (k for k in ("file", "uri", "path") if k in pipfile_dict), None
+                )
+            )
             if target_key:
                 winning_value = pipfile_dict.pop(target_key)
                 collisions = (k for k in collision_keys if k in pipfile_dict)
                 for key in collisions:
                     pipfile_dict.pop(key)
                 pipfile_dict[dict_key] = winning_value
-        elif self.is_remote_artifact or self.link.is_artifact and (self._uri_scheme and self._uri_scheme == 'file'):
+        elif (
+            self.is_remote_artifact
+            or self.link.is_artifact
+            and (self._uri_scheme and self._uri_scheme == "file")
+        ):
             dict_key = "file"
             # Look for uri first because file is a uri format and this is designed
             # to make sure we add file keys to the pipfile as a replacement of uri
-            target_key = next((k for k in ('file', 'uri', 'path') if k in pipfile_dict), None)
+            target_key = next(
+                (k for k in ("file", "uri", "path") if k in pipfile_dict), None
+            )
             winning_value = pipfile_dict.pop(target_key)
             key_to_remove = (k for k in collision_keys if k in pipfile_dict)
             for key in key_to_remove:
                 pipfile_dict.pop(key)
             pipfile_dict[dict_key] = winning_value
         else:
-            collisions = [key for key in ["path", "file", "uri",] if key in pipfile_dict]
+            collisions = [key for key in ["path", "file", "uri"] if key in pipfile_dict]
             if len(collisions) > 1:
                 for k in collisions[1:]:
                     pipfile_dict.pop(k)
@@ -456,7 +469,7 @@ class VCSRequirement(FileRequirement):
         if "+" in scheme:
             vcs_type, scheme = scheme.split("+", 1)
             vcs_type = "{0}+".format(vcs_type)
-        new_uri = urllib_parse.urlunsplit((scheme,) + rest)
+        new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",))
         new_uri = "{0}{1}".format(vcs_type, new_uri)
         self.uri = new_uri
 
@@ -548,8 +561,10 @@ class VCSRequirement(FileRequirement):
         name = link.egg_fragment
         subdirectory = link.subdirectory_fragment
         ref = None
-        if "@" in link.show_url:
+        if "@" in link.show_url and "@" in uri:
             uri, ref = uri.rsplit("@", 1)
+        if relpath and "@" in relpath:
+            relpath, ref = relpath.rsplit("@", 1)
         return cls(
             name=name,
             ref=ref,
diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py
index b95da89d..b7daf890 100644
--- a/pipenv/vendor/requirementslib/models/utils.py
+++ b/pipenv/vendor/requirementslib/models/utils.py
@@ -79,7 +79,7 @@ def get_version(pipfile_entry):
 def strip_ssh_from_git_uri(uri):
     """Return git+ssh:// formatted URI to git+git@ format"""
     if isinstance(uri, six.string_types):
-        uri = uri.replace("git+ssh://", "git+")
+        uri = uri.replace("git+ssh://", "git+", 1)
     return uri
 
 
diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py
index b5a802d7..8a50b6fd 100644
--- a/pipenv/vendor/requirementslib/utils.py
+++ b/pipenv/vendor/requirementslib/utils.py
@@ -69,9 +69,9 @@ def get_converted_relative_path(path, relative_to=os.curdir):
         start = start.resolve()
     except OSError:
         start = start.absolute()
-    path = start.joinpath('.', path).relative_to(start)
+    path = start.joinpath(".", path).relative_to(start)
     # Normalize these to use forward slashes even on windows
-    if os.name == 'nt':
+    if os.name == "nt":
         return os.altsep.join([".", path.as_posix()])
     return os.sep.join([".", path.as_posix()])
 
diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py
index a69135a5..d676e1c0 100644
--- a/tests/integration/test_install_markers.py
+++ b/tests/integration/test_install_markers.py
@@ -136,9 +136,11 @@ def test_resolver_unique_markers(PipenvInstance, pypi):
 
     This verifies that we clean that successfully.
     """
-    with PipenvInstance(chdir=True) as p:
+    with PipenvInstance(chdir=True, pypi=pypi) as p:
         c = p.pipenv('install vcrpy==1.11.0')
         assert c.return_code == 0
+        c = p.pipenv('lock')
+        assert c.return_code == 0
         assert 'yarl' in p.lockfile['default']
         yarl = p.lockfile['default']['yarl']
         assert 'markers' in yarl
diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py
index 301dbc83..45aa2501 100644
--- a/tests/integration/test_install_uri.py
+++ b/tests/integration/test_install_uri.py
@@ -14,14 +14,14 @@ except ImportError:
 @flaky
 def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
-        c = p.pipenv('install git+https://github.com/benjaminp/six.git#egg=six')
+        c = p.pipenv('install git+https://github.com/benjaminp/six.git@1.11.0#egg=six')
         assert c.return_code == 0
         # edge case where normal package starts with VCS name shouldn't be flagged as vcs
         c = p.pipenv('install gitdb2')
         assert c.return_code == 0
         assert all(package in p.pipfile['packages'] for package in ['six', 'gitdb2'])
         assert 'git' in p.pipfile['packages']['six']
-        assert p.lockfile['default']['six'] == {"git": "https://github.com/benjaminp/six.git"}
+        assert p.lockfile['default']['six'] == {"git": "https://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a"}
         assert 'gitdb2' in p.lockfile['default']
 
 
@@ -206,3 +206,32 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir):
         assert 'structlog' in p.pipfile['packages']
         assert 'structlog' in p.lockfile['default']
         assert 'six' in p.lockfile['default']
+
+
+@pytest.mark.vcs
+@pytest.mark.install
+@pytest.mark.needs_internet
+def test_vcs_entry_supersedes_non_vcs(PipenvInstance, pip_src_dir):
+    """See issue #2181 -- non-editable VCS dep was specified, but not showing up
+    in the lockfile -- due to not running pip install before locking and not locking
+    the resolution graph of non-editable vcs dependencies.
+    """
+    with PipenvInstance(chdir=True) as p:
+        with open(p.pipfile_path, 'w') as f:
+            f.write("""
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+PyUpdater = "*"
+PyInstaller = {ref = "develop", git = "https://github.com/pyinstaller/pyinstaller.git"}
+            """.strip())
+        p.pipenv('install')
+        installed_packages = ['PyUpdater', 'PyInstaller']
+        assert all([k in p.pipfile['packages'] for k in installed_packages])
+        assert all([k.lower() in p.lockfile['default'] for k in installed_packages])
+        assert all([k in p.lockfile['default']['pyinstaller'] for k in ['ref', 'git']])
+        assert p.lockfile['default']['pyinstaller'].get('ref') is not None
+        assert p.lockfile['default']['pyinstaller']['git'] == "https://github.com/pyinstaller/pyinstaller.git"