Repository: pypa/pipenv

Issue:
2018.11.14 regression: pipenv fails to install editable extras
### Issue description

```
[vagrant@cluster-01 src]$ pipenv lock
Locking [dev-packages] dependencies...
Locking [packages] dependencies...
✘ Locking Failed!
Traceback (most recent call last):
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 126, in <module>
    main()
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 119, in main
    parsed.requirements_dir, parsed.packages)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 85, in _main
    requirements_dir=requirements_dir,
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 69, in resolve
    req_dir=requirements_dir
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/utils.py", line 695, in resolve_deps
    req_dir=req_dir,
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/utils.py", line 466, in actually_resolve_deps
    deps, index_lookup, markers_lookup, project, sources,
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/utils.py", line 237, in get_resolver_metadata
    req = Requirement.from_line(dep)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/requirements.py", line 1072, in from_line
    r = FileRequirement.from_line(line_with_prefix)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/requirements.py", line 528, in from_line
    return cls.create(**arg_dict)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/requirements.py", line 460, in create
    setup_info = SetupInfo.from_ireq(ireq)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 391, in from_ireq
    created.get_info()
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 309, in get_info
    self.get_egg_metadata()
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 281, in get_egg_metadata
    metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 146, in get_metadata
    for req in _deps
  File "/home/vagrant/.local/share/virtualenvs/src-uiGXbgQW/lib/python2.7/site-packages/pkg_resources/__init__.py", line 3023, in parse
    req, = parse_requirements(s)
  File "/home/vagrant/.local/share/virtualenvs/src-uiGXbgQW/lib/python2.7/site-packages/pkg_resources/__init__.py", line 2970, in parse_requirements
    yield Requirement(line)
  File "/home/vagrant/.local/share/virtualenvs/src-uiGXbgQW/lib/python2.7/site-packages/pkg_resources/__init__.py", line 2979, in __init__
    raise RequirementParseError(str(e))
pkg_resources.RequirementParseError: Invalid requirement, parse error at "'testing'"
File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 126, in <module>
    main()
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 119, in main
    parsed.requirements_dir, parsed.packages)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 85, in _main
    requirements_dir=requirements_dir,
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/resolver.py", line 69, in resolve
    req_dir=requirements_dir
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/utils.py", line 695, in resolve_deps
    req_dir=req_dir,
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/utils.py", line 466, in actually_resolve_deps
    deps, index_lookup, markers_lookup, project, sources,
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/utils.py", line 237, in get_resolver_metadata
    req = Requirement.from_line(dep)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/requirements.py", line 1072, in from_line
    r = FileRequirement.from_line(line_with_prefix)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/requirements.py", line 528, in from_line
    return cls.create(**arg_dict)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/requirements.py", line 460, in create
    setup_info = SetupInfo.from_ireq(ireq)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 391, in from_ireq
    created.get_info()
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 309, in get_info
    self.get_egg_metadata()
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 281, in get_egg_metadata
    metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name)
  File "/home/vagrant/.local/lib/python2.7/site-packages/pipenv/vendor/requirementslib/models/setup_info.py", line 146, in get_metadata
    for req in _deps
  File "/home/vagrant/.local/share/virtualenvs/src-uiGXbgQW/lib/python2.7/site-packages/pkg_resources/__init__.py", line 3023, in parse
    req, = parse_requirements(s)
  File "/home/vagrant/.local/share/virtualenvs/src-uiGXbgQW/lib/python2.7/site-packages/pkg_resources/__init__.py", line 2970, in parse_requirements
    yield Requirement(line)
  File "/home/vagrant/.local/share/virtualenvs/src-uiGXbgQW/lib/python2.7/site-packages/pkg_resources/__init__.py", line 2979, in __init__
    raise RequirementParseError(str(e))
pkg_resources.RequirementParseError: Invalid requirement, parse error at "'testing'"
```

Downgrading to pipenv version 2018.10.13, and everything works as expected.

### Expected result

`Pipfile.lock` is generated

### Actual result

💥 

### Steps to replicate

Provide the steps to replicate (which usually at least includes the commands and the Pipfile).

1. `pipenv lock`

<details><summary>$ pipenv --support</summary>

Pipenv version: `'2018.10.13'`

Pipenv location: `'/home/vagrant/.local/lib/python2.7/site-packages/pipenv'`

Python location: `'/usr/bin/python2'`

Python installations found:

  - `2.7.5`: `/usr/bin/python2.7`

PEP 508 Information:

```
{'implementation_name': 'cpython',
 'implementation_version': '0',
 'os_name': 'posix',
 'platform_machine': 'x86_64',
 'platform_python_implementation': 'CPython',
 'platform_release': '3.10.0-862.14.4.el7.x86_64',
 'platform_system': 'Linux',
 'platform_version': '#1 SMP Wed Sep 26 15:12:11 UTC 2018',
 'python_full_version': '2.7.5',
 'python_version': '2.7',
 'sys_platform': 'linux2'}
```

System environment variables:

  - `HISTTIMEFORMAT`
  - `PYTHONDONTWRITEBYTECODE`
  - `LESSOPEN`
  - `SSH_CLIENT`
  - `SELINUX_USE_CURRENT_RANGE`
  - `LOGNAME`
  - `USER`
  - `PATH`
  - `HOME`
  - `LANG`
  - `TERM`
  - `SHELL`
  - `SHLVL`
  - `HISTSIZE`
  - `XDG_RUNTIME_DIR`
  - `SSH_AUTH_SOCK`
  - `SELINUX_ROLE_REQUESTED`
  - `PIP_PYTHON_PATH`
  - `XDG_SESSION_ID`
  - `_`
  - `SSH_CONNECTION`
  - `SSH_TTY`
  - `OLDPWD`
  - `HOSTNAME`
  - `SELINUX_LEVEL_REQUESTED`
  - `HISTCONTROL`
  - `PWD`
  - `CPATH`
  - `PIP_SHIMS_BASE_MODULE`
  - `MAIL`
  - `LS_COLORS`

Pipenv–specific environment variables:


Debug–specific environment variables:

  - `PATH`: `/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/vagrant/.local/bin:/home/vagrant/bin`
  - `SHELL`: `/bin/bash`
  - `LANG`: `en_US.UTF-8`
  - `PWD`: `/home/vagrant/src`


---------------------------

Contents of `Pipfile` ('/home/vagrant/src/Pipfile'):

```toml
[[source]]
url = "https://user:password@example.com/devpi/unstable/+simple/"
verify_ssl = true
name = "pypi"

[dev-packages]

[packages]
server = {path = "./server", extras = ["testing"], editable = true}
"e01afd4" = {path = "./lib", extras = ["testing"], editable = true}

[requires]
python_version = "2.7"

[pipenv]
allow_prereleases = true
```


Discussion:
Hey @bertjwregeer,

We did a quick check and this issue looks very darn similar to

* [#3229 - 2018.11.14 pipenv emits pythons errors when creating a virtualenv](3229)
* [#3228 - 2018.11.14 regression: &#x60;pipenv lock -r&#x60; with named Pipfile](3228)
* [#3226 - Don&#39;t disable --user when --system is passed](3226)

This could be a coincidence, but if any of these issues solves your problem then I did a good job :smile:

If not, the maintainers will get to this issue shortly.

Cheers,
Your Friendly Neighborhood ProBot

@duplicate-issues: No, they are not the same or similar.
Can't replicate here, can you try installing from a public pypi source?

Or can you provide more info about the package structure, say, does it contain `src` folder?
frostming, folder structure:

```
.git
.gitignore
server/
    setup.py
    src/
        server/
            __init__.py
            other.py
lib/
    setup.py
    src/
        lib/
            __init__.py
            somethingelse.py
Pipfile
```

The projects themselves are stock standard setup tool projects. Nothing fancy going on. The issue is that pipenv or another library is adding a second set of quotes to the extra to be installed:

```
pkg_resources.RequirementParseError: Invalid requirement, parse error at "'testing'"
```

Notice the double quotes followed by the single quotes.

It doesn't matter where I install the dependencies for the project from, private or not, this is currently broken.

@bertjwregeer Please try to prune the virtualenv `pipenv --rm` and run `pipenv lock` from a clean environment.
@frostming the issue occurred when running `pipenv` from scratch in a new environment without a `Pipfile.lock` available.

If I create the `Pipfile.lock` using an older version of `pipenv` and then upgrade to the new `pipenv` all works without issues.
@frostming the nuance here is going to be that this is a local package which has a setupfile, yet we are escaping the extras for reasons which make no discernible sense. I'm going to venture a random guess that this is related to the upstream restructuring we made to `requirementslib` around `setup.py` parsing, we could easily be mishandling extras when converting them (either in `setup_info` or `pipfile`.

@bertjwregeer -- I tried to get as close as possible to your setup as I could but I can't reproduce the issue -- are your extras in your `setup.py` or `setup.cfg` or are they configured with `pbr` or some other tooling? We jump through some hoops to try and parse that all, so it's possible in some specific cases we are pulling in some stray quotation marks

here was my test run (using setup.cfg files):

```toml
[[sources]]
url = "https://pypi.org/simple"
name = "pypi"
verify_ssl = true

[packages]
pylint = "*"
installer = {path = "./installer", editable = true, extras = ["virtualenv"] }
vistir = { path = "./vistir", editable = true, extras = ["spinner"] }

[dev-packages]

[requires]
python_version = "2.7"

[pipenv]
allow_prereleases = true
```


```console
/t/test2  pipenv install --dev
Creating a virtualenv for this project…
Pipfile: /tmp/test2/Pipfile
Using /home/hawk/.pyenv/versions/2.7.15/bin/python2 (2.7.15) to create virtualenv…
⠏ Creating virtual environment...New python executable in /home/hawk/.virtualenvs/test2-h6PFdzq5/bin/python2
Also creating executable in /home/hawk/.virtualenvs/test2-h6PFdzq5/bin/python
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /home/hawk/.pyenv/versions/2.7.15/bin/python2

✔ Successfully created virtual environment! 
Virtualenv location: /home/hawk/.virtualenvs/test2-h6PFdzq5
Pipfile.lock (a7c78d) out of date, updating to (d69bb7)…
Locking [dev-packages] dependencies…
Locking [packages] dependencies…
✔ Success! 
Updated Pipfile.lock (a7c78d)!
Installing dependencies from Pipfile.lock (a7c78d)…
  🐍   ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 47/47 — 00:00:09
To activate this project's virtualenv, run pipenv shell.
Alternatively, run a command inside the virtualenv with pipenv run.
```
My extras are defined in `setup.py`.

```
xistence$ cat server/setup.py
import os
from setuptools import setup, find_packages

thisdir = os.path.abspath(os.path.dirname(__file__))
version = open(os.path.join(thisdir, 'version.txt'), 'rb').read().strip()

testing_extras = [
    'beautifulsoup4',
    'coverage',
    'cr.lib[testing]',
    'flaky',
    'mock==1.0.1',
    'pycrunch>=0.2.1',
    'pytest',
    'pytest-cov',
    'pytest-xdist',
    'randomize',
    'requests_mock',
    'moto',
    'ming',
]

setup(
    name='cr.server',
    version=version,
    description="The Backend HTTP Server",
    long_description=open(os.path.join(thisdir, 'README.md')).read(),
    install_requires=[
        'CherryPy>=11.0.0',
        'cheroot>=5.3.8',
        'PyYAML>=3.10',
        'argparse>=1.2.1',
        'pymongo>=3.4.0',
        'pytz>=2011k',
        'repoze.who>=2.1b1',
        'zope.interface>=4.0.3',
        'docopt',
        'Dozer>=0.4',
        'openpyxl',
        'python-dateutil',
        'unicodecsv',
        'requests',
        'requests-cache',
        'six',
    ],
    tests_require=testing_extras,
    extras_require={
        'testing': testing_extras,
    },
    packages=find_packages(where='src'),
    package_dir={"": "src"},
    namespace_packages=['cr'],
    include_package_data=True,
    zip_safe=False,
)
```

Resolving patch:
diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py
--- a/pipenv/vendor/requirementslib/models/setup_info.py
+++ b/pipenv/vendor/requirementslib/models/setup_info.py
@@ -131,25 +131,41 @@ def get_metadata(path, pkg_name=None):
             None,
         )
         if dist:
-            requires = dist.requires()
-            dep_map = dist._build_dep_map()
+            try:
+                requires = dist.requires()
+            except exception:
+                requires = []
+            try:
+                dep_map = dist._build_dep_map()
+            except Exception:
+                dep_map = {}
             deps = []
+            extras = {}
             for k in dep_map.keys():
                 if k is None:
                     deps.extend(dep_map.get(k))
                     continue
                 else:
+                    extra = None
                     _deps = dep_map.get(k)
-                    k = k.replace(":", "; ")
+                    if k.startswith(":python_version"):
+                        marker = k.replace(":", "; ")
+                    else:
+                        marker = ""
+                        extra = "{0}".format(k)
                     _deps = [
-                        pkg_resources.Requirement.parse("{0}{1}".format(str(req), k))
+                        pkg_resources.Requirement.parse("{0}{1}".format(str(req), marker))
                         for req in _deps
                     ]
-                    deps.extend(_deps)
+                    if extra:
+                        extras[extra] = _deps
+                    else:
+                        deps.extend(_deps)
             return {
                 "name": dist.project_name,
                 "version": dist.version,
                 "requires": requires,
+                "extras": extras
             }
 
 
@@ -158,7 +174,6 @@ class SetupInfo(object):
     name = attr.ib(type=str, default=None)
     base_dir = attr.ib(type=Path, default=None)
     version = attr.ib(type=packaging.version.Version, default=None)
-    extras = attr.ib(type=list, default=attr.Factory(list))
     requires = attr.ib(type=dict, default=attr.Factory(dict))
     build_requires = attr.ib(type=list, default=attr.Factory(list))
     build_backend = attr.ib(type=list, default=attr.Factory(list))
@@ -247,8 +262,7 @@ def run_setup(self):
                 except NameError:
                     python = os.environ.get('PIP_PYTHON_PATH', sys.executable)
                     out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True,
-                                 combine_stderr=False, return_object=False, nospin=True,
-                                 write_to_stdout=False)
+                                 combine_stderr=False, return_object=False, nospin=True)
                 finally:
                     _setup_stop_after = None
                     sys.argv = save_argv
@@ -288,6 +302,15 @@ def get_egg_metadata(self):
                 self.requires.update(
                     {req.key: req for req in metadata.get("requires", {})}
                 )
+                if getattr(self.ireq, "extras", None):
+                    for extra in self.ireq.extras:
+                        self.requires.update(
+                            {
+                                req.key: req for req
+                                in metadata.get("extras", {}).get(extra)
+                                if req is not None
+                            }
+                        )
 
     def run_pyproject(self):
         if self.pyproject and self.pyproject.exists():


Test patch: