Repository: pypa/pipenv

Issue:
Upgrading package reinstalls old version
> Be sure to check the existing issues (both open and closed!).

Checked #1490 and other issued that mention *update*.

> Describe the issue briefly here.

Upgrading package with `pipenv update PACKAGE` upgrades given PACKAGE but then reinstalls old version.

> Please run `$ python -m pipenv.help`, and paste the results here.

<details><summary>$ python -m pipenv.help output</summary>

Pipenv version: `'11.10.4'`

Pipenv location: `'/usr/local/lib/python3.6/site-packages/pipenv'`

Python location: `'/usr/local/bin/python'`

Other Python installations in `PATH`:

  - `2.7`: `/usr/bin/python2.7`
  - `2.7`: `/usr/bin/python2.7`
  - `3.6`: `/usr/local/bin/python3.6m`
  - `3.6`: `/usr/local/bin/python3.6`
  - `3.6`: `/usr/local/bin/python3.6`

  - `3.6.5`: `/usr/local/bin/python`
  - `3.6.5`: `/usr/local/bin/python`
  - `2.7.9`: `/usr/bin/python`
  - `2.7.9`: `/usr/bin/python2`
  - `3.6.5`: `/usr/local/bin/python3`
  - `3.6.5`: `/usr/local/bin/python3`

PEP 508 Information:

```
{'implementation_name': 'cpython',
 'implementation_version': '3.6.5',
 'os_name': 'posix',
 'platform_machine': 'x86_64',
 'platform_python_implementation': 'CPython',
 'platform_release': '4.9.87-linuxkit-aufs',
 'platform_system': 'Linux',
 'platform_version': '#1 SMP Wed Mar 14 15:12:16 UTC 2018',
 'python_full_version': '3.6.5',
 'python_version': '3.6',
 'sys_platform': 'linux'}
```

System environment variables:

  - `HOSTNAME`
  - `GPG_KEY`
  - `SHELL`
  - `TERM`
  - `OLDPWD`
  - `PYTHON_VERSION`
  - `PATH`
  - `PWD`
  - `LANG`
  - `PYTHON_PIP_VERSION`
  - `SHLVL`
  - `HOME`
  - `_`
  - `PYTHONDONTWRITEBYTECODE`
  - `PIP_PYTHON_PATH`

Pipenv–specific environment variables:


Debug–specific environment variables:

  - `PATH`: `/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`
  - `SHELL`: `/bin/bash`
  - `LANG`: `C.UTF-8`
  - `PWD`: `/test`


---------------------------

Contents of `Pipfile` ('/test/Pipfile'):

```toml
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[packages]
requests = "==2.17.0"

[dev-packages]

[requires]
python_version = "3.6"

```


Contents of `Pipfile.lock` ('/test/Pipfile.lock'):

```json
{
    "_meta": {
        "hash": {
            "sha256": "d2e2282e2d97b6e5eb1ea3481d394b64d2598fb4e0c3e7e84f4a477275deacf6"
        },
        "pipfile-spec": 6,
        "requires": {
            "python_version": "3.6"
        },
        "sources": [
            {
                "name": "pypi",
                "url": "https://pypi.org/simple",
                "verify_ssl": true
            }
        ]
    },
    "default": {
        "certifi": {
            "hashes": [
                "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7",
                "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0"
            ],
            "version": "==2018.4.16"
        },
        "chardet": {
            "hashes": [
                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
            ],
            "version": "==3.0.4"
        },
        "idna": {
            "hashes": [
                "sha256:3cb5ce08046c4e3a560fc02f138d0ac63e00f8ce5901a56b32ec8b7994082aab",
                "sha256:cc19709fd6d0cbfed39ea875d29ba6d4e22c0cebc510a76d6302a28385e8bb70"
            ],
            "version": "==2.5"
        },
        "requests": {
            "hashes": [
                "sha256:73b4088c05f7fb5ca8e68651ed802df3ca40621281acf74bb321b4a8408aab7e",
                "sha256:eff227db5864238d44270cbadc8ac4133e69b69a2e7092b7b316ed1e4761cbd6"
            ],
            "index": "pypi",
            "version": "==2.17.0"
        },
        "urllib3": {
            "hashes": [
                "sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1",
                "sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5"
            ],
            "version": "==1.21.1"
        }
    },
    "develop": {}
}

```
</details>
------------

##### Expected result

Given I have `requests = "==2.17.0"` in Pipfile and I change it to `requests = "*"`, running `pipenv  update requests` I would expect to have latest version of requests.

##### Actual result

```
(test-BnvVBCnl) root@c4d4399a20d3:/test# pipenv update requests
Installing requests…
Collecting requests
  Using cached https://files.pythonhosted.org/packages/49/df/50aa1999ab9bde74656c2919d9c0c085fd2b3775fd3eca826012bef76d8c/requests-2.18.4-py2.py3-none-any.whl
Requirement not upgraded as not directly required: chardet<3.1.0,>=3.0.2 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (3.0.4)
Requirement not upgraded as not directly required: certifi>=2017.4.17 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (2018.4.16)
Requirement not upgraded as not directly required: idna<2.7,>=2.5 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (2.6)
Requirement not upgraded as not directly required: urllib3<1.23,>=1.21.1 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (1.22)
Installing collected packages: requests
  Found existing installation: requests 2.17.0
    Uninstalling requests-2.17.0:
      Successfully uninstalled requests-2.17.0
Successfully installed requests-2.18.4

Adding requests to Pipfile's [packages]…
Installing dependencies from Pipfile.lock (b14837)…
  🐍   ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 5/5 — 00:00:01
```

##### Steps to replicate

1. 

```
root@c4d4399a20d3:/test# pipenv install requests==2.17.0
Installing requests==2.17.0…
Collecting requests==2.17.0
  Using cached https://files.pythonhosted.org/packages/5b/b6/9a18db79553524246aa1b081829e6f977667ec558cef684988895c1092d9/requests-2.17.0-py2.py3-none-any.whl
Collecting certifi>=2017.4.17 (from requests==2.17.0)
  Using cached https://files.pythonhosted.org/packages/7c/e6/92ad559b7192d846975fc916b65f667c7b8c3a32bea7372340bfe9a15fa5/certifi-2018.4.16-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from requests==2.17.0)
  Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl
Collecting urllib3<1.22,>=1.21.1 (from requests==2.17.0)
  Using cached https://files.pythonhosted.org/packages/24/53/f397db567de0aa0e81b211d81c13c41a779f14893e42189cf5bdb97611b2/urllib3-1.21.1-py2.py3-none-any.whl
Collecting idna<2.6,>=2.5 (from requests==2.17.0)
  Using cached https://files.pythonhosted.org/packages/11/7d/9bbbd7bb35f34b0169542487d2a8859e44306bb2e6a4455d491800a5621f/idna-2.5-py2.py3-none-any.whl
Installing collected packages: certifi, chardet, urllib3, idna, requests
Successfully installed certifi-2018.4.16 chardet-3.0.4 idna-2.5 requests-2.17.0 urllib3-1.21.1

Adding requests==2.17.0 to Pipfile's [packages]…
Pipfile.lock not found, creating…
Locking [dev-packages] dependencies…
Locking [packages] dependencies…
Updated Pipfile.lock (deacf6)!
Installing dependencies from Pipfile.lock (deacf6)…
  🐍   ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 5/5 — 00:00:01
To activate this project's virtualenv, run the following:
 $ pipenv shell
```

2. Edit Pipfile, replace `requests = "==2.17.0"` with `requests = "*"`
```
(test-BnvVBCnl) root@c4d4399a20d3:/test# more Pipfile
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[packages]
requests = "*"

[dev-packages]

[requires]
python_version = "3.6"
```

3. Check for outdated packages

```
(test-BnvVBCnl) root@c4d4399a20d3:/test# pipenv update --outdated
Package 'idna' out–of–date: '==2.5' installed, '==2.6' available.
Package 'requests' out–of–date: '==2.17.0' installed, '==2.18.4' available.
Package 'urllib3' out–of–date: '==1.21.1' installed, '==1.22' available.
```

4. Try to update `requests`

```
(test-BnvVBCnl) root@c4d4399a20d3:/test# pipenv update requests
Installing requests…
Collecting requests
  Using cached https://files.pythonhosted.org/packages/49/df/50aa1999ab9bde74656c2919d9c0c085fd2b3775fd3eca826012bef76d8c/requests-2.18.4-py2.py3-none-any.whl
Requirement not upgraded as not directly required: certifi>=2017.4.17 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (2018.4.16)
Requirement not upgraded as not directly required: chardet<3.1.0,>=3.0.2 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (3.0.4)
Requirement not upgraded as not directly required: urllib3<1.23,>=1.21.1 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (1.21.1)
Requirement not upgraded as not directly required: idna<2.7,>=2.5 in /root/.local/share/virtualenvs/test-BnvVBCnl/lib/python3.6/site-packages (from requests) (2.5)
Installing collected packages: requests
  Found existing installation: requests 2.17.0
    Uninstalling requests-2.17.0:
      Successfully uninstalled requests-2.17.0
Successfully installed requests-2.18.4

Adding requests to Pipfile's [packages]…
Installing dependencies from Pipfile.lock (b14837)…
  🐍   ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 5/5 — 
```

5. Check `requests` version

```
(test-BnvVBCnl) root@c4d4399a20d3:/test# pipenv graph
requests==2.17.0
  - certifi [required: >=2017.4.17, installed: 2018.4.16]
  - chardet [required: <3.1.0,>=3.0.2, installed: 3.0.4]
  - idna [required: >=2.5,<2.6, installed: 2.6]
  - urllib3 [required: >=1.21.1,<1.22, installed: 1.22]
```

I found that if I run `pipenv lock` before step 4, `Pipfile.lock` would be updated and everything would work as expected for me. [Upgrade workflow](https://docs.pipenv.org/basics/#example-pipenv-upgrade-workflow) does not mention need to lock before update.

Please note that in my real project I have `"*"` version specified but I wouldn't be able to make reproducable case as it happend when new version of package is released.





Discussion:
Thanks for the incredibly thorough report. It seems like `update` isn’t locking first— it definitely is supposed to. We should look into this for sure. It may relate to our inclusion of requests in the list of `BAD_PACKAGES` or packages that shouldn’t be touched by certain pipenv processes because pipenv relies on them. 
Thanks for fast response, I was afraid all the time that I am doing something stupid and will find out just after I click Submit 😅

I have used `requests` package just to avoid adding unknown bits to test case, this also happens with other packages as well.

Resolving patch:
diff --git a/pipenv/cli.py b/pipenv/cli.py
--- a/pipenv/cli.py
+++ b/pipenv/cli.py
@@ -823,24 +823,6 @@ def update(
                 crayons.white('.', bold=True),
             )
         )
-        do_lock(
-            verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror
-        )
-        do_sync(
-            ctx=ctx,
-            install=install,
-            dev=dev,
-            three=three,
-            python=python,
-            bare=bare,
-            dont_upgrade=False,
-            user=False,
-            verbose=verbose,
-            clear=clear,
-            unused=False,
-            sequential=sequential,
-            pypi_mirror=pypi_mirror,
-        )
     else:
         for package in ([package] + list(more_packages) or []):
             if package not in project.all_packages:
@@ -853,28 +835,23 @@ def update(
                     err=True,
                 )
                 sys.exit(1)
-        ensure_lockfile(keep_outdated=project.lockfile_exists, pypi_mirror=pypi_mirror)
-        # Install the dependencies.
-        do_install(
-            package_name=package,
-            more_packages=more_packages,
-            dev=dev,
-            three=three,
-            python=python,
-            pypi_mirror=pypi_mirror,
-            system=system,
-            lock=True,
-            ignore_pipfile=False,
-            skip_lock=False,
-            verbose=verbose,
-            requirements=False,
-            sequential=sequential,
-            pre=pre,
-            code=False,
-            deploy=False,
-            keep_outdated=True,
-            selective_upgrade=True,
-        )
+    do_lock(
+        verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror
+    )
+    do_sync(
+        ctx=ctx,
+        dev=dev,
+        three=three,
+        python=python,
+        bare=bare,
+        dont_upgrade=False,
+        user=False,
+        verbose=verbose,
+        clear=clear,
+        unused=False,
+        sequential=sequential,
+        pypi_mirror=pypi_mirror,
+    )
 
 
 @command(
@@ -999,7 +976,6 @@ def sync(
 
     do_sync(
         ctx=ctx,
-        install=install,
         dev=dev,
         three=three,
         python=python,
diff --git a/pipenv/core.py b/pipenv/core.py
--- a/pipenv/core.py
+++ b/pipenv/core.py
@@ -2535,11 +2535,9 @@ def traverse(obj):
 
 def do_sync(
     ctx,
-    install,
     dev=False,
     three=None,
     python=None,
-    dry_run=False,
     bare=False,
     dont_upgrade=False,
     user=False,


Test patch:
diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py
--- a/tests/integration/test_pipenv.py
+++ b/tests/integration/test_pipenv.py
@@ -58,3 +58,24 @@ def test_deploy_works(PipenvInstance, pypi):
 
         c = p.pipenv('install --deploy')
         assert c.return_code > 0
+
+
+@pytest.mark.update
+@pytest.mark.lock
+def test_update_locks(PipenvInstance, pypi):
+
+    with PipenvInstance(pypi=pypi) as p:
+        c = p.pipenv('install requests==2.14.0')
+        assert c.return_code == 0
+        with open(p.pipfile_path, 'r') as fh:
+            pipfile_contents = fh.read()
+        pipfile_contents = pipfile_contents.replace('==2.14.0', '*')
+        with open(p.pipfile_path, 'w') as fh:
+            fh.write(pipfile_contents)
+        c = p.pipenv('update requests')
+        assert c.return_code == 0
+        assert p.lockfile['default']['requests']['version'] == '==2.18.4'
+        c = p.pipenv('run pip freeze')
+        assert c.return_code == 0
+        lines = c.out.splitlines()
+        assert 'requests==2.18.4' in [l.strip() for l in lines]