Repository: pypa/pip

Issue:
Improve speed of `pip list --outdated` via parallelization
**Environment**

* pip version: 20.0.2
* Python version: 3.6.9
* OS: Any

<!-- Feel free to add more information about your environment here -->

**Description**
`pip list --outdated` works slow in case of many packages. Currently 50% of real time is wasted on `requests` result waiting. (`real time` is much higher than `user time`)

**Expected behavior**
`pip list --outdated` sends queries in parallel (`real time` is almost equal to `user time`)

**How to Reproduce**
Linux/Mac OS X:
```
python3 -m venv venv
source venv/bin/activate
pip install -U setuptools
pip install -U pip
pip install numpy scipy pandas matplotlib seaborn statsmodels sklearn jupyter tqdm requests six catalyst PyQt5
time python3 -m pip list --outdated
deactivate
rm -rf venv
```
Watch on `time` output.

Windows 10 (not good reproduction):
In Powershell (not cmd):
```
python -m venv venv
.\venv\Scripts\activate.bat
pip install -U setuptools
python -m pip install -U pip
pip install numpy scipy pandas matplotlib seaborn statsmodels sklearn jupyter tqdm requests six catalyst PyQt5
(Measure-Command {pip list --outdated}).TotalSeconds
.\venv\Scripts\deactivate.bat
Remove-Item -path venv -recurse
```
Watch `Measure-Command` output.
Or use Windows Linux Subsystem to reproduce this.


**Output**
![image](https://user-images.githubusercontent.com/9883873/78288585-336bcb00-752a-11ea-9979-7c59c8be5dc7.png)


Discussion:

Resolving patch:
diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py
--- a/src/pip/_internal/commands/list.py
+++ b/src/pip/_internal/commands/list.py
@@ -5,8 +5,10 @@
 
 import json
 import logging
+from multiprocessing.dummy import Pool
 
 from pip._vendor import six
+from pip._vendor.requests.adapters import DEFAULT_POOLSIZE
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import IndexGroupCommand
@@ -183,7 +185,7 @@ def iter_packages_latest_infos(self, packages, options):
         with self._build_session(options) as session:
             finder = self._build_package_finder(options, session)
 
-            for dist in packages:
+            def latest_info(dist):
                 typ = 'unknown'
                 all_candidates = finder.find_all_candidates(dist.key)
                 if not options.pre:
@@ -196,7 +198,7 @@ def iter_packages_latest_infos(self, packages, options):
                 )
                 best_candidate = evaluator.sort_best_candidate(all_candidates)
                 if best_candidate is None:
-                    continue
+                    return None
 
                 remote_version = best_candidate.version
                 if best_candidate.link.is_wheel:
@@ -206,7 +208,19 @@ def iter_packages_latest_infos(self, packages, options):
                 # This is dirty but makes the rest of the code much cleaner
                 dist.latest_version = remote_version
                 dist.latest_filetype = typ
-                yield dist
+                return dist
+
+            # This is done for 2x speed up of requests to pypi.org
+            # so that "real time" of this function
+            # is almost equal to "user time"
+            pool = Pool(DEFAULT_POOLSIZE)
+
+            for dist in pool.imap_unordered(latest_info, packages):
+                if dist is not None:
+                    yield dist
+
+            pool.close()
+            pool.join()
 
     def output_package_listing(self, packages, options):
         packages = sorted(
diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py
--- a/src/pip/_internal/utils/logging.py
+++ b/src/pip/_internal/utils/logging.py
@@ -52,7 +52,6 @@
 
 
 _log_state = threading.local()
-_log_state.indentation = 0
 subprocess_logger = getLogger('pip.subprocessor')
 
 
@@ -104,6 +103,8 @@ def indent_log(num=2):
     A context manager which will cause the log output to be indented for any
     log messages emitted inside it.
     """
+    # For thread-safety
+    _log_state.indentation = get_indentation()
     _log_state.indentation += num
     try:
         yield


Test patch:
diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py
--- a/tests/unit/test_logging.py
+++ b/tests/unit/test_logging.py
@@ -2,6 +2,7 @@
 import logging
 import os
 import time
+from threading import Thread
 
 import pytest
 from mock import patch
@@ -11,6 +12,7 @@
     BrokenStdoutLoggingError,
     ColorizedStreamHandler,
     IndentingFormatter,
+    indent_log,
 )
 from pip._internal.utils.misc import captured_stderr, captured_stdout
 
@@ -108,6 +110,39 @@ def test_format_deprecated(self, level_name, expected):
         f = IndentingFormatter(fmt="%(message)s")
         assert f.format(record) == expected
 
+    def test_thread_safety_base(self):
+        record = self.make_record(
+            'DEPRECATION: hello\nworld', level_name='WARNING',
+        )
+        f = IndentingFormatter(fmt="%(message)s")
+        results = []
+
+        def thread_function():
+            results.append(f.format(record))
+
+        thread_function()
+        thread = Thread(target=thread_function)
+        thread.start()
+        thread.join()
+        assert results[0] == results[1]
+
+    def test_thread_safety_indent_log(self):
+        record = self.make_record(
+            'DEPRECATION: hello\nworld', level_name='WARNING',
+        )
+        f = IndentingFormatter(fmt="%(message)s")
+        results = []
+
+        def thread_function():
+            with indent_log():
+                results.append(f.format(record))
+
+        thread_function()
+        thread = Thread(target=thread_function)
+        thread.start()
+        thread.join()
+        assert results[0] == results[1]
+
 
 class TestColorizedStreamHandler(object):