Repository: conda/conda

Issue:
RuntimeError: can't start new thread
```
error: RuntimeError("generator didn't stop after throw()",)
command: /home/mholekev/miniconda3/bin/conda install --yes numpy scipy matplotlib jupyter git pandas
user_agent: conda/4.4.4 requests/2.18.4 CPython/3.6.3 Linux/2.6.32-696.10.3.el6.x86_64 centos/6.7 glibc/2.12
_messageid: -9223372036842075552
_messagetime: 1514766990662 / 2017-12-31 18:36:30

Traceback (most recent call last):
  File "/home/mholekev/miniconda3/lib/python3.6/site-packages/conda/common/io.py", line 399, in backdown_thread_pool
    yield ThreadPoolExecutor(max_workers)
  File "/home/mholekev/miniconda3/lib/python3.6/site-packages/conda/core/index.py", line 214, in get_reduced_index
    new_records = query_all(spec)
  File "/home/mholekev/miniconda3/lib/python3.6/site-packages/conda/core/index.py", line 187, in query_all
    return tuple(concat(future.result() for future in as_completed(futures)))
  File "/home/mholekev/miniconda3/lib/python3.6/site-packages/conda/core/index.py", line 187, in <genexpr>
    return tuple(concat(future.result() for future in as_completed(futures)))
  File "/home/mholekev/miniconda3/lib/python3.6/concurrent/futures/_base.py", line 217, in as_completed
    fs = set(fs)
  File "/home/mholekev/miniconda3/lib/python3.6/site-packages/conda/core/index.py", line 186, in <genexpr>
    futures = (executor.submit(sd.query, spec) for sd in subdir_datas)
  File "/home/mholekev/miniconda3/lib/python3.6/concurrent/futures/thread.py", line 123, in submit
    self._adjust_thread_count()
  File "/home/mholekev/miniconda3/lib/python3.6/concurrent/futures/thread.py", line 142, in _adjust_thread_count
    t.start()
  File "/home/mholekev/miniconda3/lib/python3.6/threading.py", line 846, in start
    _start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread

```


Discussion:
```
error: RuntimeError("can't start new thread",)
command: /ahg/regevdata/users/subraman/software/miniconda3/bin/conda create -n test
user_agent: conda/4.4.4 requests/2.18.4 CPython/3.6.3 Linux/2.6.32-696.16.1.el6.x86_64 rhel/6.9 glibc/2.12
_messageid: -9223372036842425559
_messagetime: 1514681157023 / 2017-12-30 18:45:57

Traceback (most recent call last):
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/exceptions.py", line 724, in __call__
    return func(*args, **kwargs)
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/cli/main.py", line 78, in _main
    exit_code = do_call(args, p)
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/cli/conda_argparse.py", line 75, in do_call
    exit_code = getattr(module, func_name)(args, parser)
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/cli/main_create.py", line 11, in execute
    install(args, parser, 'create')
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/cli/install.py", line 236, in install
    force_reinstall=context.force,
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/core/solve.py", line 514, in solve_for_transaction
    SubdirData.query_all(self.channels, self.subdirs, conda_newer_spec),
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/core/repodata.py", line 88, in query_all
    return tuple(concat(future.result() for future in as_completed(futures)))
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/core/repodata.py", line 88, in <genexpr>
    return tuple(concat(future.result() for future in as_completed(futures)))
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/concurrent/futures/_base.py", line 217, in as_completed
    fs = set(fs)
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/site-packages/conda/core/repodata.py", line 87, in <genexpr>
    ) for url in channel_urls)
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/concurrent/futures/thread.py", line 123, in submit
    self._adjust_thread_count()
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/concurrent/futures/thread.py", line 142, in _adjust_thread_count
    t.start()
  File "/ahg/regevdata/users/subraman/software/miniconda3/lib/python3.6/threading.py", line 846, in start
    _start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread

```

Resolving patch:
diff --git a/conda/common/io.py b/conda/common/io.py
--- a/conda/common/io.py
+++ b/conda/common/io.py
@@ -2,7 +2,8 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, _base, as_completed
+from concurrent.futures.thread import _WorkItem
 from contextlib import contextmanager
 from enum import Enum
 from errno import EPIPE, ESHUTDOWN
@@ -11,7 +12,6 @@
 import json
 import logging
 from logging import CRITICAL, Formatter, NOTSET, StreamHandler, WARN, getLogger
-from math import floor
 import os
 from os.path import dirname, isdir, isfile, join
 import signal
@@ -35,7 +35,7 @@
 class CaptureTarget(Enum):
     """Constants used for contextmanager captured.
 
-    Used similarily like the constants PIPE, STDOUT for stdlib's subprocess.Popen.
+    Used similarly like the constants PIPE, STDOUT for stdlib's subprocess.Popen.
     """
     STRING = -1
     STDOUT = -2
@@ -303,7 +303,6 @@ def spinner(message=None, enabled=True, json=False):
             If False, usage is a no-op.
         json (bool):
            If True, will not output non-json to stdout.
-
     """
     sp = Spinner(enabled)
     exception_raised = False
@@ -390,21 +389,48 @@ def close(self):
         self.enabled = False
 
 
-@contextmanager
-def backdown_thread_pool(max_workers=10):
-    """Tries to create an executor with max_workers, but will back down ultimately to a single
-    thread of the OS decides you can't have more than one.
-    """
-    try:
-        yield ThreadPoolExecutor(max_workers)
-    except RuntimeError as e:  # pragma: no cover
-        # RuntimeError is thrown if number of threads are limited by OS
-        log.debug(repr(e))
-        try:
-            yield ThreadPoolExecutor(floor(max_workers / 2))
-        except RuntimeError as e:
-            log.debug(repr(e))
-            yield ThreadPoolExecutor(1)
+class ThreadLimitedThreadPoolExecutor(ThreadPoolExecutor):
+
+    def __init__(self, max_workers=10):
+        super(ThreadLimitedThreadPoolExecutor, self).__init__(max_workers)
+
+    def submit(self, fn, *args, **kwargs):
+        """
+        This is an exact reimplementation of the `submit()` method on the parent class, except
+        with an added `try/except` around `self._adjust_thread_count()`.  So long as there is at
+        least one living thread, this thread pool will not throw an exception if threads cannot
+        be expanded to `max_workers`.
+
+        In the implementation, we use "protected" attributes from concurrent.futures (`_base`
+        and `_WorkItem`). Consider vendoring the whole concurrent.futures library
+        as an alternative to these protected imports.
+
+        https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131  # NOQA
+        https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124
+        """
+        with self._shutdown_lock:
+            if self._shutdown:
+                raise RuntimeError('cannot schedule new futures after shutdown')
+
+            f = _base.Future()
+            w = _WorkItem(f, fn, args, kwargs)
+
+            self._work_queue.put(w)
+            try:
+                self._adjust_thread_count()
+            except RuntimeError:
+                # RuntimeError: can't start new thread
+                # See https://github.com/conda/conda/issues/6624
+                if len(self._threads) > 0:
+                    # It's ok to not be able to start new threads if we already have at least
+                    # one thread alive.
+                    pass
+                else:
+                    raise
+            return f
+
+
+as_completed = as_completed  # anchored here for import by other modules
 
 
 class ContextDecorator(object):
@@ -427,6 +453,7 @@ def __enter__(self):
         enabled = os.environ.get('CONDA_INSTRUMENTATION_ENABLED')
         if enabled and boolify(enabled):
             self.start_time = time()
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.start_time:
diff --git a/conda/core/index.py b/conda/core/index.py
--- a/conda/core/index.py
+++ b/conda/core/index.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from concurrent.futures import as_completed
 from itertools import chain
 from logging import getLogger
 
@@ -11,7 +10,7 @@
 from .._vendor.boltons.setutils import IndexedSet
 from ..base.context import context
 from ..common.compat import iteritems, itervalues
-from ..common.io import backdown_thread_pool, time_recorder
+from ..common.io import ThreadLimitedThreadPoolExecutor, as_completed, time_recorder
 from ..exceptions import OperationNotAllowed
 from ..models.channel import Channel, all_channel_urls
 from ..models.dist import Dist
@@ -162,7 +161,7 @@ def get_reduced_index(prefix, channels, subdirs, specs):
     #                 keep_specs.append(spec)
     #         consolidated_specs.update(keep_specs)
 
-    with backdown_thread_pool() as executor:
+    with ThreadLimitedThreadPoolExecutor() as executor:
 
         channel_urls = all_channel_urls(channels, subdirs=subdirs)
         check_whitelist(channel_urls)
diff --git a/conda/core/repodata.py b/conda/core/repodata.py
--- a/conda/core/repodata.py
+++ b/conda/core/repodata.py
@@ -23,6 +23,7 @@
 from ..base.context import context
 from ..common.compat import (ensure_binary, ensure_text_type, ensure_unicode, text_type,
                              with_metaclass)
+from ..common.io import ThreadLimitedThreadPoolExecutor, as_completed
 from ..common.url import join_url, maybe_unquote
 from ..core.package_cache import PackageCache
 from ..exceptions import CondaDependencyError, CondaHTTPError, CondaIndexError, NotWritableError
@@ -75,23 +76,14 @@ class SubdirData(object):
 
     @staticmethod
     def query_all(channels, subdirs, package_ref_or_match_spec):
+        from .index import check_whitelist  # TODO: fix in-line import
         channel_urls = all_channel_urls(channels, subdirs=subdirs)
-
-        executor = None
-        try:
-            from concurrent.futures import ThreadPoolExecutor, as_completed
-            executor = ThreadPoolExecutor(10)
+        check_whitelist(channel_urls)
+        with ThreadLimitedThreadPoolExecutor() as executor:
             futures = (executor.submit(
                 SubdirData(Channel(url)).query, package_ref_or_match_spec
             ) for url in channel_urls)
             return tuple(concat(future.result() for future in as_completed(futures)))
-        except RuntimeError as e:  # pragma: no cover
-            # concurrent.futures is only available in Python >= 3.2 or if futures is installed
-            # RuntimeError is thrown if number of threads are limited by OS
-            raise
-        finally:
-            if executor:
-                executor.shutdown(wait=True)
 
     def query(self, package_ref_or_match_spec):
         if not self._loaded:


Test patch: