Repository: tqdm/tqdm (Python)

Issue:
add `nrows`
- expose a new argument `nrows=None`
    + default: auto-detect terminal height with some sensible default such as `20` (similar to `ncols`)
- hide bars `if n < total and pos >= nrows`
    + completed/`close()`d bars will ignore `nrows` and just print according to `leave`
- [ ] TODO: link related issues

This fixes issues where there are more nested bars than room for on a screen.

Cross-platform auto-detection compatibility will be at least as annoying as is for `ncols`.
tqdm.contrib.concurrent.process_map: Add 'chunksize' argument
Allow to set ProcessPoolExecutor.map 'chunksize' argument and provide
sane default value if unset.

- [x] I have marked all applicable categories:
    + [ ] exception-raising fix
    + [ ] visual output fix
    + [ ] documentation modification
    + [x] new feature
- [x] If applicable, I have mentioned the relevant/related issue(s)

Less important but also useful:

- [x] I have visited the [source website], and in particular
  read the [known issues]
- [x] I have searched through the [issue tracker] for duplicates
- [x] I have mentioned version numbers, operating system and
  environment, where applicable:
  ```python
  import tqdm, sys
  print(tqdm.__version__, sys.version, sys.platform)
  ```

  [source website]: https://github.com/tqdm/tqdm/
  [known issues]: https://github.com/tqdm/tqdm/#faq-and-known-issues
  [issue tracker]: https://github.com/tqdm/tqdm/issues?q=



Hints:

Patch:
diff --git a/tqdm/contrib/concurrent.py b/tqdm/contrib/concurrent.py
--- a/tqdm/contrib/concurrent.py
+++ b/tqdm/contrib/concurrent.py
@@ -2,8 +2,18 @@
 Thin wrappers around `concurrent.futures`.
 """
 from __future__ import absolute_import
+from tqdm import TqdmWarning
 from tqdm.auto import tqdm as tqdm_auto
 from copy import deepcopy
+try:
+    from operator import length_hint
+except ImportError:
+    def length_hint(it, default=0):
+        """Returns `len(it)`, falling back to `default`"""
+        try:
+            return len(it)
+        except TypeError:
+            return default
 try:
     from os import cpu_count
 except ImportError:
@@ -24,19 +34,27 @@ def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs):
     Parameters
     ----------
     tqdm_class  : [default: tqdm.auto.tqdm].
+    max_workers  : [default: max(32, cpu_count() + 4)].
+    chunksize  : [default: 1].
     """
     kwargs = deepcopy(tqdm_kwargs)
     if "total" not in kwargs:
         kwargs["total"] = len(iterables[0])
     tqdm_class = kwargs.pop("tqdm_class", tqdm_auto)
     max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4))
+    chunksize = kwargs.pop("chunksize", 1)
     pool_kwargs = dict(max_workers=max_workers)
-    if sys.version_info[:2] >= (3, 7):
+    sys_version = sys.version_info[:2]
+    if sys_version >= (3, 7):
         # share lock in case workers are already using `tqdm`
         pool_kwargs.update(
             initializer=tqdm_class.set_lock, initargs=(tqdm_class.get_lock(),))
+    map_args = {}
+    if not (3, 0) < sys_version < (3, 5):
+        map_args.update(chunksize=chunksize)
     with PoolExecutor(**pool_kwargs) as ex:
-        return list(tqdm_class(ex.map(fn, *iterables), **kwargs))
+        return list(tqdm_class(
+            ex.map(fn, *iterables, **map_args), **kwargs))
 
 
 def thread_map(fn, *iterables, **tqdm_kwargs):
@@ -46,7 +64,12 @@ def thread_map(fn, *iterables, **tqdm_kwargs):
 
     Parameters
     ----------
-    tqdm_class  : [default: tqdm.auto.tqdm].
+    tqdm_class : optional
+        `tqdm` class to use for bars [default: tqdm.auto.tqdm].
+    max_workers : int, optional
+        Maximum number of workers to spawn; passed to
+        `concurrent.futures.ThreadPoolExecutor.__init__`.
+        [default: max(32, cpu_count() + 4)].
     """
     from concurrent.futures import ThreadPoolExecutor
     return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)
@@ -59,7 +82,25 @@ def process_map(fn, *iterables, **tqdm_kwargs):
 
     Parameters
     ----------
-    tqdm_class  : [default: tqdm.auto.tqdm].
+    tqdm_class  : optional
+        `tqdm` class to use for bars [default: tqdm.auto.tqdm].
+    max_workers : int, optional
+        Maximum number of workers to spawn; passed to
+        `concurrent.futures.ProcessPoolExecutor.__init__`.
+        [default: max(32, cpu_count() + 4)].
+    chunksize : int, optional
+        Size of chunks sent to worker processes; passed to
+        `concurrent.futures.ProcessPoolExecutor.map`. [default: 1].
     """
     from concurrent.futures import ProcessPoolExecutor
+    if iterables and "chunksize" not in tqdm_kwargs:
+        # default `chunksize=1` has poor performance for large iterables
+        # (most time spent dispatching items to workers).
+        longest_iterable_len = max(map(length_hint, iterables))
+        if longest_iterable_len > 1000:
+            from warnings import warn
+            warn("Iterable length %d > 1000 but `chunksize` is not set."
+                 " This may seriously degrade multiprocess performance."
+                 " Set `chunksize=1` or more." % longest_iterable_len,
+                 TqdmWarning, stacklevel=2)
     return _executor_map(ProcessPoolExecutor, fn, *iterables, **tqdm_kwargs)
diff --git a/tqdm/std.py b/tqdm/std.py
--- a/tqdm/std.py
+++ b/tqdm/std.py
@@ -9,7 +9,7 @@
 """
 from __future__ import absolute_import, division
 # compatibility functions and utilities
-from .utils import _supports_unicode, _environ_cols_wrapper, _range, _unich, \
+from .utils import _supports_unicode, _screen_shape_wrapper, _range, _unich, \
     _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \
     Comparable, _is_ascii, FormatReplace, disp_len, disp_trim, \
     SimpleTextIOWrapper, CallbackIOWrapper
@@ -352,7 +352,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
             r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '
               '{rate_fmt}{postfix}]'
             Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
-              percentage, elapsed, elapsed_s, ncols, desc, unit,
+              percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,
               rate, rate_fmt, rate_noinv, rate_noinv_fmt,
               rate_inv, rate_inv_fmt, postfix, unit_divisor,
               remaining, remaining_s.
@@ -790,6 +790,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
                  unit_scale=False, dynamic_ncols=False, smoothing=0.3,
                  bar_format=None, initial=0, position=None, postfix=None,
                  unit_divisor=1000, write_bytes=None, lock_args=None,
+                 nrows=None,
                  gui=False, **kwargs):
         """
         Parameters
@@ -852,8 +853,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
             (kilo, mega, etc.) [default: False]. If any other non-zero
             number, will scale `total` and `n`.
         dynamic_ncols  : bool, optional
-            If set, constantly alters `ncols` to the environment (allowing
-            for window resizes) [default: False].
+            If set, constantly alters `ncols` and `nrows` to the
+            environment (allowing for window resizes) [default: False].
         smoothing  : float, optional
             Exponential moving average smoothing factor for speed estimates
             (ignored in GUI mode). Ranges from 0 (average speed) to 1
@@ -865,7 +866,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
             r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '
               '{rate_fmt}{postfix}]'
             Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
-              percentage, elapsed, elapsed_s, ncols, desc, unit,
+              percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,
               rate, rate_fmt, rate_noinv, rate_noinv_fmt,
               rate_inv, rate_inv_fmt, postfix, unit_divisor,
               remaining, remaining_s.
@@ -891,6 +892,10 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
         lock_args  : tuple, optional
             Passed to `refresh` for intermediate output
             (initialisation, iterating, and updating).
+        nrows  : int, optional
+            The screen height. If specified, hides nested bars outside this
+            bound. If unspecified, attempts to use environment height.
+            The fallback is 20.
         gui  : bool, optional
             WARNING: internal parameter - do not use.
             Use tqdm.gui.tqdm(...) instead. If set, will attempt to use
@@ -949,16 +954,21 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
                 TqdmKeyError("Unknown argument(s): " + str(kwargs)))
 
         # Preprocess the arguments
-        if ((ncols is None) and (file in (sys.stderr, sys.stdout))) or \
+        if ((ncols is None or nrows is None) and
+            (file in (sys.stderr, sys.stdout))) or \
                 dynamic_ncols:  # pragma: no cover
             if dynamic_ncols:
-                dynamic_ncols = _environ_cols_wrapper()
+                dynamic_ncols = _screen_shape_wrapper()
                 if dynamic_ncols:
-                    ncols = dynamic_ncols(file)
+                    ncols, nrows = dynamic_ncols(file)
             else:
-                _dynamic_ncols = _environ_cols_wrapper()
+                _dynamic_ncols = _screen_shape_wrapper()
                 if _dynamic_ncols:
-                    ncols = _dynamic_ncols(file)
+                    _ncols, _nrows = _dynamic_ncols(file)
+                    if ncols is None:
+                        ncols = _ncols
+                    if nrows is None:
+                        nrows = _nrows
 
         if miniters is None:
             miniters = 0
@@ -989,6 +999,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
         self.leave = leave
         self.fp = file
         self.ncols = ncols
+        self.nrows = nrows
         self.mininterval = mininterval
         self.maxinterval = maxinterval
         self.miniters = miniters
@@ -1248,8 +1259,9 @@ def close(self):
         pos = abs(self.pos)
         self._decr_instances(self)
 
-        # GUI mode
-        if not hasattr(self, "sp"):
+        # GUI mode or overflow
+        if not hasattr(self, "sp") or pos >= (self.nrows or 20):
+            # never printed so nothing to do
             return
 
         # annoyingly, _supports_unicode isn't good enough
@@ -1283,10 +1295,12 @@ def clear(self, nolock=False):
 
         if not nolock:
             self._lock.acquire()
-        self.moveto(abs(self.pos))
-        self.sp('')
-        self.fp.write('\r')  # place cursor back at the beginning of line
-        self.moveto(-abs(self.pos))
+        pos = abs(self.pos)
+        if pos < (self.nrows or 20):
+            self.moveto(pos)
+            self.sp('')
+            self.fp.write('\r')  # place cursor back at the beginning of line
+            self.moveto(-pos)
         if not nolock:
             self._lock.release()
 
@@ -1406,12 +1420,14 @@ def moveto(self, n):
     @property
     def format_dict(self):
         """Public API for read-only member access."""
+        if self.dynamic_ncols:
+            self.ncols, self.nrows = self.dynamic_ncols(self.fp)
+        ncols, nrows = self.ncols, self.nrows
         return dict(
             n=self.n, total=self.total,
             elapsed=self._time() - self.start_t
             if hasattr(self, 'start_t') else 0,
-            ncols=self.dynamic_ncols(self.fp)
-            if self.dynamic_ncols else self.ncols,
+            ncols=ncols, nrows=nrows,
             prefix=self.desc, ascii=self.ascii, unit=self.unit,
             unit_scale=self.unit_scale,
             rate=1 / self.avg_time if self.avg_time else None,
@@ -1434,6 +1450,12 @@ def display(self, msg=None, pos=None):
         if pos is None:
             pos = abs(self.pos)
 
+        nrows = self.nrows or 20
+        if pos >= nrows - 1:
+            if pos >= nrows:
+                return
+            msg = " ... (more hidden) ..."
+
         if pos:
             self.moveto(pos)
         self.sp(self.__repr__() if msg is None else msg)
diff --git a/tqdm/utils.py b/tqdm/utils.py
--- a/tqdm/utils.py
+++ b/tqdm/utils.py
@@ -3,6 +3,8 @@
 from platform import system as _curos
 import re
 import subprocess
+from warnings import warn
+
 CUR_OS = _curos()
 IS_WIN = CUR_OS in ['Windows', 'cli']
 IS_NIX = (not IS_WIN) and any(
@@ -203,6 +205,9 @@ def write(self, s):
         """
         return self._wrapped.write(s.encode(self.wrapper_getattr('encoding')))
 
+    def __eq__(self, other):
+        return self._wrapped == getattr(other, '_wrapped', other)
+
 
 class CallbackIOWrapper(ObjectWrapper):
     def __init__(self, callback, stream, method="read"):
@@ -260,22 +265,22 @@ def _is_ascii(s):
     return _supports_unicode(s)
 
 
-def _environ_cols_wrapper():  # pragma: no cover
+def _screen_shape_wrapper():  # pragma: no cover
     """
-    Return a function which gets width and height of console
-    (linux,osx,windows,cygwin).
+    Return a function which returns console dimensions (width, height).
+    Supported: linux, osx, windows, cygwin.
     """
-    _environ_cols = None
+    _screen_shape = None
     if IS_WIN:
-        _environ_cols = _environ_cols_windows
-        if _environ_cols is None:
-            _environ_cols = _environ_cols_tput
+        _screen_shape = _screen_shape_windows
+        if _screen_shape is None:
+            _screen_shape = _screen_shape_tput
     if IS_NIX:
-        _environ_cols = _environ_cols_linux
-    return _environ_cols
+        _screen_shape = _screen_shape_linux
+    return _screen_shape
 
 
-def _environ_cols_windows(fp):  # pragma: no cover
+def _screen_shape_windows(fp):  # pragma: no cover
     try:
         from ctypes import windll, create_string_buffer
         import struct
@@ -291,28 +296,26 @@ def _environ_cols_windows(fp):  # pragma: no cover
         csbi = create_string_buffer(22)
         res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
         if res:
-            (_bufx, _bufy, _curx, _cury, _wattr, left, _top, right, _bottom,
+            (_bufx, _bufy, _curx, _cury, _wattr, left, top, right, bottom,
              _maxx, _maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
-            # nlines = bottom - top + 1
-            return right - left  # +1
+            return right - left, bottom - top  # +1
     except:
         pass
-    return None
+    return None, None
 
 
-def _environ_cols_tput(*_):  # pragma: no cover
+def _screen_shape_tput(*_):  # pragma: no cover
     """cygwin xterm (windows)"""
     try:
         import shlex
-        cols = int(subprocess.check_call(shlex.split('tput cols')))
-        # rows = int(subprocess.check_call(shlex.split('tput lines')))
-        return cols
+        return [int(subprocess.check_call(shlex.split('tput ' + i))) - 1
+                for i in ('cols', 'lines')]
     except:
         pass
-    return None
+    return None, None
 
 
-def _environ_cols_linux(fp):  # pragma: no cover
+def _screen_shape_linux(fp):  # pragma: no cover
 
     try:
         from termios import TIOCGWINSZ
@@ -322,12 +325,31 @@ def _environ_cols_linux(fp):  # pragma: no cover
         return None
     else:
         try:
-            return array('h', ioctl(fp, TIOCGWINSZ, '\0' * 8))[1]
+            rows, cols = array('h', ioctl(fp, TIOCGWINSZ, '\0' * 8))[:2]
+            return cols, rows
         except:
             try:
-                return int(os.environ["COLUMNS"]) - 1
+                return [int(os.environ[i]) - 1 for i in ("COLUMNS", "LINES")]
             except KeyError:
-                return None
+                return None, None
+
+
+def _environ_cols_wrapper():  # pragma: no cover
+    """
+    Return a function which returns console width.
+    Supported: linux, osx, windows, cygwin.
+    """
+    warn("Use `_screen_shape_wrapper()(file)[0]` instead of"
+         " `_environ_cols_wrapper()(file)`", DeprecationWarning, stacklevel=2)
+    shape = _screen_shape_wrapper()
+    if not shape:
+        return None
+
+    @wraps(shape)
+    def inner(fp):
+        return shape(fp)[0]
+
+    return inner
 
 
 def _term_move_up():  # pragma: no cover


Test patch:
diff --git a/tqdm/tests/tests_concurrent.py b/tqdm/tests/tests_concurrent.py
--- a/tqdm/tests/tests_concurrent.py
+++ b/tqdm/tests/tests_concurrent.py
@@ -1,6 +1,7 @@
 """
 Tests for `tqdm.contrib.concurrent`.
 """
+from warnings import catch_warnings
 from tqdm.contrib.concurrent import thread_map, process_map
 from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \
     closing
@@ -34,3 +35,24 @@ def test_process_map():
             assert process_map(incr, a, file=our_file) == b
         except ImportError:
             raise SkipTest
+
+
+def test_chunksize_warning():
+    """Test contrib.concurrent.process_map chunksize warnings"""
+    try:
+        from unittest.mock import patch
+    except ImportError:
+        raise SkipTest
+
+    for iterables, should_warn in [
+        ([], False),
+        (['x'], False),
+        ([()], False),
+        (['x', ()], False),
+        (['x' * 1001], True),
+        (['x' * 100, ('x',) * 1001], True),
+    ]:
+        with patch('tqdm.contrib.concurrent._executor_map'):
+            with catch_warnings(record=True) as w:
+                process_map(incr, *iterables)
+                assert should_warn == bool(w)
diff --git a/tqdm/tests/tests_tqdm.py b/tqdm/tests/tests_tqdm.py
--- a/tqdm/tests/tests_tqdm.py
+++ b/tqdm/tests/tests_tqdm.py
@@ -194,8 +194,7 @@ def squash_ctrlchars(s):
     lines = ['']  # state of our fake terminal
 
     # Split input string by control codes
-    RE_ctrl = re.compile("(%s)" % ("|".join(CTRLCHR)), flags=re.DOTALL)
-    s_split = RE_ctrl.split(s)
+    s_split = RE_ctrlchr.split(s)
     s_split = filter(None, s_split)  # filter out empty splits
 
     # For each control character or message
@@ -1890,3 +1889,74 @@ def test_float_progress():
                         assert not w
                 assert w
                 assert "clamping frac" in str(w[-1].message)
+
+
+@with_setup(pretest, posttest)
+def test_screen_shape():
+    """Test screen shape"""
+    # ncols
+    with closing(StringIO()) as our_file:
+        with trange(10, file=our_file, ncols=50) as t:
+            list(t)
+
+        res = our_file.getvalue()
+        assert all(len(i.strip('\n')) in (0, 50) for i in res.split('\r'))
+
+    # no second bar, leave=False
+    with closing(StringIO()) as our_file:
+        kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0,
+                      mininterval=0, leave=False)
+        with trange(10, desc="one", **kwargs) as t1:
+            with trange(10, desc="two", **kwargs) as t2:
+                list(t2)
+            list(t1)
+
+        res = our_file.getvalue()
+        assert "one" in res
+        assert "two" not in res
+        assert "\n\n" not in res
+        assert "more hidden" in res
+        # double-check ncols
+        assert all(len(i) in (0, 50) for i in squash_ctrlchars(res)
+                   if "more hidden" not in i)
+
+    # no third bar, leave=True
+    with closing(StringIO()) as our_file:
+        kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0,
+                      mininterval=0)
+        with trange(10, desc="one", **kwargs) as t1:
+            with trange(10, desc="two", **kwargs) as t2:
+                assert "two" not in our_file.getvalue()
+                with trange(10, desc="three", **kwargs) as t3:
+                    list(t3)
+                list(t2)
+            list(t1)
+
+        res = our_file.getvalue()
+        assert "one" in res
+        assert "two" in res
+        assert "three" not in res
+        assert "\n\n" not in res
+        assert "more hidden" in res
+        # double-check ncols
+        assert all(len(i) in (0, 50) for i in squash_ctrlchars(res)
+                   if "more hidden" not in i)
+
+    # second bar becomes first, leave=False
+    with closing(StringIO()) as our_file:
+        kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0,
+                      mininterval=0, leave=False)
+        t1 = tqdm(total=10, desc="one", **kwargs)
+        t2 = tqdm(total=10, desc="two", **kwargs)
+        t1.update()
+        t2.update()
+        t1.close()
+        res = our_file.getvalue()
+        assert "one" in res
+        assert "two" not in res
+        assert "more hidden" in res
+        t2.update()
+        t2.close()
+
+        res = our_file.getvalue()
+        assert "two" in res