Repository: tqdm/tqdm (Python)

Issue:
Allow bar_format without total or len(iterable)
(This builds on https://github.com/tqdm/tqdm/pull/283 to allow me to test it locally.)

This PR is a solution to my issue here: https://github.com/tqdm/tqdm/issues/282 .

Allow bar_format without total
Thanks for `tqdm`!  It's a wonderful tool.

In order to use `bar_format`, `total` must be used.  I want to be able to do this without `total=int(1e100)`:

```
import tqdm, time
slow_iter = (time.sleep(1e-4) or i for i in range(int(1e5)))
with tqdm.tqdm(slow_iter, total=int(1e100), bar_format='Found {n:7} files') as prog_iter:
     for i in prog_iter:
          if i % int(1e4) == 0:
               prog_iter.write('Woohoo,{:6} files!'.format(i))
```



Hints:
## [Current coverage](https://codecov.io/gh/tqdm/tqdm/pull/284?src=pr) is 90.49% (diff: 100%)

> Merging [#284](https://codecov.io/gh/tqdm/tqdm/pull/284?src=pr) into [master](https://codecov.io/gh/tqdm/tqdm/branch/master?src=pr) will decrease coverage by **1.27%**

``` diff
@@             master       #284   diff @@
==========================================
  Files             7          7          
  Lines           535        484    -51   
  Methods           0          0          
  Messages          0          0          
  Branches         97         87    -10   
==========================================
- Hits            491        438    -53   
- Misses           43         44     +1   
- Partials          1          2     +1   
```

> Powered by [Codecov](https://codecov.io?src=pr). Last update [a65e347...fb4b525](https://codecov.io/gh/tqdm/tqdm/compare/a65e34751843141527b0af76f8133be7e92fa043...fb4b525391f41cbcd043d1ffe82f4cecb1f59e3e?src=pr)

Thank you @pjvandehaar, this will be merged with #223 which already implements bar_total when there is no total, but your PR has: an extended docstring (should also be added to the readme) + skip if coverall fails.

Alright.  Feel free to just copy that docstring and reject this PR.

Yes this will be implemented by #223 (I know it's a bit obscure, sorry about that, there are a lot of things in this PR). But thank you for your PR, we will merge it with #223.


Patch:
diff --git a/mkdocs.py b/mkdocs.py
new file mode 100644
--- /dev/null
+++ b/mkdocs.py
@@ -0,0 +1,58 @@
+from __future__ import print_function
+import tqdm
+from textwrap import dedent
+from io import open as io_open
+from os import path
+
+HEAD_ARGS = """
+Parameters
+----------
+"""
+HEAD_RETS = """
+Returns
+-------
+"""
+HEAD_CLI = """
+Extra CLI Options
+-----------------
+name  : type, optional
+    TODO: find out why this is needed.
+"""
+
+
+def doc2rst(doc, arglist=True):
+    """
+    arglist  : bool, whether to create argument lists
+    """
+    doc = dedent(doc).replace('`', '``')
+    if arglist:
+        doc = '\n'.join([i if not i or i[0] == ' ' else '* ' + i + '  '
+                         for i in doc.split('\n')])
+    return doc
+
+
+src_dir = path.abspath(path.dirname(__file__))
+README_rst = path.join(src_dir, '.readme.rst')
+with io_open(README_rst, mode='r', encoding='utf-8') as fd:
+    README_rst = fd.read()
+DOC_tqdm = doc2rst(tqdm.tqdm.__doc__, False).replace('\n', '\n      ')
+DOC_tqdm_init = doc2rst(tqdm.tqdm.__init__.__doc__)
+DOC_tqdm_init_args = DOC_tqdm_init.partition(doc2rst(HEAD_ARGS))[-1]\
+    .replace('\n      ', '\n    ')
+DOC_tqdm_init_args, _, DOC_tqdm_init_rets = DOC_tqdm_init_args\
+    .partition(doc2rst(HEAD_RETS))
+DOC_cli = doc2rst(tqdm._main.CLI_EXTRA_DOC).partition(doc2rst(HEAD_CLI))[-1]
+
+# special cases
+DOC_tqdm_init_args = DOC_tqdm_init_args.replace(' *,', ' ``*``,')
+DOC_tqdm_init_args = DOC_tqdm_init_args.partition('* gui  : bool, optional')[0]
+
+README_rst = README_rst.replace('{DOC_tqdm}', DOC_tqdm)\
+    .replace('{DOC_tqdm.tqdm.__init__.Parameters}', DOC_tqdm_init_args)\
+    .replace('{DOC_tqdm._main.CLI_EXTRA_DOC}', DOC_cli)\
+    .replace('{DOC_tqdm.tqdm.__init__.Returns}', DOC_tqdm_init_rets)
+
+if __name__ == "__main__":
+    fndoc = path.join(src_dir, 'README.rst')
+    with io_open(fndoc, mode='w', encoding='utf-8') as fd:
+        fd.write(README_rst)
diff --git a/tqdm/_main.py b/tqdm/_main.py
--- a/tqdm/_main.py
+++ b/tqdm/_main.py
@@ -99,7 +99,7 @@ def posix_pipe(fin, fout, delim='\n', buf_size=256,
         Extra CLI Options
         -----------------
         name  : type, optional
-             TODO: find out why this is needed.
+            TODO: find out why this is needed.
         delim  : chr, optional
             Delimiting character [default: '\n']. Use '\0' for null.
             N.B.: on Windows systems, Python converts '\n' to '\r\n'.
diff --git a/tqdm/_tqdm.py b/tqdm/_tqdm.py
--- a/tqdm/_tqdm.py
+++ b/tqdm/_tqdm.py
@@ -13,7 +13,7 @@
 # compatibility functions and utilities
 from ._utils import _supports_unicode, _environ_cols_wrapper, _range, _unich, \
     _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \
-    Comparable, RE_ANSI
+    Comparable, RE_ANSI, _is_ascii
 from ._monitor import TMonitor
 # native libraries
 import sys
@@ -125,6 +125,9 @@ def create_th_lock(cls):
 # context and does not allow the user to use 'spawn' or 'forkserver' methods.
 TqdmDefaultWriteLock.create_th_lock()
 
+ASCII_FMT = " 123456789#"
+UTF_FMT = u" " + u''.join(map(_unich, range(0x258F, 0x2587, -1)))
+
 
 class tqdm(Comparable):
     """
@@ -274,10 +277,10 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
         prefix  : str, optional
             Prefix message (included in total width) [default: ''].
             Use as {desc} in bar_format string.
-        ascii  : bool, optional
+        ascii  : bool, optional or str, optional
             If not set, use unicode (smooth blocks) to fill the meter
             [default: False]. The fallback is to use ASCII characters
-            (1-9 #).
+            " 123456789#".
         unit  : str, optional
             The iteration unit [default: 'it'].
         unit_scale  : bool or int or float, optional
@@ -296,7 +299,8 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
               '{rate_fmt}{postfix}]'
             Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
               percentage, rate, rate_fmt, rate_noinv, rate_noinv_fmt,
-              rate_inv, rate_inv_fmt, elapsed, remaining, desc, postfix.
+              rate_inv, rate_inv_fmt, elapsed, elapsed_s,
+              remaining, remaining_s, desc, postfix, unit.
             Note that a trailing ": " is automatically removed after {desc}
             if the latter is empty.
         postfix  : *, optional
@@ -319,14 +323,14 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
 
         # apply custom scale if necessary
         if unit_scale and unit_scale not in (True, 1):
-            total *= unit_scale
+            if total:
+                total *= unit_scale
             n *= unit_scale
             if rate:
                 rate *= unit_scale  # by default rate = 1 / self.avg_time
             unit_scale = False
 
-        format_interval = tqdm.format_interval
-        elapsed_str = format_interval(elapsed)
+        elapsed_str = tqdm.format_interval(elapsed)
 
         # if unspecified, attempt to use rate = average speed
         # (we allow manual override since predicting time is an arcane art)
@@ -345,53 +349,59 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
         if unit_scale:
             n_fmt = format_sizeof(n, divisor=unit_divisor)
             total_fmt = format_sizeof(total, divisor=unit_divisor) \
-                if total else None
+                if total is not None else '?'
         else:
             n_fmt = str(n)
-            total_fmt = str(total)
+            total_fmt = str(total) if total is not None else '?'
 
         try:
             postfix = ', ' + postfix if postfix else ''
         except TypeError:
             pass
 
+        remaining = (total - n) / rate if rate and total else 0
+        remaining_str = tqdm.format_interval(remaining) if rate else '?'
+
+        # format the stats displayed to the left and right sides of the bar
+        if prefix:
+            # old prefix setup work around
+            bool_prefix_colon_already = (prefix[-2:] == ": ")
+            l_bar = prefix if bool_prefix_colon_already else prefix + ": "
+        else:
+            l_bar = ''
+
+        r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
+            n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix)
+
+        # Custom bar formatting
+        # Populate a dict with all available progress indicators
+        format_dict = dict(
+            n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt,
+            rate=inv_rate if inv_rate and inv_rate > 1 else rate,
+            rate_fmt=rate_fmt, rate_noinv=rate,
+            rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate,
+            rate_inv_fmt=rate_inv_fmt,
+            elapsed=elapsed_str, elapsed_s=elapsed,
+            remaining=remaining_str, remaining_s=remaining,
+            l_bar=l_bar, r_bar=r_bar,
+            desc=prefix or '', postfix=postfix, unit=unit,
+            # bar=full_bar,  # replaced by procedure below
+            **extra_kwargs)
+
         # total is known: we can predict some stats
         if total:
             # fractional and percentage progress
             frac = n / total
             percentage = frac * 100
 
-            remaining_str = format_interval((total - n) / rate) \
-                if rate else '?'
-
-            # format the stats displayed to the left and right sides of the bar
-            if prefix:
-                # old prefix setup work around
-                bool_prefix_colon_already = (prefix[-2:] == ": ")
-                l_bar = prefix if bool_prefix_colon_already else prefix + ": "
-            else:
-                l_bar = ''
             l_bar += '{0:3.0f}%|'.format(percentage)
-            r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
-                n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix)
 
             if ncols == 0:
                 return l_bar[:-1] + r_bar[1:]
 
             if bar_format:
-                # Custom bar formatting
-                # Populate a dict with all available progress indicators
-                format_dict = dict(
-                    n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt,
-                    percentage=percentage,
-                    rate=inv_rate if inv_rate and inv_rate > 1 else rate,
-                    rate_fmt=rate_fmt, rate_noinv=rate,
-                    rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate,
-                    rate_inv_fmt=rate_inv_fmt, elapsed=elapsed_str,
-                    remaining=remaining_str, l_bar=l_bar, r_bar=r_bar,
-                    desc=prefix or '', postfix=postfix,
-                    # bar=full_bar,  # replaced by procedure below
-                    **extra_kwargs)
+                format_dict.update(l_bar=l_bar, percentage=percentage)
+                # , bar=full_bar  # replaced by procedure below
 
                 # auto-remove colon for empty `desc`
                 if not prefix:
@@ -415,34 +425,33 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
                 N_BARS = 10
 
             # format bar depending on availability of unicode/ascii chars
-            if ascii:
-                bar_length, frac_bar_length = divmod(
-                    int(frac * N_BARS * 10), 10)
-
-                bar = '#' * bar_length
-                frac_bar = chr(48 + frac_bar_length) if frac_bar_length \
-                    else ' '
-
-            else:
-                bar_length, frac_bar_length = divmod(int(frac * N_BARS * 8), 8)
+            if ascii is True:
+                ascii = ASCII_FMT
+            elif ascii is False:
+                ascii = UTF_FMT
+            nsyms = len(ascii) - 1
+            bar_length, frac_bar_length = divmod(
+                    int(frac * N_BARS * nsyms), nsyms)
 
-                bar = _unich(0x2588) * bar_length
-                frac_bar = _unich(0x2590 - frac_bar_length) \
-                    if frac_bar_length else ' '
+            bar = ascii[-1] * bar_length
+            frac_bar = ascii[frac_bar_length]
 
             # whitespace padding
             if bar_length < N_BARS:
                 full_bar = bar + frac_bar + \
-                    ' ' * max(N_BARS - bar_length - 1, 0)
+                    ascii[0] * (N_BARS - bar_length - 1)
             else:
                 full_bar = bar + \
-                    ' ' * max(N_BARS - bar_length, 0)
+                    ascii[0] * (N_BARS - bar_length)
 
             # Piece together the bar parts
             return l_bar + full_bar + r_bar
 
-        # no total: no progressbar, ETA, just progress stats
+        elif bar_format:
+            # user-specified bar_format but no total
+            return bar_format.format(bar='?', **format_dict)
         else:
+            # no total: no progressbar, ETA, just progress stats
             return ((prefix + ": ") if prefix else '') + \
                 '{0}{1} [{2}, {3}{4}]'.format(
                     n_fmt, unit, elapsed_str, rate_fmt, postfix)
@@ -746,9 +755,9 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
             Tweak this and `mininterval` to get very efficient loops.
             If your progress is erratic with both fast and slow iterations
             (network, skipping items, etc) you should set miniters=1.
-        ascii  : bool, optional
+        ascii  : bool or str, optional
             If unspecified or False, use unicode (smooth blocks) to fill
-            the meter. The fallback is to use ASCII characters `1-9 #`.
+            the meter. The fallback is to use ASCII characters " 123456789#".
         disable  : bool, optional
             Whether to disable the entire progressbar wrapper
             [default: False]. If set to None, disable on non-TTY.
@@ -776,7 +785,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
               '{rate_fmt}{postfix}]'
             Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
               percentage, rate, rate_fmt, rate_noinv, rate_noinv_fmt,
-              rate_inv, rate_inv_fmt, elapsed, remaining, desc, postfix.
+              rate_inv, rate_inv_fmt, elapsed, elapsed_s, remaining,
+              remaining_s, desc, postfix, unit.
             Note that a trailing ": " is automatically removed after {desc}
             if the latter is empty.
         initial  : int, optional
@@ -868,7 +878,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
         if ascii is None:
             ascii = not _supports_unicode(file)
 
-        if bar_format and not ascii:
+        if bar_format and not ((ascii is True) or _is_ascii(ascii)):
             # Convert bar format into unicode since terminal uses unicode
             bar_format = _unicode(bar_format)
 
diff --git a/tqdm/_utils.py b/tqdm/_utils.py
--- a/tqdm/_utils.py
+++ b/tqdm/_utils.py
@@ -163,6 +163,15 @@ def _supports_unicode(fp):
         return False
 
 
+def _is_ascii(s):
+    if isinstance(s, str):
+        for c in s:
+            if ord(c) > 255:
+                return False
+        return True
+    return _supports_unicode(s)
+
+
 def _environ_cols_wrapper():  # pragma: no cover
     """
     Return a function which gets width and height of console


Test patch:
diff --git a/tqdm/tests/tests_tqdm.py b/tqdm/tests/tests_tqdm.py
--- a/tqdm/tests/tests_tqdm.py
+++ b/tqdm/tests/tests_tqdm.py
@@ -69,16 +69,16 @@ def pos_line_diff(res_list, expected_list, raise_nonempty=True):
     Return differences between two bar output lists.
     To be used with `RE_pos`
     """
-    l = len(res_list)
-    if l < len(expected_list):
-        res = [(None, e) for e in expected_list[l:]]
-    elif l > len(expected_list):
-        res = [(r, None) for r in res_list[l:]]
+    ln = len(res_list)
+    if ln < len(expected_list):
+        res = [(None, e) for e in expected_list[ln:]]
+    elif ln > len(expected_list):
+        res = [(r, None) for r in res_list[ln:]]
     res = [(r, e) for r, e in zip(res_list, expected_list)
            for pos in [len(e)-len(e.lstrip('\n'))]  # bar position
            if not r.startswith(e)  # start matches
            or not (r.endswith('\x1b[A' * pos)  # move up at end
-                   or r=='\n')  # final bar
+                   or r == '\n')  # final bar
            or r[(-1-pos) * len('\x1b[A'):] == '\x1b[A']  # extra move up
     if res and raise_nonempty:
         raise AssertionError(
@@ -772,6 +772,21 @@ def test_infinite_total():
             pass
 
 
+@with_setup(pretest, posttest)
+def test_nototal():
+    """Test unknown total length"""
+    with closing(StringIO()) as our_file:
+        for i in tqdm((i for i in range(10)), file=our_file, unit_scale=10):
+            pass
+        assert "100it" in our_file.getvalue()
+
+    with closing(StringIO()) as our_file:
+        for i in tqdm((i for i in range(10)), file=our_file,
+                      bar_format="{l_bar}{bar}{r_bar}"):
+            pass
+        assert "10/?" in our_file.getvalue()
+
+
 @with_setup(pretest, posttest)
 def test_unit():
     """Test SI unit prefix"""
@@ -1152,7 +1167,6 @@ def test_position():
                  '\n\n\rpos2 bar:   0%']
         pos_line_diff(res, exres)
 
-
         t2.close()
         t4 = tqdm(total=10, file=our_file, desc='pos3 bar', mininterval=0)
         t1.update(1)