Repository: tqdm/tqdm (Python)

Issue:
Recognize wide characters when calculating bar size
Updated #410 to current master with added (and not failing) tests.

Closes #410. Fixes #803 

I also added a comment regarding using `wcswidth`, but I didn't want to add a dependency in a PR.
Use unicodedata.east_asian_width to determine actual width of provided text
Some unicode characters, notably glyphs used in Eastern Asia, are "full-width" and have the property to be rendered in two cells in most terminal emulators. Fortunately, there is a unicode property that specifies its width character and how it is to be rendered in fixed width environments. ([reference](https://en.wikipedia.org/wiki/Halfwidth_and_fullwidth_forms#In_Unicode)) Python's `unicodedata` module has this inside the `unicodedata.east_asian_width` function.

```
$ python --version
Python 3.7.4
$ tqdm --version
4.26.0
$ tqdm --desc "第4話 気づいたんですよね" --total 10
第4話 気づいたんですよね:   0%|                    | 0/10 [00:0
0<?, ?it/s]
```

![2019-09-05_14-38-24](https://user-images.githubusercontent.com/931051/64342480-e0a94f80-cfea-11e9-8f0f-de28e8274cb9.png)

My terminal emulator is alacritty. The issue was also verified in Konsole.

<details>

<summary>Example script illustrating a possible fix</summary>

```py
import unicodedata

tests = [
    "第4話 気づいたんですよね",
    "normal width ASCII text",
    "ｆｕｌｌｗｉｄｔｈ",
    "ニッポン",
    "ﾆｯﾎﾟﾝ",
]

for test in tests:
    print(f"{test} [{len(test)}]")
    cells = len(test) + sum(unicodedata.east_asian_width(c) in "FW" for c in test)
    print(f"{'#' * cells} [{cells}]")

    print("".join(map(unicodedata.category, test)))
    print("".join(map(unicodedata.east_asian_width, test)))
    print()
```

```
~ Δ python /tmp/wide_chars.py
第4話 気づいたんですよね [13]
######################## [24]
LoNdLoZsLoLoLoLoLoLoLoLoLo
WNaWNaWWWWWWWWW

normal width ASCII text [23]
####################### [23]
LlLlLlLlLlLlZsLlLlLlLlLlZsLuLuLuLuLuZsLlLlLlLl
NaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNa

ｆｕｌｌｗｉｄｔｈ [9]
################## [18]
LlLlLlLlLlLlLlLlLl
FFFFFFFFF

ニッポン [4]
######## [8]
LoLoLoLo
WWWW

ﾆｯﾎﾟﾝ [5]
##### [5]
LoLoLoLmLo
HHHHH
```

![2019-09-05_14-32-14](https://user-images.githubusercontent.com/931051/64342042-fa966280-cfe9-11e9-8d7f-c380e615fe7a.png)


</details>


Hints:

Patch:
diff --git a/tqdm/std.py b/tqdm/std.py
--- a/tqdm/std.py
+++ b/tqdm/std.py
@@ -12,7 +12,7 @@
 from __future__ import division
 # compatibility functions and utilities
 from .utils import _supports_unicode, _environ_cols_wrapper, _range, _unich, \
-    _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \
+    _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, _text_width, \
     Comparable, RE_ANSI, _is_ascii, SimpleTextIOWrapper, FormatReplace
 from ._monitor import TMonitor
 # native libraries
@@ -474,7 +474,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
             # Formatting progress bar space available for bar's display
             full_bar = Bar(
                 frac,
-                max(1, ncols - len(RE_ANSI.sub('', nobar))) if ncols else 10,
+                max(1, ncols - _text_width(RE_ANSI.sub('', nobar))) if ncols else 10,
                 charset=Bar.ASCII if ascii is True else ascii or Bar.UTF)
             if not _is_ascii(full_bar.charset) and _is_ascii(bar_format):
                 bar_format = _unicode(bar_format)
@@ -490,7 +490,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
                 return nobar
             full_bar = Bar(
                 0,
-                max(1, ncols - len(RE_ANSI.sub('', nobar))) if ncols else 10,
+                max(1, ncols - _text_width(RE_ANSI.sub('', nobar))) if ncols else 10,
                 charset=Bar.BLANK)
             return bar_format.format(bar=full_bar, **format_dict)
         else:
diff --git a/tqdm/utils.py b/tqdm/utils.py
--- a/tqdm/utils.py
+++ b/tqdm/utils.py
@@ -283,3 +283,13 @@ def _environ_cols_linux(fp):  # pragma: no cover
 
 def _term_move_up():  # pragma: no cover
     return '' if (os.name == 'nt') and (colorama is None) else '\x1b[A'
+
+
+try:
+    # TODO consider using wcswidth third-party package for 0-width characters
+    from unicodedata import east_asian_width
+except ImportError:
+    _text_width = len
+else:
+    def _text_width(s):
+        return sum(2 if east_asian_width(ch) in 'FW' else 1 for ch in _unicode(s))


Test patch:
diff --git a/tqdm/tests/tests_tqdm.py b/tqdm/tests/tests_tqdm.py
--- a/tqdm/tests/tests_tqdm.py
+++ b/tqdm/tests/tests_tqdm.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Advice: use repr(our_file.read()) to print the full output of tqdm
 # (else '\r' will replace the previous lines and you'll see only the latest.
 
@@ -277,6 +278,12 @@ def test_format_meter():
     assert format_meter(20, 100, 12, ncols=14, rate=8.1,
                         bar_format=r'{l_bar}{bar}|{n_fmt}/{total_fmt}') == \
         " 20%|" + unich(0x258d) + " |20/100"
+    # Check wide characters
+    if sys.version_info >= (3,):
+        assert format_meter(0, 1000, 13, ncols=68, prefix='ｆｕｌｌｗｉｄｔｈ: ') == \
+            "ｆｕｌｌｗｉｄｔｈ:   0%|                  | 0/1000 [00:13<?, ?it/s]"
+        assert format_meter(0, 1000, 13, ncols=68, prefix='ニッポン [ﾆｯﾎﾟﾝ]: ') == \
+            "ニッポン [ﾆｯﾎﾟﾝ]:   0%|                    | 0/1000 [00:13<?, ?it/s]"
     # Check that bar_format can print only {bar} or just one side
     assert format_meter(20, 100, 12, ncols=2, rate=8.1,
                         bar_format=r'{bar}') == \