Repository: tqdm/tqdm (Python)

Issue:
Redirect both sys.stdout and sys.stderr
I have a function from another library that occasionally prints to both stdout and stderr. I cannot hack the source code of that function. 

While [the example](https://stackoverflow.com/questions/36986929/redirect-print-command-in-python-script-through-tqdm-write) shows how to redirect sys.stdout inside a tqdm loop, how do I redirect stderr _at the same time_?


Hints:

Patch:
diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py
new file mode 100644
--- /dev/null
+++ b/examples/parallel_bars.py
@@ -0,0 +1,29 @@
+from __future__ import print_function
+from time import sleep
+from tqdm import tqdm
+from multiprocessing import Pool, freeze_support, Lock
+
+
+L = list(range(9))
+
+
+def progresser(n):
+    interval = 0.001 / (n + 2)
+    total = 5000
+    text = "#{}, est. {:<04.2}s".format(n, interval * total)
+    for _ in tqdm(range(total), desc=text, position=n):
+        sleep(interval)
+
+
+if __name__ == '__main__':
+    freeze_support()  # for Windows support
+    p = Pool(len(L),
+             initializer=tqdm.set_lock,
+             initargs=(Lock(),))
+    p.map(progresser, L)
+    print("\n" * (len(L) - 2))
+
+    # alternatively, on UNIX, just use the default internal lock
+    p = Pool(len(L))
+    p.map(progresser, L)
+    print("\n" * (len(L) - 2))
diff --git a/examples/redirect_print.py b/examples/redirect_print.py
--- a/examples/redirect_print.py
+++ b/examples/redirect_print.py
@@ -29,32 +29,36 @@ def write(self, x):
         if len(x.rstrip()) > 0:
             tqdm.write(x, file=self.file)
 
+    def flush(self):
+        return getattr(self.file, "flush", lambda: None)()
+
 
 @contextlib.contextmanager
-def stdout_redirect_to_tqdm():
-    save_stdout = sys.stdout
+def std_out_err_redirect_tqdm():
+    orig_out_err = sys.stdout, sys.stderr
     try:
-        sys.stdout = DummyTqdmFile(sys.stdout)
-        yield save_stdout
+        # sys.stdout = sys.stderr = DummyTqdmFile(orig_out_err[0])
+        sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err)
+        yield orig_out_err[0]
     # Relay exceptions
     except Exception as exc:
         raise exc
-    # Always restore sys.stdout if necessary
+    # Always restore sys.stdout/err if necessary
     finally:
-        sys.stdout = save_stdout
+        sys.stdout, sys.stderr = orig_out_err
 
 
-def blabla():
-    print("Foo blabla")
+def some_fun(i):
+    print("Fee, fi, fo,".split()[i])
 
 
-# Redirect stdout to tqdm.write() (don't forget the `as save_stdout`)
-with stdout_redirect_to_tqdm() as save_stdout:
-    # tqdm call need to specify sys.stdout, not sys.stderr (default)
+# Redirect stdout to tqdm.write()
+with std_out_err_redirect_tqdm() as orig_stdout:
+    # tqdm needs the original stdout
     # and dynamic_ncols=True to autodetect console width
-    for _ in tqdm(range(3), file=save_stdout, dynamic_ncols=True):
-        blabla()
+    for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True):
         sleep(.5)
+        some_fun(i)
 
 # After the `with`, printing is restored
-print('Done!')
+print("Done!")
diff --git a/tqdm/_tqdm.py b/tqdm/_tqdm.py
--- a/tqdm/_tqdm.py
+++ b/tqdm/_tqdm.py
@@ -20,6 +20,9 @@
 from time import time
 from time import sleep
 from contextlib import contextmanager
+# For parallelism safety
+import multiprocessing as mp
+import threading as th
 
 
 __author__ = {"github.com/": ["noamraph", "obiwanus", "kmike", "hadim",
@@ -45,6 +48,38 @@ def __init__(self, msg, fp_write=None, *a, **k):
             super(TqdmDeprecationWarning, self).__init__(msg, *a, **k)
 
 
+# Create global parallelism locks to avoid racing issues with parallel bars
+# works only if fork available (Linux, MacOSX, but not on Windows)
+mp_lock = mp.Lock()  # multiprocessing lock
+th_lock = th.Lock()  # thread lock
+
+
+class TqdmDefaultWriteLock(object):
+    """
+    Provide a default write lock for thread and multiprocessing safety.
+    Works only on platforms supporting `fork` (so Windows is excluded).
+    On Windows, you need to supply the lock from the parent to the children as
+    an argument to joblib or the parallelism lib you use.
+    """
+    def __init__(self):
+        global mp_lock, th_lock
+        self.locks = [mp_lock, th_lock]
+
+    def acquire(self):
+        for lock in self.locks:
+            lock.acquire()
+
+    def release(self):
+        for lock in self.locks[::-1]:  # Release in inverse order of acquisition
+            lock.release()
+
+    def __enter__(self):
+        self.acquire()
+
+    def __exit__(self, *exc):
+        self.release()
+
+
 class TMonitor(Thread):
     """
     Monitoring thread for tqdm bars.
@@ -98,19 +133,21 @@ def run(self):
             if self.was_killed:
                 return
             # Then monitor!
-            cur_t = self._time()
-            # Check for each tqdm instance if one is waiting too long to print
-            # NB: copy avoids size change during iteration RuntimeError
-            for instance in self.tqdm_cls._instances.copy():
-                # Only if mininterval > 1 (else iterations are just slow)
-                # and last refresh was longer than maxinterval in this instance
-                if instance.miniters > 1 and \
-                        (cur_t - instance.last_print_t) >= instance.maxinterval:
-                    # We force bypassing miniters on next iteration
-                    # dynamic_miniters should adjust mininterval automatically
-                    instance.miniters = 1
-                    # Refresh now! (works only for manual tqdm)
-                    instance.refresh()
+            # Acquire lock (to access _instances)
+            with self.tqdm_cls.get_lock():
+                cur_t = self._time()
+                # Check tqdm instances are waiting too long to print
+                for instance in self.tqdm_cls._instances:
+                    # Only if mininterval > 1 (else iterations are just slow)
+                    # and last refresh exceeded maxinterval
+                    if instance.miniters > 1 and \
+                            (cur_t - instance.last_print_t) >= \
+                            instance.maxinterval:
+                        # force bypassing miniters on next iteration
+                        # (dynamic_miniters adjusts mininterval automatically)
+                        instance.miniters = 1
+                        # Refresh now! (works only for manual tqdm)
+                        instance.refresh(nolock=True)
 
     def report(self):
         # return self.is_alive()  # TODO: does not work...
@@ -307,11 +344,11 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
 
             # format the stats displayed to the left and right sides of the bar
             if prefix:
-              # old prefix setup work around
-              bool_prefix_colon_already = (prefix[-2:] == ": ")
-              l_bar = prefix if bool_prefix_colon_already else prefix + ": "
+                # old prefix setup work around
+                bool_prefix_colon_already = (prefix[-2:] == ": ")
+                l_bar = prefix if bool_prefix_colon_already else prefix + ": "
             else:
-              l_bar = ''
+                l_bar = ''
             l_bar += '{0:3.0f}%|'.format(percentage)
             r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
                 n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt,
@@ -340,7 +377,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
                             'l_bar': l_bar,
                             'r_bar': r_bar,
                             'desc': prefix or '',
-                            'postfix': ', '+postfix if postfix else '',
+                            'postfix': ', ' + postfix if postfix else '',
                             # 'bar': full_bar  # replaced by procedure below
                             }
 
@@ -399,15 +436,19 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
                     ', ' + postfix if postfix else '')
 
     def __new__(cls, *args, **kwargs):
+        # Create default lock if none set
+        if "_lock" not in cls.__dict__:
+            cls.set_lock(TqdmDefaultWriteLock())
         # Create a new instance
         instance = object.__new__(cls)
         # Add to the list of instances
         if "_instances" not in cls.__dict__:
             cls._instances = WeakSet()
-        cls._instances.add(instance)
+        with cls._lock:
+            cls._instances.add(instance)
         # Create the monitoring thread
-        if cls.monitor_interval and (cls.monitor is None
-                                     or not cls.monitor.report()):
+        if cls.monitor_interval and (cls.monitor is None or not
+                                     cls.monitor.report()):
             cls.monitor = TMonitor(cls, cls.monitor_interval)
         # Return the instance
         return instance
@@ -430,41 +471,45 @@ def _decr_instances(cls, instance):
         so that newer bars won't overlap previous bars
         """
         try:  # in case instance was explicitly positioned, it won't be in set
-            cls._instances.remove(instance)
-            for inst in cls._instances:
-                if inst.pos > instance.pos:
-                    inst.pos -= 1
+            with cls._lock:
+                cls._instances.remove(instance)
+                for inst in cls._instances:
+                    if inst.pos > instance.pos:
+                        inst.pos -= 1
             # Kill monitor if no instances are left
             if not cls._instances and cls.monitor:
-                cls.monitor.exit()
                 try:
+                    cls.monitor.exit()
                     del cls.monitor
-                except AttributeError:
+                except AttributeError:  # pragma: nocover
                     pass
-                cls.monitor = None
+                else:
+                    cls.monitor = None
         except KeyError:
             pass
 
     @classmethod
-    def write(cls, s, file=None, end="\n"):
+    def write(cls, s, file=None, end="\n", nolock=False):
         """
         Print a message via tqdm (without overlap with bars)
         """
         fp = file if file is not None else sys.stdout
-        with cls.external_write_mode(file=file):
+        with cls.external_write_mode(file=file, nolock=nolock):
             # Write the message
             fp.write(s)
             fp.write(end)
 
     @classmethod
     @contextmanager
-    def external_write_mode(cls, file=None):
+    def external_write_mode(cls, file=None, nolock=False):
         """
         Disable tqdm within context and refresh tqdm when exits.
         Useful when writing to standard output stream
         """
         fp = file if file is not None else sys.stdout
 
+        if not nolock:
+            cls._lock.acquire()
         # Clear all bars
         inst_cleared = []
         for inst in getattr(cls, '_instances', []):
@@ -473,16 +518,26 @@ def external_write_mode(cls, file=None):
             # sys.stdout or sys.stderr (because both are mixed in terminal)
             if inst.fp == fp or all(
                     f in (sys.stdout, sys.stderr) for f in (fp, inst.fp)):
-                inst.clear()
+                inst.clear(nolock=True)
                 inst_cleared.append(inst)
         yield
         # Force refresh display of bars we cleared
         for inst in inst_cleared:
             # Avoid race conditions by checking that the instance started
             if hasattr(inst, 'start_t'):  # pragma: nocover
-                inst.refresh()
+                inst.refresh(nolock=True)
+        if not nolock:
+            cls._lock.release()
         # TODO: make list of all instances incl. absolutely positioned ones?
 
+    @classmethod
+    def set_lock(cls, lock):
+        cls._lock = lock
+
+    @classmethod
+    def get_lock(cls):
+        return cls._lock
+
     @classmethod
     def pandas(tclass, *targs, **tkwargs):
         """
@@ -716,7 +771,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
             raise (TqdmDeprecationWarning("""\
 `nested` is deprecated and automated. Use position instead for manual control.
 """, fp_write=getattr(file, 'write', sys.stderr.write)) if "nested" in kwargs
-                   else TqdmKeyError("Unknown argument(s): " + str(kwargs)))
+                else TqdmKeyError("Unknown argument(s): " + str(kwargs)))
 
         # Preprocess the arguments
         if total is None and iterable is not None:
@@ -803,14 +858,15 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
         if not gui:
             # Initialize the screen printer
             self.sp = self.status_printer(self.fp)
-            if self.pos:
-                self.moveto(self.pos)
-            self.sp(self.format_meter(self.n, total, 0,
-                    (dynamic_ncols(file) if dynamic_ncols else ncols),
-                    self.desc, ascii, unit, unit_scale, None, bar_format,
-                    self.postfix, unit_divisor))
-            if self.pos:
-                self.moveto(-self.pos)
+            with self._lock:
+                if self.pos:
+                    self.moveto(self.pos)
+                self.sp(self.format_meter(self.n, total, 0,
+                        (dynamic_ncols(file) if dynamic_ncols else ncols),
+                        self.desc, ascii, unit, unit_scale, None, bar_format,
+                        self.postfix, unit_divisor))
+                if self.pos:
+                    self.moveto(-self.pos)
 
         # Init the time counter
         self.last_print_t = self._time()
@@ -922,20 +978,21 @@ def __iter__(self):
                                 else smoothing * delta_t / delta_it + \
                                 (1 - smoothing) * avg_time
 
-                        if self.pos:
-                            self.moveto(self.pos)
+                        with self._lock:
+                            if self.pos:
+                                self.moveto(self.pos)
 
-                        # Print bar update
-                        sp(format_meter(
-                            n, self.total, elapsed,
-                            (dynamic_ncols(self.fp) if dynamic_ncols
-                             else ncols),
-                            self.desc, ascii, unit, unit_scale,
-                            1 / avg_time if avg_time else None, bar_format,
-                            self.postfix, unit_divisor))
+                            # Print bar update
+                            sp(format_meter(
+                                n, self.total, elapsed,
+                                (dynamic_ncols(self.fp) if dynamic_ncols
+                                 else ncols),
+                                self.desc, ascii, unit, unit_scale,
+                                1 / avg_time if avg_time else None, bar_format,
+                                self.postfix, unit_divisor))
 
-                        if self.pos:
-                            self.moveto(-self.pos)
+                            if self.pos:
+                                self.moveto(-self.pos)
 
                         # If no `miniters` was specified, adjust automatically
                         # to the max iteration rate seen so far between 2 prints
@@ -953,8 +1010,7 @@ def __iter__(self):
                                 # towards the timeframe of mininterval
                                 miniters = smoothing * delta_it * \
                                     (mininterval / delta_t
-                                     if mininterval and delta_t
-                                     else 1) + \
+                                     if mininterval and delta_t else 1) + \
                                     (1 - smoothing) * miniters
                             else:
                                 # Maximum nb of iterations between 2 prints
@@ -1019,17 +1075,18 @@ def update(self, n=1):
 Please use `tqdm_gui(...)` instead of `tqdm(..., gui=True)`
 """, fp_write=getattr(self.fp, 'write', sys.stderr.write))
 
-                if self.pos:
-                    self.moveto(self.pos)
+                with self._lock:
+                    if self.pos:
+                        self.moveto(self.pos)
 
-                # Print bar update
-                self.sp(self.__repr__())
+                    # Print bar update
+                    self.sp(self.__repr__())
 
-                if self.pos:
-                    self.moveto(-self.pos)
+                    if self.pos:
+                        self.moveto(-self.pos)
 
                 # If no `miniters` was specified, adjust automatically to the
-                # maximum iteration rate seen so far between 2 prints
+                # maximum iteration rate seen so far between two prints.
                 # e.g.: After running `tqdm.update(5)`, subsequent
                 # calls to `tqdm.update()` will only cause an update after
                 # at least 5 more iterations.
@@ -1083,29 +1140,30 @@ def fp_write(s):
                 return
             raise  # pragma: no cover
 
-        if pos:
-            self.moveto(pos)
-
-        if self.leave:
-            if self.last_print_n < self.n:
-                cur_t = self._time()
-                # stats for overall rate (no weighted average)
-                self.sp(self.format_meter(
-                    self.n, self.total, cur_t - self.start_t,
-                    (self.dynamic_ncols(self.fp) if self.dynamic_ncols
-                     else self.ncols),
-                    self.desc, self.ascii, self.unit, self.unit_scale, None,
-                    self.bar_format, self.postfix, self.unit_divisor))
-            if pos:
-                self.moveto(-pos)
-            else:
-                fp_write('\n')
-        else:
-            self.sp('')  # clear up last bar
+        with self._lock:
             if pos:
-                self.moveto(-pos)
+                self.moveto(pos)
+
+            if self.leave:
+                if self.last_print_n < self.n:
+                    cur_t = self._time()
+                    # stats for overall rate (no weighted average)
+                    self.sp(self.format_meter(
+                        self.n, self.total, cur_t - self.start_t,
+                        (self.dynamic_ncols(self.fp) if self.dynamic_ncols
+                         else self.ncols),
+                        self.desc, self.ascii, self.unit, self.unit_scale, None,
+                        self.bar_format, self.postfix, self.unit_divisor))
+                if pos:
+                    self.moveto(-pos)
+                else:
+                    fp_write('\n')
             else:
-                fp_write('\r')
+                self.sp('')  # clear up last bar
+                if pos:
+                    self.moveto(-pos)
+                else:
+                    fp_write('\r')
 
     def unpause(self):
         """
@@ -1179,13 +1237,15 @@ def set_postfix_str(self, s='', refresh=True):
     def moveto(self, n):
         self.fp.write(_unicode('\n' * n + _term_move_up() * -n))
 
-    def clear(self, nomove=False):
+    def clear(self, nomove=False, nolock=False):
         """
         Clear current bar display
         """
         if self.disable:
             return
 
+        if not nolock:
+            self._lock.acquire()
         if not nomove:
             self.moveto(self.pos)
         # clear up the bar (can't rely on sp(''))
@@ -1194,20 +1254,26 @@ def clear(self, nomove=False):
         self.fp.write('\r')  # place cursor back at the beginning of line
         if not nomove:
             self.moveto(-self.pos)
+        if not nolock:
+            self._lock.release()
 
-    def refresh(self):
+    def refresh(self, nolock=False):
         """
         Force refresh the display of this bar
         """
         if self.disable:
             return
 
+        if not nolock:
+            self._lock.acquire()
         self.moveto(self.pos)
         # clear up this line's content (whatever there was)
-        self.clear(nomove=True)
+        self.clear(nomove=True, nolock=True)
         # Print current/last bar state
         self.fp.write(self.__repr__())
         self.moveto(-self.pos)
+        if not nolock:
+            self._lock.release()
 
 
 def trange(*args, **kwargs):


Test patch:
diff --git a/tqdm/tests/tests_tqdm.py b/tqdm/tests/tests_tqdm.py
--- a/tqdm/tests/tests_tqdm.py
+++ b/tqdm/tests/tests_tqdm.py
@@ -1125,28 +1125,27 @@ def test_position():
 
 @with_setup(pretest, posttest)
 def test_set_description():
-   """Test set description"""
-   with closing(StringIO()) as our_file:
-       with tqdm(desc='Hello', file=our_file) as t:
-           assert t.desc == 'Hello'
-           t.set_description_str('World')
-           assert t.desc == 'World'
-           t.set_description()
-           assert t.desc == ''
-           t.set_description('Bye')
-           assert t.desc == 'Bye: '
-       assert "World" in our_file.getvalue()
-
-
-   # without refresh
-   with closing(StringIO()) as our_file:
-       with tqdm(desc='Hello', file=our_file) as t:
-           assert t.desc == 'Hello'
-           t.set_description_str('World', False)
-           assert t.desc == 'World'
-           t.set_description(None, False)
-           assert t.desc == ''
-       assert "World" not in our_file.getvalue()
+    """Test set description"""
+    with closing(StringIO()) as our_file:
+        with tqdm(desc='Hello', file=our_file) as t:
+            assert t.desc == 'Hello'
+            t.set_description_str('World')
+            assert t.desc == 'World'
+            t.set_description()
+            assert t.desc == ''
+            t.set_description('Bye')
+            assert t.desc == 'Bye: '
+        assert "World" in our_file.getvalue()
+
+    # without refresh
+    with closing(StringIO()) as our_file:
+        with tqdm(desc='Hello', file=our_file) as t:
+            assert t.desc == 'Hello'
+            t.set_description_str('World', False)
+            assert t.desc == 'World'
+            t.set_description(None, False)
+            assert t.desc == ''
+        assert "World" not in our_file.getvalue()
 
 
 @with_setup(pretest, posttest)
@@ -1602,24 +1601,21 @@ def __init__(self, file):
     def write(self, x):
         # Avoid print() second call (useless \n)
         if len(x.rstrip()) > 0:
-            tqdm.write(x, file=self.file)
+            tqdm.write(x, file=self.file, nolock=True)
 
 
 @contextmanager
-def stdout_stderr_redirect_to_tqdm(tqdm_file=sys.stderr):
-    save_stdout = sys.stdout
-    save_stderr = sys.stderr
+def std_out_err_redirect_tqdm(tqdm_file=sys.stderr):
+    orig_out_err = sys.stdout, sys.stderr
     try:
-        sys.stdout = DummyTqdmFile(tqdm_file)
-        sys.stderr = DummyTqdmFile(tqdm_file)
-        yield save_stdout, save_stderr
+        sys.stdout = sys.stderr = DummyTqdmFile(tqdm_file)
+        yield orig_out_err[0]
     # Relay exceptions
     except Exception as exc:
         raise exc
-    # Always restore sys.stdout if necessary
+    # Always restore sys.stdout/err if necessary
     finally:
-        sys.stdout = save_stdout
-        sys.stderr = save_stderr
+        sys.stdout, sys.stderr = orig_out_err
 
 
 @with_setup(pretest, posttest)
@@ -1627,8 +1623,7 @@ def test_file_redirection():
     """Test redirection of output"""
     with closing(StringIO()) as our_file:
         # Redirect stdout to tqdm.write()
-        with stdout_stderr_redirect_to_tqdm(tqdm_file=our_file):
-            # as (stdout, stderr)
+        with std_out_err_redirect_tqdm(tqdm_file=our_file):
             for _ in trange(3):
                 print("Such fun")
         res = our_file.getvalue()
@@ -1637,6 +1632,20 @@ def test_file_redirection():
         assert "3/3" in res
 
 
+@with_setup(pretest, posttest)
+def test_external_write():
+    """Test external write mode"""
+    with closing(StringIO()) as our_file:
+        # Redirect stdout to tqdm.write()
+        for _ in trange(3, file=our_file):
+            with tqdm.external_write_mode(file=our_file):
+                our_file.write("Such fun\n")
+        res = our_file.getvalue()
+        assert res.count("Such fun\n") == 3
+        assert "0/3" in res
+        assert "3/3" in res
+
+
 @with_setup(pretest, posttest)
 def test_unit_scale():
     """Test numeric `unit_scale`"""