Repository: ipython/ipython

Issue:
NBconvert fails with unicode errors when `--stdout` and file redirection is specified and HTML entities are present
I'm not sure if this is a bug, but we just got caught by this in:https://github.com/swcarpentry/bc/pull/163

To reproduce, create an IPython Notebook with any HTML entity such as:

`&times;`

Then try running nbconvert with --stdout with output to a file.

```
ipython nbconvert --stdout test.ipynb > test.html
```

This will break, although:

```
ipython nbconvert --stdout test.ipynb
```

and

```
ipython nbconvert test.ipynb
```

both work.



Discussion:
Yep, this is a bug. I can reproduce it on Python 2, but it works on Python 3.

My attempt at a fix - https://github.com/ipython/ipython/pull/4579

@jdfreder - Thanks for working on this.  I'm still able to reproduce the error against your latest patch.


Resolving patch:
diff --git a/IPython/nbconvert/writers/stdout.py b/IPython/nbconvert/writers/stdout.py
--- a/IPython/nbconvert/writers/stdout.py
+++ b/IPython/nbconvert/writers/stdout.py
@@ -13,6 +13,7 @@
 # Imports
 #-----------------------------------------------------------------------------
 
+from IPython.utils import io
 from .base import WriterBase
 
 #-----------------------------------------------------------------------------
@@ -30,5 +31,4 @@ def write(self, output, resources, **kw):
 
         See base for more...
         """
-
-        print(output)
+        io.unicode_std_stream().write(output)
diff --git a/IPython/utils/io.py b/IPython/utils/io.py
--- a/IPython/utils/io.py
+++ b/IPython/utils/io.py
@@ -10,15 +10,17 @@
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 from __future__ import print_function
+from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+import codecs
 import os
 import sys
 import tempfile
 from .capture import CapturedIO, capture_output
-from .py3compat import string_types, input
+from .py3compat import string_types, input, PY3
 
 #-----------------------------------------------------------------------------
 # Code
@@ -227,3 +229,26 @@ def raw_print_err(*args, **kw):
 # Short aliases for quick debugging, do NOT use these in production code.
 rprint = raw_print
 rprinte = raw_print_err
+
+def unicode_std_stream(stream='stdout'):
+    """Get a wrapper to write unicode to stdout/stderr as UTF-8.
+
+    This ignores environment variables and default encodings, to reliably write
+    unicode to stdout or stderr.
+
+    ::
+
+        unicode_std_stream().write(u'ł@e¶ŧ←')
+    """
+    assert stream in ('stdout', 'stderr')
+    stream  = getattr(sys, stream)
+    if PY3:
+        try:
+            stream_b = stream.buffer
+        except AttributeError:
+            # sys.stdout has been replaced - use it directly
+            return stream
+    else:
+        stream_b = stream
+
+    return codecs.getwriter('utf-8')(stream_b)


Test patch:
diff --git a/IPython/nbconvert/writers/tests/test_stdout.py b/IPython/nbconvert/writers/tests/test_stdout.py
--- a/IPython/nbconvert/writers/tests/test_stdout.py
+++ b/IPython/nbconvert/writers/tests/test_stdout.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 """
 Module with tests for stdout
 """
@@ -43,9 +44,11 @@ def test_output(self):
 
         # Create stdout writer, test output
         writer = StdoutWriter()
-        writer.write('a', {'b': 'c'})
+        writer.write(u'a×', {'b': 'c'})
         output = stream.getvalue()
-        self.fuzzy_compare(output, 'a')
+        if not PY3:
+            output = output.decode('utf-8')
+        self.fuzzy_compare(output, u'a×')
 
         # Revert stdout
         sys.stdout = stdout
\ No newline at end of file
diff --git a/IPython/utils/tests/test_io.py b/IPython/utils/tests/test_io.py
--- a/IPython/utils/tests/test_io.py
+++ b/IPython/utils/tests/test_io.py
@@ -12,7 +12,9 @@
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
+from __future__ import absolute_import
 
+import io as stdlib_io
 import sys
 
 from subprocess import Popen, PIPE
@@ -20,7 +22,8 @@
 
 import nose.tools as nt
 
-from IPython.utils.io import Tee, capture_output
+from IPython.testing.decorators import skipif
+from IPython.utils.io import Tee, capture_output, unicode_std_stream
 from IPython.utils.py3compat import doctest_refactor_print, PY3
 
 if PY3:
@@ -88,3 +91,34 @@ def test_capture_output():
     
     nt.assert_equal(io.stdout, 'hi, stdout\n')
     nt.assert_equal(io.stderr, 'hi, stderr\n')
+
+def test_UnicodeStdStream():
+    # Test wrapping a bytes-level stdout
+    if PY3:
+        stdoutb = stdlib_io.BytesIO()
+        stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii')
+    else:
+        stdout = stdoutb = stdlib_io.BytesIO()
+
+    orig_stdout = sys.stdout
+    sys.stdout = stdout
+    try:
+        sample = u"@łe¶ŧ←"
+        unicode_std_stream().write(sample)
+
+        output = stdoutb.getvalue().decode('utf-8')
+        nt.assert_equal(output, sample)
+        assert not stdout.closed
+    finally:
+        sys.stdout = orig_stdout
+
+@skipif(not PY3, "Not applicable on Python 2")
+def test_UnicodeStdStream_nowrap():
+    # If we replace stdout with a StringIO, it shouldn't get wrapped.
+    orig_stdout = sys.stdout
+    sys.stdout = StringIO()
+    try:
+        nt.assert_is(unicode_std_stream(), sys.stdout)
+        assert not sys.stdout.closed
+    finally:
+        sys.stdout = orig_stdout
\ No newline at end of file