Repository: ipython/ipython

Issue:
Fix %history -f for two-process clients on python2
Saving history has been broken for a while on python2 for the two process
terminals.

This fixes `history -f myhistfile` which used to cause a traceback complaining
about unicode that looked something like this:

```
/home/pi/code/ipython/IPython/core/magics/history.pyc in history(self, parameter_s)
       212                 if multiline:
       213                     inline = "\n...  ".join(inline.splitlines()) + "\n..."
   --> 214             print(inline, file=outfile)
       215             if get_output and output:
       216                 print(output, file=outfile)

TypeError: must be unicode, not str
```

@takluyver should direct me if I should have used a different approach here.
Part of the reason I used str_to_unicode is because then this patch can be
easily backported to 1.x, though I don't know if we'll be having another one of those releases



Discussion:
I think using `str_to_unicode` could break it for writing non-ascii characters to stdout on Python 2. I suspect that to do this reliably,  we need something like my `unicode_std_stream` function that I just introduced in #4581.

travis is angry ... @ivanov ?

> I suspect that to do this reliably, we need something like my unicode_std_stream function that I just introduced in #4581.

I agree with @takluyver here...

Actually, on reflection, it may be simpler than that, but this isn't quite right - the Travis failure is due to it calling `str_to_unicode` on something that's already unicode.

I think the actual issue is that we're putting byte strings into the history, when it should all be unicode.

> I think the actual issue is that we're putting byte strings into the history, when it should all be unicode.

Ah... it may be the problem...


Resolving patch:
diff --git a/IPython/kernel/zmq/ipkernel.py b/IPython/kernel/zmq/ipkernel.py
--- a/IPython/kernel/zmq/ipkernel.py
+++ b/IPython/kernel/zmq/ipkernel.py
@@ -341,7 +341,7 @@ def execute_request(self, stream, ident, parent):
         
         try:
             content = parent[u'content']
-            code = content[u'code']
+            code = py3compat.cast_unicode_py2(content[u'code'])
             silent = content[u'silent']
             store_history = content.get(u'store_history', not silent)
         except:
diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py
--- a/IPython/utils/py3compat.py
+++ b/IPython/utils/py3compat.py
@@ -83,6 +83,7 @@ def safe_unicode(e):
     str_to_bytes = encode
     bytes_to_str = decode
     cast_bytes_py2 = no_code
+    cast_unicode_py2 = no_code
     
     string_types = (str,)
     unicode_type = str
@@ -139,6 +140,7 @@ def u_format(s):
     str_to_bytes = no_code
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
+    cast_unicode_py2 = cast_unicode
     
     string_types = (str, unicode)
     unicode_type = unicode


Test patch:
diff --git a/IPython/kernel/tests/test_kernel.py b/IPython/kernel/tests/test_kernel.py
--- a/IPython/kernel/tests/test_kernel.py
+++ b/IPython/kernel/tests/test_kernel.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 """test the IPython Kernel"""
 
 #-------------------------------------------------------------------------------
@@ -11,6 +12,8 @@
 # Imports
 #-------------------------------------------------------------------------------
 
+import io
+import os.path
 import sys
 
 import nose.tools as nt
@@ -18,8 +21,10 @@
 from IPython.testing import decorators as dec, tools as tt
 from IPython.utils import py3compat
 from IPython.utils.path import locate_profile
+from IPython.utils.tempdir import TemporaryDirectory
 
-from .utils import new_kernel, kernel, TIMEOUT, assemble_output, execute, flush_channels
+from .utils import (new_kernel, kernel, TIMEOUT, assemble_output, execute,
+                    flush_channels, wait_for_idle)
 
 #-------------------------------------------------------------------------------
 # Tests
@@ -180,6 +185,22 @@ def test_eval_input():
         nt.assert_equal(stdout, "2\n")
 
 
+def test_save_history():
+    # Saving history from the kernel with %hist -f was failing because of
+    # unicode problems on Python 2.
+    with kernel() as kc, TemporaryDirectory() as td:
+        file = os.path.join(td, 'hist.out')
+        execute(u'a=1', kc=kc)
+        wait_for_idle(kc)
+        execute(u'b=u"abcþ"', kc=kc)
+        wait_for_idle(kc)
+        _, reply = execute("%hist -f " + file, kc=kc)
+        nt.assert_equal(reply['status'], 'ok')
+        with io.open(file, encoding='utf-8') as f:
+            content = f.read()
+        nt.assert_in(u'a=1', content)
+        nt.assert_in(u'b=u"abcþ"', content)
+
 def test_help_output():
     """ipython kernel --help-all works"""
     tt.help_all_output_test('kernel')
diff --git a/IPython/kernel/tests/utils.py b/IPython/kernel/tests/utils.py
--- a/IPython/kernel/tests/utils.py
+++ b/IPython/kernel/tests/utils.py
@@ -170,5 +170,10 @@ def assemble_output(iopub):
             pass
     return stdout, stderr
 
-
-
+def wait_for_idle(kc):
+    while True:
+        msg = kc.iopub_channel.get_msg(block=True, timeout=1)
+        msg_type = msg['msg_type']
+        content = msg['content']
+        if msg_type == 'status' and content['execution_state'] == 'idle':
+            break