Repository: Gallopsled/pwntools

Issue:
interactive() readline history chokes on UTF-8 characters
test.py (doesn't work from inside REPL):

```py
from pwn import *
r = process('cat', shell=True)
r.interactive()
```

- enter a string containing UTF-8 characters such `löl` `<ENTER>`.
- hit cursor-up to access readline history
- encounter exception:

```
  File "/lib/python2.7/site-packages/pwnlib/term/readline.py", line 131, in set_buffer
    buffer_left = unicode(left)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 0: ordinal not in range(128)
```



Discussion:
Yup.  Welcome to Python2!  If you are sending `unicode` you need to perform the appropriate `encode` / `decode` magic to turn it into a `bytes` object.

Alternately, I recommend setting a default encoding for your system, and setting an appropriate header.

https://stackoverflow.com/questions/2276200/changing-default-encoding-of-python
https://stackoverflow.com/questions/3170211/why-declare-unicode-by-string-in-python
so you don't think that pwntools should catch this error ?
After some sleep, I see that the issue you're reporting is different than I was expecting.

Resolving patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -44,6 +44,23 @@
     'sphinxcontrib.napoleon'
 ]
 
+# Disable "info" logging directly to stdout by Sphinx
+import logging
+
+class SphinxPwnlibFilter(logging.Filter):
+    def filter(self, record):
+        if record.name.startswith('pwn'):
+            return False
+        if record.name.startswith('paramiko'):
+            return False
+        return True
+
+log_filter = SphinxPwnlibFilter()
+
+for i, handler in enumerate(logging.root.handlers):
+    print("Filtering Sphinx handler", handler)
+    handler.addFilter(log_filter)
+
 # Napoleon settings
 napoleon_use_ivar = True
 napoleon_use_rtype = False
diff --git a/pwnlib/term/readline.py b/pwnlib/term/readline.py
--- a/pwnlib/term/readline.py
+++ b/pwnlib/term/readline.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
 from pwnlib.term import keyconsts as kc
@@ -7,7 +9,7 @@
 
 cursor = text.reverse
 
-buffer_left, buffer_right = u'', u''
+buffer_left, buffer_right = '', ''
 saved_buffer = None
 history = []
 history_idx = None
@@ -29,6 +31,14 @@
 
 tabs = 0
 
+def force_to_bytes(data):
+    if isinstance(data, bytes):
+        return data
+    try:
+        return data.encode('utf-8')
+    except Exception:
+        return data.encode('latin-1')
+
 def set_completer(completer):
     global complete_hook, suggest_hook
     if completer is None:
@@ -80,7 +90,7 @@ def handle_keypress(trace):
 
 def clear():
     global buffer_left, buffer_right, history_idx, search_idx
-    buffer_left, buffer_right = u'', u''
+    buffer_left, buffer_right = '', ''
     history_idx = None
     search_idx = None
     redisplay()
@@ -141,7 +151,7 @@ def cancel_search(*_):
 def commit_search():
     global search_idx
     if search_idx is not None and search_results:
-        set_buffer(history[search_results[search_idx][0]], u'')
+        set_buffer(history[search_results[search_idx][0]], '')
         search_idx = None
         redisplay()
 
@@ -169,7 +179,7 @@ def update_search_results():
 def search_history(*_):
     global buffer_left, buffer_right, history_idx, search_idx
     if search_idx is None:
-        buffer_left, buffer_right = buffer_left + buffer_right, u''
+        buffer_left, buffer_right = buffer_left + buffer_right, ''
         history_idx = None
         search_idx = 0
         update_search_results()
@@ -187,7 +197,7 @@ def history_prev(*_):
         history_idx = -1
     if history_idx < len(history) - 1:
         history_idx += 1
-        set_buffer(history[history_idx], u'')
+        set_buffer(history[history_idx], '')
 
 def history_next(*_):
     global history_idx, saved_buffer
@@ -200,7 +210,7 @@ def history_next(*_):
         saved_buffer = None
     else:
         history_idx -= 1
-        set_buffer(history[history_idx], u'')
+        set_buffer(history[history_idx], '')
 
 def backward_char(*_):
     global buffer_left, buffer_right
@@ -319,11 +329,11 @@ def forward_word(*_):
 
 def go_beginning(*_):
     commit_search()
-    set_buffer(u'', buffer_left + buffer_right)
+    set_buffer('', buffer_left + buffer_right)
 
 def go_end(*_):
     commit_search()
-    set_buffer(buffer_left + buffer_right, u'')
+    set_buffer(buffer_left + buffer_right, '')
 
 keymap = km.Keymap({
     '<nomatch>'   : self_insert,
@@ -382,10 +392,10 @@ def readline(_size = None, prompt = '', float = True, priority = 10):
                 if eof:
                     return ''
                 else:
-                    buffer = (buffer_left + buffer_right).encode('utf-8')
+                    buffer = (buffer_left + buffer_right)
                     if buffer:
                         history.insert(0, buffer)
-                    return buffer + '\n'
+                    return force_to_bytes(buffer) + '\n'
             except KeyboardInterrupt:
                 control_c()
     finally:
diff --git a/pwnlib/tubes/process.py b/pwnlib/tubes/process.py
--- a/pwnlib/tubes/process.py
+++ b/pwnlib/tubes/process.py
@@ -336,11 +336,11 @@ def __init__(self, argv = None,
 
         if self.pty is not None:
             if stdin is slave:
-                self.proc.stdin = os.fdopen(os.dup(master), 'r+')
+                self.proc.stdin = os.fdopen(os.dup(master), 'r+', 0)
             if stdout is slave:
-                self.proc.stdout = os.fdopen(os.dup(master), 'r+')
+                self.proc.stdout = os.fdopen(os.dup(master), 'r+', 0)
             if stderr is slave:
-                self.proc.stderr = os.fdopen(os.dup(master), 'r+')
+                self.proc.stderr = os.fdopen(os.dup(master), 'r+', 0)
 
             os.close(master)
             os.close(slave)


Test patch: