Repository: jquast/blessed

Issue:
Some non-ascii input echos wrong characters or fails to decode
Hi!

Running the editor example <https://github.com/jquast/blessed/blob/master/bin/editor.py> on `Windows 10 x64`, `blessed==1.18.0` and Python 3.7, 3.8 or 3.9 yields

```
Traceback (most recent call last):
  File "<snip>\blessed\bin\editor.py", line 269, in <module>
    main()
  File "<snip>\blessed\bin\editor.py", line 228, in main
    inp = term.inkey()
  File "<snip>\Python39\lib\site-packages\blessed\terminal.py", line 1386, in inkey
    ucs += self.getch()
  File "<snip>\Python39\lib\site-packages\blessed\win_terminal.py", line 39, in getch
    return super(Terminal, self).getch()
  File "<snip>\Python39\lib\site-packages\blessed\terminal.py", line 1169, in getch
    return self._keyboard_decoder.decode(byte, final=False)
  File "<snip>\Python39\lib\encodings\cp1252.py", line 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 0: character maps to <undefined>
```

when I try to input a German umlaut `ü` (using the US-International keyboard layout you can type `"`+`u`). Running `chcp` outputs `Active code page: 437` whereas the traceback shows `cp1252`.
Typing `ß` (`alt`+`s`) gives me a `á`. So someting is messing up the encodings here.


Entering either in the cmd prompt or the interactive python interpreter is not a problem and displays fine.

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index ac386ee..92e90f5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ matrix:
       env: TOXENV=py37,codecov TEST_QUICK=1 COVERAGE_ID=travis-ci
     - python: 3.8
       env: TOXENV=py38,codecov COVERAGE_ID=travis-ci TEST_RAW=yes
-    - python: 3.9-dev
+    - python: 3.9
       env: TOXENV=py39,codecov TEST_QUICK=1 COVERAGE_ID=travis-ci TOXPYTHON=3.9
     - python: 3.10-dev
       env: TOXENV=py310,codecov TEST_QUICK=1 COVERAGE_ID=travis-ci TOXPYTHON=3.10
@@ -38,7 +38,6 @@ matrix:
       env: PATH=/c/Python38:/c/Python38/Scripts:$PATH TOXENV=py38,codecov COVERAGE_ID=travis-ci TEST_KEYBOARD=no
 
   allow_failures:
-    - python: 3.9-dev
     - python: 3.10-dev
 
 install:
diff --git a/blessed/__init__.py b/blessed/__init__.py
index fb72bb7..cb372d7 100644
--- a/blessed/__init__.py
+++ b/blessed/__init__.py
@@ -19,4 +19,4 @@ if (3, 0, 0) <= _sys.version_info[:3] < (3, 2, 3):
                       'support due to http://bugs.python.org/issue10570.')
 
 __all__ = ('Terminal',)
-__version__ = '1.18.0'
+__version__ = "1.18.1"
diff --git a/blessed/_capabilities.py b/blessed/_capabilities.py
index 0b2b92f..c4df54b 100644
--- a/blessed/_capabilities.py
+++ b/blessed/_capabilities.py
@@ -3,7 +3,6 @@
 import re
 from collections import OrderedDict
 
-
 __all__ = (
     'CAPABILITY_DATABASE',
     'CAPABILITIES_RAW_MIXIN',
diff --git a/blessed/terminal.py b/blessed/terminal.py
index 192e89e..02945a7 100644
--- a/blessed/terminal.py
+++ b/blessed/terminal.py
@@ -47,8 +47,11 @@ except NameError:
 
 HAS_TTY = True
 if platform.system() == 'Windows':
+    IS_WINDOWS = True
     import jinxed as curses  # pylint: disable=import-error
+    from jinxed.win32 import get_console_input_encoding  # pylint: disable=import-error
 else:
+    IS_WINDOWS = False
     import curses
 
     try:
@@ -161,7 +164,7 @@ class Terminal(object):
         self._is_a_tty = False
         self.__init__streams()
 
-        if platform.system() == 'Windows' and self._init_descriptor is not None:
+        if IS_WINDOWS and self._init_descriptor is not None:
             self._kind = kind or curses.get_term(self._init_descriptor)
         else:
             self._kind = kind or os.environ.get('TERM', 'dumb') or 'dumb'
@@ -325,7 +328,13 @@ class Terminal(object):
 
         if self._keyboard_fd is not None:
             # set input encoding and initialize incremental decoder
-            self._encoding = locale.getpreferredencoding() or 'UTF-8'
+
+            if IS_WINDOWS:
+                self._encoding = get_console_input_encoding() \
+                    or locale.getpreferredencoding() or 'UTF-8'
+            else:
+                self._encoding = locale.getpreferredencoding() or 'UTF-8'
+
             try:
                 self._keyboard_decoder = codecs.getincrementaldecoder(self._encoding)()
             except LookupError as err:
@@ -1100,20 +1109,33 @@ class Terminal(object):
         """
         return Sequence(text, self).strip_seqs()
 
-    def split_seqs(self, text, **kwds):
+    def split_seqs(self, text, maxsplit=0):
         r"""
         Return ``text`` split by individual character elements and sequences.
 
         :arg str text: String containing sequences
-        :arg kwds: remaining keyword arguments for :func:`re.split`.
+        :arg int maxsplit: When maxsplit is nonzero, at most maxsplit splits
+            occur, and the remainder of the string is returned as the final element
+            of the list (same meaning is argument for :func:`re.split`).
         :rtype: list[str]
         :returns: List of sequences and individual characters
 
         >>> term.split_seqs(term.underline(u'xyz'))
         ['\x1b[4m', 'x', 'y', 'z', '\x1b(B', '\x1b[m']
+
+        >>> term.split_seqs(term.underline(u'xyz'), 1)
+        ['\x1b[4m', r'xyz\x1b(B\x1b[m']
         """
         pattern = self._caps_unnamed_any
-        return list(filter(None, re.split(pattern, text, **kwds)))
+        result = []
+        for idx, match in enumerate(re.finditer(pattern, text)):
+            result.append(match.group())
+            if maxsplit and idx == maxsplit:
+                remaining = text[match.end():]
+                if remaining:
+                    result[-1] += remaining
+                break
+        return result
 
     def wrap(self, text, width=None, **kwargs):
         """
diff --git a/blessed/terminal.pyi b/blessed/terminal.pyi
index 4ceb644..d528d41 100644
--- a/blessed/terminal.pyi
+++ b/blessed/terminal.pyi
@@ -88,7 +88,7 @@ class Terminal:
     def rstrip(self, text: str, chars: Optional[str] = ...) -> str: ...
     def lstrip(self, text: str, chars: Optional[str] = ...) -> str: ...
     def strip_seqs(self, text: str) -> str: ...
-    def split_seqs(self, text: str, **kwds: Any) -> List[str]: ...
+    def split_seqs(self, text: str, maxsplit: int) -> List[str]: ...
     def wrap(
         self, text: str, width: Optional[int] = ..., **kwargs: Any
     ) -> List[str]: ...
diff --git a/docs/history.rst b/docs/history.rst
index 4018bd6..fc4543b 100644
--- a/docs/history.rst
+++ b/docs/history.rst
@@ -1,6 +1,8 @@
 Version History
 ===============
 1.18
+  * bugfix: :meth:`~Terminal.split_seqs` for some sequences
+    like ``term.move_left(3)``, :ghissue:`197`.
   * introduced: type annotations, :ghissue:`192` by :ghuser:`dlax`.
   * bugfix: do not fail when ``sys.stdin`` is unset, :ghissue:`195` by
     :ghuser:`Olen`
diff --git a/requirements.txt b/requirements.txt
index 917d2a4..8f80337 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,4 @@ ordereddict==1.1; python_version < "2.7"
 # support python2.7 by using backport of 'functools.lru_cache'
 backports.functools-lru-cache>=1.2.1; python_version < "3.2"
 # Windows requires jinxed
-jinxed>=0.5.4; platform_system == "Windows"
\ No newline at end of file
+jinxed>=1.1.0; platform_system == "Windows"
\ No newline at end of file
diff --git a/run_codecov.py b/run_codecov.py
deleted file mode 100644
index 49e7d7f..0000000
--- a/run_codecov.py
+++ /dev/null
@@ -1,47 +0,0 @@
-"""
-Workaround for https://github.com/codecov/codecov-python/issues/158
-"""
-
-# std imports
-import sys
-import time
-
-# local
-import codecov
-
-RETRIES = 5
-TIMEOUT = 2
-
-
-def main():
-    """
-    Run codecov up to RETRIES times
-    On the final attempt, let it exit normally
-    """
-
-    # Make a copy of argv and make sure --required is in it
-    args = sys.argv[1:]
-    if '--required' not in args:
-        args.append('--required')
-
-    for num in range(1, RETRIES + 1):
-
-        print('Running codecov attempt %d: ' % num)
-        # On the last, let codecov handle the exit
-        if num == RETRIES:
-            codecov.main()
-
-        try:
-            codecov.main(*args)
-        except SystemExit as err:
-            # If there's no exit code, it was successful
-            if err.code:
-                time.sleep(TIMEOUT)
-            else:
-                sys.exit(err.code)
-        else:
-            break
-
-
-if __name__ == '__main__':
-    main()
diff --git a/tox.ini b/tox.ini
index 1deb809..899b9d8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,6 @@
 [tox]
 envlist = about
+          stamp
           autopep8
           docformatter
           isort
@@ -95,6 +96,7 @@ deps = pytest==4.6.9
        pytest-cov==2.8.1
        pytest-xdist==1.31.0
        mock==3.0.5
+       attrs==20.3.0
 
 [testenv:py35]
 setenv = TEST_QUICK=1
@@ -138,6 +140,9 @@ commands = python {toxinidir}/bin/display-sighandlers.py
            python {toxinidir}/bin/display-terminalinfo.py
            python {toxinidir}/bin/display-fpathconf.py
 
+[testenv:stamp]
+commands = python {toxinidir}/version.py
+
 [testenv:autopep8]
 deps = autopep8==1.4.4
 commands =
@@ -208,10 +213,8 @@ commands = {envbindir}/sphinx-build -v -W -d {toxinidir}/docs/_build/doctrees -b
 [testenv:codecov]
 basepython = python{env:TOXPYTHON:{env:TRAVIS_PYTHON_VERSION:3.8}}
 passenv = TOXENV CI TRAVIS TRAVIS_* CODECOV_*
-deps = codecov>=1.4.0
-# commands = codecov -e TOXENV
-# Workaround for https://github.com/codecov/codecov-python/issues/158
-commands = {envpython} run_codecov.py -e TOXENV
+deps = codecov>=2.1
+commands = codecov -e TOXENV
 
 [testenv:publish_static]
 # Synchronize the artifacts in docs/_static/ with https://dxtz6bzwq9sxx.cloudfront.net/
diff --git a/version.json b/version.json
index 617b2b3..01b85dc 100644
--- a/version.json
+++ b/version.json
@@ -1,1 +1,1 @@
-{"version": "1.18.0"}
+{"version": "1.18.1"}
diff --git a/version.py b/version.py
new file mode 100644
index 0000000..5587aa1
--- /dev/null
+++ b/version.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# std imports
+import os
+import re
+import json
+
+
+def main():
+    # I don't know why, we maintain __version__ in blessed, because that's
+    # how it was done a long time ago before pip, anyway we do basic
+    # code generation, version.json -> __init__.py
+    fpath_json = os.path.join(os.path.dirname(__file__), 'version.json')
+    version = json.load(open(fpath_json, 'r'))['version']
+    fpath_py = os.path.join(os.path.dirname(__file__), 'blessed', '__init__.py')
+    prev_text = open(fpath_py, 'r').read()
+    next_text = re.sub(r"(__version__ = )(.*)$", r'\1"{0}"'.format(version),
+                       prev_text, flags=re.MULTILINE)
+    if prev_text != next_text:
+        print('Updating blessed.__version__ to {}'.format(version))
+        open(fpath_py, 'w').write(next_text)
+
+
+if __name__ == '__main__':
+    main()


Test patch:
diff --git a/tests/accessories.py b/tests/accessories.py
index 91e864a..f8928ee 100644
--- a/tests/accessories.py
+++ b/tests/accessories.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 """Accessories for automated py.test runner."""
-# standard imports
 from __future__ import print_function, with_statement
 
 # std imports
@@ -11,11 +10,9 @@ import platform
 import functools
 import traceback
 import contextlib
-import subprocess
 
 # 3rd party
 import six
-import pytest
 
 # local
 from blessed import Terminal
@@ -40,31 +37,6 @@ if platform.system() == 'Windows':
 TestTerminal = functools.partial(Terminal, kind=test_kind)  # type: Callable[..., Terminal]
 SEND_SEMAPHORE = SEMAPHORE = b'SEMAPHORE\n'
 RECV_SEMAPHORE = b'SEMAPHORE\r\n'
-many_lines_params = [40, 80]
-# we must test a '1' column for conditional in _handle_long_word
-many_columns_params = [1, 10]
-
-if os.environ.get('TEST_QUICK'):
-    many_lines_params = [80, ]
-    many_columns_params = [25, ]
-
-all_terms_params = 'xterm screen ansi vt220 rxvt cons25 linux'.split()
-
-if os.environ.get('TEST_FULL'):
-    try:
-        all_terms_params = [
-            # use all values of the first column of data in output of 'toe -a'
-            _term.split(None, 1)[0] for _term in
-            subprocess.Popen(('toe', '-a'),
-                             stdout=subprocess.PIPE,
-                             close_fds=True)
-            .communicate()[0].splitlines()]
-    except OSError:
-        pass
-elif platform.system() == 'Windows':
-    all_terms_params = ['vtwin10', ]
-elif os.environ.get('TEST_QUICK'):
-    all_terms_params = 'xterm screen ansi linux'.split()
 
 
 def init_subproc_coverage(run_note):
@@ -164,8 +136,7 @@ class as_subprocess(object):
         assert os.WEXITSTATUS(status) == 0
 
 
-def read_until_semaphore(fd, semaphore=RECV_SEMAPHORE,
-                         encoding='utf8', timeout=10):
+def read_until_semaphore(fd, semaphore=RECV_SEMAPHORE, encoding='utf8'):
     """
     Read file descriptor ``fd`` until ``semaphore`` is found.
 
@@ -254,21 +225,3 @@ def unicode_parm(cap, *parms):
         if val:
             return val.decode('latin1')
     return u''
-
-
-@pytest.fixture(params=all_terms_params)
-def all_terms(request):
-    """Common kind values for all kinds of terminals."""
-    return request.param
-
-
-@pytest.fixture(params=many_lines_params)
-def many_lines(request):
-    """Various number of lines for screen height."""
-    return request.param
-
-
-@pytest.fixture(params=many_columns_params)
-def many_columns(request):
-    """Various number of columns for screen width."""
-    return request.param
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..4fe3602
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,51 @@
+# std imports
+import os
+import platform
+import subprocess
+
+# 3rd party
+import pytest
+
+all_terms_params = 'xterm screen ansi vt220 rxvt cons25 linux'.split()
+many_lines_params = [40, 80]
+# we must test a '1' column for conditional in _handle_long_word
+many_columns_params = [1, 10]
+
+if os.environ.get('TEST_FULL'):
+    try:
+        all_terms_params = [
+            # use all values of the first column of data in output of 'toe -a'
+            _term.split(None, 1)[0] for _term in
+            subprocess.Popen(('toe', '-a'),
+                             stdout=subprocess.PIPE,
+                             close_fds=True)
+            .communicate()[0].splitlines()]
+    except OSError:
+        pass
+elif platform.system() == 'Windows':
+    all_terms_params = ['vtwin10', ]
+elif os.environ.get('TEST_QUICK'):
+    all_terms_params = 'xterm screen ansi linux'.split()
+
+
+if os.environ.get('TEST_QUICK'):
+    many_lines_params = [80, ]
+    many_columns_params = [25, ]
+
+
+@pytest.fixture(params=all_terms_params)
+def all_terms(request):
+    """Common kind values for all kinds of terminals."""
+    return request.param
+
+
+@pytest.fixture(params=many_lines_params)
+def many_lines(request):
+    """Various number of lines for screen height."""
+    return request.param
+
+
+@pytest.fixture(params=many_columns_params)
+def many_columns(request):
+    """Various number of columns for screen width."""
+    return request.param
diff --git a/tests/test_core.py b/tests/test_core.py
index 609e8f3..f449274 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -18,7 +18,7 @@ import pytest
 from six.moves import reload_module
 
 # local
-from .accessories import TestTerminal, all_terms, unicode_cap, as_subprocess
+from .accessories import TestTerminal, unicode_cap, as_subprocess
 
 
 def test_export_only_Terminal():
diff --git a/tests/test_keyboard.py b/tests/test_keyboard.py
index b604293..9233c91 100644
--- a/tests/test_keyboard.py
+++ b/tests/test_keyboard.py
@@ -11,7 +11,7 @@ import mock
 import pytest
 
 # local
-from .accessories import TestTerminal, all_terms, as_subprocess
+from .accessories import TestTerminal, as_subprocess
 
 if platform.system() != 'Windows':
     import curses
diff --git a/tests/test_length_sequence.py b/tests/test_length_sequence.py
index 214f51e..d911fbc 100644
--- a/tests/test_length_sequence.py
+++ b/tests/test_length_sequence.py
@@ -10,8 +10,8 @@ import itertools
 import six
 import pytest
 
-from .accessories import (  # isort:skip
-    TestTerminal, as_subprocess, all_terms, many_lines, many_columns)
+# local
+from .accessories import TestTerminal, as_subprocess
 
 if platform.system() != 'Windows':
     import fcntl
diff --git a/tests/test_sequences.py b/tests/test_sequences.py
index 0188123..5e65ae9 100644
--- a/tests/test_sequences.py
+++ b/tests/test_sequences.py
@@ -9,7 +9,7 @@ import six
 import pytest
 
 # local
-from .accessories import TestTerminal, all_terms, unicode_cap, unicode_parm, as_subprocess
+from .accessories import TestTerminal, unicode_cap, unicode_parm, as_subprocess
 
 
 @pytest.mark.skipif(platform.system() == 'Windows', reason="requires real tty")
@@ -543,9 +543,57 @@ def test_split_seqs(all_terms):
             result = list(term.split_seqs(given_text))
             assert result == expected
 
+    child(all_terms)
+
+
+def test_split_seqs_maxsplit1(all_terms):
+    """Test Terminal.split_seqs with maxsplit=1."""
+    @as_subprocess
+    def child(kind):
+        from blessed import Terminal
+        term = Terminal(kind)
+
         if term.bold:
             given_text = term.bold + 'bbq'
-            expected = [term.bold, 'b', 'b', 'q']
+            expected = [term.bold, 'bbq']
+            result = list(term.split_seqs(given_text, 1))
+            assert result == expected
+
+    child(all_terms)
+
+
+def test_split_seqs_term_right(all_terms):
+    """Test Terminal.split_seqs with parameterized sequence"""
+    @as_subprocess
+    def child(kind):
+        from blessed import Terminal
+        term = Terminal(kind)
+
+        if term.move_up:
+            given_text = 'XY' + term.move_right + 'VK'
+            expected = ['X', 'Y', term.move_right, 'V', 'K']
+            result = list(term.split_seqs(given_text))
+            assert result == expected
+
+    child(all_terms)
+
+
+def test_split_seqs_maxsplit3_and_term_right(all_terms):
+    """Test Terminal.split_seqs with parameterized sequence."""
+    @as_subprocess
+    def child(kind):
+        from blessed import Terminal
+        term = Terminal(kind)
+
+        if term.move_right(32):
+            given_text = 'PQ' + term.move_right(32) + 'RS'
+            expected = ['P', 'Q', term.move_right(32), 'RS']
+            result = list(term.split_seqs(given_text, 3))
+            assert result == expected
+
+        if term.move_up(45):
+            given_text = 'XY' + term.move_up(45) + 'VK'
+            expected = ['X', 'Y', term.move_up(45), 'V', 'K']
             result = list(term.split_seqs(given_text))
             assert result == expected
 
diff --git a/tests/test_wrap.py b/tests/test_wrap.py
index 2953759..89f2e1e 100644
--- a/tests/test_wrap.py
+++ b/tests/test_wrap.py
@@ -6,7 +6,7 @@ import textwrap
 import pytest
 
 # local
-from .accessories import TestTerminal, many_columns, as_subprocess
+from .accessories import TestTerminal, as_subprocess
 
 TEXTWRAP_KEYWORD_COMBINATIONS = [
     dict(break_long_words=False,