Repository: spyder-ide/spyder

Issue:
IndexError: string index out of range in fallback plugin
## Description

### What steps will reproduce the problem?

<!--- You can use Markdown here --->



When I type in another say_hello to call a the function in the last line of the code listed below, this error comes up.
 
  File "C:\Users\Al\Anaconda3\lib\site-packages\spyder\plugins\completion\fallback\actor.py", line 62, in tokenize
    valid = is_prefix_valid(text, offset, language)
  File "C:\Users\Al\Anaconda3\lib\site-packages\spyder\plugins\completion\fallback\utils.py", line 122, in is_prefix_valid
    current_pos_text = text[offset - 1]
IndexError: string index out of range

```python
def say_hello(name = 'Al', emoji = '😒'):
    print(f'helllloooo {name} {emoji}')
say_hello(name = 'Linda')
say_hello(emoji = ';>|')
say_hello()
say_   
```

### Traceback
```python-traceback
  File "C:\Users\Al\Anaconda3\lib\site-packages\spyder\plugins\completion\fallback\actor.py", line 155, in handle_msg
    msg['current_word'])
  File "C:\Users\Al\Anaconda3\lib\site-packages\spyder\plugins\completion\fallback\actor.py", line 62, in tokenize
    valid = is_prefix_valid(text, offset, language)
  File "C:\Users\Al\Anaconda3\lib\site-packages\spyder\plugins\completion\fallback\utils.py", line 122, in is_prefix_valid
    current_pos_text = text[offset - 1]
IndexError: string index out of range
```

## Versions

* Spyder version: 4.1.0 
* Python version: 3.7.6
* Qt version: 5.9.6
* PyQt5 version: 5.9.2
* Operating System: Windows 10

### Dependencies

```

# Mandatory:
atomicwrites >=1.2.0           :  1.3.0 (OK)
chardet >=2.0.0                :  3.0.4 (OK)
cloudpickle >=0.5.0            :  1.3.0 (OK)
diff_match_patch >=20181111    :  20181111 (OK)
intervaltree                   :  None (OK)
IPython >=4.0                  :  7.13.0 (OK)
jedi =0.14.1                   :  0.14.1 (OK)
nbconvert >=4.0                :  5.6.1 (OK)
numpydoc >=0.6.0               :  0.9.2 (OK)
paramiko >=2.4.0               :  2.7.1 (OK)
parso =0.5.2                   :  0.5.2 (OK)
pexpect >=4.4.0                :  4.8.0 (OK)
pickleshare >=0.4              :  0.7.5 (OK)
psutil >=5.3                   :  5.7.0 (OK)
pygments >=2.0                 :  2.6.1 (OK)
pylint >=0.25                  :  2.4.3 (OK)
pyls >=0.31.2;<0.32.0          :  0.31.8 (OK)
qdarkstyle >=2.8               :  2.8 (OK)
qtawesome >=0.5.7              :  0.7.0 (OK)
qtconsole >=4.6.0              :  4.7.1 (OK)
qtpy >=1.5.0                   :  1.9.0 (OK)
rtree >=0.8.3                  :  0.9.3 (OK)
sphinx >=0.6.6                 :  2.4.0 (OK)
spyder_kernels >=1.9.0;<1.10.0 :  1.9.0 (OK)
watchdog                       :  None (OK)
zmq >=17                       :  18.1.1 (OK)

# Optional:
cython >=0.21                  :  0.29.15 (OK)
matplotlib >=2.0.0             :  3.1.3 (OK)
numpy >=1.7                    :  1.18.1 (OK)
pandas >=0.13.1                :  1.0.2 (OK)
scipy >=0.17.0                 :  1.4.1 (OK)
sympy >=0.7.3                  :  1.5.1 (OK)
```



Discussion:

Resolving patch:
diff --git a/spyder/plugins/completion/fallback/utils.py b/spyder/plugins/completion/fallback/utils.py
--- a/spyder/plugins/completion/fallback/utils.py
+++ b/spyder/plugins/completion/fallback/utils.py
@@ -14,17 +14,18 @@
 import os.path as osp
 import re
 
+# Third-party imports
+from pygments.lexer import words
+from pygments.lexers import (get_lexer_for_filename, get_lexer_by_name,
+                             TextLexer)
+
 # Local imports
 from spyder.utils.misc import memoize
-
+from spyder.utils.qstringhelpers import qstring_length
 from spyder.utils.syntaxhighlighters import (
     custom_extension_lexer_mapping
 )
 
-# Other imports
-from pygments.lexer import words
-from pygments.lexers import (get_lexer_for_filename, get_lexer_by_name,
-                             TextLexer)
 
 letter_regex = re.compile(r'\w')
 empty_regex = re.compile(r'\s')
@@ -97,7 +98,7 @@ def get_keywords(lexer):
     return keywords
 
 
-def get_words(text, exclude_offset=None, language=None):
+def get_words(text, exclude_offset=None, language=''):
     """
     Extract all words from a source code file to be used in code completion.
 
@@ -117,11 +118,17 @@ def get_words(text, exclude_offset=None, language=None):
 
 def is_prefix_valid(text, offset, language):
     """Check if current offset prefix is valid."""
-    regex = LANGUAGE_REGEX.get(language.lower(), all_regex)
-    prefix = ''
-    current_pos_text = text[offset - 1]
+    # Account for length differences in text when using characters
+    # such as emojis in the editor.
+    # Fixes spyder-ide/spyder#11862
+    utf16_diff = qstring_length(text) - len(text)
+    current_pos_text = text[offset - utf16_diff - 1]
+
     empty_start = empty_regex.match(current_pos_text) is not None
     max_end = -1
+    regex = LANGUAGE_REGEX.get(language.lower(), all_regex)
+    prefix = ''
+
     for match in regex.finditer(text):
         start, end = match.span()
         max_end = max(end, max_end)


Test patch:
diff --git a/spyder/plugins/editor/widgets/tests/test_introspection.py b/spyder/plugins/editor/widgets/tests/test_introspection.py
--- a/spyder/plugins/editor/widgets/tests/test_introspection.py
+++ b/spyder/plugins/editor/widgets/tests/test_introspection.py
@@ -928,8 +928,8 @@ def test_completion_order(lsp_codeeditor, qtbot):
 
 @pytest.mark.slow
 @pytest.mark.first
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason='Only works on Linux')
+@pytest.mark.skipif(not sys.platform.startswith('linux') or PY2,
+                    reason='Only works on Linux and Python 3')
 @flaky(max_runs=5)
 def test_fallback_completions(fallback_codeeditor, qtbot):
     code_editor, _ = fallback_codeeditor
@@ -984,6 +984,18 @@ def test_fallback_completions(fallback_codeeditor, qtbot):
     word_set = {x['insertText'] for x in sig.args[0]}
     assert 'another' not in word_set
 
+    # Check that fallback doesn't give an error with utf-16 characters.
+    # This is a regression test for issue spyder-ide/spyder#11862.
+    qtbot.keyPress(code_editor, Qt.Key_Enter, delay=300)
+    with qtbot.waitSignal(completion.sig_show_completions,
+                          timeout=10000) as sig:
+        code_editor.append("'😒 foobar'")
+        qtbot.keyPress(code_editor, Qt.Key_Enter, delay=300)
+        qtbot.keyClicks(code_editor, 'foob')
+        qtbot.keyPress(code_editor, Qt.Key_Tab, delay=300)
+    word_set = {x['insertText'] for x in sig.args[0]}
+    assert 'foobar' in word_set
+
     code_editor.toggle_automatic_completions(True)
     code_editor.toggle_code_snippets(True)
 
diff --git a/spyder/plugins/ipythonconsole/widgets/tests/test_kernelconnect.py b/spyder/plugins/ipythonconsole/widgets/tests/test_kernelconnect.py
--- a/spyder/plugins/ipythonconsole/widgets/tests/test_kernelconnect.py
+++ b/spyder/plugins/ipythonconsole/widgets/tests/test_kernelconnect.py
@@ -130,7 +130,7 @@ def test_connection_dialog_remembers_input_with_ssh_passphrase(
     assert new_dlg.un.text() == pytest.un
     assert new_dlg.pn.text() == str(pytest.pn)
     assert new_dlg.kf.text() == pytest.kf
-    if (not sys.platform.startswith('linux') or
+    if (sys.platform == 'darwin' or
             not os.environ.get('CI') is not None):
         assert new_dlg.kfp.text() == pytest.kfp
 
@@ -158,7 +158,7 @@ def test_connection_dialog_doesnt_remember_input_with_ssh_passphrase(
     assert new_dlg.un.text() == ""
     assert new_dlg.pn.text() == "22"
     assert new_dlg.kf.text() == ""
-    if (not sys.platform.startswith('linux') or
+    if (sys.platform == 'darwin' or
             not os.environ.get('CI') is not None):
         assert new_dlg.kfp.text() == ""
 
@@ -184,7 +184,7 @@ def test_connection_dialog_remembers_input_with_password(
     assert new_dlg.hn.text() == pytest.hn
     assert new_dlg.un.text() == pytest.un
     assert new_dlg.pn.text() == str(pytest.pn)
-    if (not sys.platform.startswith('linux') or
+    if (sys.platform == 'darwin' or
             not os.environ.get('CI') is not None):
         assert new_dlg.pw.text() == pytest.pw
 
@@ -211,7 +211,7 @@ def test_connection_dialog_doesnt_remember_input_with_password(
     assert new_dlg.hn.text() == ""
     assert new_dlg.un.text() == ""
     assert new_dlg.pn.text() == "22"
-    if (not sys.platform.startswith('linux') or
+    if (sys.platform == 'darwin' or
             not os.environ.get('CI') is not None):
         assert new_dlg.pw.text() == ""
 
diff --git a/spyder/widgets/github/tests/test_github_backend.py b/spyder/widgets/github/tests/test_github_backend.py
--- a/spyder/widgets/github/tests/test_github_backend.py
+++ b/spyder/widgets/github/tests/test_github_backend.py
@@ -101,10 +101,8 @@ def test_get_credentials_from_settings():
 
 
 @pytest.mark.skipif((os.environ.get('CI', None) is not None and
-                     sys.platform.startswith('linux') or
-                     sys.platform.startswith('linux') and PY2),
-                    reason=("Not possible to make it work on Linux and our "
-                            "CIs and skip it locally on Linux and Python 2"))
+                     not sys.platform == 'darwin'),
+                    reason="Only work on macOS and our CIs")
 def test_store_user_credentials():
     b = get_backend()
     b._store_token('token', True)