Repository: spyder-ide/spyder

Issue:
UnicodeDecodeError thrown when viewing Dataframes with non-ascii index
## Description

**What steps will reproduce the problem?**

1. check the data in the variable explorer
2. after several times
3. 

**What is the expected output? What do you see instead?**


**Please provide any additional information below**

  File "/home/zhangjing/anaconda2/lib/python2.7/site-packages/spyder/widgets/variableexplorer/dataframeeditor.py", line 277, in data
    return to_qvariant(to_text_string(self.df_index[row]))
  File "/home/zhangjing/anaconda2/lib/python2.7/site-packages/spyder/py3compat.py", line 123, in to_text_string
    return unicode(obj)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xb3 in position 5: ordinal not in range(128)


## Version and main components

* Spyder Version: 3.2.4 
* Python Version: 2.7.14
* Qt Versions: 5.6.2, PyQt5 5.6 on Linux

## Dependencies
```
pyflakes >=0.5.0  :  1.6.0 (OK)
pycodestyle >=2.3 :  2.3.1 (OK)
pygments >=2.0    :  2.2.0 (OK)
pandas >=0.13.1   :  0.20.3 (OK)
numpy >=1.7       :  1.13.3 (OK)
sphinx >=0.6.6    :  1.6.3 (OK)
rope >=0.9.4      :  0.10.5 (OK)
jedi >=0.9.0      :  0.10.2 (OK)
psutil >=0.3      :  5.4.0 (OK)
nbconvert >=4.0   :  5.3.1 (OK)
sympy >=0.7.3     :  1.1.1 (OK)
cython >=0.21     :  0.26.1 (OK)
qtconsole >=4.2.0 :  4.3.1 (OK)
IPython >=4.0;<6.0:  5.4.1 (OK)
pylint >=0.25     :  1.7.4 (OK)

```



Discussion:
Thanks for reporting. This means that your Dataframe index has non-ascii characters and Spyder can't handle them. We'll try to fix this in a future release.

In the meantime, you should consider using Python 3 instead of Python 2 to process data with non-ascii characters.

Resolving patch:
diff --git a/spyder/widgets/variableexplorer/dataframeeditor.py b/spyder/widgets/variableexplorer/dataframeeditor.py
--- a/spyder/widgets/variableexplorer/dataframeeditor.py
+++ b/spyder/widgets/variableexplorer/dataframeeditor.py
@@ -18,10 +18,9 @@
 from qtpy.compat import from_qvariant, to_qvariant
 from qtpy.QtCore import QAbstractTableModel, QModelIndex, Qt, Signal, Slot
 from qtpy.QtGui import QColor, QCursor
-from qtpy.QtWidgets import (QApplication, QCheckBox, QDialogButtonBox, QDialog,
-                            QGridLayout, QHBoxLayout, QInputDialog, QLineEdit,
-                            QMenu, QMessageBox, QPushButton, QTableView,
-                            QHeaderView)
+from qtpy.QtWidgets import (QApplication, QCheckBox, QDialog, QGridLayout,
+                            QHBoxLayout, QInputDialog, QLineEdit, QMenu,
+                            QMessageBox, QPushButton, QTableView, QHeaderView)
 
 from pandas import DataFrame, DatetimeIndex, Series
 try:
@@ -34,9 +33,8 @@
 from spyder.config.base import _
 from spyder.config.fonts import DEFAULT_SMALL_DELTA
 from spyder.config.gui import get_font, config_shortcut
-from spyder.py3compat import (io, is_text_string, PY2, to_text_string,
-                              TEXT_TYPES)
-from spyder.utils import encoding
+from spyder.py3compat import (io, is_text_string, is_type_text_string, PY2,
+                              to_text_string)
 from spyder.utils import icon_manager as ima
 from spyder.utils.qthelpers import (add_actions, create_action,
                                     keybinding, qapplication)
@@ -195,31 +193,26 @@ def headerData(self, section, orientation, role=Qt.DisplayRole):
             if section == 0:
                 return 'Index'
             elif section == 1 and PY2:
+                header = self.df_header[0]
                 # Get rid of possible BOM utf-8 data present at the
                 # beginning of a file, which gets attached to the first
                 # column header when headers are present in the first
                 # row.
                 # Fixes Issue 2514
                 try:
-                    header = to_text_string(self.df_header[0],
-                                            encoding='utf-8-sig')
+                    header = to_text_string(header, encoding='utf-8-sig')
                 except:
-                    header = to_text_string(self.df_header[0])
+                    # Don't perform any conversion on strings because it
+                    # leads to differences between the data present in
+                    # the dataframe and what is shown by Spyder
+                    if not is_type_text_string(header):
+                        header = to_text_string(header)
                 return to_qvariant(header)
-            elif isinstance(self.df_header[section-1], TEXT_TYPES):
-                # Get the proper encoding of the text in the header.
-                # Fixes Issue 3896
-                if not PY2:
-                    try:
-                        header = self.df_header[section-1].encode('utf-8')
-                        coding = 'utf-8-sig'
-                    except:
-                        header = self.df_header[section-1].encode('utf-8')
-                        coding = encoding.get_coding(header)
-                else:
-                    header = self.df_header[section-1]
-                    coding = encoding.get_coding(header)
-                return to_qvariant(to_text_string(header, encoding=coding))
+            elif is_type_text_string(self.df_header[section-1]):
+                # Don't perform any conversion on strings because it
+                # leads to differences between the data present in
+                # the dataframe and what is shown by Spyder
+                return to_qvariant(self.df_header[section-1])
             else:
                 return to_qvariant(to_text_string(self.df_header[section-1]))
         else:
@@ -280,7 +273,15 @@ def data(self, index, role=Qt.DisplayRole):
             column = index.column()
             row = index.row()
             if column == 0:
-                return to_qvariant(to_text_string(self.df_index[row]))
+                df_idx = self.df_index[row]
+                if is_type_text_string(df_idx):
+                    # Don't perform any conversion on strings
+                    # because it leads to differences between
+                    # the data present in the dataframe and
+                    # what is shown by Spyder
+                    return df_idx
+                else:
+                    return to_qvariant(to_text_string(df_idx))
             else:
                 value = self.get_value(row, column-1)
                 if isinstance(value, float):
@@ -290,11 +291,14 @@ def data(self, index, role=Qt.DisplayRole):
                         # may happen if format = '%d' and value = NaN;
                         # see issue 4139
                         return to_qvariant(DEFAULT_FORMAT % value)
+                elif is_type_text_string(value):
+                    # Don't perform any conversion on strings
+                    # because it leads to differences between
+                    # the data present in the dataframe and
+                    # what is shown by Spyder
+                    return value
                 else:
-                    try:
-                        return to_qvariant(to_text_string(value))
-                    except UnicodeDecodeError:
-                        return to_qvariant(encoding.to_unicode(value))
+                    return to_qvariant(to_text_string(value))
         elif role == Qt.BackgroundColorRole:
             return to_qvariant(self.get_bgcolor(index))
         elif role == Qt.FontRole:


Test patch:
diff --git a/spyder/widgets/variableexplorer/tests/issue_5833.csv b/spyder/widgets/variableexplorer/tests/issue_5833.csv
new file mode 100644
--- /dev/null
+++ b/spyder/widgets/variableexplorer/tests/issue_5833.csv
@@ -0,0 +1,2 @@
+Это,кодирование
+пример,файла
diff --git a/spyder/widgets/variableexplorer/tests/test_dataframeeditor.py b/spyder/widgets/variableexplorer/tests/test_dataframeeditor.py
--- a/spyder/widgets/variableexplorer/tests/test_dataframeeditor.py
+++ b/spyder/widgets/variableexplorer/tests/test_dataframeeditor.py
@@ -224,7 +224,7 @@ def test_header_encoding():
     model = editor.dataModel
     assert model.headerData(0, orientation=Qt.Horizontal) == "Index"
     assert model.headerData(1, orientation=Qt.Horizontal) == "Unnamed: 0"
-    assert model.headerData(2, orientation=Qt.Horizontal) == "Unieke_Idcode"
+    assert "Unieke_Idcode" in model.headerData(2, orientation=Qt.Horizontal)
     assert model.headerData(3, orientation=Qt.Horizontal) == "a"
     assert model.headerData(4, orientation=Qt.Horizontal) == "b"
     assert model.headerData(5, orientation=Qt.Horizontal) == "c"
@@ -492,5 +492,29 @@ def test_dataframeeditor_edit_bool(qtbot, monkeypatch):
                 len(expected_df))
 
 
+def test_non_ascii_index():
+    """
+    Test that there are no errors when displaying a dataframe with
+    a non-ascii index.
+    """
+    df = read_csv(os.path.join(FILES_PATH, 'issue_5833.csv'), index_col=0)
+    dfm = DataFrameModel(df)
+    assert data(dfm, 0, 0) == 'пример'
+
+
+def test_no_convert_strings_to_unicode():
+    """
+    Test that we don't apply any conversion to strings in headers,
+    indexes or data.
+    """
+    df = read_csv(os.path.join(FILES_PATH, 'issue_5833.csv'), index_col=0,
+                  encoding='koi8_r')
+    dfm = DataFrameModel(df)
+
+    assert dfm.headerData(1, orientation=Qt.Horizontal) != u"Это"
+    assert data(dfm, 0, 0) != u'пример'
+    assert data(dfm, 0, 1) != u'файла'
+
+
 if __name__ == "__main__":
     pytest.main()