Repository: spyder-ide/spyder

Issue:
Correctly report the encoding of files in the Editor using the chardet library
We're going to add `chardet` as a dependency in **3.1** (as part of PR #3640), so we can use it to correctly report the encoding of files in the status bar:

![seleccion_001](https://cloud.githubusercontent.com/assets/365293/20573117/559a74da-b17c-11e6-868e-15ecbb61f78c.png)

@rlaverde, this one is for you too :-)


Discussion:

Resolving patch:
diff --git a/spyder/utils/encoding.py b/spyder/utils/encoding.py
--- a/spyder/utils/encoding.py
+++ b/spyder/utils/encoding.py
@@ -17,6 +17,8 @@
 import sys
 from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF32, getincrementaldecoder
 
+from chardet.universaldetector import UniversalDetector
+
 # Local imports
 from spyder.py3compat import (is_string, to_text_string, is_binary_string,
                               is_unicode)
@@ -105,12 +107,29 @@ def get_coding(text):
     @return coding string
     """
     for line in text.splitlines()[:2]:
-        result = CODING_RE.search(to_text_string(line))
-        if result:
-            codec = result.group(1)
-            # sometimes we find a false encoding that can result in errors
-            if codec in CODECS:
-                return codec
+        try:
+            result = CODING_RE.search(to_text_string(line))
+        except UnicodeDecodeError:
+            # This could fail because to_text_string assume the text is utf8-like
+            # and we don't know the encoding to give it to to_text_string
+            pass
+        else:
+            if result:
+                codec = result.group(1)
+                # sometimes we find a false encoding that can result in errors
+                if codec in CODECS:
+                    return codec
+
+    # Fallback using chardet
+    if is_binary_string(text):
+        detector = UniversalDetector()
+        for line in text.splitlines()[:2]:
+            detector.feed(line)
+            if detector.done: break
+
+        detector.close()
+        return detector.result['encoding']
+
     return None
 
 def decode(text):


Test patch:
diff --git a/spyder/utils/tests/Big5.txt b/spyder/utils/tests/Big5.txt
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/Big5.txt
@@ -0,0 +1 @@
+?�O�@?�bbig5��??���ܨҤ��
diff --git a/spyder/utils/tests/KOI8-R.txt b/spyder/utils/tests/KOI8-R.txt
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/KOI8-R.txt
@@ -0,0 +1 @@
+��� ����������� ������ ����� � KOI8-R
diff --git a/spyder/utils/tests/ascii.txt b/spyder/utils/tests/ascii.txt
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/ascii.txt
@@ -0,0 +1 @@
+This is a sample file encoding in ascii
diff --git a/spyder/utils/tests/test_encoding.py b/spyder/utils/tests/test_encoding.py
--- a/spyder/utils/tests/test_encoding.py
+++ b/spyder/utils/tests/test_encoding.py
@@ -6,8 +6,12 @@
 """Tests for encodings.py"""
 
 import pytest
+import os
 
-from spyder.utils.encoding import is_text_file
+from spyder.utils.encoding import is_text_file, get_coding
+
+__location__ = os.path.realpath(os.path.join(os.getcwd(),
+                                             os.path.dirname(__file__)))
 
 
 def test_is_text_file(tmpdir):
@@ -16,5 +20,19 @@ def test_is_text_file(tmpdir):
     assert is_text_file(str(p)) == True
 
 
+@pytest.mark.parametrize(
+    'expected_encoding, text_file',
+    [('utf-8', 'utf-8.txt'),
+     ('windows-1252', 'windows-1252.txt'),
+     ('ascii', 'ascii.txt'),
+     ('Big5', 'Big5.txt'),
+     ('KOI8-R', 'KOI8-R.txt'),
+     ])
+def test_files_encodings(expected_encoding, text_file):
+    with open(os.path.join(__location__, text_file), 'rb') as f:
+        text = f.read()
+        assert get_coding(text) == expected_encoding
+
+
 if __name__ == '__main__':
     pytest.main()
diff --git a/spyder/utils/tests/utf-8.txt b/spyder/utils/tests/utf-8.txt
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/utf-8.txt
@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+This is a sample file with utf-8 header
diff --git a/spyder/utils/tests/windows-1252.txt b/spyder/utils/tests/windows-1252.txt
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/windows-1252.txt
@@ -0,0 +1,2 @@
+This is a sample file with encoded in windows-1252
+��