Repository: spyder-ide/spyder
Pull request: PR: Add all possible string prefixes for syntax highlighting
Fixes #4884 

----

As suggested in #4884 the string prefixes should be highlighted with the same color as the string itself. The list of all possible prefixes is given [here](https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals) for Python 3 and [here](https://docs.python.org/2/reference/lexical_analysis.html#string-literals) for Python 2. 
![image](https://user-images.githubusercontent.com/20270441/29490573-c5d1884a-8540-11e7-9b8e-427cf412950a.png)

I have added a small test as well but not sure it is actually necessary. Is it? 

Problem statement:
prefix 'b' of bytes is not highlighting in the editor window
## Description

**What steps will reproduce the problem?**

when inputting the bytes-type string in the editor window, the prefix 'b' is not highlighting with the same color of string.

**What is the expected output? What do you see instead?**


**Please provide any additional information below**



## Version and main components

* Spyder Version: 3.2.0 
* Python Version: 3.6.1
* Qt Versions: 5.6.2, PyQt5 5.6 on Windows

## Dependencies
```
pyflakes >=0.6.0 :  1.5.0 (OK)
pycodestyle >=2.3:  2.3.1 (OK)
pygments >=2.0   :  2.2.0 (OK)
pandas >=0.13.1  :  0.20.1 (OK)
numpy >=1.7      :  1.12.1 (OK)
sphinx >=0.6.6   :  1.6.3 (OK)
rope >=0.9.4     :  0.10.5 (OK)
jedi >=0.9.0     :  0.10.2 (OK)
nbconvert >=4.0  :  5.2.1 (OK)
sympy >=0.7.3    :  1.0 (OK)
cython >=0.21    :  0.25.2 (OK)
qtconsole >=4.2.0:  4.3.0 (OK)
IPython >=4.0    :  5.3.0 (OK)
pylint >=0.25    :  1.7.2 (OK)

```

Hints from issue discussion:
@fanstvn could you add a screenshot?
I don't think we're going to address this, but pull requests are welcome.
@ccordoba12 I think it should be fairly simple? also we should add f strings, no?

https://github.com/spyder-ide/spyder/blob/master/spyder/utils/syntaxhighlighters.py#L345

Adding `f` and `b` in there?

Commit message: added test for some string prefixes

Patch to review:
diff --git a/spyder/utils/syntaxhighlighters.py b/spyder/utils/syntaxhighlighters.py
index 13eaf7ddce7..4703072663d 100644
--- a/spyder/utils/syntaxhighlighters.py
+++ b/spyder/utils/syntaxhighlighters.py
@@ -14,6 +14,7 @@
 import keyword
 import os
 import re
+import sys
 import weakref
 
 # Third party imports
@@ -340,14 +341,20 @@ def make_python_patterns(additional_keywords=[], additional_builtins=[]):
                   r"\b[+-]?0[oO][0-7]+[lL]?\b",
                   r"\b[+-]?0[bB][01]+[lL]?\b",
                   r"\b[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?[jJ]?\b"])
-    sqstring =     r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*'?"
-    dqstring =     r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*"?'
-    uf_sqstring =  r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*(\\)$(?!')$"
-    uf_dqstring =  r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*(\\)$(?!")$'
-    sq3string =    r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
-    dq3string =    r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
-    uf_sq3string = r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(\\)?(?!''')$"
-    uf_dq3string = r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(\\)?(?!""")$'
+    if sys.version_info[0] == 3:
+        prefix = "r|u|R|U|f|F|fr|Fr|fR|FR|rf|rF|Rf|RF|b|B|br|Br|bR|BR|rb|rB|Rb|RB"
+    else:
+        prefix = "r|u|ur|R|U|UR|Ur|uR|b|B|br|Br|bR|BR"
+    sqstring =     r"(\b(%s))?'[^'\\\n]*(\\.[^'\\\n]*)*'?" % prefix
+    dqstring =     r'(\b(%s))?"[^"\\\n]*(\\.[^"\\\n]*)*"?' % prefix
+    uf_sqstring =  r"(\b(%s))?'[^'\\\n]*(\\.[^'\\\n]*)*(\\)$(?!')$" % prefix
+    uf_dqstring =  r'(\b(%s))?"[^"\\\n]*(\\.[^"\\\n]*)*(\\)$(?!")$' % prefix
+    sq3string =    r"(\b(%s))?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" % prefix
+    dq3string =    r'(\b(%s))?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' % prefix
+    uf_sq3string = r"(\b(%s))?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(\\)?(?!''')$" \
+                   % prefix
+    uf_dq3string = r'(\b(%s))?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(\\)?(?!""")$' \
+                   % prefix
     string = any("string", [sq3string, dq3string, sqstring, dqstring])
     ufstring1 = any("uf_sqstring", [uf_sqstring])
     ufstring2 = any("uf_dqstring", [uf_dqstring])
diff --git a/spyder/utils/tests/test_syntaxhighlighters.py b/spyder/utils/tests/test_syntaxhighlighters.py
index 2ee1b9d34d2..8720dda25cc 100644
--- a/spyder/utils/tests/test_syntaxhighlighters.py
+++ b/spyder/utils/tests/test_syntaxhighlighters.py
@@ -47,6 +47,26 @@ def test_HtmlSH_unclosed_commend():
     compare_formats(doc.firstBlock().layout().additionalFormats(), res, sh)
 
 
+def test_python_string_prefix():
+    txt = "[r'test', b'test', f'test', u'test']"     
+
+    doc = QTextDocument(txt)
+    sh = PythonSH(doc, color_scheme='Spyder')
+    sh.rehighlightBlock(doc.firstBlock())
+
+    res = [(0, 1, 'normal'),   # |[|
+           (1, 7, 'string'),   # |r'test'|
+           (8, 2, 'normal'),   # |, |
+           (10, 7, 'string'),  # |b'test'|
+           (17, 2, 'normal'),  # |, |
+           (19, 7, 'string'),  # |f'test'|
+           (26, 2, 'normal'),  # |, |
+           (28, 7, 'string'),  # |u'test'|
+           (35, 1, 'normal')]  # |]|
+
+    compare_formats(doc.firstBlock().layout().additionalFormats(), res, sh)
+
+
 def test_Markdown_basic():
     txt = "Some __random__ **text** with ~~different~~ [styles](link_url)"
 


Code review comments:
diff_hunk: @@ -340,14 +341,20 @@ def make_python_patterns(additional_keywords=[], additional_builtins=[]):
                   r"\b[+-]?0[oO][0-7]+[lL]?\b",
                   r"\b[+-]?0[bB][01]+[lL]?\b",
                   r"\b[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?[jJ]?\b"])
-    sqstring =     r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*'?"
-    dqstring =     r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*"?'
-    uf_sqstring =  r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*(\\)$(?!')$"
-    uf_dqstring =  r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*(\\)$(?!")$'
-    sq3string =    r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
-    dq3string =    r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
-    uf_sq3string = r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(\\)?(?!''')$"
-    uf_dq3string = r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(\\)?(?!""")$'
+    if sys.version_info[0] == 3:
path: spyder/utils/syntaxhighlighters.py
text: @user1:
Please use `PY3`

`from spyder.py3compat import PY3`