Repository: nvaccess/nvda

Issue:
Emojis Do Not Speak when Arrowing by Character
### Steps to reproduce:
Note: These steps apply when running NVDA Master 16003 with pull request #8758 included, which reads unicode emoji characters.

1. Open Notepad
2. Insert an Emoji such as 🌮 which is the taco symbol.
3. Arrow over the emoji by character.
You can also arrow over the taco emoji above to observe the same results. Using the up and down arrow keys reads it correctly.
I'm not sure if some emojis are represented by two characters, especially in browse mode.
### Actual behavior:
NVDA does not speak anything when landing on an emoji character, which is the same behavior as before Master 16003.

### Expected behavior:
The emoji character should be spoken.

### System configuration:

#### NVDA Installed/portable/running from source:
installed

#### NVDA version:
NVDA Master 16003, 

#### Windows version:
1803

#### Name and version of other software in use when reproducing the issue:
n/a
#### Other information about your system:
Eloquence speech.

### Other questions:

#### Does the issue still occur after restarting your PC?
yes
#### Have you tried any other versions of NVDA?
n/a, based off recent PR



Discussion:

Resolving patch:
diff --git a/source/textInfos/offsets.py b/source/textInfos/offsets.py
--- a/source/textInfos/offsets.py
+++ b/source/textInfos/offsets.py
@@ -11,6 +11,11 @@
 import config
 import textInfos
 
+HIGH_SURROGATE_FIRST = u"\uD800"
+HIGH_SURROGATE_LAST = u"\uDBFF"
+LOW_SURROGATE_FIRST = u"\uDC00"
+LOW_SURROGATE_LAST = u"\uDFFF"
+
 class Offsets(object):
 	"""Represents two offsets."""
 
@@ -221,7 +226,28 @@ def _getFormatFieldAndOffsets(self,offset,formatConfig,calculateOffsets=True):
 		return formatField,(startOffset,endOffset)
 
 	def _getCharacterOffsets(self,offset):
-		return [offset,offset+1]
+		# Windows Unicode is UTF-16, so a character may be two offsets for code points beyond 16 bits.
+		if offset > 0:
+			chars = self._getTextRange(offset - 1, offset + 2)
+			# Slicing avoids the need to check length. If invalid, it'll be the empty string.
+			prevChar = chars[0:1]
+			curChar = chars[1:2]
+			nextChar = chars[2:3]
+		else:
+			chars = self._getTextRange(offset, offset + 2)
+			prevChar = u"" # Empty string, any subsequent comparisons will evaluate to False.
+			# Slicing avoids the need to check length. If invalid, it'll be the empty string.
+			curChar = chars[0:1]
+			nextChar = chars[1:2]
+		if HIGH_SURROGATE_FIRST <= curChar <= HIGH_SURROGATE_LAST and LOW_SURROGATE_FIRST <= nextChar <= LOW_SURROGATE_LAST:
+			# curChar is a high (leading) surrogate;
+			# nextChar is a low surrogate and also part of this character.
+			return offset, offset + 2
+		elif HIGH_SURROGATE_FIRST <= prevChar <= HIGH_SURROGATE_LAST and LOW_SURROGATE_FIRST <= curChar <= LOW_SURROGATE_LAST:
+			# curChar is a low (trailing) surrogate;
+			# prevChar is a high surrogate and also part of this character.
+			return offset - 1, offset + 1
+		return offset, offset + 1
 
 	def _getWordOffsets(self,offset):
 		lineStart,lineEnd=self._getLineOffsets(offset)


Test patch:
diff --git a/tests/unit/test_textInfos.py b/tests/unit/test_textInfos.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/test_textInfos.py
@@ -0,0 +1,131 @@
+# -*- coding: UTF-8 -*-
+#tests/unit/test_textInfos.py
+#A part of NonVisual Desktop Access (NVDA)
+#This file is covered by the GNU General Public License.
+#See the file COPYING for more details.
+#Copyright (C) 2018 NV Access Limited, Babbage B.V.
+
+"""Unit tests for the textInfos module, its submodules and classes."""
+
+import unittest
+from .textProvider import BasicTextProvider
+import textInfos
+from textInfos.offsets import Offsets
+
+class TestCharacterOffsets(unittest.TestCase):
+	"""
+	Tests for textInfos.offsets.OffsetsTextInfo for its ability to deal with
+	UTF-16 surrogate characters (i.e. whether a surrogate pair is treated as one character).
+	"""
+
+	def test_nonSurrogateForward(self):
+		obj = BasicTextProvider(text="abc")
+		ti = obj.makeTextInfo(Offsets(0, 0))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at a
+		self.assertEqual(ti.offsets, (0, 1)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at b
+		self.assertEqual(ti.offsets, (1, 2)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at c
+		self.assertEqual(ti.offsets, (2, 3)) # One offset
+
+	def test_nonSurrogateBackward(self):
+		obj = BasicTextProvider(text="abc")
+		ti = obj.makeTextInfo(Offsets(2, 2))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at c
+		self.assertEqual(ti.offsets, (2, 3)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at b
+		self.assertEqual(ti.offsets, (1, 2)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at a
+		self.assertEqual(ti.offsets, (0, 1)) # One offset
+
+	def test_surrogatePairsForward(self):
+		obj = BasicTextProvider(text=u"\ud83e\udd26\ud83d\ude0a\ud83d\udc4d") # 🤦😊👍
+		ti = obj.makeTextInfo(Offsets(0, 0))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🤦
+		self.assertEqual(ti.offsets, (0, 2)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 😊
+		self.assertEqual(ti.offsets, (2, 4)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 👍
+		self.assertEqual(ti.offsets, (4, 6)) # Two offsets
+
+	def test_surrogatePairsBackward(self):
+		obj = BasicTextProvider(text=u"\ud83e\udd26\ud83d\ude0a\ud83d\udc4d") # 🤦😊👍
+		ti = obj.makeTextInfo(Offsets(5, 5))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 👍
+		self.assertEqual(ti.offsets, (4, 6)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 😊
+		self.assertEqual(ti.offsets, (2, 4)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🤦
+		self.assertEqual(ti.offsets, (0, 2)) # Two offsets
+
+	def test_mixedSurrogatePairsAndNonSurrogatesForward(self):
+		obj = BasicTextProvider(text=u"a\ud83e\udd26b") # a🤦b
+		ti = obj.makeTextInfo(Offsets(0, 0))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at a
+		self.assertEqual(ti.offsets, (0, 1)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🤦
+		self.assertEqual(ti.offsets, (1, 3)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at c
+		self.assertEqual(ti.offsets, (3, 4)) # One offset
+
+	def test_mixedSurrogatePairsAndNonSurrogatesBackward(self):
+		obj = BasicTextProvider(text=u"a\ud83e\udd26b") # a🤦b
+		ti = obj.makeTextInfo(Offsets(3, 3))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at c
+		self.assertEqual(ti.offsets, (3, 4)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🤦
+		self.assertEqual(ti.offsets, (1, 3)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at a
+		self.assertEqual(ti.offsets, (0, 1)) # One offset
+
+	def test_mixedSurrogatePairsNonSurrogatesAndSingleSurrogatesForward(self):
+		"""
+		Tests surrogate pairs, non surrogates as well as
+		single surrogate characters (i.e. incomplete pairs)
+		"""
+		obj = BasicTextProvider(text=u"a\ud83e\ud83e\udd26\udd26b")
+		ti = obj.makeTextInfo(Offsets(0, 0))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at a
+		self.assertEqual(ti.offsets, (0, 1)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🠀
+		self.assertEqual(ti.offsets, (1, 2)) # Leading surrogate without a trailing surrogate
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🤦
+		self.assertEqual(ti.offsets, (2, 4)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 񙠀
+		self.assertEqual(ti.offsets, (4, 5)) # Trailing surrogate without a leading surrogate.
+		ti.move(textInfos.UNIT_CHARACTER, 1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at c
+		self.assertEqual(ti.offsets, (5, 6)) # One offset
+
+	def test_mixedSurrogatePairsNonSurrogatesAndSingleSurrogatesBackward(self):
+		obj = BasicTextProvider(text=u"a\ud83e\ud83e\udd26\udd26b")
+		ti = obj.makeTextInfo(Offsets(5, 5))
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at c
+		self.assertEqual(ti.offsets, (5, 6)) # One offset
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 񙠀
+		self.assertEqual(ti.offsets, (4, 5)) # Trailing surrogate without a leading surrogate.
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🤦
+		self.assertEqual(ti.offsets, (2, 4)) # Two offsets
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at 🠀
+		self.assertEqual(ti.offsets, (1, 2)) # Leading surrogate without a trailing surrogate
+		ti.move(textInfos.UNIT_CHARACTER, -1)
+		ti.expand(textInfos.UNIT_CHARACTER) # Range at a
+		self.assertEqual(ti.offsets, (0, 1)) # One offset