Repository: nvaccess/nvda

Issue:
Adding support for group matches in complexSymbols?
Hello,

In e.g. `source/locale/fr/symbols.dic`, we have 

```
dates .	((?<=\b\d\d)\.(?=\d\d.(\d{2}|\d{4})\b))|((?<=\b\d\d.\d\d)\.(?=(\d{2}|\d{4})\b))
dates .	 	all	always	# point de date
```

so as to remove dots from the dates. This is relatively complex, and could be much better written with matching groups:

```
dates .	\b(\d\d)\.(\d\d)\.(\d{2}|\d{4})\b
dates .	\1 \2 \3 	all	always	# point de date
```

It would allow to write more easily such rules like:

```
price	\b(\d\+),(\d\+)\s?€\b
price	\1 euros \2	all	always	# prix à virgules
```

etc.

I would be just a few lines more in _regexpRepl to perform the group replacement, I can work on it if the principle is Ok? (basically it means making the replacement string interpret `\` characters, thus allowing for this feature, and possibly even more in the future)

Samuel
Adding support for group matches in complexSymbols?
Hello,

In e.g. `source/locale/fr/symbols.dic`, we have 

```
dates .	((?<=\b\d\d)\.(?=\d\d.(\d{2}|\d{4})\b))|((?<=\b\d\d.\d\d)\.(?=(\d{2}|\d{4})\b))
dates .	 	all	always	# point de date
```

so as to remove dots from the dates. This is relatively complex, and could be much better written with matching groups:

```
dates .	\b(\d\d)\.(\d\d)\.(\d{2}|\d{4})\b
dates .	\1 \2 \3 	all	always	# point de date
```

It would allow to write more easily such rules like:

```
price	\b(\d\+),(\d\+)\s?€\b
price	\1 euros \2	all	always	# prix à virgules
```

etc.

I would be just a few lines more in _regexpRepl to perform the group replacement, I can work on it if the principle is Ok? (basically it means making the replacement string interpret `\` characters, thus allowing for this feature, and possibly even more in the future)

Samuel


Discussion:

Resolving patch:
diff --git a/source/characterProcessing.py b/source/characterProcessing.py
--- a/source/characterProcessing.py
+++ b/source/characterProcessing.py
@@ -519,6 +519,35 @@ def __init__(self, locale):
 			log.error("Invalid complex symbol regular expression in locale %s: %s" % (locale, e))
 			raise LookupError
 
+	def _replaceGroups(self, m: re.Match, string: str) -> str:
+		"""Replace matching group references (\\1, \\2, ...) with the corresponding matched groups.
+		Also replace \\\\ with \\ and reject other escapes, for escaping coherency.
+		@param m: The currently-matched group
+		@param string: The match replacement string which may contain group references
+		"""
+		result = ''
+
+		in_escape = False
+		for char in string:
+			if not in_escape:
+				if char == '\\':
+					in_escape = True
+				else:
+					result += char
+			else:
+				if char == '\\':
+					result += '\\'
+				elif char >= '0' and char <= '9':
+					result += m.group(m.lastindex + ord(char) - ord('0'))
+				else:
+					log.error("Invalid reference \\%string" % char)
+					raise LookupError
+				in_escape = False
+		if in_escape:
+			log.error("Unterminated backslash")
+			raise LookupError
+		return result
+
 	def _regexpRepl(self, m):
 		group = m.lastgroup
 
@@ -540,16 +569,19 @@ def _regexpRepl(self, m):
 			if group == "simple":
 				# Simple symbol.
 				symbol = self.computedSymbols[text]
+				replacement = symbol.replacement
 			else:
 				# Complex symbol.
 				index = int(group[1:])
 				symbol = self._computedComplexSymbolsList[index]
+				replacement = self._replaceGroups(m, symbol.replacement)
+
 			if symbol.preserve == SYMPRES_ALWAYS or (symbol.preserve == SYMPRES_NOREP and self._level < symbol.level):
 				suffix = text
 			else:
 				suffix = " "
-			if self._level >= symbol.level and symbol.replacement:
-				return u" {repl}{suffix}".format(repl=symbol.replacement, suffix=suffix)
+			if self._level >= symbol.level and replacement:
+				return u" {repl}{suffix}".format(repl=replacement, suffix=suffix)
 			else:
 				return suffix
 


Test patch:
diff --git a/tests/unit/test_characterProcessing.py b/tests/unit/test_characterProcessing.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/test_characterProcessing.py
@@ -0,0 +1,120 @@
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2020 NV Access Limited
+
+"""Unit tests for the characterProcessing module.
+"""
+
+import unittest
+import re
+from characterProcessing import SpeechSymbolProcessor
+
+
+class TestComplex(unittest.TestCase):
+	"""Test the complex symbols rules.
+	"""
+
+	def _replace_cb(self, replacement, name=None):
+		"""Return a regexp callback which replaces matches of the given
+		group name (or all groups if no name is given) with the
+		replacement string, with support for replacement of group
+		references.
+		"""
+		def replace(m):
+			if name is None or m.lastgroup == name:
+				return SpeechSymbolProcessor._replaceGroups(self, m, replacement)
+			return m.group()
+		return replace
+
+	def _replace(self, string, pattern, replacement, name=None):
+		"""Perform a pattern-based replacement on a string, for the
+		given named group (or all groups if no name is given), with
+		support for replacement of group references.
+		"""
+		regexp = re.compile(pattern, re.UNICODE)
+		return regexp.sub(self._replace_cb(replacement, name), string)
+
+	def test_group_replacement(self):
+		"""Test that plain text gets properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement="a"
+		)
+		self.assertEqual(replaced, "a")
+
+	def test_backslash_replacement(self):
+		"""Test that backslashes get properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement=r"\\"
+		)
+		self.assertEqual(replaced, "\\")
+
+	def test_double_backslash_replacement(self):
+		"""Test that double backslashes get properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement=r"\\\\"
+		)
+		self.assertEqual(replaced, r"\\")
+
+	def test_unknown_escape(self):
+		"""Test that a non-supported escaped character (i.e. not \\1,
+		\\2, ... \\9 and \\\\) in the replacement raises an error
+		"""
+		with self.assertRaises(LookupError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement=r"\a"
+			)
+
+	def test_missing_group(self):
+		"""Test that a reference in the replacement to an non-existing
+		group raises an error
+		"""
+		with self.assertRaises(IndexError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement=r"\2"
+			)
+
+	def test_unterminated_escape(self):
+		"""Test that an escape at the end of replacement raises an
+		error, since there is nothing to be escaped there
+		"""
+		with self.assertRaises(LookupError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement="\\"
+			)
+
+	def test_group_replacements(self):
+		"""Test that group references get properly replaced
+		"""
+		replaced = self._replace(
+			string="bar.BAT",
+			pattern=r"(([a-z]*)\.([A-Z]*))",
+			replacement=r"\2>\1"
+		)
+		self.assertEqual(replaced, "BAT>bar")
+
+	def test_multiple_group_replacement(self):
+		"""Test that group indexing is correct with multiple groups
+		"""
+		replaced = self._replace(
+			string="bar.BAT",
+			pattern=r"(baz)|(?P<foo>([a-z]*)\.([A-Z]*))",
+			replacement=r"\2>\1",
+			name="foo"
+		)
+		self.assertEqual(replaced, "BAT>bar")