Repository: translate/translate

Issue:
Double quote mismatch when there is XML markup inside them (Android)
I was asked to file an issue here from: https://github.com/WeblateOrg/weblate/issues/14214

This string has double quote at the end but not at the start: https://hosted.weblate.org/translate/microg/play-services-nearby-core-strings/en/?q=exposure_confirm_start_summary
While the original string has both: https://github.com/microg/GmsCore/blob/3d54307cc02aed50c0f546962437c8f49606b1f7/play-services-nearby/core/src/main/res/values/strings.xml#L54

Specifications: https://developer.android.com/guide/topics/resources/string-resource?hl=en#FormattingAndStyling

Discussion:
@nijel: The specification is not obvious on when the quotes are processed when XML markup is present. translate-toolkit currently looks at quotes outside XML elements, what is probably wrong.

https://github.com/translate/translate/pull/5542 added xfail test for this.

Can you please verify how does the string look like in Android? Does it preserve whitespace in whole string (before and after the XML tag)?@ale5000-git: > Can you please verify how does the string look like in Android? Does it preserve whitespace in whole string (before and after the XML tag)?

I will do it soon but I'm a bit busy on these days (if there is someone else working on Android that can do it sooner, please step in).

---

From this part:

`From XML parser's perspective, there is no difference between <string>"Test this"</string> and <string>&quot;Test this&quot;</string> whatsoever. Both forms will not show any quotes but trigger Android whitespace-preserving quoting (that will have no practical effect in this case).`

it seems that there is a very special handling of the double quotes.@nijel: To clarify it, let's add some examples:

This currently works just fine in translate-toolkit:

```
"Text   with    spaces"<g>%1$s</g>"Another   with    spaces"
```

While this not:

```
"Text   with    spaces<g>%1$s</g>Another   with    spaces"
```

This is because translate-toolkit looks for quotes in individual string elements and not in whole content including XML tags. This is easier because it avoids the need to detect which quotes should be ignored because of being part of XML tag.

In case the second example works fine in Android and preserves whitespace, the behavior should be adjusted.@nijel: https://github.com/translate/translate/pull/5574 should address this. What is still unclear to me is how Android deals with unpaired quotes in the string, so I've chosen behavior that was easier to implement.

Solution patch:
diff --git a/translate/storage/aresource.py b/translate/storage/aresource.py
index 6c4c992e..575eadf2 100644
--- a/translate/storage/aresource.py
+++ b/translate/storage/aresource.py
@@ -73,6 +73,7 @@ class DecodingXMLParser:
         self.character_data = False
         self.raw_string = True
         self.in_string = False
+        self.inside_quoted = False
         self.do_cleanup = False
         self.depth = 0
         self.parser = parser = ParserCreate()
@@ -105,16 +106,27 @@ class DecodingXMLParser:
         # All others, remove, like Android does as well.
         return ""
 
-    @staticmethod
-    def strip_part(text: str) -> str:
+    def strip_part(self, text: str) -> tuple[str, bool]:
+        # Detect leading quote
         if text.startswith('"'):
-            # Missing closing quote is gracefully ignored
-            return text.removeprefix('"').removesuffix('"')
-        # Consolidate whitespace
-        return WHITESPACE_RE.sub(" ", text)
+            text = text[1:]
+            self.inside_quoted ^= True
 
-    @classmethod
-    def process_string(cls, content: str) -> tuple[str, bool, bool]:
+        # Check if whitespace should be stripped
+        strip_whitespace = not self.inside_quoted
+
+        # Detect trailing quote
+        if text.endswith('"') and not text.endswith('\\"'):
+            text = text[:-1]
+            self.inside_quoted ^= True
+
+        # Consolidate whitespace if needed
+        if strip_whitespace:
+            text = WHITESPACE_RE.sub(" ", text)
+
+        return text, strip_whitespace
+
+    def process_string(self, content: str) -> tuple[str, bool, bool]:
         # Skip processing for a blank string
         if not content:
             return "", False, False
@@ -123,18 +135,25 @@ class DecodingXMLParser:
         parts = [part for part in QUOTED_STRING.split(content) if part]
 
         # Detect possible cleanups
-        cleanup_start = not parts[0].startswith('"')
-        cleanup_end = not parts[-1].startswith('"')
+        cleanup_start = False
+        cleanup_end = False
 
         # Collapse whitespace
-        stripped_parts = [cls.strip_part(part) for part in parts]
+        stripped_parts: list[str] = []
+        for i, part in enumerate(parts):
+            stripped, was_stripped = self.strip_part(part)
+            if i == 0:
+                cleanup_start = was_stripped
+            cleanup_end = was_stripped and not self.inside_quoted
+            stripped_parts.append(stripped)
+
         # Merge this back to a single string
         text = "".join(stripped_parts)
 
         # Handle unicode escapes
-        unescaped_text = UNICODE_ESCAPE.sub(cls.decode_unicode_escapes, text)
+        unescaped_text = UNICODE_ESCAPE.sub(self.decode_unicode_escapes, text)
         # Handle other escapes
-        unescaped_text = CHAR_ESCAPE.sub(cls.decode_escapes, unescaped_text)
+        unescaped_text = CHAR_ESCAPE.sub(self.decode_escapes, unescaped_text)
 
         # Detect escaping at the start and end of a string (we actually care only for
         # whitespace escapes, but detecting more does not matter)
@@ -213,8 +232,7 @@ class EncodingXMLParser(DecodingXMLParser):
     EMIT_DEPTH = 0
     FOREGIN_DTD = False
 
-    @staticmethod
-    def process_string(text: str) -> tuple[str, bool, bool]:
+    def process_string(self, text: str) -> tuple[str, bool, bool]:
         return (
             AndroidResourceUnit.escape(text, quote_wrapping_whitespaces=False),
             False,
@@ -319,22 +337,21 @@ class AndroidResourceUnit(base.TranslationUnit):
 
     def get_xml_text_value(self, xmltarget):
         raw_xml = etree.tostring(xmltarget, encoding="unicode", with_tail=False)
+        # Use decoding parser to keep XML entitities, CDATA while processing Android escaping
+        parser = DecodingXMLParser(raw_xml)
+
         # Unescape as plain text in case there is no XML markup. Ufortunately
         # CDATA elements are not listed as childs in lxml, so test for it in raw XML.
         if len(xmltarget) == 0 and "<![CDATA[" not in raw_xml:
             if xmltarget.text is None:
                 return ""
-            text, cleanup_start, cleanup_end = DecodingXMLParser.process_string(
-                xmltarget.text
-            )
+            text, cleanup_start, cleanup_end = parser.process_string(xmltarget.text)
             if cleanup_start:
                 text = text.lstrip()
             if cleanup_end:
                 text = text.rstrip()
             return text
 
-        # Use decoding parser to keep XML entitities, CDATA while processing Android escaping
-        parser = DecodingXMLParser(raw_xml)
         return parser.parse()
 
     def set_xml_text_plain(self, target, xmltarget):


Test patch:
diff --git a/tests/translate/storage/test_aresource.py b/tests/translate/storage/test_aresource.py
index fc15abe7..6881e294 100644
--- a/tests/translate/storage/test_aresource.py
+++ b/tests/translate/storage/test_aresource.py
@@ -1,6 +1,5 @@
 from copy import copy, deepcopy
 
-import pytest
 from lxml import etree
 
 from translate.misc.multistring import multistring
@@ -507,11 +506,11 @@ class TestAndroidResourceUnit(test_monolingual.TestMonolingualUnit):
 
     def test_parse_unparied_quote(self):
         self.__check_parse(
-            """You can view it here soon: %s""",
+            "You can view it here soon: %s",
             """<string name="cmd_camera_response_success">"You can view it here soon: %s</string>""",
         )
         self.__check_parse(
-            'You can view it here soon: %s"',
+            "You can view it here soon: %s",
             """<string name="cmd_camera_response_success">You can view it here soon: %s"</string>""",
         )
 
@@ -533,7 +532,6 @@ class TestAndroidResourceUnit(test_monolingual.TestMonolingualUnit):
             r"""<string name="restore_backup_file_not_found">No file %1$s found in archive \"%2$s\"</string>""",
         )
 
-    @pytest.mark.xfail(reason="Currently undefined behavior")
     def test_parse_quoted_xml(self):
         self.__check_parse(
             """Your phone needs to use Bluetooth to securely collect and share IDs with other phones that are nearby.