Repository: mozilla/pontoon

Issue:
Revisit sync process for XLIFF files
*This issue was created automatically by a [script](https://github.com/mathjazz/bugzilla2github/).*
## [Bug 1676336](https://bugzilla.mozilla.org/show_bug.cgi?id=1676336)
Bug Reporter: @flodolo
CC:   @guerojeff, @mathjazz, sstreich@mozilla.com

So far we've been using XLIFF only for iOS projects. For both projects (Firefox, Focus) we built an infrastructure that updates all locales when adding new strings to the reference file (en-US).

Currently, if you add new strings only to the reference file:
- Strings show up as missing for all locales
- The new translations are not submitted back to the repository, since the XLIFF is missing them
- "Download Translations" gives you a file that is missing translations, like the one in the repository.

The downside of this approach, besides being very confusing, is that we need to replicate the infrastructure for every repository. For example, this is the work happening for the new VPN client
https://github.com/mozilla-l10n/mozilla-vpn-client-l10n/pull/39

Unlike GetText, XLIFF doesn't come with native commands to propagate new strings to all locales. 

I think it would make more sense to be able to add new strings only to the reference locale, and have Pontoon add them to the resulting XLIFF file when serializing it.


Discussion:
Comment Author: @guerojeff

Is this problem unique to XLIFF? I assumed all bilingual file types were affected by this issue due to bidi sync.


Comment Author: @mathjazz

In essence, the problem is the same across all bilingual file types (xliff, po, lang). The only difference is that at least po (not sure about lang) comes with tools to help with the merge of reference and l10n files.


Comment Author: @flodolo

That tool for .lang files is part of langchecker. The expectation there is that new string need to be added to all languages.
https://github.com/mozilla-l10n/langchecker/wiki/CLI-scripts-syntax#lang_update


This is a problem also when a brand new file is added to an existing project (VPN uses multiple XLIFF files).

Resolving patch:
diff --git a/pontoon/base/utils.py b/pontoon/base/utils.py
--- a/pontoon/base/utils.py
+++ b/pontoon/base/utils.py
@@ -260,7 +260,7 @@ def get_download_content(slug, code, part):
 
         # Get source file if needed
         source_path = None
-        if resource.is_asymmetric:
+        if resource.is_asymmetric or resource.format == "xliff":
             dirnames = VCSProject.SOURCE_DIR_NAMES
             source_path = _download_file(
                 source_prefixes, dirnames, vcs_project, resource.path
diff --git a/pontoon/sync/formats/xliff.py b/pontoon/sync/formats/xliff.py
--- a/pontoon/sync/formats/xliff.py
+++ b/pontoon/sync/formats/xliff.py
@@ -3,8 +3,9 @@
 """
 from lxml import etree
 from translate.storage import xliff
+import copy
 
-from pontoon.sync.exceptions import ParseError
+from pontoon.sync.exceptions import ParseError, SyncError
 from pontoon.sync.formats.base import ParsedResource
 from pontoon.sync.vcs.models import VCSTranslation
 
@@ -14,99 +15,136 @@ class XLIFFEntity(VCSTranslation):
     Interface for modifying a single entity in an xliff file.
     """
 
-    def __init__(self, order, unit):
-        self.order = order
-        self.unit = unit
-        self.strings = {None: self.target_string} if self.target_string else {}
-
-    @property
-    def key(self):
-        return self.unit.getid()
-
-    @property
-    def context(self):
-        return self.unit.xmlelement.get("id") or ""
-
-    @property
-    def source_string(self):
-        return str(self.unit.rich_source[0])
-
-    @property
-    def source_string_plural(self):
-        return ""
-
-    @property
-    def comments(self):
-        notes = self.unit.getnotes()
-        return notes.split("\n") if notes else []
-
-    @property
-    def fuzzy(self):
-        return False
-
-    @fuzzy.setter
-    def fuzzy(self, fuzzy):
-        pass  # We don't use fuzzy in XLIFF
-
-    @property
-    def source(self):
-        return []
-
-    @property
-    def target_string(self):
-        return str(self.unit.get_rich_target()[0])
-
-    @target_string.setter
-    def target_string(self, value):
-        self.unit.settarget(value)
-
-    def sync_changes(self):
-        """
-        Apply any changes made to this object to the backing unit in the
-        xliff file.
-        """
-        if None in self.strings:
-            self.target_string = self.strings[None]
-            # Store updated nodes
-            xml = self.unit.xmlelement
-            target = xml.find(self.unit.namespaced("target"))
-
-        else:
-            # Read stored nodes
-            xml = self.unit.xmlelement
-            target = xml.find(self.unit.namespaced("target"))
-            if target is not None:
-                xml.remove(target)
-
-        # Clear unused approved tag
-        if "approved" in xml.attrib:
-            del xml.attrib["approved"]
-
-        # Clear unused state tag
-        if target is not None and "state" in target.attrib:
-            del target.attrib["state"]
+    def __init__(
+        self,
+        key,
+        context,
+        source_string,
+        source_string_plural,
+        strings,
+        comments=None,
+        order=None,
+    ):
+        super().__init__(
+            key=key,
+            context=context,
+            source_string=source_string,
+            source_string_plural=source_string_plural,
+            strings=strings,
+            comments=comments or [],
+            fuzzy=False,
+            order=order,
+        )
+
+    def __repr__(self):
+        return f"<XLIFFEntity {self.key}>"
 
 
 class XLIFFResource(ParsedResource):
-    def __init__(self, path, xliff_file):
+    def __init__(self, path, locale, source_resource=None):
         self.path = path
-        self.xliff_file = xliff_file
-        self.entities = [
-            XLIFFEntity(order, unit) for order, unit in enumerate(self.xliff_file.units)
-        ]
+        self.locale = locale
+        self.source_resource = source_resource
+        self.entities = {}
+
+        # Copy entities from the source_resource if it's available.
+        if source_resource:
+            for key, entity in source_resource.entities.items():
+                self.entities[key] = XLIFFEntity(
+                    entity.key, "", "", "", {}, copy.copy(entity.comments), entity.order
+                )
+
+        # Open the file at the provided path
+        with open(path) as f:
+            # Read the contents of the file and encode it
+            xml = f.read().encode("utf-8")
+
+            try:
+                # Parse the xml content of the file into an XLIFF file object
+                self.xliff_file = xliff.xlifffile(xml)
+            except etree.XMLSyntaxError as err:
+                # If there is an error parsing the file, raise a ParseError
+                raise ParseError(f"Failed to parse {path}: {err}")
+
+            # Loop through each unit in the XLIFF file
+            for order, unit in enumerate(self.xliff_file.units):
+                # Get the unit's ID and source string
+                key = unit.getid()
+                context = unit.xmlelement.get("id")
+                source_string = str(unit.rich_source[0])
+                source_string_plural = ""
+
+                # Get the translated string for the unit. If there's no target string, this will be an empty dictionary
+                target_string = (
+                    str(unit.get_rich_target()[0]) if unit.get_rich_target() else None
+                )
+                strings = {None: target_string} if target_string else {}
+
+                # Get the unit's comments, split by newline characters
+                comments = unit.getnotes().split("\n") if unit.getnotes() else []
+
+                # Create a new XLIFFEntity from the unit
+                entity = XLIFFEntity(
+                    key,
+                    context,
+                    source_string,
+                    source_string_plural,
+                    strings,
+                    comments,
+                    order,
+                )
+                # Add the entity to the entities dictionary using its key as the dictionary key
+                self.entities[entity.key] = entity
 
     @property
     def translations(self):
-        return self.entities
+        return sorted(self.entities.values(), key=lambda e: e.order)
 
     def save(self, locale):
-        for entity in self.entities:
-            entity.sync_changes()
+        if not self.source_resource:
+            raise SyncError(
+                "Cannot save XLIFF resource {}: No source resource given.".format(
+                    self.path
+                )
+            )
+        # Open the file at the provided path
+        with open(self.source_resource.path) as f:
+            # Read the contents of the file and encode it
+            xml = f.read().encode("utf-8")
+
+            try:
+                # Parse the xml content of the file into an XLIFF file object
+                self.xliff_file = xliff.xlifffile(xml)
+            except etree.XMLSyntaxError as err:
+                # If there is an error parsing the file, raise a ParseError
+                raise ParseError(f"Failed to parse {self.source_resource.path}: {err}")
+
+            # Loop through each unit in the XLIFF file
+            for order, unit in enumerate(self.xliff_file.units):
+                # Apply any changes made to this object to the backing unit in the xliff file.
+                key = unit.getid()
+                entity = self.entities.get(key)
+                if None in entity.strings:
+                    # Store updated nodes
+                    unit.settarget(entity.strings[None])
+                    xml = unit.xmlelement
+                    target = xml.find(unit.namespaced("target"))
+
+                else:
+                    # Read stored nodes
+                    xml = unit.xmlelement
+                    target = xml.find(unit.namespaced("target"))
+                    if target is not None:
+                        xml.remove(target)
+
+                # Clear unused approved tag
+                if "approved" in xml.attrib:
+                    del xml.attrib["approved"]
+
+                # Clear unused state tag
+                if target is not None and "state" in target.attrib:
+                    del target.attrib["state"]
 
-        locale_code = locale.code
-
-        # TODO: should be made iOS specific
-        # Map locale codes for iOS: http://www.ibabbleon.com/iOS-Language-Codes-ISO-639.html
         locale_mapping = {
             "bn-IN": "bn",
             "ga-IE": "ga",
@@ -114,7 +152,7 @@ def save(self, locale):
             "nn-NO": "nn",
             "sv-SE": "sv",
         }
-
+        locale_code = locale.code
         if locale_code in locale_mapping:
             locale_code = locale_mapping[locale_code]
 
@@ -124,17 +162,15 @@ def save(self, locale):
             if not node.get("target-language"):
                 node.set("target-language", locale_code)
 
+        # Serialize and save the updated XLIFF file.
         with open(self.path, "wb") as f:
             f.write(bytes(self.xliff_file))
 
 
 def parse(path, source_path=None, locale=None):
-    with open(path) as f:
-        xml = f.read().encode("utf-8")
-
-        try:
-            xliff_file = xliff.xlifffile(xml)
-        except etree.XMLSyntaxError as err:
-            raise ParseError(f"Failed to parse {path}: {err}")
+    if source_path is not None:
+        source_resource = XLIFFResource(source_path, locale)
+    else:
+        source_resource = None
 
-    return XLIFFResource(path, xliff_file)
+    return XLIFFResource(path, locale, source_resource)


Test patch:
diff --git a/pontoon/sync/tests/formats/test_xliff.py b/pontoon/sync/tests/formats/test_xliff.py
--- a/pontoon/sync/tests/formats/test_xliff.py
+++ b/pontoon/sync/tests/formats/test_xliff.py
@@ -138,7 +138,9 @@ def test_save_basic(self):
             attr=attr,
         )
 
-        super().run_save_basic(input_string, expected_string)
+        super().run_save_basic(
+            input_string, expected_string, source_string=input_string
+        )
 
     def test_save_preserve_target_language(self):
         """
@@ -172,7 +174,9 @@ def test_save_preserve_target_language(self):
             attr=attr,
         )
 
-        super().run_save_basic(input_string, expected_string)
+        super().run_save_basic(
+            input_string, expected_string, source_string=input_string
+        )
 
     def test_save_remove(self):
         attr = f'target-language="{self.locale.code}"'
@@ -201,4 +205,6 @@ def test_save_remove(self):
             attr=attr,
         )
 
-        super().run_save_remove(input_string, expected_string)
+        super().run_save_remove(
+            input_string, expected_string, source_string=input_string
+        )