Repository: digitalfabrik/integreat-cms

Issue:
Allow unchanged XLIFF files to be uploaded
### Motivation
<!-- A clear and concise description of what the motivation for the new feature is, and what problem it is solving. -->
When uploading XLIFF files, it currently causes an error when the files does not change the existing content.
However, it is quite common that translation were already up to date when exporting, so no changes are a perfectly valid result.

### Proposed Solution
<!-- A clear and concise description of the feature you would like to add, and how it solves the motivating problem. -->
Instead of showing the error, do the following:
- Remove the `currently_in_translation` flag of the existing translation
- Update the `last_updated` date of the existing translation
- Show an info message instead of the error



Discussion:

Resolving patch:
diff --git a/integreat_cms/cms/models/pages/abstract_base_page_translation.py b/integreat_cms/cms/models/pages/abstract_base_page_translation.py
--- a/integreat_cms/cms/models/pages/abstract_base_page_translation.py
+++ b/integreat_cms/cms/models/pages/abstract_base_page_translation.py
@@ -67,7 +67,7 @@ def readable_title(self):
             # Start with translation title if exists
             # pylint: disable=no-member
             readable_title = f'"{self.title}"'
-            if best_translation != self:
+            if best_translation.title != self.title:
                 readable_title += " (" + _("Title in") + best_translation_title + ")"
         else:
             # Start directly with the title of the best translation
diff --git a/integreat_cms/xliff/utils.py b/integreat_cms/xliff/utils.py
--- a/integreat_cms/xliff/utils.py
+++ b/integreat_cms/xliff/utils.py
@@ -8,8 +8,6 @@
 import os
 import uuid
 
-from copy import deepcopy
-
 from django.conf import settings
 from django.contrib import messages
 from django.core import serializers
@@ -271,6 +269,12 @@ def get_xliff_import_diff(request, xliff_dir):
     for xliff_file, deserialized_objects in xliffs_to_pages(request, xliff_dir).items():
         for deserialized in deserialized_objects:
             page_translation = deserialized.object
+            # The prefetched translations now also contain the new deserialized object with id None, so we have to delete
+            # the cached property and query it from the database again.
+            try:
+                del page_translation.page.prefetched_translations_by_language_slug
+            except AttributeError:
+                pass
             existing_translation = page_translation.latest_version or PageTranslation(
                 page=page_translation.page,
                 language=page_translation.language,
@@ -302,7 +306,9 @@ def get_xliff_import_diff(request, xliff_dir):
                     },
                     "right_to_left": page_translation.language.text_direction
                     == text_directions.RIGHT_TO_LEFT,
-                    "errors": get_xliff_import_errors(request, page_translation),
+                    "errors": get_xliff_import_errors(
+                        request, page_translation, add_message_if_unchanged=True
+                    )[0],
                 }
             )
     return diff
@@ -328,7 +334,7 @@ def xliff_import_confirm(request, xliff_dir):
         # (typically, one xliff file contains exactly one page translation)
         for deserialized in deserialized_objects:
             page_translation = deserialized.object
-            errors = get_xliff_import_errors(request, page_translation)
+            errors, has_changed = get_xliff_import_errors(request, page_translation)
             if errors:
                 logger.warning(
                     "XLIFF import of %r not possible because validation of %r failed with the errors: %r",
@@ -347,6 +353,23 @@ def xliff_import_confirm(request, xliff_dir):
                     ).format(page_translation.readable_title, error_list),
                 )
                 success = False
+            elif not has_changed:
+                # Update existing translation
+                existing_translation = page_translation.latest_version
+                existing_translation.currently_in_translation = False
+                existing_translation.save()
+                logger.info(
+                    "%r of XLIFF file %r was imported without changes by %r",
+                    existing_translation,
+                    xliff_file,
+                    request.user,
+                )
+                messages.info(
+                    request,
+                    _("Page {} was imported without changes.").format(
+                        page_translation.readable_title
+                    ),
+                )
             else:
                 # Confirm import and write changes to the database
                 page_translation.save()
@@ -365,7 +388,7 @@ def xliff_import_confirm(request, xliff_dir):
     return success
 
 
-def get_xliff_import_errors(request, page_translation):
+def get_xliff_import_errors(request, page_translation, add_message_if_unchanged=False):
     """
     Validate an imported page translation and return all errors
 
@@ -375,6 +398,9 @@ def get_xliff_import_errors(request, page_translation):
     :param page_translation: The page translation which is being imported
     :type page_translation: ~integreat_cms.cms.models.pages.page_translation.PageTranslation
 
+    :param add_message_if_unchanged: Whether a message should be added if no changes are detected
+    :type add_message_if_unchanged: bool
+
     :return: All errors of this XLIFF import
     :rtype: list [ dict ]
     """
@@ -402,8 +428,17 @@ def get_xliff_import_errors(request, page_translation):
             }
         )
     # Retrieve existing page translation in the target language
-    # (using model instances in forms may change them, so we need to copy it in order to not change the cached property)
-    existing_translation = deepcopy(page_translation.latest_version)
+    # The prefetched translations now also contain the new deserialized object with id None, so we have to delete
+    # the cached property and query it from the database again.
+    try:
+        del page_translation.latest_version
+    except AttributeError:
+        pass
+    try:
+        del page_translation.page.prefetched_translations_by_language_slug
+    except AttributeError:
+        pass
+    existing_translation = page_translation.latest_version
     # Validate page translation
     page_translation_form = PageTranslationForm(
         data=model_to_dict(page_translation),
@@ -421,11 +456,12 @@ def get_xliff_import_errors(request, page_translation):
                 for message in page_translation_form.get_error_messages()
             ]
         )
-    if not page_translation_form.has_changed():
-        error_messages.append(
-            {
-                "level_tag": "info",
-                "message": _("No changes detected."),
-            }
-        )
-    return error_messages
+    if add_message_if_unchanged:
+        if not page_translation_form.has_changed():
+            error_messages.append(
+                {
+                    "level_tag": "info",
+                    "message": _("No changes detected."),
+                }
+            )
+    return error_messages, page_translation_form.has_changed()


Test patch:
diff --git a/tests/xliff/__init__.py b/tests/xliff/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/xliff/__init__.py
@@ -0,0 +1,3 @@
+"""
+This package contains tests of the :mod:`integreat_cms.xliff` app
+"""
diff --git a/tests/xliff/dummy_django_app/__init__.py b/tests/xliff/dummy_django_app/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/xliff/dummy_django_app/__init__.py
@@ -0,0 +1,3 @@
+"""
+This package contains a dummy Django app to simulate the serving of static files during tests with DEBUG=False
+"""
diff --git a/tests/xliff/dummy_django_app/static_urls.py b/tests/xliff/dummy_django_app/static_urls.py
new file mode 100644
--- /dev/null
+++ b/tests/xliff/dummy_django_app/static_urls.py
@@ -0,0 +1,24 @@
+from django.conf import settings
+from django.urls import include, path
+from django.views.static import serve
+
+from integreat_cms.core.urls import urlpatterns
+
+#: Add the debug serve view to test the download of XLIFF files
+urlpatterns += [
+    path(
+        "",
+        include(
+            (
+                [
+                    path(
+                        f"{settings.XLIFF_URL}<path:path>".lstrip("/"),
+                        serve,
+                        {"document_root": settings.XLIFF_DOWNLOAD_DIR},
+                    )
+                ],
+                "xliff_files",
+            )
+        ),
+    ),
+]
diff --git a/tests/xliff/files/export/augsburg_de_en_1_2_willkommen.xliff b/tests/xliff/files/export/augsburg_de_en_1_2_willkommen.xliff
new file mode 100644
--- /dev/null
+++ b/tests/xliff/files/export/augsburg_de_en_1_2_willkommen.xliff
@@ -0,0 +1,21 @@
+<?xml version="1.0" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
+	<file datatype="plaintext" original="1" source-language="de" target-language="en">
+		<header>
+			<phase-group>
+				<phase phase-name="shortcodes" process-name="Shortcodes identification"/>
+				<phase phase-name="post_type" process-name="Post type"/>
+			</phase-group>
+		</header>
+		<body>
+			<trans-unit datatype="html" id="title" resname="title" restype="string">
+				<source><![CDATA[Willkommen]]></source>
+				<target><![CDATA[Welcome]]></target>
+			</trans-unit>
+			<trans-unit datatype="html" id="body" resname="body" restype="string">
+				<source><![CDATA[<p>Willkommen in Augsburg</p>]]></source>
+				<target><![CDATA[<p>Welcome to Augsburg</p>]]></target>
+			</trans-unit>
+		</body>
+	</file>
+</xliff>
diff --git a/tests/xliff/files/export/augsburg_de_en_2_1_willkommen-in-augsburg.xliff b/tests/xliff/files/export/augsburg_de_en_2_1_willkommen-in-augsburg.xliff
new file mode 100644
--- /dev/null
+++ b/tests/xliff/files/export/augsburg_de_en_2_1_willkommen-in-augsburg.xliff
@@ -0,0 +1,21 @@
+<?xml version="1.0" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
+	<file datatype="plaintext" original="2" source-language="de" target-language="en">
+		<header>
+			<phase-group>
+				<phase phase-name="shortcodes" process-name="Shortcodes identification"/>
+				<phase phase-name="post_type" process-name="Post type"/>
+			</phase-group>
+		</header>
+		<body>
+			<trans-unit datatype="html" id="title" resname="title" restype="string">
+				<source><![CDATA[Willkommen in Augsburg]]></source>
+				<target/>
+			</trans-unit>
+			<trans-unit datatype="html" id="body" resname="body" restype="string">
+				<source><![CDATA[<p>Liebe Neub&uuml;rgerin, lieber Neub&uuml;rger, herzlich willkommen in Augsburg! Sie sind in einer vielf&auml;ltigen und weltoffenen Stadt angekommen. Fast 300.000 Menschen aus rund 170 verschiedene Nationen leben hier gut zusammen. Ich freue mich, dass Sie in der Friedensstadt Augsburg einen neuen Lebensmittelpunkt gefunden haben. Das eigene Land zu verlassen und der Umzug in ein anderes Land bedeuten eine gro&szlig;e Umstellung. Nach der Ankunft stellen sich viele allt&auml;gliche Fragen zu Wohnung oder Unterkunft, Schule und Arbeit, sozialen und kulturellen Kontakten. Auch die Orientierung in einer fremden Kultur stellt eine gro&szlig;e Herausforderung dar. Die Integreat-App ist ein Gemeinschaftsprojekt der Stadt Augsburg in Zusammenarbeit mit T&uuml;r an T&uuml;r &ndash; Digital Factory gGmbH, die ihr technisches Know-how zur Verf&uuml;gung stellt. Unterst&uuml;tzt wird das Vorhaben durch den Integrationsbeirat und die AFF- association des familles francophones. Mit der App m&ouml;chten wir Ihnen helfen, das Leben in Augsburg besser zu verstehen und kennenzulernen. Sie soll ein Beitrag dazu sein, dass Sie sich bald angekommen f&uuml;hlen und willkommen in unserer Stadtgesellschaft sind. Hier finden Sie wichtige Adressen, allgemeine Anlaufstellen sowie Hilfsangebote in besonderen Lebenslagen. Ich hoffe sehr, dass wir Sie beim Einleben in Augsburg mit der App unterst&uuml;tzen k&ouml;nnen. Ich w&uuml;nsche Ihnen, dass Sie bald erste Kontakte kn&uuml;pfen und voller Zuversicht auf Ihre pers&ouml;nliche Zukunft in unserer gemeinsamen Stadt blicken werden. Dr. Kurt Gribl Oberb&uuml;rgermeister der Stadt Augsburg</p>]]></source>
+				<target/>
+			</trans-unit>
+		</body>
+	</file>
+</xliff>
diff --git a/tests/xliff/files/import/augsburg_de_en_1_2_willkommen.xliff b/tests/xliff/files/import/augsburg_de_en_1_2_willkommen.xliff
new file mode 100644
--- /dev/null
+++ b/tests/xliff/files/import/augsburg_de_en_1_2_willkommen.xliff
@@ -0,0 +1,21 @@
+<?xml version="1.0" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
+	<file datatype="plaintext" original="1" source-language="de" target-language="en">
+		<header>
+			<phase-group>
+				<phase phase-name="shortcodes" process-name="Shortcodes identification"/>
+				<phase phase-name="post_type" process-name="Post type"/>
+			</phase-group>
+		</header>
+		<body>
+			<trans-unit datatype="html" id="title" resname="title" restype="string">
+				<source><![CDATA[Willkommen]]></source>
+				<target><![CDATA[Updated title]]></target>
+			</trans-unit>
+			<trans-unit datatype="html" id="body" resname="body" restype="string">
+				<source><![CDATA[<p>Willkommen in Augsburg</p>]]></source>
+				<target><![CDATA[<p>Updated content</p>]]></target>
+			</trans-unit>
+		</body>
+	</file>
+</xliff>
diff --git a/tests/xliff/files/import/augsburg_de_en_2_1_willkommen-in-augsburg.xliff b/tests/xliff/files/import/augsburg_de_en_2_1_willkommen-in-augsburg.xliff
new file mode 100644
--- /dev/null
+++ b/tests/xliff/files/import/augsburg_de_en_2_1_willkommen-in-augsburg.xliff
@@ -0,0 +1,21 @@
+<?xml version="1.0" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
+	<file datatype="plaintext" original="2" source-language="de" target-language="en">
+		<header>
+			<phase-group>
+				<phase phase-name="shortcodes" process-name="Shortcodes identification"/>
+				<phase phase-name="post_type" process-name="Post type"/>
+			</phase-group>
+		</header>
+		<body>
+			<trans-unit datatype="html" id="title" resname="title" restype="string">
+				<source><![CDATA[Willkommen in Augsburg]]></source>
+				<target><![CDATA[Updated title]]></target>
+			</trans-unit>
+			<trans-unit datatype="html" id="body" resname="body" restype="string">
+				<source><![CDATA[<p>Liebe Neub&uuml;rgerin, lieber Neub&uuml;rger, herzlich willkommen in Augsburg! Sie sind in einer vielf&auml;ltigen und weltoffenen Stadt angekommen. Fast 300.000 Menschen aus rund 170 verschiedene Nationen leben hier gut zusammen. Ich freue mich, dass Sie in der Friedensstadt Augsburg einen neuen Lebensmittelpunkt gefunden haben. Das eigene Land zu verlassen und der Umzug in ein anderes Land bedeuten eine gro&szlig;e Umstellung. Nach der Ankunft stellen sich viele allt&auml;gliche Fragen zu Wohnung oder Unterkunft, Schule und Arbeit, sozialen und kulturellen Kontakten. Auch die Orientierung in einer fremden Kultur stellt eine gro&szlig;e Herausforderung dar. Die Integreat-App ist ein Gemeinschaftsprojekt der Stadt Augsburg in Zusammenarbeit mit T&uuml;r an T&uuml;r &ndash; Digital Factory gGmbH, die ihr technisches Know-how zur Verf&uuml;gung stellt. Unterst&uuml;tzt wird das Vorhaben durch den Integrationsbeirat und die AFF- association des familles francophones. Mit der App m&ouml;chten wir Ihnen helfen, das Leben in Augsburg besser zu verstehen und kennenzulernen. Sie soll ein Beitrag dazu sein, dass Sie sich bald angekommen f&uuml;hlen und willkommen in unserer Stadtgesellschaft sind. Hier finden Sie wichtige Adressen, allgemeine Anlaufstellen sowie Hilfsangebote in besonderen Lebenslagen. Ich hoffe sehr, dass wir Sie beim Einleben in Augsburg mit der App unterst&uuml;tzen k&ouml;nnen. Ich w&uuml;nsche Ihnen, dass Sie bald erste Kontakte kn&uuml;pfen und voller Zuversicht auf Ihre pers&ouml;nliche Zukunft in unserer gemeinsamen Stadt blicken werden. Dr. Kurt Gribl Oberb&uuml;rgermeister der Stadt Augsburg</p>]]></source>
+				<target><![CDATA[<p>Updated content</p>]]></target>
+			</trans-unit>
+		</body>
+	</file>
+</xliff>
diff --git a/tests/xliff/xliff_test.py b/tests/xliff/xliff_test.py
new file mode 100644
--- /dev/null
+++ b/tests/xliff/xliff_test.py
@@ -0,0 +1,164 @@
+import filecmp
+import io
+import zipfile
+
+import pytest
+
+from lxml.html import fromstring
+from django.urls import reverse
+
+from integreat_cms.cms.models import Page
+
+from ..conftest import ANONYMOUS, STAFF_ROLES, PRIV_STAFF_ROLES, MANAGEMENT, EDITOR
+
+
+@pytest.mark.django_db
+# Override urls to serve XLIFF files
+@pytest.mark.urls("tests.xliff.dummy_django_app.static_urls")
+def test_xliff_export(login_role_user, settings, tmp_path):
+    """
+    This test checks whether the xliff export works as expected
+
+    :param login_role_user: The fixture providing the http client and the current role (see :meth:`~tests.conftest.login_role_user`)
+    :type login_role_user: tuple
+
+    :param settings: The fixture providing the django settings
+    :type settings: :fixture:`settings`
+
+    :param tmp_path: The fixture providing the directory for temporary files for this test case
+    :type tmp_path: :fixture:`tmp_path`
+    """
+    client, role = login_role_user
+    export_xliff = reverse(
+        "download_xliff", kwargs={"region_slug": "augsburg", "language_slug": "en"}
+    )
+    response = client.post(export_xliff, data={"selected_ids[]": [1, 2]})
+    print(response.headers)
+    if role in STAFF_ROLES + [MANAGEMENT, EDITOR]:
+        # If the role should be allowed to access the view, we expect a successful result
+        assert response.status_code == 302
+        page_tree = reverse(
+            "pages", kwargs={"region_slug": "augsburg", "language_slug": "en"}
+        )
+        assert response.headers.get("Location") == page_tree
+        response = client.get(page_tree)
+        print(response.headers)
+        # The download link to the xliff file is contained in the message
+        parsed_content = fromstring(response.content.decode("utf-8"))
+        download_links = [
+            link.attrib["href"]
+            for link in parsed_content.iter("a")
+            if "data-auto-download" in link.attrib
+        ]
+        print(download_links)
+        assert len(download_links) == 1
+        response = client.get(download_links[0])
+        print(response.headers)
+        print(response.status_code)
+        assert response.status_code == 200
+        # Check if zip can be unzipped and contains the correct xliff files
+        file = io.BytesIO(response.getvalue())
+        with zipfile.ZipFile(file, "r") as zipped_file:
+            assert zipped_file.testzip() is None
+            assert zipped_file.namelist() == [
+                "augsburg_de_en_1_2_willkommen.xliff",
+                "augsburg_de_en_2_1_willkommen-in-augsburg.xliff",
+            ]
+            zipped_file.extractall(path=tmp_path)
+            for xliff_file in zipped_file.namelist():
+                assert filecmp.cmp(
+                    f"{tmp_path}/{xliff_file}", f"tests/xliff/files/export/{xliff_file}"
+                )
+        # Check if existing translations are now "currently in translation"
+        for page in Page.objects.filter(id__in=[1, 2]):
+            translation = page.get_translation("en")
+            if translation:
+                assert translation.currently_in_translation
+    elif role == ANONYMOUS:
+        # For anonymous users, we want to redirect to the login form instead of showing an error
+        assert response.status_code == 302
+        assert (
+            response.headers.get("location")
+            == f"{settings.LOGIN_URL}?next={export_xliff}"
+        )
+    else:
+        # For logged in users, we want to show an error if they get a permission denied
+        assert response.status_code == 403
+
+
+# pylint: disable=too-many-locals
+@pytest.mark.django_db
+def test_xliff_import(login_role_user, settings):
+    """
+    This test checks whether the xliff import works as expected
+
+    :param login_role_user: The fixture providing the http client and the current role (see :meth:`~tests.conftest.login_role_user`)
+    :type login_role_user: tuple
+
+    :param settings: The fixture providing the django settings
+    :type settings: :fixture:`settings`
+    """
+    # Test for english messages
+    settings.LANGUAGE_CODE = "en"
+    client, role = login_role_user
+    upload_xliff = reverse(
+        "upload_xliff", kwargs={"region_slug": "augsburg", "language_slug": "en"}
+    )
+    import_path = "tests/xliff/files/import"
+    file_1 = "augsburg_de_en_1_2_willkommen.xliff"
+    file_2 = "augsburg_de_en_2_1_willkommen-in-augsburg.xliff"
+    with open(f"{import_path}/{file_1}", encoding="utf-8") as f1:
+        with open(f"{import_path}/{file_2}", encoding="utf-8") as f2:
+            response = client.post(
+                upload_xliff, data={"xliff_file": [f1, f2]}, format="multipart"
+            )
+    print(response.headers)
+
+    if role in PRIV_STAFF_ROLES + [MANAGEMENT, EDITOR]:
+        # If the role should be allowed to access the view, we expect a successful result
+        assert response.status_code == 302
+        page_tree = reverse(
+            "pages", kwargs={"region_slug": "augsburg", "language_slug": "en"}
+        )
+        redirect_location = response.headers.get("Location")
+        # If errors occur, we get redirected to the page tree
+        assert redirect_location != page_tree
+        # Check if xliff import view is correctly rendered
+        response = client.get(redirect_location)
+        print(response.headers)
+        assert response.status_code == 200
+        assert file_1 in response.content.decode("utf-8")
+        assert file_2 in response.content.decode("utf-8")
+        # Check if XLIFF import is correctly confirmed
+        # (perform test twice to check whether unchanged diffs can be imported as well)
+        for msg in ["successfully", "without changes"]:
+            response = client.post(redirect_location)
+            print(response.headers)
+            assert response.status_code == 302
+            # If the import has been successfully confirmed, we get redirected to the page tree
+            assert response.headers.get("Location") == page_tree
+            # Check if xliff import view is correctly rendered
+            response = client.get(page_tree)
+            print(response.headers)
+            assert response.status_code == 200
+            # Check if existing translations are now updated
+            for page in Page.objects.filter(id__in=[1, 2]):
+                translation = page.get_translation("en")
+                if translation:
+                    assert translation.title == "Updated title"
+                    assert translation.content == "<p>Updated content</p>"
+                    assert not translation.currently_in_translation
+                    assert (
+                        f'Page "{translation.title}" was imported {msg}.'
+                        in response.content.decode("utf-8")
+                    )
+    elif role == ANONYMOUS:
+        # For anonymous users, we want to redirect to the login form instead of showing an error
+        assert response.status_code == 302
+        assert (
+            response.headers.get("location")
+            == f"{settings.LOGIN_URL}?next={upload_xliff}"
+        )
+    else:
+        # For logged in users, we want to show an error if they get a permission denied
+        assert response.status_code == 403