Repository: mozilla/pontoon

Issue:
Transifex API wishlist
Transifex API has improved dramatically in terms of what we need to make it work smoothly with Pontoon. We can now get and set translations as JSON list instead of getting and parsing and creating and setting PO files. There are still two issues remaining.
## 1. Getting originals & translations at once

We would really appreciate fetch all strings with available translations in certain language _with one request_. Right now we have to do two requests: 1st to get the original strings and 2nd to get translations. Or, we can do one request to get the PO file, but then we have to parse it.
## 2. Deleting translations

It's not possible at all to delete existing translations. We need this feature and the API should be the same as for PUT-ting new translation strings, but with e.g. "" provided as translation:

http://help.transifex.com/features/api/index.html#translations-for-a-collection-of-strings



Discussion:
Hi @mathjazz 

Regarding the above #1 issue, the translations API in Transifex now also returns the source strings along with the translations. Are you speaking of this one?

I am having a look at issue #2.

Indeed, I can now get source strings together with translations! I wasn't aware of that change, sorry.

Thanks!


Resolving patch:
diff --git a/pontoon/base/models.py b/pontoon/base/models.py
--- a/pontoon/base/models.py
+++ b/pontoon/base/models.py
@@ -530,7 +530,7 @@ def adjust_stats(
         fuzzy_strings_diff,
         strings_with_errors_diff,
         strings_with_warnings_diff,
-        unreviewed_strings_diff
+        unreviewed_strings_diff,
     ):
         self.total_strings = F('total_strings') + total_strings_diff
         self.approved_strings = F('approved_strings') + approved_strings_diff
@@ -2330,6 +2330,86 @@ def cleaned_key(self):
     def __str__(self):
         return self.string
 
+    def get_stats(self, locale):
+        """
+        Get stats for a single (entity, locale) pair.
+
+        :arg Locale locale: filter translations for this locale.
+        :return: a dictionary with stats for an Entity, all keys are suffixed with `_diff` to
+            make them easier to pass into adjust_all_stats.
+        """
+        translations = list(
+            self.translation_set
+                .filter(locale=locale)
+                .prefetch_related(
+                    'errors',
+                    'warnings',
+                )
+        )
+
+        approved_strings_count = len([
+            t for t in translations
+            if t.approved and not(t.errors.exists() or t.warnings.exists())
+        ])
+
+        fuzzy_strings_count = len([
+            t for t in translations
+            if t.fuzzy and not(t.errors.exists() or t.warnings.exists())
+        ])
+
+        if self.string_plural:
+            approved = int(approved_strings_count == locale.nplurals)
+            fuzzy = int(fuzzy_strings_count == locale.nplurals)
+
+        else:
+            approved = int(approved_strings_count > 0)
+            fuzzy = int(fuzzy_strings_count > 0)
+
+        if not (approved or fuzzy):
+            has_errors = bool([
+                t for t in translations
+                if (t.approved or t.fuzzy) and t.errors.exists()
+            ])
+            has_warnings = bool([
+                t for t in translations
+                if (t.approved or t.fuzzy) and t.warnings.exists()
+            ])
+
+            errors = int(has_errors)
+            warnings = int(has_warnings)
+
+        else:
+            errors = 0
+            warnings = 0
+
+        unreviewed_count = len([
+            t for t in translations
+            if not (t.approved or t.fuzzy or t.rejected)
+        ])
+
+        return {
+            'total_strings_diff': 0,
+            'approved_strings_diff': approved,
+            'fuzzy_strings_diff': fuzzy,
+            'strings_with_errors_diff': errors,
+            'strings_with_warnings_diff': warnings,
+            'unreviewed_strings_diff': unreviewed_count,
+        }
+
+    @classmethod
+    def get_stats_diff(cls, stats_before, stats_after):
+        """
+        Return stat difference between the two states of the entity.
+
+        :arg dict stats_before: dict returned by get_stats() for the initial state.
+        :arg dict stats_after: dict returned by get_stats() for the current state.
+        :return: dictionary with differences between provided stats.
+        """
+        return {
+            stat_name: stats_after[stat_name] - stats_before[stat_name]
+            for stat_name in stats_before
+        }
+
     def has_changed(self, locale):
         """
         Check if translations in the given locale have changed since the
@@ -2762,7 +2842,11 @@ def latest_activity(self):
     def __str__(self):
         return self.string
 
-    def save(self, *args, **kwargs):
+    def save(self, update_stats=True, *args, **kwargs):
+        # We parametrize update of stats to make testing easier.
+        if update_stats:
+            stats_before = self.entity.get_stats(self.locale)
+
         super(Translation, self).save(*args, **kwargs)
 
         # Only one translation can be approved at a time for any
@@ -2798,21 +2882,28 @@ def save(self, *args, **kwargs):
                     project=self.entity.resource.project,
                 )
 
-        # Update stats AFTER changing approval status.
-        translatedresource, _ = TranslatedResource.objects.get_or_create(
-            resource=self.entity.resource, locale=self.locale
-        )
-        translatedresource.calculate_stats()
-
         # Whenever a translation changes, mark the entity as having
         # changed in the appropriate locale. We could be smarter about
         # this but for now this is fine.
         if self.approved:
             self.entity.mark_changed(self.locale)
 
+        # We use get_or_create() instead of just get() to make it easier to test.
+        translatedresource, _ = TranslatedResource.objects.get_or_create(
+            resource=self.entity.resource,
+            locale=self.locale
+        )
+
         # Update latest translation where necessary
         self.update_latest_translation()
 
+        # We parametrize update of stats to make testing easier.
+        if update_stats:
+            # Update stats AFTER changing approval status.
+            stats_after = self.entity.get_stats(self.locale)
+            stats_diff = Entity.get_stats_diff(stats_before, stats_after)
+            translatedresource.adjust_all_stats(**stats_diff)
+
     def update_latest_translation(self):
         """
         Set `latest_translation` to this translation if its more recent than
@@ -3126,6 +3217,25 @@ class TranslatedResource(AggregatedStats):
     class Meta(object):
         unique_together = (('locale', 'resource'), )
 
+    def adjust_all_stats(self, *args, **kwargs):
+        project = self.resource.project
+        locale = self.locale
+
+        project_locale = utils.get_object_or_none(
+            ProjectLocale,
+            project=project,
+            locale=locale,
+        )
+
+        self.adjust_stats(*args, **kwargs)
+        project.adjust_stats(*args, **kwargs)
+
+        if not project.system_project:
+            locale.adjust_stats(*args, **kwargs)
+
+        if project_locale:
+            project_locale.adjust_stats(*args, **kwargs)
+
     def calculate_stats(self, save=True):
         """Update stats, including denormalized ones."""
         resource = self.resource
@@ -3173,8 +3283,6 @@ def calculate_stats(self, save=True):
             rejected=False,
         ).count()
 
-        missing = resource.total_strings - approved - fuzzy - errors - warnings
-
         # Plural
         nplurals = locale.nplurals or 1
         for e in translated_entities.exclude(string_plural='').values_list('pk'):
@@ -3218,8 +3326,6 @@ def calculate_stats(self, save=True):
                     errors += 1
                 elif plural_warnings_count:
                     warnings += 1
-                else:
-                    missing += 1
 
             plural_unreviewed_count = translations.filter(
                 approved=False,
@@ -3247,8 +3353,7 @@ def calculate_stats(self, save=True):
         strings_with_warnings_diff = warnings - self.strings_with_warnings
         unreviewed_strings_diff = unreviewed - self.unreviewed_strings
 
-        # Translated Resource
-        self.adjust_stats(
+        self.adjust_all_stats(
             total_strings_diff,
             approved_strings_diff,
             fuzzy_strings_diff,
@@ -3256,41 +3361,3 @@ def calculate_stats(self, save=True):
             strings_with_warnings_diff,
             unreviewed_strings_diff,
         )
-
-        # Project
-        project = resource.project
-        project.adjust_stats(
-            total_strings_diff,
-            approved_strings_diff,
-            fuzzy_strings_diff,
-            strings_with_errors_diff,
-            strings_with_warnings_diff,
-            unreviewed_strings_diff,
-        )
-
-        # Locale
-        if not project.system_project:
-            locale.adjust_stats(
-                total_strings_diff,
-                approved_strings_diff,
-                fuzzy_strings_diff,
-                strings_with_errors_diff,
-                strings_with_warnings_diff,
-                unreviewed_strings_diff,
-            )
-
-        # ProjectLocale
-        project_locale = utils.get_object_or_none(
-            ProjectLocale,
-            project=project,
-            locale=locale
-        )
-        if project_locale:
-            project_locale.adjust_stats(
-                total_strings_diff,
-                approved_strings_diff,
-                fuzzy_strings_diff,
-                strings_with_errors_diff,
-                strings_with_warnings_diff,
-                unreviewed_strings_diff,
-            )


Test patch:
diff --git a/pontoon/base/tests/managers/test_entity.py b/pontoon/base/tests/managers/test_entity.py
--- a/pontoon/base/tests/managers/test_entity.py
+++ b/pontoon/base/tests/managers/test_entity.py
@@ -5,6 +5,7 @@
 from pontoon.base.models import (
     Entity,
     Translation,
+    TranslatedResource,
 )
 from pontoon.test.factories import (
     EntityFactory,
@@ -281,6 +282,10 @@ def test_mgr_entity_filter_warnings(resource_a, locale_a):
     WarningFactory.create(
         translation=translations[2]
     )
+    TranslatedResource.objects.get(
+        resource=translations[2].entity.resource,
+        locale=translations[2].locale,
+    ).calculate_stats()
 
     translations[2].fuzzy = False
     translations[2].save()
diff --git a/pontoon/base/tests/models/test_stats.py b/pontoon/base/tests/models/test_stats.py
new file mode 100644
--- /dev/null
+++ b/pontoon/base/tests/models/test_stats.py
@@ -0,0 +1,204 @@
+"""
+Test consistency of calculations between `calculate_stats` and `translation.save()`.
+"""
+import pytest
+from pontoon.base.models import TranslatedResource
+from pontoon.checks.models import (
+    Error,
+    Warning,
+)
+
+
+@pytest.fixture
+def translation_with_error(translation_a):
+    Error.objects.create(
+        translation=translation_a,
+        library='p',
+        message='error'
+    )
+    return translation_a
+
+
+@pytest.fixture
+def translation_with_warning(translation_a):
+    Warning.objects.create(
+        translation=translation_a,
+        library='p',
+        message='warning',
+    )
+    return translation_a
+
+
+def recalculate_stats(translation):
+    """
+    Make the full recalculate stats on a TranslatedResource.
+    """
+    translation.save(update_stats=False)
+    translated_resource = TranslatedResource.objects.get(
+        resource=translation.entity.resource,
+        locale=translation.locale,
+    )
+    translated_resource.calculate_stats()
+
+
+def diff_stats(t):
+    """
+    Update only necessary stats by calculating difference between stats.
+    """
+    t.save()
+
+
+@pytest.fixture(
+    params=(
+        recalculate_stats,
+        diff_stats,
+    )
+)
+def stats_update(db, request):
+    """
+    Wrapper fixture which allows to test both implementations of stats calculations.
+    """
+    return request.param
+
+
+@pytest.fixture
+def get_stats():
+    def f(translation):
+        return (
+            TranslatedResource.objects.filter(
+                resource=translation.entity.resource,
+                locale=translation.locale,
+            ).aggregated_stats()
+        )
+    return f
+
+
+def test_translation_approved(stats_update, get_stats, translation_a):
+    translation_a.approved = True
+    stats_update(translation_a)
+
+    assert get_stats(translation_a) == {
+        'total': 1,
+        'approved': 1,
+        'fuzzy': 0,
+        'unreviewed': 0,
+        'warnings': 0,
+        'errors': 0,
+    }
+
+    translation_a.approved = False
+    stats_update(translation_a)
+
+    assert get_stats(translation_a) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 1,
+        'warnings': 0,
+        'errors': 0,
+    }
+
+
+def test_translation_fuzzy(stats_update, get_stats, translation_a):
+    translation_a.fuzzy = True
+    stats_update(translation_a)
+
+    assert get_stats(translation_a) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 1,
+        'unreviewed': 0,
+        'warnings': 0,
+        'errors': 0,
+    }
+
+    translation_a.fuzzy = False
+    translation_a.rejected = True
+    stats_update(translation_a)
+
+    assert get_stats(translation_a) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 0,
+        'warnings': 0,
+        'errors': 0,
+    }
+
+
+def test_translation_with_error(stats_update, get_stats, translation_with_error):
+    translation_with_error.approved = True
+    stats_update(translation_with_error)
+
+    assert get_stats(translation_with_error) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 0,
+        'warnings': 0,
+        'errors': 1,
+    }
+
+    translation_with_error.approved = False
+    translation_with_error.fuzzy = True
+    stats_update(translation_with_error)
+
+    assert get_stats(translation_with_error) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 0,
+        'warnings': 0,
+        'errors': 1,
+    }
+
+    translation_with_error.fuzzy = False
+    stats_update(translation_with_error)
+
+    assert get_stats(translation_with_error) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 1,
+        'warnings': 0,
+        'errors': 0,
+    }
+
+
+def test_translation_with_warning(stats_update, get_stats, translation_with_warning):
+    translation_with_warning.approved = True
+    stats_update(translation_with_warning)
+
+    assert get_stats(translation_with_warning) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 0,
+        'warnings': 1,
+        'errors': 0,
+    }
+
+    translation_with_warning.approved = False
+    translation_with_warning.fuzzy = True
+    stats_update(translation_with_warning)
+
+    assert get_stats(translation_with_warning) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 0,
+        'warnings': 1,
+        'errors': 0,
+    }
+
+    translation_with_warning.fuzzy = False
+    stats_update(translation_with_warning)
+
+    assert get_stats(translation_with_warning) == {
+        'total': 1,
+        'approved': 0,
+        'fuzzy': 0,
+        'unreviewed': 1,
+        'warnings': 0,
+        'errors': 0,
+    }