Repository: translate/pootle

Issue:
Improve performance of inserting/deleting units in the store
_(Original issue and discussion at evernote/pootle#145)_

Currently each unit stores its position (index) in the .po file. Consider a .po file with 1000 units. If a new unit is added to the very top of the .po file, and then `update_stores` is run, the new unit will be created in the database, but in addition to that, 1000 units' position values will be updated (this is another 1000 DB updates). Inserting one unit into a large store can take several seconds. And this happens for each unit that is being inserted or deleted, which slows down the sync cycle.

We'll try to overcome this by implementing the following:
- Update the store updating algorithm, so that when we're trying to insert `N` units after unit with index `X`, and the gap between `X` and next index `Y` is smaller than `N`, then run a query that shifts all indexes starting with `Y` at once by the value of `N-(Y-X)` (which is cheap). Then do an insert.
- Deleting a unit (or making it an obsolete one) shouldn't affect other units' indexes. We just introduce a natural gap in numeration that can be reused later. Deleting units should be done **before** we insert new units, because the most common scenario is when the string is simply altered, which leads to deleting the old unit and creating a new one at exactly the same index. In this case, no index-shifting queries are necessary.

The code resulting out of this issue must come with unit tests.

Also let's measure the old & new methods over the same operations to see the performance gains we get out of this.



Discussion:

Resolving patch:
diff --git a/pootle/apps/pootle_store/models.py b/pootle/apps/pootle_store/models.py
--- a/pootle/apps/pootle_store/models.py
+++ b/pootle/apps/pootle_store/models.py
@@ -8,7 +8,9 @@
 # AUTHORS file for copyright and authorship information.
 
 import datetime
+import difflib
 import logging
+import operator
 import os
 
 from hashlib import md5
@@ -18,6 +20,7 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.db import models, transaction, IntegrityError
+from django.db.models import F
 from django.template.defaultfilters import escape, truncatechars
 from django.utils import timezone
 from django.utils.functional import cached_property
@@ -971,6 +974,7 @@ def makeobsolete(self):
             self._save_action = UNIT_OBSOLETE
 
             self.state = OBSOLETE
+            self.index = 0
 
     def resurrect(self, is_fuzzy=False):
         if self.state > OBSOLETE:
@@ -1461,7 +1465,7 @@ def makeobsolete(self):
         for unit_id in unit_ids:
             action_log(user='system', action=UNIT_OBSOLETE, lang=lang,
                        unit=unit_id, translation='', path=self.pootle_path)
-        unit_query.update(state=OBSOLETE)
+        unit_query.update(state=OBSOLETE, index=0)
         self.obsolete = True
         self.save()
         self.clear_all_cache(parents=False, children=False)
@@ -1576,7 +1580,12 @@ def get_file_mtime(self):
 
         return disk_mtime
 
-    @transaction.atomic
+    def update_index(self, start, delta):
+        op = operator.add if delta > 0 else operator.sub
+        Unit.objects.filter(store_id=self.id, index__gte=start).update(
+            index=op(F('index'), delta)
+        )
+
     def update(self, overwrite=False, store=None, only_newer=False):
         """Update DB with units from file.
 
@@ -1626,20 +1635,81 @@ def update(self, overwrite=False, store=None, only_newer=False):
                 'added': 0,
             }
 
-            # Force a rebuild of the unit ID <-> DB ID index and get IDs for
-            # in-DB (old) and on-disk (new) stores
-            self.require_dbid_index(update=True, obsolete=True)
-            old_ids = set(self.dbid_index.keys())
-            new_ids = set(store.getids())
+            old_unitid_list = []  # unitid list for comparing (DB)
+            old_unitids = {}  # unitids dict (DB)
+            old_unitid_set = set()
+            old_obsolete_unitid_set = set()  # obsolete unitids in DB
+            update_unitids = {}
+
+            for (unitid, state, dbid, index) in \
+                self.unit_set.order_by('index') \
+                    .values_list('unitid', 'state', 'id', 'index'):
+                if state == OBSOLETE:
+                    old_obsolete_unitid_set.add(unitid)
+                else:
+                    old_unitid_list.append(unitid)
+                old_unitids[unitid] = {'dbid': dbid, 'index': index}
+                old_unitid_set.add(unitid)
+
+            new_unitid_list = sorted(store.getids(), key=lambda x: store.findid(x).index)
+            new_unitid_set = set(new_unitid_list)
+
+            sm = difflib.SequenceMatcher(None, old_unitid_list, new_unitid_list)
+            User = get_user_model()
+            system = User.objects.get_system_user()
+
+            common_dbids = set()
+
+            def insert(i1, j1, j2):
+                # Add new units to the store
+                new_units = (store.findid(uid) for uid in new_unitid_list[j1:j2])
+                for index, unit in enumerate(new_units):
+                    new_unit_index = i1 + index + 1
+                    uid = unit.getid()
+                    if uid not in old_unitid_set:
+                        self.addunit(unit, new_unit_index, user=system)
+                        changes['added'] += 1
+                    else:
+                        update_unitids[uid] = {'dbid': old_unitids[uid]['dbid'],
+                                               'index': new_unit_index}
+
+            for (tag, i1, i2, j1, j2) in sm.get_opcodes():
+                if tag == 'delete':
+                    continue
+                elif tag == 'insert':
+                    previous_index = 0
+                    if i1 > 0:
+                        previous = old_unitids[old_unitid_list[i1 - 1]]
+                        previous_index = previous['index']
+                    if i1 < len(old_unitid_list):
+                        next = old_unitids[old_unitid_list[i1]]
+                        delta = j2 - j1
+                        if previous_index + delta >= next['index']:
+                            delta = next['index'] - previous_index + delta - 1
+                            self.update_index(start=next['index'], delta=delta)
+                    insert(previous_index, j1, j2)
+                elif tag == 'replace':
+                    i1_index = old_unitids[old_unitid_list[i1 - 1]]['index']
+                    i2_index = old_unitids[old_unitid_list[i2 - 1]]['index']
+                    delta = j2 - j1 - i2_index + i1_index
+                    if delta > 0:
+                        self.update_index(start=i2_index, delta=delta)
+                    insert(i1_index, j1, j2)
+                else:
+                    common_dbids.update(set(old_unitids[uid]['dbid']
+                                            for uid in old_unitid_list[i1:i2]))
+
+            update_dbids = set([x['dbid'] for x in update_unitids.values()])
+            common_dbids.update(update_dbids)
 
-            # Remove old units or make them obsolete if they were already
-            # translated
-            obsolete_dbids = [self.dbid_index.get(uid)
-                              for uid in old_ids - new_ids]
+            obsolete_dbids = [old_unitids[uid]['dbid']
+                              for uid in old_unitid_set -
+                                         old_obsolete_unitid_set -
+                                         new_unitid_set]
             for unit in self.findid_bulk(obsolete_dbids):
-                # Use the same (parent) object since units will accumulate
-                # the list of cache attributes to clear in the parent Store
-                # object
+                # Use the same (parent) object since units will
+                # accumulate the list of cache attributes to clear
+                # in the parent Store object
                 unit.store = self
                 if not unit.isobsolete():
                     unit.makeobsolete()
@@ -1647,18 +1717,6 @@ def update(self, overwrite=False, store=None, only_newer=False):
                     unit.save()
                     changes['obsolete'] += 1
 
-            User = get_user_model()
-            system = User.objects.get_system_user()
-
-            # Add new units to the store
-            new_units = (store.findid(uid) for uid in new_ids - old_ids)
-            for unit in new_units:
-                newunit = self.addunit(unit, unit.index, user=system)
-                changes['added'] += 1
-
-            common_dbids = set(self.dbid_index.get(uid)
-                               for uid in old_ids & new_ids)
-
             if not overwrite:
                 # Get units that were modified after last sync
                 filter_by = {'store': self}
@@ -1682,15 +1740,15 @@ def update(self, overwrite=False, store=None, only_newer=False):
                 # the list of cache attributes to clear in the parent Store
                 # object
                 unit.store = self
-                newunit = store.findid(unit.getid())
+                uid = unit.getid()
+                newunit = store.findid(uid)
                 old_target_f = unit.target_f
                 old_unit_state = unit.state
 
                 changed = unit.update(newunit, user=system)
 
-                # Unit's index within the store might have changed
-                if unit.index != newunit.index:
-                    unit.index = newunit.index
+                if uid in update_unitids:
+                    unit.index = update_unitids[uid]['index']
                     changed = True
 
                 if changed:
@@ -1974,7 +2032,7 @@ def findid(self, id):
 
         unitid_hash = md5(id.encode("utf-8")).hexdigest()
         try:
-            return self.units.get(unitid_hash=unitid_hash)
+            return self.unit_set.get(unitid_hash=unitid_hash)
         except Unit.DoesNotExist:
             return None
 


Test patch:
diff --git a/tests/data/po/tutorial/ru/tutorial.po b/tests/data/po/tutorial/ru/tutorial.po
new file mode 100644
--- /dev/null
+++ b/tests/data/po/tutorial/ru/tutorial.po
@@ -0,0 +1,20 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"X-Generator: Pootle Tests\n"
+
+#: first
+msgid "first"
+msgstr ""
+
+#: test.c
+msgid "second"
+msgstr ""
+
+#: fish.c
+msgid "third"
+msgstr ""
diff --git a/tests/data/po/tutorial/ru/tutorial_updated.po b/tests/data/po/tutorial/ru/tutorial_updated.po
new file mode 100644
--- /dev/null
+++ b/tests/data/po/tutorial/ru/tutorial_updated.po
@@ -0,0 +1,20 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"X-Generator: Pootle Tests\n"
+
+#: fish.c
+msgid "before first"
+msgstr ""
+
+#: fish.c
+msgid "first"
+msgstr ""
+
+#: fish.c
+msgid "third"
+msgstr ""
diff --git a/tests/fixtures/models/language.py b/tests/fixtures/models/language.py
--- a/tests/fixtures/models/language.py
+++ b/tests/fixtures/models/language.py
@@ -74,6 +74,12 @@ def italian(english):
     return _require_language('it', 'Italian')
 
 
+@pytest.fixture
+def russian(english):
+    """Require the Russian language."""
+    return _require_language('ru', 'Russian')
+
+
 @pytest.fixture
 def fish(english):
     """Require the Fish language ><(((º>"""
diff --git a/tests/fixtures/models/store.py b/tests/fixtures/models/store.py
--- a/tests/fixtures/models/store.py
+++ b/tests/fixtures/models/store.py
@@ -110,3 +110,10 @@ def es_tutorial_subdir_remove_po(settings, spanish_tutorial, system):
     """Require the /es/tutorial/subdir/remove_tutorial.po store."""
     po_directory = settings.POOTLE_TRANSLATION_DIRECTORY
     return _require_store(spanish_tutorial, po_directory, 'subdir/remove_tutorial.po')
+
+
+@pytest.fixture
+def ru_tutorial_po(settings, russian_tutorial, system):
+    """Require the /ru/tutorial/tutorial.po store."""
+    po_directory = settings.POOTLE_TRANSLATION_DIRECTORY
+    return _require_store(russian_tutorial, po_directory, 'tutorial.po')
diff --git a/tests/fixtures/models/translation_project.py b/tests/fixtures/models/translation_project.py
--- a/tests/fixtures/models/translation_project.py
+++ b/tests/fixtures/models/translation_project.py
@@ -55,3 +55,9 @@ def spanish_tutorial(spanish, tutorial):
 def italian_tutorial(italian, tutorial):
     """Require Italian Tutorial."""
     return _require_tp(italian, tutorial)
+
+
+@pytest.fixture
+def russian_tutorial(russian, tutorial):
+    """Require Russian Tutorial."""
+    return _require_tp(russian, tutorial)
diff --git a/tests/models/store.py b/tests/models/store.py
--- a/tests/models/store.py
+++ b/tests/models/store.py
@@ -66,3 +66,31 @@ def test_sync(fr_tutorial_remove_sync_po):
     assert not store.file.exists()
     store.sync()
     assert store.file.exists()
+
+
+@pytest.mark.django_db
+def test_update(ru_tutorial_po):
+    """Tests unit order after a specific update.
+    """
+
+    tp = ru_tutorial_po.translation_project
+    pootle_path = ru_tutorial_po.pootle_path
+
+    # Parse stores
+    ru_tutorial_po.update(overwrite=False, only_newer=False)
+
+    print ru_tutorial_po.file
+    assert ru_tutorial_po.file.exists()
+
+    # check if initial indexing is correct
+    old_unit_order = {'first': 1, 'second': 2, 'third': 3}
+    for unit in ru_tutorial_po.units:
+        assert old_unit_order[unit.unitid] == unit.index
+
+    ru_tutorial_po.file = 'tutorial/ru/tutorial_updated.po'
+    ru_tutorial_po.update(overwrite=False, only_newer=False)
+
+    # check if updated indexing is correct
+    new_unit_order = {'before first': 1, 'first': 2, 'third': 4}
+    for unit in ru_tutorial_po.units:
+        assert new_unit_order[unit.unitid] == unit.index