Repository: learningequality/kolibri

Issue:
Database is locked errors when registering a discovered server from zeroconf

When starting up the server on 0.15.0a2, sometimes seeing the following traceback:
```
INFO     Enqueuing a sync task for user 41ce810ca577c88ef8d26d9215d0d480 in job e054d01b068d491dbfc696eda33286d8
Exception in thread zeroconf-ServiceBrowser_Kolibri._sub._http._tcp.local.:
Traceback (most recent call last):
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/query.py", line 480, in update_or_create
    obj = self.select_for_update().get(**lookup)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/query.py", line 378, in get
    raise self.model.DoesNotExist(
kolibri.core.discovery.models.DoesNotExist: DynamicNetworkLocation matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/backends/utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/backends/sqlite3/base.py", line 328, in execute
    return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
    self.run()
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/zeroconf.py", line 1444, in run
    handler(self.zc)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/zeroconf.py", line 1381, in <lambda>
    lambda zeroconf: self._service_state_changed.fire(
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/zeroconf.py", line 1281, in fire
    h(**kwargs)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/zeroconf.py", line 1354, in on_change
    listener.add_service(*args)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/core/discovery/utils/network/search.py", line 173, in add_service
    DynamicNetworkLocation.objects.update_or_create(
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/core/discovery/models.py", line 95, in update_or_create
    return super(DynamicNetworkLocationManager, self).update_or_create(
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/manager.py", line 85, in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/query.py", line 484, in update_or_create
    obj, created = self._create_object_from_params(lookup, params, lock=True)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/query.py", line 500, in _create_object_from_params
    obj = self.create(**params)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/query.py", line 394, in create
    obj.save(force_insert=True, using=self.db)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/core/discovery/models.py", line 115, in save
    return super(DynamicNetworkLocation, self).save(*args, **kwargs)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/base.py", line 807, in save
    self.save_base(using=using, force_insert=force_insert,
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/base.py", line 838, in save_base
    updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/base.py", line 924, in _save_table
    result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/base.py", line 962, in _do_insert
    return manager._insert([self], fields=fields, return_id=update_pk,
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/manager.py", line 85, in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/query.py", line 1079, in _insert
    return query.get_compiler(using=using).execute_sql(return_id)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/models/sql/compiler.py", line 1112, in execute_sql
    cursor.execute(sql, params)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/backends/utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/utils.py", line 94, in __exit__
    six.reraise(dj_exc_type, dj_exc_value, traceback)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/utils/six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/backends/utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
INFO     Enqueuing a sync task for user 9d66f1cd2557d2dab4c103b2af46b173 in job 79cdfd8064f344c5b80dcf100a02e111
  File "/home/jamalex/.pex/install/kolibri-0.15.0a2-py2.py3-none-any.whl.cec1f081fd1cbbd85f2d51b8bd2ec20de32fe99d/kolibri-0.15.0a2-py2.py3-none-any.whl/kolibri/dist/django/db/backends/sqlite3/base.py", line 328, in execute
    return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked
```


Discussion:

Resolving patch:
diff --git a/kolibri/core/discovery/utils/network/search.py b/kolibri/core/discovery/utils/network/search.py
--- a/kolibri/core/discovery/utils/network/search.py
+++ b/kolibri/core/discovery/utils/network/search.py
@@ -1,9 +1,11 @@
 import json
 import logging
 import socket
+import time
 
 from django.core.exceptions import ValidationError
 from django.db import connection
+from django.db.utils import OperationalError
 from six import integer_types
 from six import string_types
 from zeroconf import get_all_addresses
@@ -168,43 +170,55 @@ def add_service(self, zeroconf, type, name):
 
         if not is_self:
 
-            try:
+            logger.info(
+                "Kolibri instance '%s' joined zeroconf network; service info: %s"
+                % (id, self.instances[id])
+            )
 
-                DynamicNetworkLocation.objects.update_or_create(
-                    dict(base_url=base_url, **device_info), id=id
-                )
+            db_locked = self.store_service(id, base_url, device_info)
 
-                logger.info(
-                    "Kolibri instance '%s' joined zeroconf network; service info: %s"
-                    % (id, self.instances[id])
-                )
+            if get_device_setting(
+                "subset_of_users_device", False
+            ) and not device_info.get("subset_of_users_device", False):
+                server = base_url[:-1]  # removes ending slash
+                for user in FacilityUser.objects.all().values("id"):
+                    begin_request_soud_sync(server=server, user=user["id"])
+
+            attempts = 0
+            while db_locked and attempts < 5:
+                db_locked = self.store_service(id, base_url, device_info)
+                attempts += 1
+                time.sleep(0.1)
+
+    def store_service(self, id, base_url, device_info):
+        db_locked = False
+        try:
+
+            DynamicNetworkLocation.objects.update_or_create(
+                dict(base_url=base_url, **device_info), id=id
+            )
 
-            except ValidationError:
-                import traceback
+        except ValidationError:
+            import traceback
 
-                logger.warn(
+            logger.warn(
+                """
+                    A new Kolibri instance '%s' was seen on the zeroconf network,
+                    but we had trouble getting the information we needed about it.
+                    Service info:
+                    %s
+                    The following exception was raised:
+                    %s
                     """
-                        A new Kolibri instance '%s' was seen on the zeroconf network,
-                        but we had trouble getting the information we needed about it.
-                        Service info:
-                        %s
-                        The following exception was raised:
-                        %s
-                        """
-                    % (id, self.instances[id], traceback.format_exc(limit=1))
-                )
-            finally:
-                connection.close()
-
-            if (
-                "subset_of_users_device" in device_info
-            ):  # discards previous versions of Kolibri
-                if get_device_setting(
-                    "subset_of_users_device", False
-                ) and not device_info.get("subset_of_users_device", False):
-                    server = base_url[:-1]  # removes ending slash
-                    for user in FacilityUser.objects.all().values("id"):
-                        begin_request_soud_sync(server=server, user=user["id"])
+                % (id, self.instances[id], traceback.format_exc(limit=1))
+            )
+        except OperationalError as e:
+            if "database is locked" not in str(e):
+                raise
+            db_locked = True
+        finally:
+            connection.close()
+        return db_locked
 
     def remove_service(self, zeroconf, type, name):
         id = _id_from_name(name)


Test patch: