Repository: learningequality/kolibri

Issue:
Possibility of unhandled exception during diskcache usage
## Observed behavior
A very odd behaviour whereby diskcache seems to lose track of its transaction id.

## Errors and logs
```
ERROR 2024-01-23 15:39:39,832 django.request Internal Server Error: /coach/api/notifications/
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/kolibri/dist/diskcache/core.py", line 762, in _transact
    yield sql, filenames.append
  File "/usr/lib/python3/dist-packages/kolibri/dist/diskcache/core.py", line 827, in set
    rows = sql(
sqlite3.OperationalError: disk I/O error

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/core/handlers/exception.py", line 41, in inner
    response = get_response(request)
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/core/handlers/base.py", line 172, in _get_response
    resolver_match = resolver.resolve(request.path_info)
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/urls/resolvers.py", line 366, in resolve
    sub_match = pattern.resolve(new_path)
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/urls/resolvers.py", line 366, in resolve
    sub_match = pattern.resolve(new_path)
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/urls/resolvers.py", line 366, in resolve
    sub_match = pattern.resolve(new_path)
  [Previous line repeated 1 more time]
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/urls/resolvers.py", line 361, in resolve
    match = self.regex.search(path)
  File "/usr/lib/python3/dist-packages/kolibri/core/device/translation.py", line 158, in regex
    device_language = get_device_language() or get_settings_language()
  File "/usr/lib/python3/dist-packages/kolibri/core/device/translation.py", line 26, in get_device_language
    if not device_provisioned():
  File "/usr/lib/python3/dist-packages/kolibri/core/device/utils.py", line 68, in device_provisioned
    return get_device_setting("is_provisioned")
  File "/usr/lib/python3/dist-packages/kolibri/core/device/utils.py", line 54, in get_device_setting
    device_settings = DeviceSettings.objects.get()
  File "/usr/lib/python3/dist-packages/kolibri/core/device/models.py", line 73, in get
    cache.set(DEVICE_SETTINGS_CACHE_KEY, model, 600)
  File "/usr/lib/python3/dist-packages/kolibri/deployment/default/custom_django_cache.py", line 76, in set
    return super(CustomDjangoCache, self).set(
  File "/usr/lib/python3/dist-packages/kolibri/dist/diskcache/djangocache.py", line 147, in set
    return self._cache.set(key, value, timeout, read, tag, retry)
  File "/usr/lib/python3/dist-packages/kolibri/dist/diskcache/fanout.py", line 97, in set
    return shard.set(key, value, expire, read, tag, retry)
  File "/usr/lib/python3/dist-packages/kolibri/dist/diskcache/core.py", line 842, in set
    return True
  File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/lib/python3/dist-packages/kolibri/dist/diskcache/core.py", line 765, in _transact
    assert self._txn_id == tid
AssertionError
```

## Expected behavior
Our CustomDjangoCache object should also handle AssertionErrors: https://github.com/learningequality/kolibri/blob/fc94f279c8affe2a5c69fc8f4c0879b9fb804b41/kolibri/deployment/default/custom_django_cache.py#L7

Might be simplest to do a small refactor to create a class level `handled errors` tuple, and then use that in each try/except block to handle.

## User-facing consequences
Very limited, has only been observed once in testing.

## Context
First reported here: https://github.com/learningequality/kolibri/pull/11765#issuecomment-1906241734


Discussion:
May i work on this?

Hi @thesujai! Sure! I will assign this to you. Thanks! :open_hands: 

Resolving patch:
diff --git a/kolibri/deployment/default/custom_django_cache.py b/kolibri/deployment/default/custom_django_cache.py
--- a/kolibri/deployment/default/custom_django_cache.py
+++ b/kolibri/deployment/default/custom_django_cache.py
@@ -13,6 +13,30 @@ class CustomDjangoCache(DjangoCache):
     https://github.com/grantjenks/python-diskcache/blob/v4.1.0/diskcache/djangocache.py
     """
 
+    ERRORS_TO_HANDLE = (sqlite3.OperationalError, AssertionError)
+
+    def try_execute(self, method_name, error_return_value, *args, **kwargs):
+        """
+        Safely executes a method with error handling.
+            :param method_name --> (str): name of method to execute
+            :param error_return_value --> (any): value to return if error occur
+            :param *args: positional arguments for method
+            :param *kwargs: keyword arguments for method
+            :return: The return value of the executed method if successful,
+                     otherwise returns error_return_value
+        """
+        try:
+            method = getattr(super(CustomDjangoCache, self), method_name)
+            if method is None:
+                raise ValueError(
+                    "{method_name} is not a valid method".format(
+                        method_name=method_name
+                    )
+                )
+            return method(*args, **kwargs)
+        except self.ERRORS_TO_HANDLE:
+            return error_return_value
+
     def add(
         self,
         key,
@@ -23,12 +47,17 @@ def add(
         tag=None,
         retry=True,
     ):
-        try:
-            return super(CustomDjangoCache, self).add(
-                key, value, timeout, version, read, tag, retry
-            )
-        except sqlite3.OperationalError:
-            return False
+        return self.try_execute(
+            "add",
+            False,
+            key=key,
+            value=value,
+            timeout=timeout,
+            version=version,
+            read=read,
+            tag=tag,
+            retry=retry,
+        )
 
     def has_key(self, key, version=None):
         """Returns True if the key is in the cache and has not expired.
@@ -38,12 +67,7 @@ def has_key(self, key, version=None):
         :return: True if key is found
 
         """
-        try:
-            return super(CustomDjangoCache, self).has_key(  # noqa: W601
-                key, version=version
-            )
-        except sqlite3.OperationalError:
-            return False
+        return self.try_execute("has_key", False, key=key, version=version)
 
     def get(
         self,
@@ -55,12 +79,17 @@ def get(
         tag=False,
         retry=False,
     ):
-        try:
-            return super(CustomDjangoCache, self).get(
-                key, default, version, read, expire_time, tag, retry
-            )
-        except sqlite3.OperationalError:
-            return None
+        return self.try_execute(
+            "get",
+            None,
+            key=key,
+            default=default,
+            version=version,
+            read=read,
+            expire_time=expire_time,
+            tag=tag,
+            retry=retry,
+        )
 
     def set(
         self,
@@ -72,95 +101,93 @@ def set(
         tag=None,
         retry=True,
     ):
-        try:
-            return super(CustomDjangoCache, self).set(
-                key, value, timeout, version, read, tag, retry
-            )
-        except sqlite3.OperationalError:
-            return False
+        return self.try_execute(
+            "set",
+            False,
+            key=key,
+            value=value,
+            timeout=timeout,
+            version=version,
+            read=read,
+            tag=tag,
+            retry=retry,
+        )
 
     def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True):
-        try:
-            return super(CustomDjangoCache, self).touch(key, timeout, version, retry)
-        except sqlite3.OperationalError:
-            return False
+        return self.try_execute(
+            "touch",
+            False,
+            key=key,
+            timeout=timeout,
+            version=version,
+            retry=retry,
+        )
 
     def pop(
         self, key, default=None, version=None, expire_time=False, tag=False, retry=True
     ):
-        try:
-            return super(CustomDjangoCache, self).pop(
-                key, default, version, expire_time, tag, retry
-            )
-        except sqlite3.OperationalError:
-            return None
+        return self.try_execute(
+            "pop",
+            None,
+            key=key,
+            default=default,
+            version=version,
+            expire_time=expire_time,
+            tag=tag,
+            retry=retry,
+        )
 
     def delete(self, key, version=None, retry=True):
-        try:
-            super(CustomDjangoCache, self).delete(key, version, retry)
-        except sqlite3.OperationalError:
-            pass
+        self.try_execute(
+            "delete",
+            None,
+            key=key,
+            version=version,
+            retry=retry,
+        )
 
     def incr(self, key, delta=1, version=None, default=None, retry=True):
-        try:
-            return super(CustomDjangoCache, self).incr(
-                key, delta, version, default, retry
-            )
-        except sqlite3.OperationalError:
-            return None
+        return self.try_execute(
+            "incr",
+            None,
+            key=key,
+            delta=delta,
+            version=version,
+            default=default,
+            retry=retry,
+        )
 
     def decr(self, key, delta=1, version=None, default=None, retry=True):
-        try:
-            return super(CustomDjangoCache, self).decr(
-                key, delta, version, default, retry
-            )
-        except sqlite3.OperationalError:
-            return None
+        return self.try_execute(
+            "decr",
+            None,
+            key=key,
+            delta=delta,
+            version=version,
+            default=default,
+            retry=retry,
+        )
 
     def expire(self, retry=False):
-        try:
-            return super(CustomDjangoCache, self).expire(retry)
-        except sqlite3.OperationalError:
-            return 0
+        return self.try_execute("expire", 0, retry=retry)
 
     def stats(self, enable=True, reset=False):
-        try:
-            return super(CustomDjangoCache, self).stats(enable, reset)
-        except sqlite3.OperationalError:
-            return 0, 0
+        return self.try_execute("stats", (0, 0), enable=enable, reset=reset)
 
     def create_tag_index(self):
-        try:
-            super(CustomDjangoCache, self).create_tag_index()
-        except sqlite3.OperationalError:
-            pass
+        return self.try_execute("create_tag_index", None)
 
     def drop_tag_index(self):
-        try:
-            super(CustomDjangoCache, self).drop_tag_index()
-        except sqlite3.OperationalError:
-            pass
+        return self.try_execute("drop_tag_index", None)
 
     def evict(self, tag):
-        try:
-            return super(CustomDjangoCache, self).evict(tag)
-        except sqlite3.OperationalError:
-            return 0
+        return self.try_execute("evict", 0, tag=tag)
 
     def cull(self):
-        try:
-            return super(CustomDjangoCache, self).cull()
-        except sqlite3.OperationalError:
-            return 0
+        return self.try_execute("cull", 0)
 
     def clear(self):
-        try:
-            return super(CustomDjangoCache, self).clear()
-        except sqlite3.OperationalError:
-            return 0
+        return self.try_execute("clear", 0)
 
     def close(self, **kwargs):
-        try:
-            super(CustomDjangoCache, self).close(**kwargs)
-        except sqlite3.OperationalError:
-            pass
+        return self.try_execute("close", None, **kwargs)


Test patch: