Repository: paperless-ngx/paperless-ngx

Issue:
[BUG] v1.8.0 RC1 Bug - Error uploading documents (related to PaperlessTask)
### Description

When uploading multiple documents through the web ui, some documents will produce an error in the post_document API endpoint

### Steps to reproduce

1. Upload a number of documents through web ui
2. observe logs for the below

### Webserver logs

```bash
paperless-ngx-dev-webserver  | [2022-07-18 08:10:37,986] [ERROR] [django.request] Internal Server Error: /api/documents/post_document/
paperless-ngx-dev-webserver  | Traceback (most recent call last):
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/query.py", line 657, in get_or_create
paperless-ngx-dev-webserver  |     return self.get(**kwargs), False
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/query.py", line 496, in get
paperless-ngx-dev-webserver  |     raise self.model.DoesNotExist(
paperless-ngx-dev-webserver  | documents.models.PaperlessTask.DoesNotExist: PaperlessTask matching query does not exist.
paperless-ngx-dev-webserver  |
paperless-ngx-dev-webserver  | During handling of the above exception, another exception occurred:
paperless-ngx-dev-webserver  |
paperless-ngx-dev-webserver  | Traceback (most recent call last):
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 89, in _execute
paperless-ngx-dev-webserver  |     return self.cursor.execute(sql, params)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/sqlite3/base.py", line 477, in execute
paperless-ngx-dev-webserver  |     return Database.Cursor.execute(self, query, params)
paperless-ngx-dev-webserver  | sqlite3.OperationalError: database is locked
paperless-ngx-dev-webserver  |
paperless-ngx-dev-webserver  | The above exception was the direct cause of the following exception:
paperless-ngx-dev-webserver  |
paperless-ngx-dev-webserver  | Traceback (most recent call last):
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/asgiref/sync.py", line 472, in thread_handler
paperless-ngx-dev-webserver  |     raise exc_info[1]
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 42, in inner
paperless-ngx-dev-webserver  |     response = await get_response(request)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 253, in _get_response_async
paperless-ngx-dev-webserver  |     response = await wrapped_callback(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/asgiref/sync.py", line 435, in __call__
paperless-ngx-dev-webserver  |     ret = await asyncio.wait_for(future, timeout=None)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/asyncio/tasks.py", line 442, in wait_for
paperless-ngx-dev-webserver  |     return await fut
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/asgiref/current_thread_executor.py", line 22, in run
paperless-ngx-dev-webserver  |     result = self.fn(*self.args, **self.kwargs)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/asgiref/sync.py", line 476, in thread_handler
paperless-ngx-dev-webserver  |     return func(*args, **kwargs)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
paperless-ngx-dev-webserver  |     return view_func(*args, **kwargs)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/views/generic/base.py", line 84, in view
paperless-ngx-dev-webserver  |     return self.dispatch(request, *args, **kwargs)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 509, in dispatch
paperless-ngx-dev-webserver  |     response = self.handle_exception(exc)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 469, in handle_exception
paperless-ngx-dev-webserver  |     self.raise_uncaught_exception(exc)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
paperless-ngx-dev-webserver  |     raise exc
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 506, in dispatch
paperless-ngx-dev-webserver  |     response = handler(request, *args, **kwargs)
paperless-ngx-dev-webserver  |   File "/usr/src/paperless/src/documents/views.py", line 544, in post
paperless-ngx-dev-webserver  |     async_task(
paperless-ngx-dev-webserver  |   File "/usr/src/paperless/src/docker/src/django-q/django_q/tasks.py", line 67, in async_task
paperless-ngx-dev-webserver  |     pre_enqueue.send(sender="django_q", task=task)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/dispatch/dispatcher.py", line 176, in send
paperless-ngx-dev-webserver  |     return [
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/dispatch/dispatcher.py", line 177, in <listcomp>
paperless-ngx-dev-webserver  |     (receiver, receiver(signal=self, sender=sender, **named))
paperless-ngx-dev-webserver  |   File "/usr/src/paperless/src/documents/signals/handlers.py", line 509, in init_paperless_task
paperless-ngx-dev-webserver  |     paperless_task, created = PaperlessTask.objects.get_or_create(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/manager.py", line 85, in manager_method
paperless-ngx-dev-webserver  |     return getattr(self.get_queryset(), name)(*args, **kwargs)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/query.py", line 664, in get_or_create
paperless-ngx-dev-webserver  |     return self.create(**params), True
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/query.py", line 514, in create
paperless-ngx-dev-webserver  |     obj.save(force_insert=True, using=self.db)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/base.py", line 806, in save
paperless-ngx-dev-webserver  |     self.save_base(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/base.py", line 857, in save_base
paperless-ngx-dev-webserver  |     updated = self._save_table(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/base.py", line 1000, in _save_table
paperless-ngx-dev-webserver  |     results = self._do_insert(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/base.py", line 1041, in _do_insert
paperless-ngx-dev-webserver  |     return manager._insert(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/manager.py", line 85, in manager_method
paperless-ngx-dev-webserver  |     return getattr(self.get_queryset(), name)(*args, **kwargs)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/query.py", line 1434, in _insert
paperless-ngx-dev-webserver  |     return query.get_compiler(using=using).execute_sql(returning_fields)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/models/sql/compiler.py", line 1621, in execute_sql
paperless-ngx-dev-webserver  |     cursor.execute(sql, params)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 67, in execute
paperless-ngx-dev-webserver  |     return self._execute_with_wrappers(
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 80, in _execute_with_wrappers
paperless-ngx-dev-webserver  |     return executor(sql, params, many, context)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 89, in _execute
paperless-ngx-dev-webserver  |     return self.cursor.execute(sql, params)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/utils.py", line 91, in __exit__
paperless-ngx-dev-webserver  |     raise dj_exc_value.with_traceback(traceback) from exc_value
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/utils.py", line 89, in _execute
paperless-ngx-dev-webserver  |     return self.cursor.execute(sql, params)
paperless-ngx-dev-webserver  |   File "/usr/local/lib/python3.9/site-packages/django/db/backends/sqlite3/base.py", line 477, in execute
paperless-ngx-dev-webserver  |     return Database.Cursor.execute(self, query, params)
paperless-ngx-dev-webserver  | django.db.utils.OperationalError: database is locked
```


### Paperless-ngx version

beta rc1

### Host OS

Ubuntu 20.04

### Installation method

Docker - official image

### Browser

_No response_

### Configuration changes

_No response_

### Other

_No response_


Discussion:
This is probably coming from https://github.com/paperless-ngx/paperless-ngx/blob/4c697ab50e3a4ecc92291659c9ca93921421d61d/src/documents/signals/handlers.py#L507

It may be related to my using SQLite, but presumably something else (django-q?) has a lock on the DB at this point, but only sometimes.  Or maybe it's due to the uploading probably happening in almost parallels?
Ah, so even though its `PaperlessTask.objects.get_or_create` it cant create so in effect it doesnt exist. What do we think is the solution here, can we catch the locked DB error and try again or is there somehow a way to wait for db? Not sure what best practice is there. This from Django docs isnt too encouraging: https://docs.djangoproject.com/en/dev/ref/databases/#database-is-locked-errors

Related, on the next signals do we consider changing them to `get_or_create` as well, in case creation was missed before?
I'm not finding a lot of details either so far.  If the other signal handlers are changed to a `get_or_create`, they might also happen to fail, if enough documents are being processed?
Yes, they might though I have no idea how likely that is. The other two are already wrapped in a try in case it failed in the first signal handler, so I think wrapping the first one also in a try would at least prevent this issue. Of course, in that case the `PaperlessTask` never gets created. Obviously not the end of the world but sorta puts a big dent in the usefulness of the feature which is why I was wondering if using `get_or_create` later would at least salvage _some_ cases.

Hope its obvious but Im totally open to a better solution, not sure whats best here.
> What do we think is the solution here, can we catch the locked DB error and try again or is there somehow a way to wait for db?

I think that's what the timeout is referencing here, waiting for the db. Could we simply increase the timeout as suggested in the docs?
I'll test increasing the timeout when I can.  It looks like the default for Python is 5s, unless django changes that, but it doesn't sound like they do.
Yea presumably that would change the timeout for all db types, which shouldn’t be a problem I suppose?

thanks stumpy! let us know what you find, hopefully a combo of the above ideas will smooth this over.

Btw, as the Django docs note, SQLite might be a little out of its league so perhaps we could also add some kind of warning, eg we catch a db locked error and note to the user “sqlite may not be ideal for this”, etc

Resolving patch:
diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py
--- a/src/documents/signals/handlers.py
+++ b/src/documents/signals/handlers.py
@@ -11,6 +11,7 @@
 from django.db import DatabaseError
 from django.db import models
 from django.db.models import Q
+from django.db.utils import OperationalError
 from django.dispatch import receiver
 from django.utils import termcolors
 from django.utils import timezone
@@ -506,21 +507,28 @@ def add_to_index(sender, document, **kwargs):
 @receiver(django_q.signals.pre_enqueue)
 def init_paperless_task(sender, task, **kwargs):
     if task["func"] == "documents.tasks.consume_file":
-        paperless_task, created = PaperlessTask.objects.get_or_create(
-            task_id=task["id"],
-        )
-        paperless_task.name = task["name"]
-        paperless_task.created = task["started"]
-        paperless_task.save()
+        try:
+            paperless_task, created = PaperlessTask.objects.get_or_create(
+                task_id=task["id"],
+            )
+            paperless_task.name = task["name"]
+            paperless_task.created = task["started"]
+            paperless_task.save()
+        except OperationalError as e:
+            logger.error(f"Creating PaperlessTask failed: {e}")
 
 
 @receiver(django_q.signals.pre_execute)
 def paperless_task_started(sender, task, **kwargs):
     try:
         if task["func"] == "documents.tasks.consume_file":
-            paperless_task = PaperlessTask.objects.get(task_id=task["id"])
+            paperless_task, created = PaperlessTask.objects.get_or_create(
+                task_id=task["id"],
+            )
             paperless_task.started = timezone.now()
             paperless_task.save()
+    except OperationalError as e:
+        logger.error(f"Creating PaperlessTask failed: {e}")
     except PaperlessTask.DoesNotExist:
         pass
 
@@ -529,8 +537,12 @@ def paperless_task_started(sender, task, **kwargs):
 def update_paperless_task(sender, instance, **kwargs):
     try:
         if instance.func == "documents.tasks.consume_file":
-            paperless_task = PaperlessTask.objects.get(task_id=instance.id)
+            paperless_task, created = PaperlessTask.objects.get_or_create(
+                task_id=instance.id,
+            )
             paperless_task.attempted_task = instance
             paperless_task.save()
+    except OperationalError as e:
+        logger.error(f"Creating PaperlessTask failed: {e}")
     except PaperlessTask.DoesNotExist:
         pass
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -327,6 +327,13 @@ def __get_float(key: str, default: float) -> float:
     if os.getenv("PAPERLESS_DBPORT"):
         DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")
 
+if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
+    _new_opts = {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}
+    if "OPTIONS" in DATABASES["default"]:
+        DATABASES["default"]["OPTIONS"].update(_new_opts)
+    else:
+        DATABASES["default"]["OPTIONS"] = _new_opts
+
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 ###############################################################################


Test patch: