Repository: paperless-ngx/paperless-ngx

Issue:
[BUG] [ERROR] [django.request] Internal Server Error: /api/search/
### Description

After updating to the latest version v2.8.1 and when I make a global search nothing happens and the small charging ball then moves continuously:

<img width="1413" alt="screenshot_204" src="https://github.com/paperless-ngx/paperless-ngx/assets/6528507/ac9df9ee-af2f-44c9-9274-81ca20feb6df">

After I checked the paperless-webserver logfiles the following error occurs:

> [2024-05-07 13:14:44,607] [ERROR] [django.request] Internal Server Error: /api/search/
> Traceback (most recent call last):
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute
>     return self.cursor.execute(sql, params)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/mysql/base.py", line 75, in execute
>     return self.cursor.execute(query, args)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 179, in execute
>     res = self._query(mogrified_query)
>           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 330, in _query
>     db.query(q)
>   File "/usr/local/lib/python3.11/site-packages/MySQLdb/connections.py", line 261, in query
>     _mysql.connection.query(self, query)
> MySQLdb.NotSupportedError: (1235, "This version of MariaDB doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery'")
> The above exception was the direct cause of the following exception:
> Traceback (most recent call last):
>   File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 518, in thread_handler
>     raise exc_info[1]
>   File "/usr/local/lib/python3.11/site-packages/django/core/handlers/exception.py", line 42, in inner
>     response = await get_response(request)
>                ^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 518, in thread_handler
>     raise exc_info[1]
>   File "/usr/local/lib/python3.11/site-packages/django/core/handlers/base.py", line 253, in _get_response_async
>     response = await wrapped_callback(
>                ^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 468, in __call__
>     ret = await asyncio.shield(exec_coro)
>           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/asgiref/current_thread_executor.py", line 40, in run
>     result = self.fn(*self.args, **self.kwargs)
>              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 522, in thread_handler
>     return func(*args, **kwargs)
>            ^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/views/decorators/csrf.py", line 56, in wrapper_view
>     return view_func(*args, **kwargs)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/views/generic/base.py", line 104, in view
>     return self.dispatch(request, *args, **kwargs)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 509, in dispatch
>     response = self.handle_exception(exc)
>                ^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 469, in handle_exception
>     self.raise_uncaught_exception(exc)
>   File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
>     raise exc
>   File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 506, in dispatch
>     response = handler(request, *args, **kwargs)
>                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/src/paperless/src/documents/views.py", line 1296, in get
>     "total": len(docs)
>              ^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/models/query.py", line 380, in __len__
>     self._fetch_all()
>   File "/usr/local/lib/python3.11/site-packages/django/db/models/query.py", line 1881, in _fetch_all
>     self._result_cache = list(self._iterable_class(self))
>                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/models/query.py", line 91, in __iter__
>     results = compiler.execute_sql(
>               ^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/models/sql/compiler.py", line 1562, in execute_sql
>     cursor.execute(sql, params)
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 67, in execute
>     return self._execute_with_wrappers(
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 80, in _execute_with_wrappers
>     return executor(sql, params, many, context)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 84, in _execute
>     with self.db.wrap_database_errors:
>   File "/usr/local/lib/python3.11/site-packages/django/db/utils.py", line 91, in __exit__
>     raise dj_exc_value.with_traceback(traceback) from exc_value
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute
>     return self.cursor.execute(sql, params)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/django/db/backends/mysql/base.py", line 75, in execute
>     return self.cursor.execute(query, args)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 179, in execute
>     res = self._query(mogrified_query)
>           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 330, in _query
>     db.query(q)
>   File "/usr/local/lib/python3.11/site-packages/MySQLdb/connections.py", line 261, in query
>     _mysql.connection.query(self, query)
> django.db.utils.NotSupportedError: (1235, "This version of MariaDB doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery'")

### Steps to reproduce

I used the following Stack:

```
version: "1.0"
services:
  broker:
    image: docker.io/library/redis:7
    restart: always
    volumes:
      - /share/CACHEDEV1_DATA/docker/paperless/redisdata:/data

  db:
    image: docker.io/library/mariadb:10
    restart: always
    volumes:
      - /share/CACHEDEV1_DATA/docker/paperless/dbdata:/var/lib/mysql
    env_file:
      - stack.env

  webserver:
    image: ghcr.io/paperless-ngx/paperless-ngx:latest
    restart: always
    depends_on:
      - db
      - broker
      - gotenberg
      - tika
    ports:
      - "8010:8000"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000"]
      interval: 30s
      timeout: 10s
      retries: 5
    volumes:
      - /share/CACHEDEV1_DATA/docker/paperless/data:/usr/src/paperless/data
      - /share/CACHEDEV1_DATA/docker/paperless/media:/usr/src/paperless/media
      - /share/CACHEDEV1_DATA/docker/paperless/export:/usr/src/paperless/export
      - /share/CACHEDEV1_DATA/docker/paperless/scripts:/usr/src/paperless/scripts
      - /share/CACHEDEV2_DATA/Scans/consume:/usr/src/paperless/consume

    env_file:
      - stack.env

  gotenberg:
    image: docker.io/gotenberg/gotenberg:7.8
    restart: always
    # The gotenberg chromium route is used to convert .eml files. We do not
    # want to allow external content like tracking pixels or even javascript.
    command:
      - "gotenberg"
      - "--chromium-disable-javascript=true"
      - "--chromium-allow-list=file:///tmp/.*"

  tika:
    image: ghcr.io/paperless-ngx/tika:latest
    restart: always

```

### Webserver logs

```bash
[2024-05-07 13:14:44,607] [ERROR] [django.request] Internal Server Error: /api/search/
Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute
    return self.cursor.execute(sql, params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/mysql/base.py", line 75, in execute
    return self.cursor.execute(query, args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 179, in execute
    res = self._query(mogrified_query)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 330, in _query
    db.query(q)
  File "/usr/local/lib/python3.11/site-packages/MySQLdb/connections.py", line 261, in query
    _mysql.connection.query(self, query)
MySQLdb.NotSupportedError: (1235, "This version of MariaDB doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery'")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 518, in thread_handler
    raise exc_info[1]
  File "/usr/local/lib/python3.11/site-packages/django/core/handlers/exception.py", line 42, in inner
    response = await get_response(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 518, in thread_handler
    raise exc_info[1]
  File "/usr/local/lib/python3.11/site-packages/django/core/handlers/base.py", line 253, in _get_response_async
    response = await wrapped_callback(
               ^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 468, in __call__
    ret = await asyncio.shield(exec_coro)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/asgiref/current_thread_executor.py", line 40, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/asgiref/sync.py", line 522, in thread_handler
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/views/decorators/csrf.py", line 56, in wrapper_view
    return view_func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/views/generic/base.py", line 104, in view
    return self.dispatch(request, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 509, in dispatch
    response = self.handle_exception(exc)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 469, in handle_exception
    self.raise_uncaught_exception(exc)
  File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
    raise exc
  File "/usr/local/lib/python3.11/site-packages/rest_framework/views.py", line 506, in dispatch
    response = handler(request, *args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/src/paperless/src/documents/views.py", line 1296, in get
    "total": len(docs)
             ^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/models/query.py", line 380, in __len__
    self._fetch_all()
  File "/usr/local/lib/python3.11/site-packages/django/db/models/query.py", line 1881, in _fetch_all
    self._result_cache = list(self._iterable_class(self))
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/models/query.py", line 91, in __iter__
    results = compiler.execute_sql(
              ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/models/sql/compiler.py", line 1562, in execute_sql
    cursor.execute(sql, params)
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 80, in _execute_with_wrappers
    return executor(sql, params, many, context)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 84, in _execute
    with self.db.wrap_database_errors:
  File "/usr/local/lib/python3.11/site-packages/django/db/utils.py", line 91, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute
    return self.cursor.execute(sql, params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/django/db/backends/mysql/base.py", line 75, in execute
    return self.cursor.execute(query, args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 179, in execute
    res = self._query(mogrified_query)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/MySQLdb/cursors.py", line 330, in _query
    db.query(q)
  File "/usr/local/lib/python3.11/site-packages/MySQLdb/connections.py", line 261, in query
    _mysql.connection.query(self, query)
django.db.utils.NotSupportedError: (1235, "This version of MariaDB doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery'")
```


### Browser logs

_No response_

### Paperless-ngx version

2.8.1

### Host OS

Linux 5.10.60-qnap

### Installation method

Docker - official image

### Browser

Chrome

### Configuration changes

_No response_

### Other

_No response_

### Please confirm the following

- [X] I believe this issue is a bug that affects all users of Paperless-ngx, not something specific to my installation.
- [X] I have already searched for relevant existing issues and discussions before opening this report.
- [X] I have updated the title field above with a concise description.


Discussion:
Supporting multiple database types is fun.

What version of mariadb are you using?
I am using the image: docker.io/library/mariadb:10 and if I check the docker paperless-db container it seems to be Version: '10.11.7-MariaDB-1:10.11.7+maria~ubu2204
The error message here is a little misleading because no version of mysql or mariadb does support this type of query.
[MariaDB subquery limitationss](https://mariadb.com/kb/en/subquery-limitations/)

This is a common problem with django ignoring those limitations.
But what does it mean for the latest version v2.8.1? It seems that the new global search functions are using subqueries which are not supporting, isn't it? Why else would the search not work properly and the loading bank load endlessly?

I am also using the official supported mariadb version like described in the https://github.com/paperless-ngx/paperless-ngx/blob/dev/docker/compose/docker-compose.mariadb.yml
At least that's what I thought. Or what is currently officially supported? 😅


The new global search is (at this moment) not compatible with mysql/mariadb.
This is because django orm is creating queries which are not supported by mysql/mariadb no matter what version.

Resolving patch:
diff --git a/src/documents/views.py b/src/documents/views.py
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -1160,7 +1160,7 @@ def get(self, request, *args, **kwargs):
                 Document,
             )
             # First search by title
-            docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
+            docs = all_docs.filter(title__icontains=query)
             if not db_only and len(docs) < OBJECT_LIMIT:
                 # If we don't have enough results, search by content
                 from documents import index
@@ -1174,77 +1174,87 @@ def get(self, request, *args, **kwargs):
                     )._get_query()
                     results = s.search(q, limit=OBJECT_LIMIT)
                     docs = docs | all_docs.filter(id__in=[r["id"] for r in results])
+            docs = docs[:OBJECT_LIMIT]
         saved_views = (
-            SavedView.objects.filter(owner=request.user, name__icontains=query)[
-                :OBJECT_LIMIT
-            ]
+            SavedView.objects.filter(owner=request.user, name__icontains=query)
             if request.user.has_perm("documents.view_savedview")
             else []
         )
+        saved_views = saved_views[:OBJECT_LIMIT]
         tags = (
             get_objects_for_user_owner_aware(request.user, "view_tag", Tag).filter(
                 name__icontains=query,
-            )[:OBJECT_LIMIT]
+            )
             if request.user.has_perm("documents.view_tag")
             else []
         )
+        tags = tags[:OBJECT_LIMIT]
         correspondents = (
             get_objects_for_user_owner_aware(
                 request.user,
                 "view_correspondent",
                 Correspondent,
-            ).filter(name__icontains=query)[:OBJECT_LIMIT]
+            ).filter(name__icontains=query)
             if request.user.has_perm("documents.view_correspondent")
             else []
         )
+        correspondents = correspondents[:OBJECT_LIMIT]
         document_types = (
             get_objects_for_user_owner_aware(
                 request.user,
                 "view_documenttype",
                 DocumentType,
-            ).filter(name__icontains=query)[:OBJECT_LIMIT]
+            ).filter(name__icontains=query)
             if request.user.has_perm("documents.view_documenttype")
             else []
         )
+        document_types = document_types[:OBJECT_LIMIT]
         storage_paths = (
             get_objects_for_user_owner_aware(
                 request.user,
                 "view_storagepath",
                 StoragePath,
-            ).filter(name__icontains=query)[:OBJECT_LIMIT]
+            ).filter(name__icontains=query)
             if request.user.has_perm("documents.view_storagepath")
             else []
         )
+        storage_paths = storage_paths[:OBJECT_LIMIT]
         users = (
-            User.objects.filter(username__icontains=query)[:OBJECT_LIMIT]
+            User.objects.filter(username__icontains=query)
             if request.user.has_perm("auth.view_user")
             else []
         )
+        users = users[:OBJECT_LIMIT]
         groups = (
-            Group.objects.filter(name__icontains=query)[:OBJECT_LIMIT]
+            Group.objects.filter(name__icontains=query)
             if request.user.has_perm("auth.view_group")
             else []
         )
+        groups = groups[:OBJECT_LIMIT]
         mail_rules = (
-            MailRule.objects.filter(name__icontains=query)[:OBJECT_LIMIT]
+            MailRule.objects.filter(name__icontains=query)
             if request.user.has_perm("paperless_mail.view_mailrule")
             else []
         )
+        mail_rules = mail_rules[:OBJECT_LIMIT]
         mail_accounts = (
-            MailAccount.objects.filter(name__icontains=query)[:OBJECT_LIMIT]
+            MailAccount.objects.filter(name__icontains=query)
             if request.user.has_perm("paperless_mail.view_mailaccount")
             else []
         )
+        mail_accounts = mail_accounts[:OBJECT_LIMIT]
         workflows = (
-            Workflow.objects.filter(name__icontains=query)[:OBJECT_LIMIT]
+            Workflow.objects.filter(name__icontains=query)
             if request.user.has_perm("documents.view_workflow")
             else []
         )
+        workflows = workflows[:OBJECT_LIMIT]
         custom_fields = (
-            CustomField.objects.filter(name__icontains=query)[:OBJECT_LIMIT]
+            CustomField.objects.filter(name__icontains=query)
             if request.user.has_perm("documents.view_customfield")
             else []
         )
+        custom_fields = custom_fields[:OBJECT_LIMIT]
 
         context = {
             "request": request,


Test patch: