Repository: learningequality/kolibri

Issue:
Search - Getting a 500 error after having applied too many filters
## Observed behavior
When searching I'm getting a 500 error after having applied too many filters. The issue could be caused by the fake data used but should be further investigated.

## Expected behavior
Fully functional search functionality. 

## Steps to reproduce the issue
1. Install the following [build.](https://github.com/learningequality/kolibri/releases/tag/v0.15.0-beta2)
2. Import some channels and run the following management command: `kolibri manage labeltestdata `
3. Filter by as many criteria as possible

## Additional information

![2021-11-18_19-35-32](https://user-images.githubusercontent.com/79847249/142470762-279635ad-7bba-456b-8a6f-514730c49e84.png)

## Logs

[logs.zip](https://github.com/learningequality/kolibri/files/7564984/logs.zip)

## Usage Details
 - OS: Windows 10
 - Browser: Chrome


Discussion:
@radinamatic 
Oh wow, that's interesting, this actually caused a too many SQL variables error!

```
ERROR 2021-11-18 19:34:18,963 django.request Internal Server Error: /api/content/contentnode/
Traceback (most recent call last):
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\backends\utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\backends\sqlite3\base.py", line 328, in execute
    return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: too many SQL variables

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "c:\python36\lib\site-packages\kolibri\dist\django\core\handlers\exception.py", line 41, in inner
    response = get_response(request)
  File "c:\python36\lib\site-packages\kolibri\dist\django\core\handlers\base.py", line 187, in _get_response
    response = self.process_exception_by_middleware(e, request)
  File "c:\python36\lib\site-packages\kolibri\dist\django\core\handlers\base.py", line 185, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "c:\python36\lib\site-packages\kolibri\dist\django\views\decorators\csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "c:\python36\lib\site-packages\kolibri\dist\rest_framework\viewsets.py", line 116, in view
    return self.dispatch(request, *args, **kwargs)
  File "c:\python36\lib\site-packages\kolibri\dist\django\utils\decorators.py", line 67, in _wrapper
    return bound_func(*args, **kwargs)
  File "c:\python36\lib\site-packages\kolibri\core\auth\middleware.py", line 107, in wrapper_func
    return view(*args, **kwargs)
  File "c:\python36\lib\site-packages\kolibri\core\content\api.py", line 90, in wrapper_func
    response = some_func(*args, **kwargs)
  File "c:\python36\lib\site-packages\kolibri\dist\django\utils\decorators.py", line 63, in bound_func
    return func.__get__(self, type(self))(*args2, **kwargs2)
  File "c:\python36\lib\site-packages\kolibri\dist\rest_framework\views.py", line 495, in dispatch
    response = self.handle_exception(exc)
  File "c:\python36\lib\site-packages\kolibri\dist\rest_framework\views.py", line 455, in handle_exception
    self.raise_uncaught_exception(exc)
  File "c:\python36\lib\site-packages\kolibri\dist\rest_framework\views.py", line 492, in dispatch
    response = handler(request, *args, **kwargs)
  File "c:\python36\lib\site-packages\kolibri\core\api.py", line 329, in list
    return self.get_paginated_response(self.serialize(queryset))
  File "c:\python36\lib\site-packages\kolibri\dist\rest_framework\generics.py", line 180, in get_paginated_response
    return self.paginator.get_paginated_response(data)
  File "c:\python36\lib\site-packages\kolibri\core\content\api.py", line 474, in get_paginated_response
    ("labels", get_available_metadata_labels(self.queryset)),
  File "c:\python36\lib\site-packages\kolibri\core\content\utils\search.py", line 109, in get_available_metadata_labels
    result = queryset.values(*all_values).first()
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\query.py", line 567, in first
    objects = list((self if self.ordered else self.order_by('pk'))[:1])
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\query.py", line 250, in __iter__
    self._fetch_all()
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\query.py", line 1121, in _fetch_all
    self._result_cache = list(self._iterable_class(self))
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\query.py", line 106, in __iter__
    for row in compiler.results_iter(chunked_fetch=self.chunked_fetch):
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\sql\compiler.py", line 841, in results_iter
    results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch)
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\sql\compiler.py", line 899, in execute_sql
    raise original_exception
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\models\sql\compiler.py", line 889, in execute_sql
    cursor.execute(sql, params)
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\backends\utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\utils.py", line 94, in __exit__
    six.reraise(dj_exc_type, dj_exc_value, traceback)
  File "c:\python36\lib\site-packages\kolibri\dist\django\utils\six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\backends\utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "c:\python36\lib\site-packages\kolibri\dist\django\db\backends\sqlite3\base.py", line 328, in execute
    return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: too many SQL variables
```

Resolving patch:
diff --git a/kolibri/core/content/api.py b/kolibri/core/content/api.py
--- a/kolibri/core/content/api.py
+++ b/kolibri/core/content/api.py
@@ -135,13 +135,36 @@ def filter_available(self, queryset, name, value):
 
 
 @method_decorator(cache_forever, name="dispatch")
-class ChannelMetadataViewSet(viewsets.ReadOnlyModelViewSet):
-    serializer_class = serializers.ChannelMetadataSerializer
+class ChannelMetadataViewSet(ReadOnlyValuesViewset):
     filter_backends = (DjangoFilterBackend,)
     filter_class = ChannelMetadataFilter
 
+    values = (
+        "author",
+        "description",
+        "tagline",
+        "id",
+        "last_updated",
+        "root__lang__lang_code",
+        "root__lang__lang_name",
+        "name",
+        "root",
+        "thumbnail",
+        "version",
+        "root__available",
+        "root__num_coach_contents",
+        "public",
+    )
+
+    field_map = {
+        "num_coach_contents": "root__num_coach_contents",
+        "available": "root__available",
+        "lang_code": "root__lang__lang_code",
+        "lang_name": "root__lang__lang_name",
+    }
+
     def get_queryset(self):
-        return models.ChannelMetadata.objects.all().select_related("root__lang")
+        return models.ChannelMetadata.objects.all()
 
 
 class IdFilter(FilterSet):
diff --git a/kolibri/core/content/utils/search.py b/kolibri/core/content/utils/search.py
--- a/kolibri/core/content/utils/search.py
+++ b/kolibri/core/content/utils/search.py
@@ -4,10 +4,17 @@
 """
 import hashlib
 
+try:
+    from django.contrib.postgres.aggregates import BitOr
+except ImportError:
+    BitOr = None
+
+from django.db import connections
+from django.db.models import Aggregate
 from django.db.models import Case
-from django.db.models import Exists
 from django.db.models import Value
 from django.db.models import When
+from django.db.models.fields import IntegerField
 from le_utils.constants.labels.accessibility_categories import (
     ACCESSIBILITYCATEGORIESLIST,
 )
@@ -53,34 +60,35 @@
 
 
 def _get_available_languages(base_queryset):
-    from kolibri.core.content.models import Language
-
     lang_ids = (
-        base_queryset.exclude(lang=None)
-        .order_by("lang_id")
-        .values_list("lang_id", flat=True)
-    )
-    return list(
-        Language.objects.filter(id__in=lang_ids)
-        .distinct()
-        .order_by("id")
-        .values("id", "lang_name")
+        base_queryset.exclude(lang=None).values_list("lang_id", flat=True).distinct()
     )
+    return list(lang_ids)
 
 
 def _get_available_channels(base_queryset):
-    from kolibri.core.content.models import ChannelMetadata
+    return list(base_queryset.values_list("channel_id", flat=True).distinct())
+
+
+class SQLiteBitwiseORAggregate(Aggregate):
+    name = "BitwiseOR"
 
-    return list(
-        ChannelMetadata.objects.filter(
-            id__in=base_queryset.values_list("channel_id", flat=True).distinct()
+    def __init__(self, expression, num_bits=None, **extra):
+        if not num_bits:
+            raise ValueError("num_bits must be a positive integer")
+        self.num_bits = num_bits
+        super(SQLiteBitwiseORAggregate, self).__init__(
+            expression, output_field=IntegerField(), **extra
+        )
+
+    @property
+    def template(self):
+        return " + ".join(
+            "max(%(expressions)s&{})".format(2 ** i) for i in range(0, self.num_bits)
         )
-        .order_by("order")
-        .values("id", "name", "thumbnail")
-    )
 
 
-def get_available_metadata_labels(base_queryset, limit_to_known_fields=True):
+def get_available_metadata_labels(base_queryset):
     from kolibri.core.device.models import ContentCacheKey
 
     content_cache_key = ContentCacheKey.get_cache_key()
@@ -89,27 +97,25 @@ def get_available_metadata_labels(base_queryset, limit_to_known_fields=True):
         hashlib.md5(str(base_queryset.query).encode("utf8")).hexdigest(),
     )
     if cache_key not in cache:
-        base_queryset = base_queryset.values("id").order_by()
-        queryset = base_queryset
-        all_values = []
-        if limit_to_known_fields:
-            lookup = get_all_contentnode_label_metadata()
-            lookup.pop("channels")
-            lookup.pop("languages")
-        else:
-            lookup = metadata_lookup
-        for field, values in lookup.items():
-            queryset = queryset.annotate(
-                **{
-                    value: Exists(base_queryset.has_all_labels(field, [value]))
-                    for value in values
-                }
-            )
-            all_values += values
-        result = queryset.values(*all_values).first()
+        base_queryset = base_queryset.order_by()
+        aggregates = {}
+        for field in bitmask_fieldnames:
+            field_agg = field + "_agg"
+            if connections[base_queryset.db].vendor == "sqlite" or BitOr is None:
+                aggregates[field_agg] = SQLiteBitwiseORAggregate(
+                    field, num_bits=len(bitmask_fieldnames[field])
+                )
+            elif connections[base_queryset.db].vendor == "postgresql":
+                aggregates[field_agg] = BitOr(field)
         output = {}
-        for field, values in lookup.items():
-            output[field] = [v for v in values if result and result.get(v)]
+        agg = base_queryset.aggregate(**aggregates)
+        for field, values in bitmask_fieldnames.items():
+            bit_value = agg[field + "_agg"]
+            for value in values:
+                if value["field_name"] not in output:
+                    output[value["field_name"]] = []
+                if bit_value is not None and bit_value & value["bits"]:
+                    output[value["field_name"]].append(value["label"])
         output["languages"] = _get_available_languages(base_queryset)
         output["channels"] = _get_available_channels(base_queryset)
         cache.set(cache_key, output, timeout=None)
@@ -119,9 +125,7 @@ def get_available_metadata_labels(base_queryset, limit_to_known_fields=True):
 def get_all_contentnode_label_metadata():
     from kolibri.core.content.models import ContentNode
 
-    return get_available_metadata_labels(
-        ContentNode.objects.filter(available=True), limit_to_known_fields=False
-    )
+    return get_available_metadata_labels(ContentNode.objects.filter(available=True))
 
 
 def annotate_label_bitmasks(queryset):
diff --git a/kolibri/plugins/learn/kolibri_plugin.py b/kolibri/plugins/learn/kolibri_plugin.py
--- a/kolibri/plugins/learn/kolibri_plugin.py
+++ b/kolibri/plugins/learn/kolibri_plugin.py
@@ -2,13 +2,11 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from django.db.models import F
 from django.urls import reverse
 
 from kolibri.core.auth.constants.user_kinds import ANONYMOUS
 from kolibri.core.auth.constants.user_kinds import LEARNER
 from kolibri.core.content.hooks import ContentNodeDisplayHook
-from kolibri.core.content.utils.search import get_all_contentnode_label_metadata
 from kolibri.core.device.utils import allow_learner_unassigned_resource_access
 from kolibri.core.device.utils import get_device_setting
 from kolibri.core.device.utils import is_landing_page
@@ -52,34 +50,21 @@ class LearnAsset(webpack_hooks.WebpackBundleHook):
 
     @property
     def plugin_data(self):
-        from kolibri.core.content.models import ChannelMetadata
-
-        channels = list(
-            ChannelMetadata.objects.filter(root__available=True)
-            .annotate(
-                lang_code=F("root__lang__lang_code"),
-                lang_name=F("root__lang__lang_name"),
-                available=F("root__available"),
-                num_coach_contents=F("root__num_coach_contents"),
-            )
-            .values(
-                "author",
-                "description",
-                "tagline",
-                "id",
-                "last_updated",
-                "lang_code",
-                "lang_name",
-                "name",
-                "root",
-                "thumbnail",
-                "version",
-                "available",
-                "num_coach_contents",
-                "public",
-            )
+        from kolibri.core.content.utils.search import get_all_contentnode_label_metadata
+        from kolibri.core.content.api import ChannelMetadataViewSet
+        from kolibri.core.content.models import Language
+
+        channel_viewset = ChannelMetadataViewSet()
+
+        channels = channel_viewset.serialize(
+            channel_viewset.get_queryset().filter(root__available=True)
         )
         label_metadata = get_all_contentnode_label_metadata()
+        languages = list(
+            Language.objects.filter(id__in=label_metadata["languages"]).values(
+                "id", "lang_name"
+            )
+        )
         return {
             "allowGuestAccess": get_device_setting("allow_guest_access"),
             "allowLearnerUnassignedResourceAccess": allow_learner_unassigned_resource_access(),
@@ -91,7 +76,7 @@ def plugin_data(self):
             ),
             "categories": label_metadata["categories"],
             "learningActivities": label_metadata["learning_activities"],
-            "languages": label_metadata["languages"],
+            "languages": languages,
             "channels": channels,
             "gradeLevels": label_metadata["grade_levels"],
             "accessibilityLabels": label_metadata["accessibility_labels"],


Test patch:
diff --git a/kolibri/core/content/test/test_search.py b/kolibri/core/content/test/test_search.py
--- a/kolibri/core/content/test/test_search.py
+++ b/kolibri/core/content/test/test_search.py
@@ -1,20 +1,150 @@
+from uuid import uuid4
+
 from django.test import TestCase
+from le_utils.constants import content_kinds
+from parameterized import parameterized
 
 from kolibri.core.content.models import ContentNode
 from kolibri.core.content.test.test_channel_upgrade import ChannelBuilder
 from kolibri.core.content.utils.search import annotate_label_bitmasks
+from kolibri.core.content.utils.search import get_available_metadata_labels
 from kolibri.core.content.utils.search import metadata_lookup
 
 
-class BitMaskTestCase(TestCase):
-    def test_ancestors(self):
+class RandomBitMaskTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        builder = ChannelBuilder()
+        builder.insert_into_default_db()
+        annotate_label_bitmasks(ContentNode.objects.all())
+
+    @parameterized.expand(
+        (field, label)
+        for field in metadata_lookup.keys()
+        for label in metadata_lookup[field]
+    )
+    def test_bitmasks(self, field, label):
+        self.assertEqual(
+            ContentNode.objects.filter(**{field + "__contains": label}).count(),
+            ContentNode.objects.has_all_labels(field, [label]).count(),
+            "{} {}".format(field, label),
+        )
+
+
+class ConstrainedBitMaskTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        for field in metadata_lookup.keys():
+            for label in metadata_lookup[field]:
+                ContentNode.objects.create(
+                    content_id=uuid4().hex,
+                    channel_id=uuid4().hex,
+                    description="Blah blah blah",
+                    id=uuid4().hex,
+                    license_name="GNU",
+                    license_owner="",
+                    license_description=None,
+                    lang_id=None,
+                    author="",
+                    title="Test{}_{}".format(field, label),
+                    parent_id=None,
+                    kind=content_kinds.VIDEO,
+                    coach_content=False,
+                    available=False,
+                    **{field: label}
+                )
+        annotate_label_bitmasks(ContentNode.objects.all())
+
+    @parameterized.expand(
+        (field, label)
+        for field in metadata_lookup.keys()
+        for label in metadata_lookup[field]
+    )
+    def test_bitmasks(self, field, label):
+        self.assertEqual(
+            ContentNode.objects.filter(**{field + "__contains": label}).count(),
+            ContentNode.objects.has_all_labels(field, [label]).count(),
+            "{} {}".format(field, label),
+        )
+
+
+class RandomMetadataLabelsTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
         builder = ChannelBuilder()
         builder.insert_into_default_db()
         annotate_label_bitmasks(ContentNode.objects.all())
-        for field, LIST in metadata_lookup.items():
-            for label in LIST:
-                self.assertEqual(
-                    ContentNode.objects.filter(**{field + "__contains": label}).count(),
-                    ContentNode.objects.has_all_labels(field, [label]).count(),
-                    "{} {}".format(field, label),
+
+    def _run_test_for_qs(self, queryset):
+        metadata_labels = get_available_metadata_labels(queryset)
+        for field, LIST in metadata_labels.items():
+            if field != "channels" and field != "languages":
+                all_labels = metadata_lookup[field]
+                for label in all_labels:
+                    if label in LIST:
+                        self.assertTrue(
+                            queryset.filter(**{field + "__contains": label}).exists(),
+                            "{} {}".format(field, label),
+                        )
+                    else:
+                        self.assertFalse(
+                            queryset.filter(**{field + "__contains": label}).exists(),
+                            "{} {}".format(field, label),
+                        )
+
+    def test_all(self):
+        self._run_test_for_qs(ContentNode.objects.all())
+
+    @parameterized.expand(content_kinds.choices)
+    def test_filtered_kind(self, kind, kind_name):
+        self._run_test_for_qs(ContentNode.objects.filter(kind=kind))
+
+    @parameterized.expand(
+        (field, label)
+        for field in metadata_lookup.keys()
+        for label in metadata_lookup[field]
+    )
+    def test_labels(self, field, label):
+        self._run_test_for_qs(ContentNode.objects.has_all_labels(field, [label]))
+
+
+class ConstrainedMetadataLabelsTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        for field in metadata_lookup.keys():
+            for label in metadata_lookup[field]:
+                ContentNode.objects.create(
+                    content_id=uuid4().hex,
+                    channel_id=uuid4().hex,
+                    description="Blah blah blah",
+                    id=uuid4().hex,
+                    license_name="GNU",
+                    license_owner="",
+                    license_description=None,
+                    lang_id=None,
+                    author="",
+                    title="Test{}_{}".format(field, label),
+                    parent_id=None,
+                    kind=content_kinds.VIDEO,
+                    coach_content=False,
+                    available=False,
+                    **{field: label}
                 )
+        annotate_label_bitmasks(ContentNode.objects.all())
+
+    @parameterized.expand(
+        (field, label)
+        for field in metadata_lookup.keys()
+        for label in metadata_lookup[field]
+    )
+    def test_labels(self, field, label):
+        metadata_labels = get_available_metadata_labels(
+            ContentNode.objects.filter(**{field: label})
+        )
+        split_labels = label.split(".")
+        expected = [
+            ".".join(split_labels[0:i]) for i in range(1, len(split_labels) + 1)
+        ]
+        self.assertEqual(
+            set(metadata_labels[field]), set(expected), "{} {}".format(field, label)
+        )
diff --git a/requirements/test.txt b/requirements/test.txt
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -11,3 +11,4 @@ pytest-django==3.3.3
 pytest-env==0.6.2
 pytest-pythonpath==0.7.2
 attrs==20.3.0
+parameterized==0.8.1