Repository: learningequality/kolibri

Issue:
Import/export issues

### Observed behavior

Observed by a [user in the forums](https://community.learningequality.org/t/import-export-issues/1396):

> I am experiencing al multitude of issues with Kolibri.
> 
>1. When I import several resources at the same time the Task Manager gets stuck at 100%. Cancelling the import doesn’t work. Only after restarting the computer does the task disappear. But afterwards I get the same problem with every task. So completely unable to import anything. I have to uninstall (deleting all imports) and reinstall Kolibri to be able to import again.
>2. When importing resources (Khan Academy) via a local drive I get the error message “Channel not found” even though Kolibri recognizes the exported resources on the drive.
>3. Kolibri often refuses to start up. Again uninstall - reinstall seems to be the only way to solve this.

### Expected behavior

import runs correctly

### User-facing consequences

cannot get content

### Errors and logs

```
ERROR 2020-02-01 13:32:26,810 exception Internal Server Error: /api/content/contentnode_granular/1ceff53605e55bef987d88e0908658c5/
Traceback (most recent call last):
File “C:\Python34\lib\site-packages\kolibri\dist\sqlalchemy\engine\base.py”, line 1193, in _execute_context
context)
File “C:\Python34\lib\site-packages\kolibri\dist\sqlalchemy\engine\default.py”, line 509, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: too many SQL variables
```

### Steps to reproduce

Unknown

### Context

Kolibri 0.13.0
Windows 10 Home
Windows Explorer


Discussion:
might be related to https://github.com/learningequality/kolibri/issues/6157 and https://github.com/learningequality/kolibri/pull/6378

Resolving patch:
diff --git a/kolibri/core/content/utils/annotation.py b/kolibri/core/content/utils/annotation.py
--- a/kolibri/core/content/utils/annotation.py
+++ b/kolibri/core/content/utils/annotation.py
@@ -14,6 +14,7 @@
 from .paths import get_content_file_name
 from .paths import get_content_storage_file_path
 from .sqlalchemybridge import Bridge
+from .sqlalchemybridge import filter_by_uuids
 from kolibri.core.content.apps import KolibriContentConfig
 from kolibri.core.content.errors import InvalidStorageFilenameError
 from kolibri.core.content.models import ChannelMetadata
@@ -33,7 +34,7 @@ def _MPTT_descendant_ids_statement(ContentNodeTable, node_ids):
     node_include = ContentNodeTable.alias()
     return select([ContentNodeTable.c.id]).where(
         and_(
-            node_include.c.id.in_(node_ids),
+            filter_by_uuids(node_include.c.id, node_ids),
             ContentNodeTable.c.lft >= node_include.c.lft,
             ContentNodeTable.c.lft <= node_include.c.rght,
             ContentNodeTable.c.tree_id == node_include.c.tree_id,
diff --git a/kolibri/core/content/utils/importability_annotation.py b/kolibri/core/content/utils/importability_annotation.py
--- a/kolibri/core/content/utils/importability_annotation.py
+++ b/kolibri/core/content/utils/importability_annotation.py
@@ -10,6 +10,7 @@
 from sqlalchemy import select
 
 from .sqlalchemybridge import Bridge
+from .sqlalchemybridge import filter_by_uuids
 from kolibri.core.content.apps import KolibriContentConfig
 from kolibri.core.content.models import ContentNode
 from kolibri.core.content.models import File
@@ -34,14 +35,15 @@ def get_channel_annotation_stats(channel_id, checksums=None):
     ContentNodeTable = bridge.get_table(ContentNode)
     FileTable = bridge.get_table(File)
     LocalFileTable = bridge.get_table(LocalFile)
-
     if checksums is not None:
         file_table = FileTable.join(
             LocalFileTable,
             and_(
                 FileTable.c.local_file_id == LocalFileTable.c.id,
                 or_(
-                    LocalFileTable.c.id.in_(checksums),
+                    # checksums are not uuids and have been got from
+                    # get_channel_stats_from_disk, so no need to validate them:
+                    filter_by_uuids(LocalFileTable.c.id, checksums, validate=False),
                     LocalFileTable.c.available == True,  # noqa
                 ),
             ),
diff --git a/kolibri/core/content/utils/sqlalchemybridge.py b/kolibri/core/content/utils/sqlalchemybridge.py
--- a/kolibri/core/content/utils/sqlalchemybridge.py
+++ b/kolibri/core/content/utils/sqlalchemybridge.py
@@ -1,9 +1,11 @@
 import logging
 import os
 import pickle
+from uuid import UUID
 
 from django.apps import apps
 from django.conf import settings
+from django.db import connection as django_connection
 from sqlalchemy import ColumnDefault
 from sqlalchemy import create_engine
 from sqlalchemy import event
@@ -13,6 +15,8 @@
 from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.pool import NullPool
+from sqlalchemy.sql import operators
+from sqlalchemy.sql.elements import UnaryExpression
 
 from .check_schema_db import db_matches_schema
 from .check_schema_db import DBSchemaError
@@ -331,3 +335,35 @@ def end(self):
         self.session.close()
         if self.connection:
             self.connection.close()
+
+
+def filter_by_uuids(field, ids, validate=True):
+    return _by_uuids(field, ids, validate, True)
+
+
+def exclude_by_uuids(field, ids, validate=True):
+    return _by_uuids(field, ids, validate, False)
+
+
+def _by_uuids(field, ids, validate, include):
+    ids_list = list(ids)
+    query = "IN (" if include else "NOT IN ("
+    # trick to workaround postgresql, it does not allow returning ():
+    empty_query = "IS NULL" if include else "IS NOT NULL"
+    for (idx, identifier) in enumerate(ids_list):
+        if validate:
+            try:
+                UUID(identifier, version=4)
+            except (TypeError, ValueError):
+                # the value is not a valid hex code for a UUID, so we don't return any results
+                return UnaryExpression(field, modifier=operators.custom_op(empty_query))
+        # wrap the uuids in string quotations
+        if django_connection.vendor == "postgresql" and validate:
+            ids_list[idx] = "'{}'::uuid".format(identifier)
+        else:  # sqlite or it's not an uuid
+            ids_list[idx] = "'{}'".format(identifier)
+    if ids_list:
+        placeholder = query + ",".join(ids_list) + ")"
+    else:
+        placeholder = empty_query
+    return UnaryExpression(field, modifier=operators.custom_op(placeholder))


Test patch: