Repository: learningequality/kolibri

Issue:
Channel metadata import: broken order of table importing
## Observed behavior

When importing from the UNW staging server, we keep getting this error during channel metadata import.

The root cause turned out to be that the `File` table is imported before the `LocalFile` table, while `File` rows had foreign keys pointed to the `LocalFile` table.

The fix is to ensure that `LocalFile` is imported into the postgres database before `File`. 



## Errors and logs

```
❯ docker-compose exec kolibri kolibri manage importchannel network 1455135935065c3e9b3abf3e7e135e00
INFO     Option DATABASE_ENGINE in section [Database] being overridden by environment variable KOLIBRI_DATABASE_ENGINE
INFO     Option DATABASE_NAME in section [Database] being overridden by environment variable KOLIBRI_DATABASE_NAME
INFO     Option DATABASE_PASSWORD in section [Database] being overridden by environment variable KOLIBRI_DATABASE_PASSWORD
INFO     Option DATABASE_USER in section [Database] being overridden by environment variable KOLIBRI_DATABASE_USER
INFO     Option DATABASE_HOST in section [Database] being overridden by environment variable KOLIBRI_DATABASE_HOST
INFO     Option DATABASE_PORT in section [Database] being overridden by environment variable KOLIBRI_DATABASE_PORT
INFO     Option RUN_MODE in section [Deployment] being overridden by environment variable KOLIBRI_RUN_MODE
INFO     Option REMOTE_CONTENT in section [Deployment] being overridden by environment variable KOLIBRI_REMOTE_CONTENT
INFO     Running Kolibri with the following settings: kolibri.deployment.default.settings.base
INFO     Invoking command importchannel network 1455135935065c3e9b3abf3e7e135e00
INFO     Downloading data for channel id 1455135935065c3e9b3abf3e7e135e00
INFO     Importing ContentTag data
INFO     Importing ContentNode_has_prerequisite data
INFO     Importing ContentNode_related data
INFO     Importing ContentNode_tags data
INFO     Importing ContentNode data
INFO     Importing Language data
INFO     Importing File data
Traceback (most recent call last):
  File "/usr/local/bin/kolibri", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/click/core.py", line 764, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/click/core.py", line 717, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/click/core.py", line 1137, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.6/site-packages/kolibri/utils/cli.py", line 193, in invoke
    return super(KolibriDjangoCommand, self).invoke(ctx)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/click/core.py", line 956, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/click/core.py", line 555, in invoke
    return callback(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/click/decorators.py", line 17, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/kolibri/utils/cli.py", line 361, in manage
    execute_from_command_line(["kolibri manage"] + ctx.args)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/django/core/management/__init__.py", line 364, in execute_from_command_line
    utility.execute()
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/django/core/management/__init__.py", line 356, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/django/core/management/base.py", line 283, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/usr/local/lib/python3.6/site-packages/kolibri/dist/django/core/management/base.py", line 330, in execute
    output = self.handle(*args, **options)
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/tasks/management/commands/base.py", line 119, in handle
    return self.handle_async(*args, **options)
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/management/commands/importchannel.py", line 185, in handle_async
    options["channel_id"], options["baseurl"], options["no_upgrade"]
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/management/commands/importchannel.py", line 94, in download_channel
    self._transfer(DOWNLOAD_METHOD, channel_id, baseurl, no_upgrade=no_upgrade)
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/management/commands/importchannel.py", line 134, in _transfer
    filetransfer, channel_id, dest, no_upgrade=no_upgrade
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/management/commands/importchannel.py", line 171, in _start_file_transfer
    import_ran = import_channel_by_id(channel_id, self.is_cancelled)
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/management/commands/importchannel.py", line 27, in import_channel_by_id
    channel_id, cancel_check=cancel_check
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/utils/channel_import.py", line 1013, in import_channel_from_local_db
    import_ran = import_manager.import_channel_data()
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/utils/channel_import.py", line 813, in import_channel_data
    self.table_import(model, row_mapper, table_mapper)
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/utils/channel_import.py", line 602, in table_import
    result = self.postgres_table_import(model, row_mapper, table_mapper)
  File "/usr/local/lib/python3.6/site-packages/kolibri/core/content/utils/channel_import.py", line 458, in postgres_table_import
    columns=column_names,
psycopg2.errors.ForeignKeyViolation: insert or update on table "content_file" violates foreign key constraint "content_file_local_file_id_fkey"
DETAIL:  Key (local_file_id)=(3641693a88b37e8d0484c340a83f9364) is not present in table "content_localfile".
```


## Expected behavior
No error during channel metadata imports.

## User-facing consequences
No imports can be done in any affected server.

## Steps to reproduce

Get a dump of the UNW staging server data by pinging Aron.

Load the data onto a postgres database, and then connect a kolibri instance to that postgres database.

Import channel metadata by running:

```sh
kolibri manage importchannel network 1455135935065c3e9b3abf3e7e135e00
```
## Context

This was discovered with @radinamatic trying to import a channel in the UNW staging server in the process of replicating another bug.


Discussion:

Resolving patch:
diff --git a/kolibri/core/content/utils/channel_import.py b/kolibri/core/content/utils/channel_import.py
--- a/kolibri/core/content/utils/channel_import.py
+++ b/kolibri/core/content/utils/channel_import.py
@@ -68,6 +68,50 @@ def convert_to_sqlite_value(python_value):
         return repr(python_value)
 
 
+def _get_dependencies(content_models):
+    references = {}
+    for model in content_models:
+        meta = model._meta
+        for f in meta.concrete_fields:
+            if f.is_relation and f.many_to_one:
+                if f.related_model not in references:
+                    references[f.related_model] = set()
+                if f.related_model is not model:
+                    references[f.related_model].add(model)
+    return references
+
+
+def topological_sort(content_models):
+    """
+    Carries out a depth first search topological sort of content models to ensure we
+    import them in the correct order.
+    ref: https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search
+    """
+    # First collect all non-self referential foreign key references
+    # in the set of models. We can't resolve the self-referentiality in a topological
+    # sort, so we simply ignore it.
+    references = _get_dependencies(content_models)
+
+    sorted_models = []
+    visiting = set()
+
+    def visit(n):
+        if n in sorted_models:
+            return
+        if n in visiting:
+            raise ReferenceError(n)
+        visiting.add(n)
+        for m in references.get(n, set()):
+            visit(m)
+        visiting.remove(n)
+        sorted_models.insert(0, n)
+
+    for model in content_models:
+        visit(model)
+
+    return sorted_models
+
+
 class ChannelImport(object):
     """
     The ChannelImport class has two functions:
@@ -147,6 +191,8 @@ def __init__(
             if blacklisted_model in self.content_models:
                 self.content_models.remove(blacklisted_model)
 
+        self.content_models = topological_sort(self.content_models)
+
         # Get the next available tree_id in our database
         self.available_tree_id = self.find_unique_tree_id()
 


Test patch:
diff --git a/kolibri/core/content/test/test_channel_import.py b/kolibri/core/content/test/test_channel_import.py
--- a/kolibri/core/content/test/test_channel_import.py
+++ b/kolibri/core/content/test/test_channel_import.py
@@ -35,10 +35,18 @@
 from kolibri.core.content.utils.annotation import update_content_metadata
 from kolibri.core.content.utils.channel_import import ChannelImport
 from kolibri.core.content.utils.channel_import import import_channel_from_local_db
+from kolibri.core.content.utils.channel_import import topological_sort
 from kolibri.core.content.utils.sqlalchemybridge import get_default_db_string
 from kolibri.core.content.utils.sqlalchemybridge import load_metadata
 
 
+class UtilityTestCase(TestCase):
+    def test_topological_sort(self):
+        sorted_models = topological_sort([File, LocalFile, ContentNode])
+        self.assertGreater(sorted_models.index(File), sorted_models.index(ContentNode))
+        self.assertGreater(sorted_models.index(File), sorted_models.index(LocalFile))
+
+
 @patch("kolibri.core.content.utils.channel_import.Bridge")
 @patch("kolibri.core.content.utils.channel_import.ChannelImport.find_unique_tree_id")
 @patch("kolibri.core.content.utils.channel_import.apps")
diff --git a/requirements/test.txt b/requirements/test.txt
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -11,3 +11,4 @@ pytest-django==3.3.3
 pytest-env==0.6.2
 pytest-pythonpath==0.7.2
 sh==1.12.14
+attrs==20.3.0