Repository: learningequality/kolibri

Issue:
Importing big channels in RPi is not possible due to db lock errors
### Observed behavior
Trying to import Khan Academy (English - US curriculum) > Math > Arithmetic in 0.14.4.dev0-0ubuntu1  in a RPI 4 (4G RAM) has been impossible after trying it several times.
This same channel has been previously imported in 0.13.3. 

Importing in 0.14.4 was much faster, but sooner or later the process is canceled by kolibri after a database error happens

This is probably related to #7669

### Expected behavior
Importing the channel should finish correctly

### User-facing consequences
No way to import the channel, sometimes it stops at 19% others at 80%
![image](https://user-images.githubusercontent.com/1008178/98409781-bed6ad00-2073-11eb-9c1f-dad1bc7588e7.png)

### Errors and logs

```
ERROR 2020-11-06 20:56:46,694 kolibri.core.tasks.worker Job 8a620aa2028a495fb43329282fc9e89e raised an exception: Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/base.py", line 1284, in _execute_context
    cursor, statement, parameters, context
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/default.py", line 590, in do_execute
    cursor.execute(statement, parameters)
sqlite3.OperationalError: database is locked

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/worker.py", line 72, in handle_finished_future
    result = future.result()
  File "/usr/lib/python3.7/concurrent/futures/_base.py", line 425, in result
    return self.__get_result()
  File "/usr/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/usr/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/worker.py", line 217, in wrap
    return f(*args, **kwargs)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/job.py", line 194, in y
    result = func(*args, **kwargs)
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/core/management/__init__.py", line 131, in call_command
    return command.execute(*args, **defaults)
  File "/usr/lib/python3/dist-packages/kolibri/dist/django/core/management/base.py", line 330, in execute
    output = self.handle(*args, **options)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/management/commands/base.py", line 119, in handle
    return self.handle_async(*args, **options)
  File "/usr/lib/python3/dist-packages/kolibri/core/content/management/commands/importcontent.py", line 468, in handle_async
    import_updates=options["import_updates"],
  File "/usr/lib/python3/dist-packages/kolibri/core/content/management/commands/importcontent.py", line 168, in download_content
    import_updates=import_updates,
  File "/usr/lib/python3/dist-packages/kolibri/core/content/management/commands/importcontent.py", line 422, in _transfer
    raise self.exception
  File "/usr/lib/python3/dist-packages/kolibri/core/content/management/commands/importcontent.py", line 361, in _transfer
    status = future.result()
  File "/usr/lib/python3.7/concurrent/futures/_base.py", line 425, in result
    return self.__get_result()
  File "/usr/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/usr/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/usr/lib/python3/dist-packages/kolibri/core/content/management/commands/importcontent.py", line 437, in _start_file_transfer
    overall_progress_update(length)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/management/commands/base.py", line 62, in update_progress
    self.update_callback(p.progress_fraction, p)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/management/commands/base.py", line 115, in _update_all_progress
    self.update_progress(progress_list[0].progress_fraction, 1.0)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/management/commands/base.py", line 123, in update_progress
    self.job.update_progress(progress_fraction, total_progress)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/job.py", line 132, in update_progress
    self.update_progress_method(self.job_id, progress, total_progress)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/worker.py", line 131, in update_progress
    self.storage.update_job_progress(job_id, progress, total_progress)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/storage.py", line 223, in update_job_progress
    self._update_job(job_id, progress=progress, total_progress=total_progress)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/storage.py", line 281, in _update_job
    job_id
  File "/usr/lib/python3.7/contextlib.py", line 119, in __exit__
    next(self.gen)
  File "/usr/lib/python3/dist-packages/kolibri/core/tasks/storage.py", line 67, in session_scope
    session.commit()
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/session.py", line 1042, in commit
    self.transaction.commit()
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/session.py", line 504, in commit
    self._prepare_impl()
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/session.py", line 483, in _prepare_impl
    self.session.flush()
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/session.py", line 2523, in flush
    self._flush(objects)
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/session.py", line 2664, in _flush
    transaction.rollback(_capture_exception=True)
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/util/langhelpers.py", line 69, in __exit__
    exc_value, with_traceback=exc_tb,
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/util/compat.py", line 178, in raise_
    raise exception
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/session.py", line 2624, in _flush
    flush_context.execute()
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/unitofwork.py", line 422, in execute
    rec.execute(self)
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/unitofwork.py", line 589, in execute
    uow,
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/persistence.py", line 236, in save_obj
    update,
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/orm/persistence.py", line 995, in _emit_update_statements
    statement, multiparams
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/base.py", line 1020, in execute
    return meth(self, multiparams, params)
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/base.py", line 1139, in _execute_clauseelement
    distilled_params,
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/base.py", line 1324, in _execute_context
    e, statement, parameters, cursor, context
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/base.py", line 1518, in _handle_dbapi_exception
    sqlalchemy_exception, with_traceback=exc_info[2], from_=e
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/util/compat.py", line 178, in raise_
    raise exception
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/base.py", line 1284, in _execute_context
    cursor, statement, parameters, context
  File "/usr/lib/python3/dist-packages/kolibri/dist/sqlalchemy/engine/default.py", line 590, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked
[SQL: UPDATE jobs SET obj=? WHERE jobs.id = ?]
[parameters: (<memory at 0xb0453308>, '8a620aa2028a495fb43329282fc9e89e')]
(Background on this error at: http://sqlalche.me/e/e3q8)

```


…

### Steps to reproduce
https://www.notion.so/learningequality/Kolibri-0-14-4-weekly7-bug-bash-c6ac33f8b1db44dc87ef4fbad877307d

### Context

- RPI 4 , with Debian 10.6 (Raspbian)
- kolibri-server installed and running
- options.ini with this setup:

```
[Cache]
CACHE_BACKEND = redis
CACHE_REDIS_MAXMEMORY = 536870912
CACHE_REDIS_MAXMEMORY_POLICY = allkeys-lru
[Database]
[Server]
CHERRYPY_START = False
```




Discussion:
@jredrejo This is the same error I reported on Android yesterday (but I forgot to file an issue!), I'm going to try and spend some time on this today. 
One quick fix might be to only update progress in the main thread, and not call it from any of the worker threads?
(downside will be that big files will make the progress appear to stall a bit, but it's better than it stalling completely because of an error!)

Resolving patch:
diff --git a/kolibri/core/content/management/commands/importcontent.py b/kolibri/core/content/management/commands/importcontent.py
--- a/kolibri/core/content/management/commands/importcontent.py
+++ b/kolibri/core/content/management/commands/importcontent.py
@@ -346,10 +346,7 @@ def _transfer(  # noqa: max-complexity=16
                         if len(file_transfers) > 0:
                             f, filetransfer = file_transfers.pop()
                             future = executor.submit(
-                                self._start_file_transfer,
-                                f,
-                                filetransfer,
-                                overall_progress_update,
+                                self._start_file_transfer, f, filetransfer
                             )
                             future_file_transfers[future] = (f, filetransfer)
 
@@ -358,7 +355,8 @@ def _transfer(  # noqa: max-complexity=16
                     ):
                         f, filetransfer = future_file_transfers[future]
                         try:
-                            status = future.result()
+                            status, data_transferred = future.result()
+                            overall_progress_update(data_transferred)
                             if self.is_cancelled():
                                 break
 
@@ -424,24 +422,25 @@ def _transfer(  # noqa: max-complexity=16
             if self.is_cancelled():
                 self.cancel()
 
-    def _start_file_transfer(self, f, filetransfer, overall_progress_update):
+    def _start_file_transfer(self, f, filetransfer):
         """
         Start to transfer the file from network/disk to the destination.
         Return value:
             * FILE_TRANSFERRED - successfully transfer the file.
             * FILE_SKIPPED - the file does not exist so it is skipped.
         """
+        data_transferred = 0
+
         with filetransfer:
             for chunk in filetransfer:
-                length = len(chunk)
-                overall_progress_update(length)
+                data_transferred += len(chunk)
 
             # Ensure that if for some reason the total file size for the transfer
             # is less than what we have marked in the database that we make up
             # the difference so that the overall progress is never incorrect.
             # This could happen, for example for a local transfer if a file
             # has been replaced or corrupted (which we catch below)
-            overall_progress_update(f.file_size - filetransfer.total_size)
+            data_transferred += f.file_size - filetransfer.total_size
 
             # If checksum of the destination file is different from the localfile
             # id indicated in the database, it means that the destination file
@@ -452,9 +451,9 @@ def _start_file_transfer(self, f, filetransfer, overall_progress_update):
                 e = "File {} is corrupted.".format(filetransfer.source)
                 logger.error("An error occurred during content import: {}".format(e))
                 os.remove(filetransfer.dest)
-                return FILE_SKIPPED
+                return FILE_SKIPPED, data_transferred
 
-        return FILE_TRANSFERRED
+        return FILE_TRANSFERRED, data_transferred
 
     def handle_async(self, *args, **options):
         if options["command"] == "network":


Test patch:
diff --git a/kolibri/core/content/test/test_import_export.py b/kolibri/core/content/test/test_import_export.py
--- a/kolibri/core/content/test/test_import_export.py
+++ b/kolibri/core/content/test/test_import_export.py
@@ -761,7 +761,6 @@ def test_local_import_source_corrupted_full_progress(
         local_src_path = tempfile.mkstemp()[1]
         with open(local_src_path, "w") as f:
             f.write("This is just a test")
-        src_file_size = os.path.getsize(local_src_path)
         expected_file_size = 10000
         local_dest_path = tempfile.mkstemp()[1]
         os.remove(local_dest_path)
@@ -798,7 +797,7 @@ def test_local_import_source_corrupted_full_progress(
                 node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
             )
 
-            mock_overall_progress.assert_any_call(expected_file_size - src_file_size)
+            mock_overall_progress.assert_any_call(expected_file_size)
 
     @patch(
         "kolibri.core.content.management.commands.importcontent.transfer.FileDownload.finalize"