Repository: learningequality/kolibri

Issue:
Add sanity check to ensure that Job Storage tables are ready
## Observed behavior
In 0.15 the deletion and recreation of Job Storage tables at every restart was removed. This means that there are some possible scenarios where Kolibri could attempt to start, but the storage tables do not exist.

## Errors and logs
```
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/kolibri/dist/magicbus/base.py", line 215, in _transition
    return self.publish(newstate, *args, **kwargs)
  File "/usr/local/lib/python3.8/site-packages/kolibri/dist/magicbus/base.py", line 291, in publish
    raise exc
magicbus.base.ChannelFailures: ProgrammingError('(psycopg2.errors.UndefinedColumn) column scheduledjobs.saved_job does not exist\nLINE 1: ...peat, scheduledjobs.queue AS scheduledjobs_queue, scheduledj...\n       
```

## Expected behavior
Kolibri should check that these tables are accessible and recreate them if not.


Discussion:

Patch:
diff --git a/kolibri/core/tasks/scheduler.py b/kolibri/core/tasks/scheduler.py
index d106ee5890..c0b05d7954 100644
--- a/kolibri/core/tasks/scheduler.py
+++ b/kolibri/core/tasks/scheduler.py
@@ -182,6 +182,13 @@ class Scheduler(StorageMixin):
             scheduled_jobs = self._ns_query(s).all()
             return [Job.from_json(o.saved_job) for o in scheduled_jobs]
 
+    def test_table_readable(self):
+        """
+        Do a quick query to raise errors if the database is unusable.
+        """
+        with self.session_scope() as s:
+            self._ns_query(s).first()
+
     def count(self):
         with self.session_scope() as s:
             return self._ns_query(s).count()
diff --git a/kolibri/core/tasks/storage.py b/kolibri/core/tasks/storage.py
index 35892b8c2f..0379ee58e6 100644
--- a/kolibri/core/tasks/storage.py
+++ b/kolibri/core/tasks/storage.py
@@ -246,6 +246,15 @@ class Storage(StorageMixin):
 
             return [self._orm_to_job(o) for o in orm_jobs]
 
+    def test_table_readable(self):
+        """
+        Do a quick query to raise errors if the database is unusable.
+        """
+        with self.session_scope() as s:
+            q = s.query(ORMJob)
+
+            q.first()
+
     def count_all_jobs(self, queue=None):
         with self.session_scope() as s:
             q = s.query(ORMJob)
diff --git a/kolibri/utils/main.py b/kolibri/utils/main.py
index 2d27c63394..839d9b2c8a 100644
--- a/kolibri/utils/main.py
+++ b/kolibri/utils/main.py
@@ -42,6 +42,7 @@ from kolibri.utils.sanity_checks import check_django_stack_ready
 from kolibri.utils.sanity_checks import check_log_file_location
 from kolibri.utils.sanity_checks import DatabaseInaccessible
 from kolibri.utils.sanity_checks import DatabaseNotMigrated
+from kolibri.utils.sanity_checks import ensure_job_tables_created
 from kolibri.utils.server import get_status
 from kolibri.utils.server import NotRunning
 
@@ -255,14 +256,14 @@ def _upgrades_after_django_setup(updated, version):
             logging.error(e)
 
 
-def initialize(
+def initialize(  # noqa C901
     skip_update=False,
     settings=None,
     debug=False,
     debug_database=False,
     no_input=False,
     pythonpath=None,
-):  # noqa: max-complexity=12
+):
     """
     This should be called before starting the Kolibri app, it initializes Kolibri plugins
     and sets up Django.
@@ -307,6 +308,16 @@ def initialize(
 
         check_django_stack_ready()
 
+        try:
+            ensure_job_tables_created()
+        except Exception as e:
+            logging.error(
+                "The job tables were not fully migrated. Tried to "
+                "create them in the database and an error occurred: "
+                "{}".format(e)
+            )
+            raise
+
         try:
             check_database_is_migrated()
         except DatabaseNotMigrated:
diff --git a/kolibri/utils/sanity_checks.py b/kolibri/utils/sanity_checks.py
index a6a199b7ac..e617dcc1f6 100644
--- a/kolibri/utils/sanity_checks.py
+++ b/kolibri/utils/sanity_checks.py
@@ -6,6 +6,8 @@ import sys
 from django.apps import apps
 from django.db.utils import OperationalError
 from django.db.utils import ProgrammingError
+from sqlalchemy.exc import OperationalError as SQLAlchemyOperationalError
+from sqlalchemy.exc import ProgrammingError as SQLAlchemyProgrammingError
 
 from .conf import KOLIBRI_HOME
 from .conf import OPTIONS
@@ -116,6 +118,29 @@ def check_database_is_migrated():
         raise DatabaseInaccessible(db_exception=e)
 
 
+def ensure_job_tables_created():
+    from kolibri.core.tasks.main import job_storage
+    from kolibri.core.tasks.main import scheduler
+
+    try:
+        job_storage.test_table_readable()
+    except (SQLAlchemyOperationalError, SQLAlchemyProgrammingError):
+        logger.warning("Database table for job storage was not accessible, recreating.")
+        job_storage.recreate_tables()
+    except Exception as e:
+        raise DatabaseInaccessible(db_exception=e)
+
+    try:
+        scheduler.test_table_readable()
+    except (SQLAlchemyOperationalError, SQLAlchemyProgrammingError):
+        logger.warning(
+            "Database table for scheduled jobs was not accessible, recreating."
+        )
+        scheduler.recreate_tables()
+    except Exception as e:
+        raise DatabaseInaccessible(db_exception=e)
+
+
 def check_default_options_exist():
     options_path = os.path.join(KOLIBRI_HOME, "options.ini")
     if not os.path.exists(options_path):


Test patch:
diff --git a/kolibri/utils/tests/test_sanity_check.py b/kolibri/utils/tests/test_sanity_check.py
index e2feae3bb6..edf62caa93 100644
--- a/kolibri/utils/tests/test_sanity_check.py
+++ b/kolibri/utils/tests/test_sanity_check.py
@@ -4,6 +4,8 @@ import tempfile
 from django.db.utils import OperationalError
 from django.test import TestCase
 from mock import patch
+from sqlalchemy.exc import OperationalError as SQLAlchemyOperationalError
+from sqlalchemy.exc import ProgrammingError as SQLAlchemyProgrammingError
 
 from kolibri.utils import sanity_checks
 from kolibri.utils.sanity_checks import DatabaseNotMigrated
@@ -52,3 +54,31 @@ class SanityCheckTestCase(TestCase):
             get_or_create_current_instance.side_effect = OperationalError("Test")
             with self.assertRaises(DatabaseNotMigrated):
                 sanity_checks.check_database_is_migrated()
+
+    def test_ensure_job_tables_created_operational_error(self):
+        with patch("kolibri.core.tasks.main.job_storage") as job_storage, patch(
+            "kolibri.core.tasks.main.scheduler"
+        ) as scheduler:
+            job_storage.test_table_readable.side_effect = SQLAlchemyOperationalError(
+                "Test", "", ""
+            )
+            scheduler.test_table_readable.side_effect = SQLAlchemyOperationalError(
+                "Test", "", ""
+            )
+            sanity_checks.ensure_job_tables_created()
+            job_storage.recreate_tables.assert_called_once()
+            scheduler.recreate_tables.assert_called_once()
+
+    def test_ensure_job_tables_created_programming_error(self):
+        with patch("kolibri.core.tasks.main.job_storage") as job_storage, patch(
+            "kolibri.core.tasks.main.scheduler"
+        ) as scheduler:
+            job_storage.test_table_readable.side_effect = SQLAlchemyProgrammingError(
+                "Test", "", ""
+            )
+            scheduler.test_table_readable.side_effect = SQLAlchemyProgrammingError(
+                "Test", "", ""
+            )
+            sanity_checks.ensure_job_tables_created()
+            job_storage.recreate_tables.assert_called_once()
+            scheduler.recreate_tables.assert_called_once()