Repository: learningequality/kolibri

Issue:
Deleted content shows as 'Already on Device'
<!--
Instructions:
 * Fill out the sections below, replace …'s with information about your issue
 * Use the 'preview' function above this text box to verify formatting before submitting
-->

### Observed behavior
<!--
Description of the behavior that was observed, including screenshots or other references when applicable
-->

Deleted content still showing as 'Already on Device'

![unable_to_import](https://user-images.githubusercontent.com/2577060/74585995-e5961480-5008-11ea-9322-8b7c517718c1.gif)

See [this video with subtitles](https://drive.google.com/file/d/1Q4TMfZ0asvxmsfWZJzbCbCbZQRM6lEpM/view?usp=sharing) till 2:40 for quicker understanding.

…

### Expected behavior
<!--
Description of what behavior was expected but did not occur
-->

Deleted content should not show as 'Already on Device'.

### User-facing consequences
<!--
Implications and real-world consequences for learners, coaches, admins, and other users of the application
-->

Confusion.

### Errors and logs
<!--
Relevant logs from:
 * the command line
 * ~/.kolibri/logs/kolibri.txt
 * the browser console

Please wrap errors in triple backticks for clean formatting like this:
```
01:10 info: something happened
01:12 error: something bad happened
```
-->

Shall share if any needed.

### Steps to reproduce
<!--
Precise steps that someone else can follow in order to see this behavior
-->

1. Delete existing content from a channel. Here I am using token `zapok-dofil`.
2. Try to re-import the content.

### Context
<!--
Tell us about your environment, including:
 * Kolibri version
 * Operating system
 * Browser
-->

Kolibri version: 0.13.1rc1, 0.13.0
OS: Linux / Windows
Content channel: multiple tried.


Discussion:
Hi @intelliant01, after you delete content, does the deleted content disappear from the Export/Delete page (where you select items that should be installed on the server)? Or are they there still there to export and delete as if they were "already on the device" (and thus consistent from the "Import more..." Page)?

@rtibbles Made a PR #6533 that fixed a similar resource-availability-mismatch after _upgrading_ a channel (and that channel had new items that weren't showing up in the "Import more..." page after the upgrade). The issue with the just-deleted resources being unavailable to re-download deals with a similar area of the code, namely something called `channel_stats`, so it might be related.
Yeah, we should probably clear the cache on deletion as well.

Resolving patch:
diff --git a/kolibri/core/content/management/commands/deletecontent.py b/kolibri/core/content/management/commands/deletecontent.py
--- a/kolibri/core/content/management/commands/deletecontent.py
+++ b/kolibri/core/content/management/commands/deletecontent.py
@@ -8,6 +8,7 @@
 from kolibri.core.content.models import LocalFile
 from kolibri.core.content.utils.annotation import propagate_forced_localfile_removal
 from kolibri.core.content.utils.annotation import set_content_invisible
+from kolibri.core.content.utils.importability_annotation import clear_channel_stats
 from kolibri.core.content.utils.import_export_content import get_files_to_transfer
 from kolibri.core.content.utils.paths import get_content_database_file_path
 from kolibri.core.tasks.management.commands.base import AsyncCommand
@@ -43,6 +44,9 @@ def delete_metadata(channel, node_ids, exclude_node_ids, force_delete):
         with db_task_write_lock:
             channel.delete_content_tree_and_files()
 
+    # Clear any previously set channel availability stats for this channel
+    clear_channel_stats(channel.id)
+
     return delete_all_metadata
 
 


Test patch:
diff --git a/kolibri/core/content/test/test_delete_content.py b/kolibri/core/content/test/test_delete_content.py
--- a/kolibri/core/content/test/test_delete_content.py
+++ b/kolibri/core/content/test/test_delete_content.py
@@ -114,111 +114,78 @@ class DeleteContentTestCase(TransactionTestCase):
     """
 
     fixtures = ["content_test.json"]
-    the_channel_id = "6199dde695db4ee4ab392222d5af1e5c"
 
-    @patch("kolibri.core.content.models.paths.get_content_storage_file_path")
-    @patch("kolibri.core.content.models.os.remove")
-    def test_include_all_nodes_all_deleted(self, os_remove_mock, content_file_path):
-        ContentNode.objects.all().update(available=True)
-        include_ids = list(
+    def _get_node_ids(self):
+        return list(
             ContentNode.objects.exclude(kind=content_kinds.TOPIC).values_list(
                 "id", flat=True
             )
         )
-        call_command("deletecontent", test_channel_id, node_ids=include_ids)
-        self.assertEqual(ContentNode.objects.all().count(), 0)
 
-    @patch("kolibri.core.content.models.paths.get_content_storage_file_path")
-    @patch("kolibri.core.content.models.os.remove")
-    def test_include_all_nodes_other_channel_node_still_available(
-        self, os_remove_mock, content_file_path
-    ):
-        include_ids = list(
-            ContentNode.objects.exclude(kind=content_kinds.TOPIC).values_list(
-                "id", flat=True
-            )
-        )
-        test = ContentNode.objects.filter(kind=content_kinds.VIDEO).first()
-        original_id = test.id
-        test.id = uuid.uuid4().hex
-        test.channel_id = uuid.uuid4().hex
-        test.available = True
-        test.parent = None
-        test.save()
-        original = ContentNode.objects.get(id=original_id)
+    def _setup_test_node(self, node, available=True):
+        node.id = uuid.uuid4().hex
+        node.channel_id = uuid.uuid4().hex
+        node.available = available
+        node.parent = None
+        node.save()
+        return node
+
+    def _setup_test_files(self, original_node_id, test_node):
+        original = ContentNode.objects.get(id=original_node_id)
         for file in original.files.all():
             file.id = uuid.uuid4().hex
-            file.contentnode = test
+            file.contentnode = test_node
             file.supplementary = False
             file.save()
-        call_command("deletecontent", test_channel_id, node_ids=include_ids)
+        return original
+
+    def _call_delete_command(self, **kwargs):
+        call_command("deletecontent", test_channel_id, **kwargs)
+
+    def test_include_all_nodes_all_deleted(self):
+        ContentNode.objects.all().update(available=True)
+        call_command("deletecontent", test_channel_id, node_ids=self._get_node_ids())
+        self.assertEqual(ContentNode.objects.all().count(), 0)
+
+    def test_include_all_nodes_other_channel_node_still_available(self):
+        test = ContentNode.objects.filter(kind=content_kinds.VIDEO).first()
+        original_id = test.id
+        test = self._setup_test_node(test)
+        self._setup_test_files(original_id, test)
+        self._call_delete_command(node_ids=self._get_node_ids())
         test.refresh_from_db()
         self.assertTrue(test.available)
 
-    @patch("kolibri.core.content.models.paths.get_content_storage_file_path")
-    @patch("kolibri.core.content.models.os.remove")
-    def test_include_all_nodes_force_delete_other_channel_node_not_available(
-        self, os_remove_mock, content_file_path
-    ):
-        include_ids = list(
-            ContentNode.objects.exclude(kind=content_kinds.TOPIC).values_list(
-                "id", flat=True
-            )
-        )
+    def test_include_all_nodes_force_delete_other_channel_node_not_available(self):
         test = ContentNode.objects.filter(kind=content_kinds.VIDEO).first()
         original_id = test.id
-        test.id = uuid.uuid4().hex
-        test.channel_id = uuid.uuid4().hex
-        test.available = True
-        test.parent = None
-        test.save()
-        original = ContentNode.objects.get(id=original_id)
-        for file in original.files.all():
-            file.id = uuid.uuid4().hex
-            file.contentnode = test
-            file.supplementary = False
-            file.save()
-        call_command(
-            "deletecontent", test_channel_id, node_ids=include_ids, force_delete=True
-        )
+        test = self._setup_test_node(test)
+        self._setup_test_files(original_id, test)
+        self._call_delete_command(node_ids=self._get_node_ids(), force_delete=True)
         test.refresh_from_db()
         self.assertFalse(test.available)
 
-    @patch("kolibri.core.content.models.paths.get_content_storage_file_path")
-    @patch("kolibri.core.content.models.os.remove")
     def test_exclude_all_nodes_force_delete_other_channel_node_not_available_no_delete(
-        self, os_remove_mock, content_file_path
+        self,
     ):
-        exclude_ids = list(
-            ContentNode.objects.exclude(kind=content_kinds.TOPIC).values_list(
-                "id", flat=True
-            )
-        )
         test = ContentNode.objects.filter(kind=content_kinds.VIDEO).first()
         original_id = test.id
-        test.id = uuid.uuid4().hex
-        test.channel_id = uuid.uuid4().hex
-        test.available = False
-        test.parent = None
-        test.save()
-        original = ContentNode.objects.get(id=original_id)
-        for file in original.files.all():
-            file.id = uuid.uuid4().hex
-            file.contentnode = test
-            file.supplementary = False
-            file.save()
-        call_command(
-            "deletecontent",
-            test_channel_id,
-            exclude_node_ids=exclude_ids,
-            force_delete=True,
+        test = self._setup_test_node(test, available=False)
+        original = self._setup_test_files(original_id, test)
+        self._call_delete_command(
+            exclude_node_ids=self._get_node_ids(), force_delete=True
         )
         try:
-            original = ContentNode.objects.get(id=original_id)
             self.assertTrue(original.available)
         except ContentNode.DoesNotExist:
             self.fail("Content node has been deleted")
 
+    @patch("kolibri.core.content.management.commands.deletecontent.clear_channel_stats")
+    def test_deleting_channel_clears_stats_cache(self, channel_stats_clear_mock):
+        self.assertFalse(channel_stats_clear_mock.called)
+        self._call_delete_command(node_ids=self._get_node_ids())
+        self.assertTrue(channel_stats_clear_mock.called)
+
     def tearDown(self):
         call_command("flush", interactive=False)
         super(DeleteContentTestCase, self).tearDown()