Repository: learningequality/kolibri

Issue:
partially-completed transfers do not show how much succeeded

### Observed behavior

After canceling a transfer, metadata does not give any indication that some of the content was successfully imported and is now on the device:

![completion-after-cancel](https://user-images.githubusercontent.com/2367265/69189446-6158f180-0aec-11ea-9fdb-58f1c783c36e.gif)


### Expected behavior

metadata should indicate how much content was successfully imported before the transfer stopped

### User-facing consequences

confusion about what actually happens after a transfer gets canceled. Is everything deleted, or did some resources get saved?



### Context

0.13.0 alpha 0


Discussion:
Can handle this in the same way as #6098 by updating the task metadata after cancellation.
do we in fact need new strings for this?
Could do it without by just changing the numbers - do we use different strings for when the job is finished versus in progress?
When a task is completed/canceled, we don't show a progress bar. We can both keep the progress bar when the task is canceled (so we have a stopped progress bar at e.g. 75%) and cobble together some copy from the existing strings to display the completed # resources/filesize
chatted with @jonboiser and @khangmach. @rtibbles will it be possible to use these strings?

For ongoing transfers and failures/cancelations:

> 50 of 100 resources (10MB of 20MB) imported

```
      importPartialRatio:
        '{currentResources} of {totalResources} {totalResources, plural, one {resource} other {resources}} ({currentSize} of {totalSize}) imported',
      exportPartialRatio:
        '{currentResources} of {totalResources} {totalResources, plural, one {resource} other {resources}} ({currentSize} of {totalSize}) exported',
      deletePartialRatio:
        '{currentResources} of {totalResources} {totalResources, plural, one {resource} other {resources}} ({currentSize} of {totalSize}) deleted',
```


For successes:

> 100 resources (20MB) successfully imported



```
      importSuccess:
        '{totalResources} {totalResources, plural, one {resource} other {resources}} ({totalSize}) successfully imported',
      exportSuccess:
        '{totalResources} {totalResources, plural, one {resource} other {resources}} ({totalSize}) successfully exported',
      deleteSuccess:
        '{totalResources} {totalResources, plural, one {resource} other {resources}} ({totalSize}) successfully deleted',
```

Seems fine to me.

Resolving patch:
diff --git a/kolibri/core/content/management/commands/exportcontent.py b/kolibri/core/content/management/commands/exportcontent.py
--- a/kolibri/core/content/management/commands/exportcontent.py
+++ b/kolibri/core/content/management/commands/exportcontent.py
@@ -1,11 +1,15 @@
 import logging
 import os
 
-from ...utils import import_export_content
+from le_utils.constants import content_kinds
+
 from ...utils import paths
 from ...utils import transfer
 from kolibri.core.content.errors import InvalidStorageFilenameError
+from kolibri.core.content.utils.import_export_content import calculate_files_to_transfer
+from kolibri.core.content.utils.import_export_content import get_nodes_to_transfer
 from kolibri.core.tasks.management.commands.base import AsyncCommand
+from kolibri.core.tasks.utils import get_current_job
 
 logger = logging.getLogger(__name__)
 
@@ -51,6 +55,13 @@ def add_arguments(self, parser):
         parser.add_argument("channel_id", type=str)
         parser.add_argument("destination", type=str)
 
+    def update_job_metadata(self, total_bytes_to_transfer, total_resource_count):
+        job = get_current_job()
+        if job:
+            job.extra_metadata["file_size"] = total_bytes_to_transfer
+            job.extra_metadata["total_resources"] = total_resource_count
+            job.save_meta()
+
     def handle_async(self, *args, **options):
         channel_id = options["channel_id"]
         data_dir = os.path.realpath(options["destination"])
@@ -60,10 +71,23 @@ def handle_async(self, *args, **options):
             "Exporting content for channel id {} to {}".format(channel_id, data_dir)
         )
 
-        files, total_bytes_to_transfer = import_export_content.get_files_to_transfer(
+        nodes_for_transfer = get_nodes_to_transfer(
             channel_id, node_ids, exclude_node_ids, True
         )
 
+        total_resource_count = (
+            nodes_for_transfer.exclude(kind=content_kinds.TOPIC)
+            .values("content_id")
+            .distinct()
+            .count()
+        )
+
+        (files, total_bytes_to_transfer,) = calculate_files_to_transfer(
+            nodes_for_transfer, True
+        )
+
+        self.update_job_metadata(total_bytes_to_transfer, total_resource_count)
+
         exported_files = []
 
         with self.start_progress(
diff --git a/kolibri/core/content/management/commands/importcontent.py b/kolibri/core/content/management/commands/importcontent.py
--- a/kolibri/core/content/management/commands/importcontent.py
+++ b/kolibri/core/content/management/commands/importcontent.py
@@ -4,13 +4,20 @@
 
 import requests
 from django.core.management.base import CommandError
+from le_utils.constants import content_kinds
 
 from ...utils import annotation
-from ...utils import import_export_content
 from ...utils import paths
 from ...utils import transfer
 from kolibri.core.content.errors import InvalidStorageFilenameError
+from kolibri.core.content.models import ContentNode
+from kolibri.core.content.utils.file_availability import LocationError
+from kolibri.core.content.utils.import_export_content import calculate_files_to_transfer
+from kolibri.core.content.utils.import_export_content import compare_checksums
+from kolibri.core.content.utils.import_export_content import get_nodes_to_transfer
+from kolibri.core.content.utils.import_export_content import retry_import
 from kolibri.core.tasks.management.commands.base import AsyncCommand
+from kolibri.core.tasks.utils import get_current_job
 from kolibri.utils import conf
 
 # constants to specify the transfer method to be used
@@ -168,22 +175,57 @@ def _transfer(  # noqa: max-complexity=16
         renderable_only=True,
     ):
 
-        (
-            files_to_download,
-            total_bytes_to_transfer,
-        ) = import_export_content.get_files_to_transfer(
-            channel_id,
-            node_ids,
-            exclude_node_ids,
-            False,
-            renderable_only=renderable_only,
-            drive_id=drive_id,
-            peer_id=peer_id,
+        try:
+            nodes_for_transfer = get_nodes_to_transfer(
+                channel_id,
+                node_ids,
+                exclude_node_ids,
+                False,
+                renderable_only=renderable_only,
+                drive_id=drive_id,
+                peer_id=peer_id,
+            )
+        except LocationError:
+            if drive_id:
+                raise CommandError(
+                    "The external drive with given drive id {} does not exist.".format(
+                        drive_id
+                    )
+                )
+            if peer_id:
+                raise CommandError(
+                    "The network location with the id {} does not exist".format(peer_id)
+                )
+        total_resource_count = (
+            nodes_for_transfer.exclude(kind=content_kinds.TOPIC)
+            .values("content_id")
+            .distinct()
+            .count()
         )
 
+        (files_to_download, total_bytes_to_transfer,) = calculate_files_to_transfer(
+            nodes_for_transfer, False
+        )
+
+        job = get_current_job()
+
+        if job:
+            job.extra_metadata["file_size"] = total_bytes_to_transfer
+            job.extra_metadata["total_resources"] = total_resource_count
+            job.save_meta()
+
         number_of_skipped_files = 0
+        transferred_file_size = 0
         file_checksums_to_annotate = []
 
+        resources_before_transfer = (
+            ContentNode.objects.filter(channel_id=channel_id, available=True)
+            .exclude(kind=content_kinds.TOPIC)
+            .values("content_id")
+            .distinct()
+            .count()
+        )
+
         with self.start_progress(
             total=total_bytes_to_transfer
         ) as overall_progress_update:
@@ -209,6 +251,7 @@ def _transfer(  # noqa: max-complexity=16
                 if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
                     overall_progress_update(f.file_size)
                     file_checksums_to_annotate.append(f.id)
+                    transferred_file_size += f.file_size
                     continue
 
                 # determine where we're downloading/copying from, and create appropriate transfer object
@@ -240,6 +283,7 @@ def _transfer(  # noqa: max-complexity=16
 
                         if status == FILE_TRANSFERRED:
                             file_checksums_to_annotate.append(f.id)
+                            transferred_file_size += f.file_size
                         elif status == FILE_SKIPPED:
                             number_of_skipped_files += 1
                 except Exception as e:
@@ -253,6 +297,21 @@ def _transfer(  # noqa: max-complexity=16
                 exclude_node_ids=exclude_node_ids,
             )
 
+            resources_after_transfer = (
+                ContentNode.objects.filter(channel_id=channel_id, available=True)
+                .exclude(kind=content_kinds.TOPIC)
+                .values("content_id")
+                .distinct()
+                .count()
+            )
+
+            if job:
+                job.extra_metadata["transferred_file_size"] = transferred_file_size
+                job.extra_metadata["transferred_resources"] = (
+                    resources_after_transfer - resources_before_transfer
+                )
+                job.save_meta()
+
             if number_of_skipped_files > 0:
                 logger.warning(
                     "{} files are skipped, because errors occurred during the import.".format(
@@ -306,9 +365,7 @@ def _start_file_transfer(self, f, filetransfer, overall_progress_update):
                 # id indicated in the database, it means that the destination file
                 # is corrupted, either from origin or during import. Skip importing
                 # this file.
-                checksum_correctness = import_export_content.compare_checksums(
-                    filetransfer.dest, f.id
-                )
+                checksum_correctness = compare_checksums(filetransfer.dest, f.id)
                 if not checksum_correctness:
                     e = "File {} is corrupted.".format(filetransfer.source)
                     logger.error(
@@ -321,7 +378,7 @@ def _start_file_transfer(self, f, filetransfer, overall_progress_update):
 
         except Exception as e:
             logger.error("An error occurred during content import: {}".format(e))
-            retry = import_export_content.retry_import(e, skip_404=True)
+            retry = retry_import(e, skip_404=True)
 
             if retry:
                 # Restore the previous progress so that the progress bar will
diff --git a/kolibri/core/tasks/api.py b/kolibri/core/tasks/api.py
--- a/kolibri/core/tasks/api.py
+++ b/kolibri/core/tasks/api.py
@@ -50,18 +50,10 @@ def validate_content_task(request, task_description, require_channel=False):
     try:
         channel = ChannelMetadata.objects.get(id=channel_id)
         channel_name = channel.name
-        file_size = channel.published_size
-        total_resources = channel.total_resource_count
     except ChannelMetadata.DoesNotExist:
         if require_channel:
             raise serializers.ValidationError("This channel does not exist")
         channel_name = ""
-        file_size = None
-        total_resources = None
-
-    file_size = task_description.get("file_size", file_size)
-
-    total_resources = task_description.get("total_resources", total_resources)
 
     node_ids = task_description.get("node_ids", None)
     exclude_node_ids = task_description.get("exclude_node_ids", None)
@@ -75,8 +67,6 @@ def validate_content_task(request, task_description, require_channel=False):
     return {
         "channel_id": channel_id,
         "channel_name": channel_name,
-        "file_size": file_size,
-        "total_resources": total_resources,
         "exclude_node_ids": exclude_node_ids,
         "node_ids": node_ids,
         "started_by": request.user.pk,


Test patch:
diff --git a/kolibri/core/content/test/test_import_export.py b/kolibri/core/content/test/test_import_export.py
--- a/kolibri/core/content/test/test_import_export.py
+++ b/kolibri/core/content/test/test_import_export.py
@@ -179,7 +179,7 @@ def test_remote_import_readtimeout(
 
 
 @patch(
-    "kolibri.core.content.management.commands.importcontent.import_export_content.get_files_to_transfer",
+    "kolibri.core.content.management.commands.importcontent.calculate_files_to_transfer",
 )
 @patch("kolibri.core.content.management.commands.importcontent.annotation")
 @override_option("Paths", "CONTENT_DIR", tempfile.mkdtemp())
@@ -267,7 +267,7 @@ def test_remote_cancel_during_transfer(
         annotation_mock.recurse_annotation_up_tree.assert_not_called()
 
     @patch(
-        "kolibri.core.content.management.commands.importcontent.import_export_content.compare_checksums",
+        "kolibri.core.content.management.commands.importcontent.compare_checksums",
         return_value=True,
     )
     @patch(