Repository: django-import-export/django-import-export

Issue:
Better result reporting
At the moment `import_data()` is a monolithic function with no easy customization points. In order to provide better feedback to users, I would like to extend it to report the total number of rows in each result type (NEW, UPDATE, SKIP, DELETE), including the total number of skipped rows even when `report_skipped == False`

There is no easy way to do this without copying the whole `import_data()` function into the subclass because there is no way to override Result class used, or the handling of individual rows.

I could achieve my requirements by making a slight change to import_data() to count the total number of each result type.

If it is desirable, I could also:
- replace the `result = Result()` with `result = self.get_result_class()()` to allow people to customize the Result if necessary
- move the inner part of the `for row in dataset.dict:` to a new `import_row()` method that would allow people to customize what happens before and after the row import is attempted
- add a `report_error()` and a `report_row_result()` to hold the code that appends the results to the result, once again providing an additional extension point for people that need additional reporting



Discussion:

Resolving patch:
diff --git a/import_export/admin.py b/import_export/admin.py
--- a/import_export/admin.py
+++ b/import_export/admin.py
@@ -142,8 +142,6 @@ def process_import(self, request, *args, **kwargs):
         '''
         opts = self.model._meta
         resource = self.get_import_resource_class()()
-        total_imports = 0
-        total_updates = 0
 
         confirm_form = ConfirmImportForm(request.POST)
         if confirm_form.is_valid():
@@ -180,14 +178,14 @@ def process_import(self, request, *args, **kwargs):
                             action_flag=logentry_map[row.import_type],
                             change_message="%s through import_export" % row.import_type,
                         )
-                    if row.import_type == row.IMPORT_TYPE_NEW:
-                        total_imports += 1
-                    elif row.import_type == row.IMPORT_TYPE_UPDATE:
-                        total_updates += 1
 
             success_message = u'Import finished, with {} new {}{} and ' \
-                              u'{} updated {}{}.'.format(total_imports, opts.model_name, pluralize(total_imports),
-                                                         total_updates, opts.model_name, pluralize(total_updates))
+                              u'{} updated {}{}.'.format(result.totals[RowResult.IMPORT_TYPE_NEW],
+                                                         opts.model_name,
+                                                         pluralize(result.totals[RowResult.IMPORT_TYPE_NEW]),
+                                                         result.totals[RowResult.IMPORT_TYPE_UPDATE],
+                                                         opts.model_name,
+                                                         pluralize(result.totals[RowResult.IMPORT_TYPE_UPDATE]))
 
             messages.success(request, success_message)
             tmp_storage.remove()
diff --git a/import_export/resources.py b/import_export/resources.py
--- a/import_export/resources.py
+++ b/import_export/resources.py
@@ -47,7 +47,7 @@
     from django.utils.datastructures import SortedDict as OrderedDict
 
 # Set default logging handler to avoid "No handler found" warnings.
-import logging # isort:skip
+import logging  # isort:skip
 try:  # Python 2.7+
     from logging import NullHandler
 except ImportError:
@@ -171,6 +171,27 @@ class Resource(six.with_metaclass(DeclarativeMetaclass)):
     representations and handle importing and exporting data.
     """
 
+    @classmethod
+    def get_result_class(self):
+        """
+        Returns the class used to store the result of an import.
+        """
+        return Result
+
+    @classmethod
+    def get_row_result_class(self):
+        """
+        Returns the class used to store the result of a row import.
+        """
+        return RowResult
+
+    @classmethod
+    def get_error_result_class(self):
+        """
+        Returns the class used to store an error resulting from an import.
+        """
+        return Error
+
     def get_use_transactions(self):
         if self._meta.use_transactions is None:
             return USE_TRANSACTIONS
@@ -357,6 +378,69 @@ def before_import(self, dataset, dry_run, **kwargs):
         """
         pass
 
+    def after_import(self, dataset, result, dry_run, **kwargs):
+        """
+        Override to add additional logic. Does nothing by default.
+
+        This method receives the ``dataset`` that's just been imported, the
+        ``result`` of the import and the ``dry_run`` parameter which determines
+        whether changes will be saved to the database, and any additional
+        keyword arguments passed to ``import_data`` in a ``kwargs`` dict. This
+        method runs after the main import finishes but before the changes are
+        committed or rolled back.
+        """
+        pass
+
+    def import_row(self, row, instance_loader, dry_run=False, **kwargs):
+        """
+        Imports data from ``tablib.Dataset``. Refer to :doc:`import_workflow`
+        for a more complete description of the whole import process.
+
+        :param row: A ``dict`` of the row to import
+
+        :param instance_loader: The instance loader to be used to load the row
+
+        :param dry_run: If ``dry_run`` is set, or error occurs, transaction
+            will be rolled back.
+        """
+        try:
+            row_result = self.get_row_result_class()()
+            instance, new = self.get_or_init_instance(instance_loader, row)
+            if new:
+                row_result.import_type = RowResult.IMPORT_TYPE_NEW
+            else:
+                row_result.import_type = RowResult.IMPORT_TYPE_UPDATE
+            row_result.new_record = new
+            original = deepcopy(instance)
+            if self.for_delete(row, instance):
+                if new:
+                    row_result.import_type = RowResult.IMPORT_TYPE_SKIP
+                    row_result.diff = self.get_diff(None, None, dry_run)
+                else:
+                    row_result.import_type = RowResult.IMPORT_TYPE_DELETE
+                    self.delete_instance(instance, dry_run)
+                    row_result.diff = self.get_diff(original, None, dry_run)
+            else:
+                self.import_obj(instance, row, dry_run)
+                if self.skip_row(instance, original):
+                    row_result.import_type = RowResult.IMPORT_TYPE_SKIP
+                else:
+                    with transaction.atomic():
+                        self.save_instance(instance, dry_run)
+                    self.save_m2m(instance, row, dry_run)
+                    # Add object info to RowResult for LogEntry
+                    row_result.object_repr = force_text(instance)
+                    row_result.object_id = instance.pk
+                row_result.diff = self.get_diff(original, instance, dry_run)
+        except Exception as e:
+            # There is no point logging a transaction error for each row
+            # when only the original error is likely to be relevant
+            if not isinstance(e, TransactionManagementError):
+                logging.exception(e)
+            tb_info = traceback.format_exc()
+            row_result.errors.append(self.get_error_result_class()(e, tb_info, row))
+        return row_result
+
     @atomic()
     def import_data(self, dataset, dry_run=False, raise_errors=False,
                     use_transactions=None, **kwargs):
@@ -375,8 +459,14 @@ def import_data(self, dataset, dry_run=False, raise_errors=False,
         :param dry_run: If ``dry_run`` is set, or error occurs, transaction
             will be rolled back.
         """
-        result = Result()
+        result = self.get_result_class()()
         result.diff_headers = self.get_diff_headers()
+        result.totals = OrderedDict([(RowResult.IMPORT_TYPE_NEW, 0),
+                                     (RowResult.IMPORT_TYPE_UPDATE, 0),
+                                     (RowResult.IMPORT_TYPE_DELETE, 0),
+                                     (RowResult.IMPORT_TYPE_SKIP, 0),
+                                     (RowResult.IMPORT_TYPE_ERROR, 0),
+                                     ('total', len(dataset))])
 
         if use_transactions is None:
             use_transactions = self.get_use_transactions()
@@ -394,7 +484,7 @@ def import_data(self, dataset, dry_run=False, raise_errors=False,
         except Exception as e:
             logging.exception(e)
             tb_info = traceback.format_exc()
-            result.base_errors.append(Error(e, tb_info))
+            result.base_errors.append(self.get_error_result_class()(e, tb_info))
             if raise_errors:
                 if use_transactions:
                     savepoint_rollback(sp1)
@@ -402,63 +492,33 @@ def import_data(self, dataset, dry_run=False, raise_errors=False,
 
         instance_loader = self._meta.instance_loader_class(self, dataset)
 
+        # Update the total in case the dataset was altered by before_import()
+        result.totals['total'] = len(dataset)
+
         for row in dataset.dict:
-            try:
-                row_result = RowResult()
-                instance, new = self.get_or_init_instance(instance_loader, row)
-                if new:
-                    row_result.import_type = RowResult.IMPORT_TYPE_NEW
-                else:
-                    row_result.import_type = RowResult.IMPORT_TYPE_UPDATE
-                row_result.new_record = new
-                original = deepcopy(instance)
-                if self.for_delete(row, instance):
-                    if new:
-                        row_result.import_type = RowResult.IMPORT_TYPE_SKIP
-                        row_result.diff = self.get_diff(None, None,
-                                                        real_dry_run)
-                    else:
-                        row_result.import_type = RowResult.IMPORT_TYPE_DELETE
-                        self.delete_instance(instance, real_dry_run)
-                        row_result.diff = self.get_diff(original, None,
-                                                        real_dry_run)
-                else:
-                    self.import_obj(instance, row, real_dry_run)
-                    if self.skip_row(instance, original):
-                        row_result.import_type = RowResult.IMPORT_TYPE_SKIP
-                    else:
-                        with transaction.atomic():
-                            self.save_instance(instance, real_dry_run)
-                        self.save_m2m(instance, row, real_dry_run)
-                        # Add object info to RowResult for LogEntry
-                        row_result.object_repr = force_text(instance)
-                        row_result.object_id = instance.pk
-                    row_result.diff = self.get_diff(original, instance,
-                                                    real_dry_run)
-            except Exception as e:
-                # There is no point logging a transaction error for each row
-                # when only the original error is likely to be relevant
-                if not isinstance(e, TransactionManagementError):
-                    logging.exception(e)
-                tb_info = traceback.format_exc()
-                row_result.errors.append(Error(e, tb_info, row))
+            row_result = self.import_row(row, instance_loader, real_dry_run, **kwargs)
+            if row_result.errors:
+                result.totals[row_result.IMPORT_TYPE_ERROR] += 1
                 if raise_errors:
                     if use_transactions:
                         savepoint_rollback(sp1)
                     six.reraise(*sys.exc_info())
+            else:
+                result.totals[row_result.import_type] += 1
             if (row_result.import_type != RowResult.IMPORT_TYPE_SKIP or
                     self._meta.report_skipped):
                 result.rows.append(row_result)
 
-        # Reset the SQL sequences when new objects are imported
-        # Adapted from django's loaddata
-        if not dry_run and any(r.import_type == RowResult.IMPORT_TYPE_NEW for r in result.rows):
-            connection = connections[DEFAULT_DB_ALIAS]
-            sequence_sql = connection.ops.sequence_reset_sql(no_style(), [self.Meta.model])
-            if sequence_sql:
-                with connection.cursor() as cursor:
-                    for line in sequence_sql:
-                        cursor.execute(line)
+        try:
+            self.after_import(dataset, result, real_dry_run, **kwargs)
+        except Exception as e:
+            logging.exception(e)
+            tb_info = traceback.format_exc()
+            result.base_errors.append(self.get_error_result_class()(e, tb_info))
+            if raise_errors:
+                if use_transactions:
+                    savepoint_rollback(sp1)
+                raise
 
         if use_transactions:
             if dry_run or result.has_errors():
@@ -669,6 +729,22 @@ def init_instance(self, row=None):
         """
         return self._meta.model()
 
+    def after_import(self, dataset, result, dry_run, **kwargs):
+        """
+        Reset the SQL sequences after new objects are imported
+        """
+        # Adapted from django's loaddata
+        if not dry_run and any(r.import_type == RowResult.IMPORT_TYPE_NEW for r in result.rows):
+            connection = connections[DEFAULT_DB_ALIAS]
+            sequence_sql = connection.ops.sequence_reset_sql(no_style(), [self._meta.model])
+            if sequence_sql:
+                cursor = connection.cursor()
+                try:
+                    for line in sequence_sql:
+                        cursor.execute(line)
+                finally:
+                    cursor.close()
+
 
 def modelresource_factory(model, resource_class=ModelResource):
     """
@@ -681,7 +757,7 @@ def modelresource_factory(model, resource_class=ModelResource):
 
     class_attrs = {
         'Meta': Meta,
-        }
+    }
 
     metaclass = ModelDeclarativeMetaclass
     return metaclass(class_name, (resource_class,), class_attrs)
diff --git a/import_export/results.py b/import_export/results.py
--- a/import_export/results.py
+++ b/import_export/results.py
@@ -14,6 +14,7 @@ class RowResult(object):
     IMPORT_TYPE_NEW = 'new'
     IMPORT_TYPE_DELETE = 'delete'
     IMPORT_TYPE_SKIP = 'skip'
+    IMPORT_TYPE_ERROR = 'error'
 
     def __init__(self):
         self.errors = []


Test patch: