Repository: django-import-export/django-import-export

Issue:
Double insert of rows while importing
When importing a CSV via the admin interface, the rows in the sheet are being imported twice. Once when loading preview and once when confirming import. Importing with the `id` column left empty. I've used the library before and I've never faced this issue before. Super confused!

Django==2.1.3
django-import-export==2.1.0

```
class ReviewsResource(resources.ModelResource):
    class Meta:
        model = Review
        fields = ('id',  the remaining fields in the model...)
```

```
class Review(models.Model):
    .. few field definitions

    class Meta:
        managed = False
        db_table = 'review'

    def __str__(self):
        return str(self.id)
```


Discussion:
Sorry that you are facing this issue. Im not sure what could be the problem, but could it have something to do with the `managed = False`? 
hi, @andrewgy8 I'm a collaborator on the same project and I just tried your solution. But I think it doesn't have to do anything with the `manged = False` statement. Is there a way to manually override the preview functionality?
I just tried an import from xlsx with the [test application](https://github.com/django-import-export/django-import-export/tree/master/tests), and didn't see a duplicate row following an import.  You could try importing using the test application, and then compare the differences with your app.

If you can post any reproducible steps here, we can look into it.
AFAIK you can not override the preview functionality. Have you tried with a different format? Can you try with xlsx? 
@andrewgy8 I have tried with xlsx too. But facing the same issue.
@andrewgy8 Found the issue.

The way I understand it, import preview works by importing all the data in a transaction and then rolling it back.
When you have multiple dbs connected and when import is performed for a model that's on the non-default database, the savepoint & rollback are still being executed against the default database. I think there should be a resource option to override this.

Interesting, I dont think that is something we had considered. 

Perhaps you can leverage the use of [`using()` in your queryset](https://docs.djangoproject.com/en/3.0/topics/db/multi-db/#manually-selecting-a-database-for-a-queryset). Have you tried that? 
I wrote a subclass for ModelResouce to fix this and its working fine.
```
class DatabaseAwareAtomicIfUsingTransaction(atomic_if_using_transaction):
    def __init__(self, using_transactions, using):
        self.using_transactions = using_transactions
        if using_transactions:
            self.context_manager = transaction.atomic(using)


class DatabaseAwareResource(resources.ModelResource):
    """
    DatabaseAwareResource class is a subclass of resources.ModelResource of the django_import_export library
    that fixes a double import bug in it. The bug is caused by the resource class not being aware of multiple databases
    When rows are imported for preview, a transaction is created and rolled back. This is to identify all the potential
    errors in the data being imported. However, the transaction and its associated operations are always run
    against the default database instead of the specific db to which data is being imported

    This class reads the :using resource meta parameter to determine the database against which operations should be run

    TODO: Stop using this hack when this issue is fixes: https://github.com/django-import-export/django-import-export/issues/1137
    """

    def import_data(self, dataset, dry_run=False, raise_errors=False,
                    use_transactions=None, collect_failed_rows=False, **kwargs):

        if use_transactions is None:
            use_transactions = self.get_use_transactions()

        db_alias = self._meta.using
        if db_alias is None:
            db_alias = DEFAULT_DB_ALIAS

        connection = connections[db_alias]
        supports_transactions = getattr(connection.features, "supports_transactions", False)

        if use_transactions and not supports_transactions:
            raise ImproperlyConfigured

        using_transactions = (use_transactions or dry_run) and supports_transactions

        with DatabaseAwareAtomicIfUsingTransaction(using_transactions, db_alias):
            return self.import_data_inner(dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)

    def import_data_inner(self, dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs):
        result = self.get_result_class()()
        result.diff_headers = self.get_diff_headers()
        result.total_rows = len(dataset)

        db_alias = self._meta.using
        if db_alias is None:
            db_alias = DEFAULT_DB_ALIAS

        if using_transactions:
            # when transactions are used we want to create/update/delete object
            # as transaction will be rolled back if dry_run is set
            sp1 = savepoint(db_alias)

        try:
            with DatabaseAwareAtomicIfUsingTransaction(using_transactions, db_alias):
                self.before_import(dataset, using_transactions, dry_run, **kwargs)
        except Exception as e:
            logger.debug(e, exc_info=e)
            tb_info = traceback.format_exc()
            result.append_base_error(self.get_error_result_class()(e, tb_info))
            if raise_errors:
                raise

        instance_loader = self._meta.instance_loader_class(self, dataset)
        # Update the total in case the dataset was altered by before_import()
        result.total_rows = len(dataset)

        if collect_failed_rows:
            result.add_dataset_headers(dataset.headers)

        for i, row in enumerate(dataset.dict, 1):
            with DatabaseAwareAtomicIfUsingTransaction(using_transactions, db_alias):
                row_result = self.import_row(
                    row,
                    instance_loader,
                    using_transactions=using_transactions,
                    dry_run=dry_run,
                    **kwargs
                )
            result.increment_row_result_total(row_result)

            if row_result.errors:
                if collect_failed_rows:
                    result.append_failed_row(row, row_result.errors[0])
                if raise_errors:
                    raise row_result.errors[-1].error
            elif row_result.validation_error:
                result.append_invalid_row(i, row, row_result.validation_error)
                if collect_failed_rows:
                    result.append_failed_row(row, row_result.validation_error)
                if raise_errors:
                    raise row_result.validation_error
            if (row_result.import_type != RowResult.IMPORT_TYPE_SKIP or
                    self._meta.report_skipped):
                result.append_row_result(row_result)

        try:
            with DatabaseAwareAtomicIfUsingTransaction(using_transactions, db_alias):
                self.after_import(dataset, result, using_transactions, dry_run, **kwargs)
        except Exception as e:
            logger.debug(e, exc_info=e)
            tb_info = traceback.format_exc()
            result.append_base_error(self.get_error_result_class()(e, tb_info))
            if raise_errors:
                raise

        if using_transactions:
            if dry_run or result.has_errors():
                savepoint_rollback(sp1, db_alias)
            else:
                savepoint_commit(sp1, db_alias)

        return result
```

Great job at figuring it out!
@andrewgy8  Shouldn't support for this added to the library itself?
ATM, I would say not unless this is a common use case that many people are running into. Otherwise, I think they can benefit from your snippet. 

Resolving patch:
diff --git a/import_export/resources.py b/import_export/resources.py
--- a/import_export/resources.py
+++ b/import_export/resources.py
@@ -11,12 +11,11 @@
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.core.management.color import no_style
 from django.core.paginator import Paginator
-from django.db import DEFAULT_DB_ALIAS, connections
+from django.db import connections, router
 from django.db.models.fields.related import ForeignObjectRel
 from django.db.models.query import QuerySet
 from django.db.transaction import (
     TransactionManagementError,
-    atomic,
     savepoint,
     savepoint_commit,
     savepoint_rollback,
@@ -159,6 +158,13 @@ class ResourceOptions:
     contain new instances.
     """
 
+    using_db = None
+    """
+    DB Connection name to use for db transactions. If not provided,
+    ``router.db_for_write(model)`` will be evaluated and if it's missing,
+    DEFAULT_DB_ALIAS constant ("default") is used.
+    """
+
 
 class DeclarativeMetaclass(type):
 
@@ -272,6 +278,12 @@ def get_diff_class(self):
         """
         return Diff
 
+    def get_db_connection_name(self):
+        if self._meta.using_db is None:
+            return router.db_for_write(self._meta.model)
+        else:
+            return self._meta.using_db
+
     def get_use_transactions(self):
         if self._meta.use_transactions is None:
             return getattr(settings, 'IMPORT_EXPORT_USE_TRANSACTIONS', True)
@@ -726,7 +738,8 @@ def import_data(self, dataset, dry_run=False, raise_errors=False,
         if use_transactions is None:
             use_transactions = self.get_use_transactions()
 
-        connection = connections[DEFAULT_DB_ALIAS]
+        db_connection = self.get_db_connection_name()
+        connection = connections[db_connection]
         supports_transactions = getattr(connection.features, "supports_transactions", False)
 
         if use_transactions and not supports_transactions:
@@ -737,21 +750,22 @@ def import_data(self, dataset, dry_run=False, raise_errors=False,
         if self._meta.batch_size is not None and (not isinstance(self._meta.batch_size, int) or self._meta.batch_size < 0):
             raise ValueError("Batch size must be a positive integer")
 
-        with atomic_if_using_transaction(using_transactions):
+        with atomic_if_using_transaction(using_transactions, using=db_connection):
             return self.import_data_inner(dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)
 
     def import_data_inner(self, dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs):
         result = self.get_result_class()()
         result.diff_headers = self.get_diff_headers()
         result.total_rows = len(dataset)
+        db_connection = self.get_db_connection_name()
 
         if using_transactions:
             # when transactions are used we want to create/update/delete object
             # as transaction will be rolled back if dry_run is set
-            sp1 = savepoint()
+            sp1 = savepoint(using=db_connection)
 
         try:
-            with atomic_if_using_transaction(using_transactions):
+            with atomic_if_using_transaction(using_transactions, using=db_connection):
                 self.before_import(dataset, using_transactions, dry_run, **kwargs)
         except Exception as e:
             logger.debug(e, exc_info=e)
@@ -769,7 +783,7 @@ def import_data_inner(self, dataset, dry_run, raise_errors, using_transactions,
             result.add_dataset_headers(dataset.headers)
 
         for i, row in enumerate(dataset.dict, 1):
-            with atomic_if_using_transaction(using_transactions):
+            with atomic_if_using_transaction(using_transactions, using=db_connection):
                 row_result = self.import_row(
                     row,
                     instance_loader,
@@ -798,13 +812,13 @@ def import_data_inner(self, dataset, dry_run, raise_errors, using_transactions,
 
         if self._meta.use_bulk:
             # bulk persist any instances which are still pending
-            with atomic_if_using_transaction(using_transactions):
+            with atomic_if_using_transaction(using_transactions, using=db_connection):
                 self.bulk_create(using_transactions, dry_run, raise_errors)
                 self.bulk_update(using_transactions, dry_run, raise_errors)
                 self.bulk_delete(using_transactions, dry_run, raise_errors)
 
         try:
-            with atomic_if_using_transaction(using_transactions):
+            with atomic_if_using_transaction(using_transactions, using=db_connection):
                 self.after_import(dataset, result, using_transactions, dry_run, **kwargs)
         except Exception as e:
             logger.debug(e, exc_info=e)
@@ -815,9 +829,9 @@ def import_data_inner(self, dataset, dry_run, raise_errors, using_transactions,
 
         if using_transactions:
             if dry_run or result.has_errors():
-                savepoint_rollback(sp1)
+                savepoint_rollback(sp1, using=db_connection)
             else:
-                savepoint_commit(sp1)
+                savepoint_commit(sp1, using=db_connection)
 
         return result
 
@@ -1102,7 +1116,8 @@ def after_import(self, dataset, result, using_transactions, dry_run, **kwargs):
         """
         # Adapted from django's loaddata
         if not dry_run and any(r.import_type == RowResult.IMPORT_TYPE_NEW for r in result.rows):
-            connection = connections[DEFAULT_DB_ALIAS]
+            db_connection = self.get_db_connection_name()
+            connection = connections[db_connection]
             sequence_sql = connection.ops.sequence_reset_sql(no_style(), [self._meta.model])
             if sequence_sql:
                 cursor = connection.cursor()
diff --git a/import_export/utils.py b/import_export/utils.py
--- a/import_export/utils.py
+++ b/import_export/utils.py
@@ -7,14 +7,14 @@ class atomic_if_using_transaction:
     Replaces code::
 
         if using_transactions:
-            with transaction.atomic():
+            with transaction.atomic(using=using):
                 return something()
         return something()
     """
-    def __init__(self, using_transactions):
+    def __init__(self, using_transactions, using):
         self.using_transactions = using_transactions
         if using_transactions:
-            self.context_manager = transaction.atomic()
+            self.context_manager = transaction.atomic(using=using)
 
     def __enter__(self):
         if self.using_transactions:


Test patch:
diff --git a/tests/core/tests/test_resources.py b/tests/core/tests/test_resources.py
--- a/tests/core/tests/test_resources.py
+++ b/tests/core/tests/test_resources.py
@@ -11,8 +11,9 @@
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
 from django.core.paginator import Paginator
-from django.db import DatabaseError, IntegrityError
+from django.db import IntegrityError
 from django.db.models import Count
+from django.db.utils import ConnectionDoesNotExist
 from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature
 from django.utils.encoding import force_str
 from django.utils.html import strip_tags
@@ -972,6 +973,22 @@ class Meta:
         self.assertEqual(WithFloatField.objects.all()[0].f, None)
         self.assertEqual(WithFloatField.objects.all()[1].f, None)
 
+    def test_get_db_connection_name(self):
+        class BookResource(resources.ModelResource):
+            class Meta:
+                using_db = 'other_db'
+
+        self.assertEqual(BookResource().get_db_connection_name(), 'other_db')
+        self.assertEqual(CategoryResource().get_db_connection_name(), 'default')
+
+    def test_import_data_raises_field_for_wrong_db(self):
+        class BookResource(resources.ModelResource):
+            class Meta:
+                using_db = 'wrong_db'
+
+        with self.assertRaises(ConnectionDoesNotExist):
+            BookResource().import_data(self.dataset)
+
 
 class ModelResourceTransactionTest(TransactionTestCase):
     @skipUnlessDBFeature('supports_transactions')
@@ -1041,29 +1058,6 @@ def save_m2m(self, instance, *args, **kwargs):
         )
         self.assertTrue(result.has_errors())
 
-    @skipUnlessDBFeature('supports_transactions')
-    def test_multiple_database_errors(self):
-
-        class CategoryResourceDbErrorsResource(CategoryResource):
-
-            def before_import(self, *args, **kwargs):
-                raise DatabaseError()
-
-            def save_instance(self):
-                raise DatabaseError()
-
-        resource = CategoryResourceDbErrorsResource()
-        headers = ['id', 'name']
-        rows = [
-            [None, 'foo'],
-        ]
-        dataset = tablib.Dataset(*rows, headers=headers)
-        result = resource.import_data(
-            dataset,
-            use_transactions=True,
-        )
-        self.assertTrue(result.has_errors())
-
 
 class ModelResourceFactoryTest(TestCase):