Repository: nautobot/nautobot

Issue:
[2.2] Job-based CSV import should optionally be atomic
### As ...

Nelly - Network Engineer

### I want ...

To have the option for CSV import (via system Job) to be either non-atomic (each individual CSV row succeeds or fails in creating an object on its own) or atomic (the entire CSV file creates objects successfully or else no objects are created at all).

Atomic import was the only option in 2.1 and earlier, but may or may not be the desirable default in 2.2.

### So that ...

If I have a large data file that may contain errors on some rows, I have the option to repeatedly retry it, cleaning up errors as I go, until the entire file is successfully importable.

### I know this is done when...

I have an option when submitting the Import Objects job to choose how I want errors to be handled.

### Optional - Feature groups this request pertains to.

- [X] Automation
- [ ] Circuits
- [ ] DCIM
- [ ] IPAM
- [X] Misc (including Data Sources)
- [ ] Organization
- [ ] Plugins (and other Extensibility)
- [ ] Security (Secrets, etc)
- [ ] Image Management
- [ ] UI/UX
- [ ] Documentation
- [ ] Other (not directly a platform feature)

### Database Changes

None

### External Dependencies

None


Discussion:

Resolving patch:
diff --git a/nautobot/core/jobs/__init__.py b/nautobot/core/jobs/__init__.py
--- a/nautobot/core/jobs/__init__.py
+++ b/nautobot/core/jobs/__init__.py
@@ -1,3 +1,4 @@
+import contextlib
 from io import BytesIO
 
 from django.conf import settings
@@ -16,7 +17,7 @@
 from nautobot.core.utils.lookup import get_filterset_for_model
 from nautobot.core.utils.requests import get_filterable_params_from_filter_params
 from nautobot.extras.datasources import ensure_git_repository, git_repository_dry_run, refresh_datasource_content
-from nautobot.extras.jobs import ChoiceVar, FileVar, Job, ObjectVar, RunJobTaskFailed, StringVar, TextVar
+from nautobot.extras.jobs import BooleanVar, ChoiceVar, FileVar, Job, ObjectVar, RunJobTaskFailed, StringVar, TextVar
 from nautobot.extras.models import ExportTemplate, GitRepository
 
 name = "System Jobs"
@@ -199,6 +200,12 @@ class ImportObjects(Job):
     )
     csv_data = TextVar(label="CSV Data", required=False)
     csv_file = FileVar(label="CSV File", required=False)
+    roll_back_if_error = BooleanVar(
+        label="All or nothing import",
+        required=False,
+        default=False,
+        description="Enable so that if any error occurs while importing, even successfully imported objects will be discarded. Disable if a partial import is acceptable.",
+    )
 
     template_name = "system_jobs/import_objects.html"
 
@@ -209,7 +216,45 @@ class Meta:
         soft_time_limit = 1800
         time_limit = 2000
 
-    def run(self, *, content_type, csv_data=None, csv_file=None):
+    def _perform_atomic_operation(self, data, serializer_class, queryset):
+        new_objs = []
+        with contextlib.suppress(AbortTransaction):
+            with transaction.atomic():
+                new_objs, validation_failed = self._perform_operation(data, serializer_class, queryset)
+                if validation_failed:
+                    raise AbortTransaction
+                return new_objs, validation_failed
+        # If validation failed return an empty list, since all objs created where rolled back
+        self.logger.warning("Rolling back all %s records.", len(new_objs))
+        return [], validation_failed
+
+    def _perform_operation(self, data, serializer_class, queryset):
+        new_objs = []
+        validation_failed = False
+        for row, entry in enumerate(data, start=1):
+            serializer = serializer_class(data=entry, context={"request": None})
+            if serializer.is_valid():
+                try:
+                    with transaction.atomic():
+                        new_obj = serializer.save()
+                        if not queryset.filter(pk=new_obj.pk).exists():
+                            raise AbortTransaction()
+                    self.logger.info('Row %d: Created record "%s"', row, new_obj, extra={"object": new_obj})
+                    new_objs.append(new_obj)
+                except AbortTransaction:
+                    self.logger.error(
+                        'Row %d: User "%s" does not have permission to create an object with these attributes',
+                        row,
+                        self.user,
+                    )
+                    validation_failed = True
+            else:
+                validation_failed = True
+                for field, err in serializer.errors.items():
+                    self.logger.error("Row %d: `%s`: `%s`", row, field, err[0])
+        return new_objs, validation_failed
+
+    def run(self, *, content_type, csv_data=None, csv_file=None, roll_back_if_error=False):
         if not self.user.has_perm(f"{content_type.app_label}.add_{content_type.model}"):
             self.logger.error('User "%s" does not have permission to create %s objects', self.user, content_type.model)
             raise PermissionDenied("User does not have create permissions on the requested content-type")
@@ -247,28 +292,10 @@ def run(self, *, content_type, csv_data=None, csv_file=None):
                 parser_context={"request": None, "serializer_class": serializer_class},
             )
             self.logger.info("Processing %d rows of data", len(data))
-            validation_failed = False
-            for row, entry in enumerate(data, start=1):
-                serializer = serializer_class(data=entry, context={"request": None})
-                if serializer.is_valid():
-                    try:
-                        with transaction.atomic():
-                            new_obj = serializer.save()
-                            if not queryset.filter(pk=new_obj.pk).exists():
-                                raise AbortTransaction()
-                        self.logger.info('Row %d: Created record "%s"', row, new_obj, extra={"object": new_obj})
-                        new_objs.append(new_obj)
-                    except AbortTransaction:
-                        self.logger.error(
-                            'Row %d: User "%s" does not have permission to create an object with these attributes',
-                            row,
-                            self.user,
-                        )
-                        validation_failed = True
-                else:
-                    validation_failed = True
-                    for field, err in serializer.errors.items():
-                        self.logger.error("Row %d: `%s`: `%s`", row, field, err[0])
+            if roll_back_if_error:
+                new_objs, validation_failed = self._perform_atomic_operation(data, serializer_class, queryset)
+            else:
+                new_objs, validation_failed = self._perform_operation(data, serializer_class, queryset)
         except drf_exceptions.ParseError as exc:
             validation_failed = True
             self.logger.error("`%s`", exc)
@@ -281,6 +308,8 @@ def run(self, *, content_type, csv_data=None, csv_file=None):
             self.logger.warning("No %s objects were created", content_type.model)
 
         if validation_failed:
+            if roll_back_if_error:
+                raise RunJobTaskFailed("CSV import not successful, all imports were rolled back, see logs")
             raise RunJobTaskFailed("CSV import not fully successful, see logs")
 
 


Test patch:
diff --git a/nautobot/core/tests/test_jobs.py b/nautobot/core/tests/test_jobs.py
--- a/nautobot/core/tests/test_jobs.py
+++ b/nautobot/core/tests/test_jobs.py
@@ -205,28 +205,54 @@ def test_csv_import_with_permission(self):
         self.assertEqual(4, Status.objects.filter(name__startswith="test_status").count())
 
     def test_csv_import_bad_row(self):
-        """A row of incorrect data should fail validation for that object but import all others successfully."""
+        """A row of incorrect data should fail validation for that object but import all others successfully if `roll_back_if_error` is False."""
         csv_data = self.csv_data.split("\n")
         csv_data.insert(1, "test_status0,notacolor,dcim.device")
         csv_data = "\n".join(csv_data)
-        job_result = create_job_result_and_run_job(
-            "nautobot.core.jobs",
-            "ImportObjects",
-            content_type=ContentType.objects.get_for_model(Status).pk,
-            csv_data=csv_data,
-        )
-        self.assertEqual(job_result.status, JobResultStatusChoices.STATUS_FAILURE)
-        log_errors = JobLogEntry.objects.filter(job_result=job_result, log_level=LogLevelChoices.LOG_ERROR)
-        self.assertEqual(log_errors[0].message, "Row 1: `color`: `Enter a valid hexadecimal RGB color code.`")
-        log_successes = JobLogEntry.objects.filter(
-            job_result=job_result, log_level=LogLevelChoices.LOG_INFO, message__icontains="created"
-        )
-        self.assertEqual(log_successes[0].message, 'Row 2: Created record "test_status1"')
-        self.assertTrue(Status.objects.filter(name="test_status1").exists())
-        self.assertEqual(log_successes[1].message, 'Row 3: Created record "test_status2"')
-        self.assertTrue(Status.objects.filter(name="test_status2").exists())
-        self.assertEqual(log_successes[2].message, 'Row 4: Created record "test_status3"')
-        self.assertTrue(Status.objects.filter(name="test_status3").exists())
-        self.assertEqual(log_successes[3].message, 'Row 5: Created record "test_status4"')
-        self.assertTrue(Status.objects.filter(name="test_status4").exists())
-        self.assertEqual(log_successes[4].message, "Created 4 status object(s) from 5 row(s) of data")
+
+        with self.subTest("Assert `roll_back_if_error`: if error all records are rolled back"):
+            job_result = create_job_result_and_run_job(
+                "nautobot.core.jobs",
+                "ImportObjects",
+                content_type=ContentType.objects.get_for_model(Status).pk,
+                csv_data=csv_data,
+                roll_back_if_error=True,
+            )
+            self.assertEqual(job_result.status, JobResultStatusChoices.STATUS_FAILURE)
+            log_info = JobLogEntry.objects.filter(
+                job_result=job_result, log_level=LogLevelChoices.LOG_INFO, message__icontains="created"
+            )
+            for idx, status_name in enumerate(("test_status1", "test_status2", "test_status3", "test_status4")):
+                self.assertIn(f'Created record "{status_name}"', log_info[idx].message)
+                self.assertFalse(Status.objects.filter(name=status_name).exists())
+
+            log_errors = JobLogEntry.objects.filter(job_result=job_result, log_level=LogLevelChoices.LOG_ERROR)
+            self.assertEqual(log_errors[0].message, "Row 1: `color`: `Enter a valid hexadecimal RGB color code.`")
+
+            log_warning = JobLogEntry.objects.filter(job_result=job_result, log_level=LogLevelChoices.LOG_WARNING)
+            self.assertEqual(log_warning[0].message, "Rolling back all 4 records.")
+            self.assertEqual(log_warning[1].message, "No status objects were created")
+
+        with self.subTest("Assert all other data are imported successfully if `roll_back_if_error` is False"):
+            job_result = create_job_result_and_run_job(
+                "nautobot.core.jobs",
+                "ImportObjects",
+                content_type=ContentType.objects.get_for_model(Status).pk,
+                csv_data=csv_data,
+            )
+            self.assertEqual(job_result.status, JobResultStatusChoices.STATUS_FAILURE)
+            log_errors = JobLogEntry.objects.filter(job_result=job_result, log_level=LogLevelChoices.LOG_ERROR)
+            self.assertEqual(log_errors[0].message, "Row 1: `color`: `Enter a valid hexadecimal RGB color code.`")
+            self.assertFalse(Status.objects.filter(name="test_status0").exists())
+            log_successes = JobLogEntry.objects.filter(
+                job_result=job_result, log_level=LogLevelChoices.LOG_INFO, message__icontains="created"
+            )
+            self.assertEqual(log_successes[0].message, 'Row 2: Created record "test_status1"')
+            self.assertTrue(Status.objects.filter(name="test_status1").exists())
+            self.assertEqual(log_successes[1].message, 'Row 3: Created record "test_status2"')
+            self.assertTrue(Status.objects.filter(name="test_status2").exists())
+            self.assertEqual(log_successes[2].message, 'Row 4: Created record "test_status3"')
+            self.assertTrue(Status.objects.filter(name="test_status3").exists())
+            self.assertEqual(log_successes[3].message, 'Row 5: Created record "test_status4"')
+            self.assertTrue(Status.objects.filter(name="test_status4").exists())
+            self.assertEqual(log_successes[4].message, "Created 4 status object(s) from 5 row(s) of data")