Repository: nautobot/nautobot

Issue:
[LTM] Deleting/Restoring git repo with jobs results in database error
### Environment
* Nautobot version (Docker tag too if applicable): 1.4.5
* Python version: 3.8
* Database platform, version: PostgreSQL 12

### Steps to Reproduce
1. Add a repository with a job.
2. Delete the repository.
3. Add the repository one more time.

### Expected Behavior
The job should be re-enabled after the repository is re-added.

### Observed Behavior
When the repo is re-added and then sync'd there is a database constraint violation as the sync tries to add the job back (the job already exists in the database, but is marked as not installed).
[LTM] Deleting/Restoring git repo with jobs results in database error
### Environment
* Nautobot version (Docker tag too if applicable): 1.4.5
* Python version: 3.8
* Database platform, version: PostgreSQL 12

### Steps to Reproduce
1. Add a repository with a job.
2. Delete the repository.
3. Add the repository one more time.

### Expected Behavior
The job should be re-enabled after the repository is re-added.

### Observed Behavior
When the repo is re-added and then sync'd there is a database constraint violation as the sync tries to add the job back (the job already exists in the database, but is marked as not installed).


Discussion:
This may be easier to implement once #2611 is implemented.
For the time being, you will have to manually delete the jobs (please check hidden and installed filters to see them) to get this to work.


Ken's workaround worked for me. To see the uninstalled jobs which are still lingering, you can follow the process:
`Jobs` dropdown menu > Select `Jobs`, on the top right of the page click `Filter`, make no changes and click `Apply`.  

Additionally, I ran into an issue where this process failed until a restart of the containers happened.
Confirmed not an issue in 2.2. Will test with 1.6 next.
This may be easier to implement once #2611 is implemented.
For the time being, you will have to manually delete the jobs (please check hidden and installed filters to see them) to get this to work.


Ken's workaround worked for me. To see the uninstalled jobs which are still lingering, you can follow the process:
`Jobs` dropdown menu > Select `Jobs`, on the top right of the page click `Filter`, make no changes and click `Apply`.  

Additionally, I ran into an issue where this process failed until a restart of the containers happened.
Confirmed not an issue in 2.2. Will test with 1.6 next.

Resolving patch:
diff --git a/nautobot/extras/utils.py b/nautobot/extras/utils.py
--- a/nautobot/extras/utils.py
+++ b/nautobot/extras/utils.py
@@ -9,6 +9,7 @@
 from django.apps import apps
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
+from django.db import IntegrityError
 from django.db.models import Q
 from django.template.loader import get_template, TemplateDoesNotExist
 from django.utils.deconstruct import deconstructible
@@ -365,21 +366,37 @@ def refresh_job_model_from_job_class(job_model_class, job_source, job_class, *,
             JOB_MAX_NAME_LENGTH,
         )
 
-    job_model, created = job_model_class.objects.get_or_create(
-        source=job_source[:JOB_MAX_SOURCE_LENGTH],
-        git_repository=git_repository,
-        module_name=job_class.__module__[:JOB_MAX_NAME_LENGTH],
-        job_class_name=job_class.__name__[:JOB_MAX_NAME_LENGTH],
-        defaults={
-            "slug": default_slug[:JOB_MAX_SLUG_LENGTH],
-            "grouping": job_class.grouping[:JOB_MAX_GROUPING_LENGTH],
-            "name": job_class.name[:JOB_MAX_NAME_LENGTH],
-            "is_job_hook_receiver": issubclass(job_class, JobHookReceiver),
-            "is_job_button_receiver": issubclass(job_class, JobButtonReceiver),
-            "installed": True,
-            "enabled": False,
-        },
-    )
+    try:
+        job_model, created = job_model_class.objects.get_or_create(
+            source=job_source[:JOB_MAX_SOURCE_LENGTH],
+            git_repository=git_repository,
+            module_name=job_class.__module__[:JOB_MAX_NAME_LENGTH],
+            job_class_name=job_class.__name__[:JOB_MAX_NAME_LENGTH],
+            defaults={
+                "slug": default_slug[:JOB_MAX_SLUG_LENGTH],
+                "grouping": job_class.grouping[:JOB_MAX_GROUPING_LENGTH],
+                "name": job_class.name[:JOB_MAX_NAME_LENGTH],
+                "is_job_hook_receiver": issubclass(job_class, JobHookReceiver),
+                "is_job_button_receiver": issubclass(job_class, JobButtonReceiver),
+                "installed": True,
+                "enabled": False,
+            },
+        )
+    except IntegrityError:
+        # can occur in the case where we've deleted a GitRepository, resulting in a Job record with
+        # source="git" but git_repository=None, and are now creating a new GitRepository to "re-claim" the Job.
+        if git_repository is not None:
+            created = False
+            job_model = job_model_class.objects.get(
+                source=job_source[:JOB_MAX_SOURCE_LENGTH],
+                git_repository=None,
+                module_name=job_class.__module__[:JOB_MAX_NAME_LENGTH],
+                job_class_name=job_class.__name__[:JOB_MAX_NAME_LENGTH],
+            )
+            job_model.git_repository = git_repository
+            job_model.save()
+        else:
+            raise
 
     for field_name in JOB_OVERRIDABLE_FIELDS:
         # Was this field directly inherited from the job before, or was it overridden in the database?


Test patch:
diff --git a/nautobot/extras/tests/test_datasources.py b/nautobot/extras/tests/test_datasources.py
--- a/nautobot/extras/tests/test_datasources.py
+++ b/nautobot/extras/tests/test_datasources.py
@@ -27,6 +27,7 @@
     ConfigContextSchema,
     ExportTemplate,
     GitRepository,
+    Job,
     JobLogEntry,
     JobResult,
     Secret,
@@ -119,13 +120,13 @@ def setUp(self):
 
     def populate_repo(self, path, url, *args, **kwargs):
         os.makedirs(path)
-        # TODO(Glenn): populate Jobs as well?
         os.makedirs(os.path.join(path, "config_contexts"))
         os.makedirs(os.path.join(path, "config_contexts", "devices"))
         os.makedirs(os.path.join(path, "config_contexts", "locations"))
         os.makedirs(os.path.join(path, "config_context_schemas"))
         os.makedirs(os.path.join(path, "export_templates", "dcim", "device"))
         os.makedirs(os.path.join(path, "export_templates", "ipam", "vlan"))
+        os.makedirs(os.path.join(path, "jobs"))
 
         with open(os.path.join(path, "config_contexts", "context.yaml"), "w") as fd:
             yaml.dump(
@@ -167,6 +168,19 @@ def populate_repo(self, path, url, *args, **kwargs):
         with open(os.path.join(path, "export_templates", "ipam", "vlan", "template.j2"), "w") as fd:
             fd.write("{% for vlan in queryset %}\n{{ vlan.name }}\n{% endfor %}")
 
+        with open(os.path.join(path, "jobs", "__init__.py"), "w") as fd:
+            pass
+
+        with open(os.path.join(path, "jobs", "job.py"), "w") as fd:
+            fd.write(
+                """\
+from nautobot.extras.jobs import Job
+
+class MyJob(Job):
+    def run(self, data, commit):
+        pass"""
+            )
+
         return mock.DEFAULT
 
     def empty_repo(self, path, url, *args, **kwargs):
@@ -177,6 +191,8 @@ def empty_repo(self, path, url, *args, **kwargs):
         os.remove(os.path.join(path, "export_templates", "dcim", "device", "template.j2"))
         os.remove(os.path.join(path, "export_templates", "dcim", "device", "template2.html"))
         os.remove(os.path.join(path, "export_templates", "ipam", "vlan", "template.j2"))
+        os.remove(os.path.join(path, "jobs", "__init__.py"))
+        os.remove(os.path.join(path, "jobs", "job.py"))
         return mock.DEFAULT
 
     def assert_config_context_schema_record_exists(self, name):
@@ -263,6 +279,11 @@ def assert_export_template_vlan_exists(self, name):
         )
         self.assertIsNotNone(export_template_vlan)
 
+    def assert_job_exists(self, installed=True):
+        """Helper function to assert Job exists."""
+        job = Job.objects.get(git_repository=self.repo, job_class_name="MyJob")
+        self.assertEqual(job.installed, installed)
+
     def test_pull_git_repository_and_refresh_data_with_no_data(self, MockGitRepo):
         """
         The pull_git_repository_and_refresh_data job should succeed if the given repo is empty.
@@ -482,6 +503,8 @@ def test_pull_git_repository_and_refresh_data_with_valid_data(self, MockGitRepo)
                 # Case when ContentType.model != ContentType.name, template was added and deleted during sync (#570)
                 self.assert_export_template_vlan_exists("template.j2")
 
+                self.assert_job_exists()
+
                 # Now "resync" the repository, but now those files no longer exist in the repository
                 MockGitRepo.side_effect = self.empty_repo
                 # For verisimilitude, don't re-use the old request and job_result
@@ -525,6 +548,9 @@ def test_pull_git_repository_and_refresh_data_with_valid_data(self, MockGitRepo)
                 self.assertIsNone(device.local_context_data)
                 self.assertIsNone(device.local_context_data_owner)
 
+                # Job should still be present in the database but no longer installed
+                self.assert_job_exists(installed=False)
+
     def test_pull_git_repository_and_refresh_data_with_bad_data(self, MockGitRepo):
         """
         The test_pull_git_repository_and_refresh_data job should gracefully handle bad data in the Git repository
@@ -654,11 +680,11 @@ def test_delete_git_repository_cleanup(self, MockGitRepo):
 
                 def populate_repo(path, url):
                     os.makedirs(path)
-                    # Just make config_contexts and export_templates directories as we don't load jobs
                     os.makedirs(os.path.join(path, "config_contexts"))
                     os.makedirs(os.path.join(path, "config_contexts", "devices"))
                     os.makedirs(os.path.join(path, "config_context_schemas"))
                     os.makedirs(os.path.join(path, "export_templates", "dcim", "device"))
+                    os.makedirs(os.path.join(path, "jobs"))
                     with open(os.path.join(path, "config_contexts", "context.yaml"), "w") as fd:
                         yaml.dump(
                             {
@@ -685,6 +711,18 @@ def populate_repo(path, url):
                         "w",
                     ) as fd:
                         fd.write("{% for device in queryset %}\n{{ device.name }}\n{% endfor %}")
+                    with open(os.path.join(path, "jobs", "__init__.py"), "w") as fd:
+                        pass
+                    with open(os.path.join(path, "jobs", "job.py"), "w") as fd:
+                        fd.write(
+                            """\
+from nautobot.extras.jobs import Job
+
+class MyJob(Job):
+    def run(self, data, commit):
+        pass"""
+                        )
+
                     return mock.DEFAULT
 
                 MockGitRepo.side_effect = populate_repo
@@ -744,6 +782,8 @@ def populate_repo(path, url):
                 )
                 self.assertIsNotNone(export_template)
 
+                self.assert_job_exists()
+
                 # Now delete the GitRepository
                 self.repo.delete()
 
@@ -768,6 +808,36 @@ def populate_repo(path, url):
                 device = Device.objects.get(name=self.device.name)
                 self.assertIsNone(device.local_context_data)
                 self.assertIsNone(device.local_context_data_owner)
+                self.assert_job_exists(installed=False)
+
+                # Now recreate the repository (https://github.com/nautobot/nautobot/issues/2974)
+                self.repo = GitRepository(
+                    name="Test Git Repository",
+                    slug="test_git_repo",
+                    remote_url="http://localhost/git.git",
+                    # Provide everything we know we can provide
+                    provided_contents=[
+                        entry.content_identifier for entry in get_datasource_contents("extras.gitrepository")
+                    ],
+                )
+                self.repo.save(trigger_resync=False)
+
+                self.job_result = JobResult.objects.create(
+                    name=self.repo.name,
+                    obj_type=ContentType.objects.get_for_model(GitRepository),
+                    job_id=uuid.uuid4(),
+                )
+
+                pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk)
+                self.job_result.refresh_from_db()
+
+                self.assertEqual(
+                    self.job_result.status,
+                    JobResultStatusChoices.STATUS_COMPLETED,
+                    self.job_result.data,
+                )
+
+                self.assert_job_exists(installed=True)
 
     def test_git_dry_run(self, MockGitRepo):
         with tempfile.TemporaryDirectory() as tempdir: