Repository: TOMToolkit/tom_base

Issue:
Add robust Target Name Recognition when ingesting target data
When ingesting new targets (see issue #70 ) or trying to ingest photometry from a TOM or a stream such as HERMES, we want to be able to make sure that the new target does not match the names or aliases of any existing targets. 

For this we should have a robust matching function that does the following:

- [x] Check if the object name matches any target name or aliases
- [x] Performs all checks with case insensitivity and regardless of spaces/special characters
- [ ] Checks object name against trusted catalog to find other known aliases. to be checked against target names in DB

Checks should be performed at both target and Data creation


Discussion:
Use existing Search capabilities on target page.

Resolving patch:
diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py
--- a/tom_dataproducts/alertstreams/hermes.py
+++ b/tom_dataproducts/alertstreams/hermes.py
@@ -117,10 +117,15 @@ def hermes_alert_handler(alert, metadata):
     photometry_table = alert_as_dict['data'].get('photometry', None)
     if photometry_table:
         hermes_alert = AlertStreamMessage(topic=alert_as_dict['topic'], exchange_status='ingested')
+        target_name = ''
+        query = []
         for row in photometry_table:
-            try:
-                target = Target.objects.get(name=row['target_name'])
-            except Target.DoesNotExist:
+            if row['target_name'] != target_name:
+                target_name = row['target_name']
+                query = Target.matches.check_for_fuzzy_match(target_name)
+            if query:
+                target = query[0]
+            else:
                 continue
 
             try:
diff --git a/tom_targets/models.py b/tom_targets/models.py
--- a/tom_targets/models.py
+++ b/tom_targets/models.py
@@ -34,6 +34,28 @@
 }
 
 
+class TargetMatchManager(models.Manager):
+    """Search for matches amongst Target names and Aliases"""
+    def check_for_fuzzy_match(self, name):
+        """
+        Check for case-insensitive names ignoring spaces, dashes, underscore, and parentheses.
+        :param name: The string against which target names and aliases will be matched.
+        :return: queryset containing matching Targets. Will return targets even when matched value is an alias.
+        """
+        simple_name = self.make_simple_name(name)
+        matching_names = []
+        for target in Target.objects.all():
+            for alias in target.names:
+                if self.make_simple_name(alias) == simple_name:
+                    matching_names.append(target.name)
+        queryset = Target.objects.filter(name__in=matching_names)
+        return queryset
+
+    def make_simple_name(self, name):
+        """Create a simplified name to be used for comparison in check_for_fuzzy_match."""
+        return name.lower().replace(" ", "").replace("-", "").replace("_", "").replace("(", "").replace(")", "")
+
+
 class Target(models.Model):
     """
     Class representing a target in a TOM
@@ -228,6 +250,9 @@ class Target(models.Model):
         null=True, blank=True, verbose_name='Perihelion Distance', help_text='AU'
     )
 
+    objects = models.Manager()
+    matches = TargetMatchManager()
+
     @transaction.atomic
     def save(self, *args, **kwargs):
         """
@@ -241,6 +266,7 @@ def save(self, *args, **kwargs):
         names = kwargs.pop('names', [])
 
         created = False if self.id else True
+
         super().save(*args, **kwargs)
 
         if created:
@@ -255,6 +281,7 @@ def save(self, *args, **kwargs):
 
         for name in names:
             name, _ = TargetName.objects.get_or_create(target=self, name=name)
+            name.full_clean()
             name.save()
 
         if not created:
@@ -265,6 +292,11 @@ def validate_unique(self, *args, **kwargs):
         Ensures that Target.name and all aliases of the target are unique. Called automatically on save.
         """
         super().validate_unique(*args, **kwargs)
+        # Check DB for similar target/alias names.
+        matches = Target.matches.check_for_fuzzy_match(self.name)
+        for match in matches:
+            if match.id is not self.id:
+                raise ValidationError(f'Target with Name or alias similar to {self.name} already exists')
         # Alias Check only necessary when updating target existing target. Reverse relationships require Primary Key.
         if self.pk:
             for alias in self.aliases.all():
@@ -386,9 +418,14 @@ def validate_unique(self, *args, **kwargs):
         Ensures that Target.name and all aliases of the target are unique. Called automatically on save.
         """
         super().validate_unique(*args, **kwargs)
-        if self.name == self.target.name:
-            raise ValidationError(f'''Alias {self.name} has a conflict with the primary name of the target
-                                      {self.target.name} (id={self.target.id})''')
+        # Check DB for similar target/alias names.
+        matches = Target.matches.check_for_fuzzy_match(self.name)
+        if matches:
+            if matches[0] == self.target:
+                raise ValidationError(f'Alias {self.name} has a conflict with the primary name of the target. '
+                                      f'(target_id={self.target.id})')
+            else:
+                raise ValidationError(f'Target with Name or alias similar to {self.name} already exists')
 
 
 class TargetExtra(models.Model):
diff --git a/tom_targets/views.py b/tom_targets/views.py
--- a/tom_targets/views.py
+++ b/tom_targets/views.py
@@ -190,12 +190,14 @@ def form_valid(self, form):
         :type form: subclass of TargetCreateForm
         """
         super().form_valid(form)
+
         extra = TargetExtraFormset(self.request.POST)
+        extra.instance = self.object
         names = TargetNamesFormset(self.request.POST)
+        names.instance = self.object
+
         if extra.is_valid() and names.is_valid():
-            extra.instance = self.object
             extra.save()
-            names.instance = self.object
             names.save()
         else:
             form.add_error(None, extra.errors)


Test patch:
diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py
--- a/tom_targets/tests/tests.py
+++ b/tom_targets/tests/tests.py
@@ -477,6 +477,144 @@ def test_create_targets_with_conflicting_aliases(self):
         second_response = self.client.post(reverse('targets:create'), data=target_data, follow=True)
         self.assertContains(second_response, 'Target name with this Alias already exists.')
 
+    def test_create_target_name_conflicting_with_existing_aliases(self):
+        target_data = {
+            'name': 'multiple_names_target',
+            'type': Target.SIDEREAL,
+            'ra': 113.456,
+            'dec': -22.1,
+            'groups': [self.group.id],
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': '',
+            'targetextra_set-0-value': '',
+            'aliases-TOTAL_FORMS': 2,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+        }
+        names = ['John', 'Doe']
+        for i, name in enumerate(names):
+            target_data[f'aliases-{i}-name'] = name
+        self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        target_data['name'] = 'John'
+        target_data['aliases-TOTAL_FORMS'] = 0
+        for i, name in enumerate(names):
+            target_data.pop(f'aliases-{i}-name')
+        second_response = self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        self.assertContains(second_response, f'Target with Name or alias similar to {target_data["name"]} '
+                                             f'already exists')
+        self.assertFalse(Target.objects.filter(name=target_data['name']).exists())
+
+    def test_create_target_alias_conflicting_with_existing_target_name(self):
+        target_data = {
+            'name': 'John',
+            'type': Target.SIDEREAL,
+            'ra': 113.456,
+            'dec': -22.1,
+            'groups': [self.group.id],
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': '',
+            'targetextra_set-0-value': '',
+            'aliases-TOTAL_FORMS': 0,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+        }
+        self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        names = ['John', 'Doe']
+        for i, name in enumerate(names):
+            target_data[f'aliases-{i}-name'] = name
+        target_data['name'] = 'multiple_names_target'
+        target_data['aliases-TOTAL_FORMS'] = 2
+        second_response = self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        self.assertContains(second_response, f'Target with Name or alias similar to {names[0]} already exists')
+
+    def test_create_target_alias_conflicting_with_target_name(self):
+        target_data = {
+            'name': 'John',
+            'type': Target.SIDEREAL,
+            'ra': 113.456,
+            'dec': -22.1,
+            'groups': [self.group.id],
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': '',
+            'targetextra_set-0-value': '',
+            'aliases-TOTAL_FORMS': 2,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+        }
+        names = ['John', 'Doe']
+        for i, name in enumerate(names):
+            target_data[f'aliases-{i}-name'] = name
+        response = self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        self.assertTrue(Target.objects.filter(name=target_data['name']).exists())
+        target = Target.objects.get(name=target_data['name'])
+        self.assertContains(response, f'Alias {names[0]} has a conflict with the primary name of the target. '
+                                      f'(target_id={target.id})')
+        self.assertFalse(TargetName.objects.filter(name=target_data['name']).exists())
+
+    def test_create_targets_with_fuzzy_conflicting_names(self):
+        target_data = {
+            'name': 'fuzzy_name_Target',
+            'type': Target.SIDEREAL,
+            'ra': 113.456,
+            'dec': -22.1,
+            'groups': [self.group.id],
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': '',
+            'targetextra_set-0-value': '',
+            'aliases-TOTAL_FORMS': 0,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+        }
+        names = ['FuzzyNameTarget', 'Fuzzy name target', 'FUZZY_NAME-TARGET', 'fuzzynametarget', '(Fuzzy) Name-Target']
+        self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        for name in names:
+            target_data['name'] = name
+            second_response = self.client.post(reverse('targets:create'), data=target_data, follow=True)
+            self.assertContains(second_response, f'Target with Name or alias similar to {name} already exists')
+
+    def test_create_alias_fuzzy_conflict_with_existing_target_name(self):
+        target_data = {
+            'name': 'John',
+            'type': Target.SIDEREAL,
+            'ra': 113.456,
+            'dec': -22.1,
+            'groups': [self.group.id],
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': '',
+            'targetextra_set-0-value': '',
+            'aliases-TOTAL_FORMS': 0,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+        }
+        self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        names = ['jo-hn', 'Doe']
+        for i, name in enumerate(names):
+            target_data[f'aliases-{i}-name'] = name
+        target_data['name'] = 'multiple_names_target'
+        target_data['aliases-TOTAL_FORMS'] = 2
+        second_response = self.client.post(reverse('targets:create'), data=target_data, follow=True)
+        self.assertContains(second_response, f'Target with Name or alias similar to {names[0]} already exists')
+
 
 class TestTargetUpdate(TestCase):
     def setUp(self):