Repository: netbox-community/netbox

Issue:
Initial NetBox 3.4.x migration fails when a plugin using SearchIndex is installed
### NetBox version

3.4.1

### Python version

3.8

### Steps to Reproduce

1. Install NetBox 3.4.1
2. Install any plugin that uses SearchIndex functionality (e.g. netbox-dns) and configure it in PLUGINS
3. Run the initial migration

### Expected Behavior

The migration succeeds

### Observed Behavior

The migration fails with ProgrammingError exception:
```
Operations to perform:
  Apply all migrations: admin, auth, circuits, contenttypes, dcim, django_rq, extras, ipam, netbox_dns, sessions, social_django, taggit, tenancy, users, virtualization, wireless
Running migrations:
  Applying extras.0083_search...Reindexing 67 models.
Clearing cached values... 0 entries deleted.
Indexing models
  netbox_dns.nameserver... Traceback (most recent call last):
  File "/opt/netbox/lib/python3.8/site-packages/django/db/backends/utils.py", line 89, in _execute
    return self.cursor.execute(sql, params)
psycopg2.errors.UndefinedTable: relation "netbox_dns_nameserver" does not exist
LINE 1: ...name", "netbox_dns_nameserver"."description" FROM "netbox_dn...
                                                             ^


The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/opt/netbox/netbox/manage.py", line 10, in <module>
    execute_from_command_line(sys.argv)
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/__init__.py", line 446, in execute_from_command_line
    utility.execute()
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/__init__.py", line 440, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/base.py", line 402, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/base.py", line 448, in execute
    output = self.handle(*args, **options)
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/base.py", line 96, in wrapped
    res = handle_func(*args, **kwargs)
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/commands/migrate.py", line 349, in handle
    post_migrate_state = executor.migrate(
  File "/opt/netbox/lib/python3.8/site-packages/django/db/migrations/executor.py", line 135, in migrate
    state = self._migrate_all_forwards(
  File "/opt/netbox/lib/python3.8/site-packages/django/db/migrations/executor.py", line 167, in _migrate_all_forwards
    state = self.apply_migration(
  File "/opt/netbox/lib/python3.8/site-packages/django/db/migrations/executor.py", line 252, in apply_migration
    state = migration.apply(state, schema_editor)
  File "/opt/netbox/lib/python3.8/site-packages/django/db/migrations/migration.py", line 130, in apply
    operation.database_forwards(
  File "/opt/netbox/lib/python3.8/site-packages/django/db/migrations/operations/special.py", line 193, in database_forwards
    self.code(from_state.apps, schema_editor)
  File "/opt/netbox/netbox/extras/migrations/0083_search.py", line 13, in reindex
    management.call_command('reindex')
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/__init__.py", line 198, in call_command
    return command.execute(*args, **defaults)
  File "/opt/netbox/lib/python3.8/site-packages/django/core/management/base.py", line 448, in execute
    output = self.handle(*args, **options)
  File "/opt/netbox/netbox/extras/management/commands/reindex.py", line 68, in handle
    i = search_backend.cache(model.objects.iterator(), remove_existing=False)
  File "/opt/netbox/netbox/netbox/search/backends.py", line 148, in cache
    for instance in instances:
  File "/opt/netbox/lib/python3.8/site-packages/django/db/models/query.py", line 512, in _iterator
    yield from iterable
  File "/opt/netbox/lib/python3.8/site-packages/django/db/models/query.py", line 87, in __iter__
    results = compiler.execute_sql(
  File "/opt/netbox/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1398, in execute_sql
    cursor.execute(sql, params)
  File "/opt/netbox/lib/python3.8/site-packages/django/db/backends/utils.py", line 103, in execute
    return super().execute(sql, params)
  File "/opt/netbox/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(
  File "/opt/netbox/lib/python3.8/site-packages/django/db/backends/utils.py", line 80, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/opt/netbox/lib/python3.8/site-packages/django/db/backends/utils.py", line 89, in _execute
    return self.cursor.execute(sql, params)
  File "/opt/netbox/lib/python3.8/site-packages/django/db/utils.py", line 91, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/opt/netbox/lib/python3.8/site-packages/django/db/backends/utils.py", line 89, in _execute
    return self.cursor.execute(sql, params)
django.db.utils.ProgrammingError: relation "netbox_dns_nameserver" does not exist
LINE 1: ...name", "netbox_dns_nameserver"."description" FROM "netbox_dn...
```

The `netbox_dns_nameserver` relation for the `netbox_dns.models.NameServer` model uses `SearchIndex`:
```
@register_search
class NameServerIndex(SearchIndex):
    model = NameServer
    fields = (
        ("name", 100),
        ("description", 500),
    )
```

That results in the NetBox migration `extras.0083_search` trying to reindex the plugin's relation, which does not exist yet at that point.

The obvious workaround is to disable the plugin, run the migration, re-enable the plugin and then run the migration again. 


Discussion:
We can tweak the migration to simply skip any indexers whose tables don't exist yet, which should address the problem cited here. (This should be non-impacting, as if the table doesn't exist there's no data to index anyway.) However, there's also a scenario where indexing gets skipped if a plugin adds a new field to an existing table, which would require that the user manually re-generate the search cache for the plugin model.
> However, there's also a scenario where indexing gets skipped if a plugin adds a new field to an existing table, which would require that the user manually re-generate the search cache for the plugin model.

Exactly that happens with my plugin ... but I don't think it's a major issue, I can add the reindexing code to the migration for the plugin.
Can the `reindex` command be added as a post-migration signal call? With that, it runs only after the migration for the plugin has finished. Although, the side-effect of this approach that I can think of is the overhead of calling the command several times for big migrations.

Resolving patch:
diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py
--- a/netbox/extras/management/commands/reindex.py
+++ b/netbox/extras/management/commands/reindex.py
@@ -27,17 +27,28 @@ def _get_indexers(self, *model_names):
         # Return only indexers for the specified models
         else:
             for label in model_names:
-                try:
-                    app_label, model_name = label.lower().split('.')
-                except ValueError:
+                labels = label.lower().split('.')
+
+                # Label specifies an exact model
+                if len(labels) == 2:
+                    app_label, model_name = labels
+                    try:
+                        idx = registry['search'][f'{app_label}.{model_name}']
+                        indexers[idx.model] = idx
+                    except KeyError:
+                        raise CommandError(f"No indexer registered for {label}")
+
+                # Label specifies all the models of an app
+                elif len(labels) == 1:
+                    app_label = labels[0] + '.'
+                    for indexer_label, idx in registry['search'].items():
+                        if indexer_label.startswith(app_label):
+                            indexers[idx.model] = idx
+
+                else:
                     raise CommandError(
-                        f"Invalid model: {label}. Model names must be in the format <app_label>.<model_name>."
+                        f"Invalid model: {label}. Model names must be in the format <app_label> or <app_label>.<model_name>."
                     )
-                try:
-                    idx = registry['search'][f'{app_label}.{model_name}']
-                    indexers[idx.model] = idx
-                except KeyError:
-                    raise CommandError(f"No indexer registered for {label}")
 
         return indexers
 
diff --git a/netbox/extras/migrations/0083_search.py b/netbox/extras/migrations/0083_search.py
--- a/netbox/extras/migrations/0083_search.py
+++ b/netbox/extras/migrations/0083_search.py
@@ -10,7 +10,16 @@
 def reindex(apps, schema_editor):
     # Build the search index (except during tests)
     if 'test' not in sys.argv:
-        management.call_command('reindex')
+        management.call_command(
+            'reindex',
+            'circuits',
+            'dcim',
+            'extras',
+            'ipam',
+            'tenancy',
+            'virtualization',
+            'wireless',
+        )
 
 
 class Migration(migrations.Migration):


Test patch: