Repository: netbox-community/netbox

Issue:
Make botocore dependency (AWS S3 backend) optional
### NetBox version

v3.5.3

### Feature type

Change to existing functionality

### Proposed functionality

[botocore](https://pypi.org/project/botocore/) is a really huge library, it consumes ~88 MiB of disk space. Since the data synchronization feature is optional and S3 is only one of the multiple backends, not all users will need it. The v3.5 release increased the disk size by a factor of 1.5 due to this dependency.

Please consider making this dependency optional.

### Use case

Saving disk space / making smaller container images, decreasing the number of dependencies.

### Database changes

_No response_

### External dependencies

_No response_


Discussion:
IMO this should apply to the `dulwich` library (used for the git backend) as well, and any additional backends we introduce in the future.

Resolving patch:
diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py
--- a/netbox/core/data_backends.py
+++ b/netbox/core/data_backends.py
@@ -6,13 +6,9 @@
 from pathlib import Path
 from urllib.parse import urlparse
 
-import boto3
-from botocore.config import Config as Boto3Config
 from django import forms
 from django.conf import settings
 from django.utils.translation import gettext as _
-from dulwich import porcelain
-from dulwich.config import ConfigDict
 
 from netbox.registry import registry
 from .choices import DataSourceTypeChoices
@@ -43,9 +39,20 @@ class DataBackend:
     parameters = {}
     sensitive_parameters = []
 
+    # Prevent Django's template engine from calling the backend
+    # class when referenced via DataSource.backend_class
+    do_not_call_in_templates = True
+
     def __init__(self, url, **kwargs):
         self.url = url
         self.params = kwargs
+        self.config = self.init_config()
+
+    def init_config(self):
+        """
+        Hook to initialize the instance's configuration.
+        """
+        return
 
     @property
     def url_scheme(self):
@@ -58,6 +65,7 @@ def fetch(self):
 
 @register_backend(DataSourceTypeChoices.LOCAL)
 class LocalBackend(DataBackend):
+
     @contextmanager
     def fetch(self):
         logger.debug(f"Data source type is local; skipping fetch")
@@ -89,14 +97,28 @@ class GitBackend(DataBackend):
     }
     sensitive_parameters = ['password']
 
+    def init_config(self):
+        from dulwich.config import ConfigDict
+
+        # Initialize backend config
+        config = ConfigDict()
+
+        # Apply HTTP proxy (if configured)
+        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
+            if proxy := settings.HTTP_PROXIES.get(self.url_scheme):
+                config.set("http", "proxy", proxy)
+
+        return config
+
     @contextmanager
     def fetch(self):
+        from dulwich import porcelain
+
         local_path = tempfile.TemporaryDirectory()
 
-        config = ConfigDict()
         clone_args = {
             "branch": self.params.get('branch'),
-            "config": config,
+            "config": self.config,
             "depth": 1,
             "errstream": porcelain.NoneStream(),
             "quiet": True,
@@ -110,10 +132,6 @@ def fetch(self):
                 }
             )
 
-        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
-            if proxy := settings.HTTP_PROXIES.get(self.url_scheme):
-                config.set("http", "proxy", proxy)
-
         logger.debug(f"Cloning git repo: {self.url}")
         try:
             porcelain.clone(self.url, local_path.name, **clone_args)
@@ -141,15 +159,20 @@ class S3Backend(DataBackend):
 
     REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com'
 
-    @contextmanager
-    def fetch(self):
-        local_path = tempfile.TemporaryDirectory()
+    def init_config(self):
+        from botocore.config import Config as Boto3Config
 
-        # Build the S3 configuration
-        s3_config = Boto3Config(
+        # Initialize backend config
+        return Boto3Config(
             proxies=settings.HTTP_PROXIES,
         )
 
+    @contextmanager
+    def fetch(self):
+        import boto3
+
+        local_path = tempfile.TemporaryDirectory()
+
         # Initialize the S3 resource and bucket
         aws_access_key_id = self.params.get('aws_access_key_id')
         aws_secret_access_key = self.params.get('aws_secret_access_key')
@@ -158,7 +181,7 @@ def fetch(self):
             region_name=self._region_name,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
-            config=s3_config
+            config=self.config
         )
         bucket = s3.Bucket(self._bucket_name)
 
diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py
--- a/netbox/core/models/data.py
+++ b/netbox/core/models/data.py
@@ -104,6 +104,10 @@ def get_status_color(self):
     def url_scheme(self):
         return urlparse(self.source_url).scheme.lower()
 
+    @property
+    def backend_class(self):
+        return registry['data_backends'].get(self.type)
+
     @property
     def is_local(self):
         return self.type == DataSourceTypeChoices.LOCAL
@@ -139,17 +143,15 @@ def enqueue_sync_job(self, request):
         )
 
     def get_backend(self):
-        backend_cls = registry['data_backends'].get(self.type)
         backend_params = self.parameters or {}
-
-        return backend_cls(self.source_url, **backend_params)
+        return self.backend_class(self.source_url, **backend_params)
 
     def sync(self):
         """
         Create/update/delete child DataFiles as necessary to synchronize with the remote source.
         """
         if self.status == DataSourceStatusChoices.SYNCING:
-            raise SyncError(f"Cannot initiate sync; syncing already in progress.")
+            raise SyncError("Cannot initiate sync; syncing already in progress.")
 
         # Emit the pre_sync signal
         pre_sync.send(sender=self.__class__, instance=self)
@@ -158,7 +160,12 @@ def sync(self):
         DataSource.objects.filter(pk=self.pk).update(status=self.status)
 
         # Replicate source data locally
-        backend = self.get_backend()
+        try:
+            backend = self.get_backend()
+        except ModuleNotFoundError as e:
+            raise SyncError(
+                f"There was an error initializing the backend. A dependency needs to be installed: {e}"
+            )
         with backend.fetch() as local_path:
 
             logger.debug(f'Syncing files from source root {local_path}')


Test patch: