Repository: CenterForOpenScience/gravyvalet
Pull request #247: avoid N+1 request when display projects' guid related to each AuthorizedStorageAccount
Author: login: ihorsokhanexoft
Created: 2025-03-03T14:40:39Z | Merged: 2025-03-20T14:05:29Z

As we want to display projects' guid for each authorized addon account, it'll create N+1 issue, thus using configured_storage_addons as ResourceRelatedField would be inefficient plus creates other issues whose solution wouldn't follow JSONAPI approach


Diff:
diff --git a/addon_service/authorized_account/citation/models.py b/addon_service/authorized_account/citation/models.py
index 00d40886..cbdef655 100644
--- a/addon_service/authorized_account/citation/models.py
+++ b/addon_service/authorized_account/citation/models.py
@@ -2,6 +2,7 @@
 
 from addon_service.addon_imp.instantiation import get_citation_addon_instance
 from addon_service.authorized_account.models import AuthorizedAccount
+from addon_service.configured_addon.citation.models import ConfiguredCitationAddon
 from addon_toolkit.interfaces.citation import CitationConfig
 
 
@@ -31,6 +32,12 @@ async def execute_post_auth_hook(self, auth_extras: dict | None = None):
         self.external_account_id = await imp.get_external_account_id(auth_extras or {})
         await self.asave()
 
+    @property
+    def configured_citation_addons(self):
+        return ConfiguredCitationAddon.objects.filter(base_account=self).select_related(
+            "authorized_resource"
+        )
+
     @property
     def config(self) -> CitationConfig:
         return CitationConfig(
diff --git a/addon_service/authorized_account/citation/serializers.py b/addon_service/authorized_account/citation/serializers.py
index 82407d3f..678c29ae 100644
--- a/addon_service/authorized_account/citation/serializers.py
+++ b/addon_service/authorized_account/citation/serializers.py
@@ -56,6 +56,13 @@ class AuthorizedCitationAccountSerializer(AuthorizedAccountSerializer):
         "authorized_operations": "addon_service.serializers.AddonOperationSerializer",
     }
 
+    configured_addons_uris = serializers.SerializerMethodField()
+
+    def get_configured_addons_uris(self, obj):
+        return obj.configured_citation_addons.values_list(
+            "authorized_resource__resource_uri", flat=True
+        )
+
     class Meta:
         model = AuthorizedCitationAccount
         fields = [
@@ -74,4 +81,5 @@ class Meta:
             "external_citation_service",
             "initiate_oauth",
             "credentials_available",
+            "configured_addons_uris",
         ]
diff --git a/addon_service/authorized_account/computing/models.py b/addon_service/authorized_account/computing/models.py
index cef4ca29..0208ffa7 100644
--- a/addon_service/authorized_account/computing/models.py
+++ b/addon_service/authorized_account/computing/models.py
@@ -1,5 +1,6 @@
 from addon_service.addon_imp.instantiation import get_computing_addon_instance
 from addon_service.authorized_account.models import AuthorizedAccount
+from addon_service.configured_addon.computing.models import ConfiguredComputingAddon
 from addon_toolkit.interfaces.computing import ComputingConfig
 
 
@@ -27,6 +28,12 @@ async def execute_post_auth_hook(self, auth_extras: dict | None = None):
         self.external_account_id = await imp.get_external_account_id(auth_extras or {})
         await self.asave()
 
+    @property
+    def configured_computing_addons(self):
+        return ConfiguredComputingAddon.objects.filter(
+            base_account=self
+        ).select_related("authorized_resource")
+
     @property
     def config(self) -> ComputingConfig:
         return ComputingConfig(
diff --git a/addon_service/authorized_account/computing/serializers.py b/addon_service/authorized_account/computing/serializers.py
index 5480c3dc..9dd7258b 100644
--- a/addon_service/authorized_account/computing/serializers.py
+++ b/addon_service/authorized_account/computing/serializers.py
@@ -56,6 +56,13 @@ class AuthorizedComputingAccountSerializer(AuthorizedAccountSerializer):
         "authorized_operations": "addon_service.serializers.AddonOperationSerializer",
     }
 
+    configured_addons_uris = serializers.SerializerMethodField()
+
+    def get_configured_addons_uris(self, obj):
+        return obj.configured_computing_addons.values_list(
+            "authorized_resource__resource_uri", flat=True
+        )
+
     class Meta:
         model = AuthorizedComputingAccount
         fields = [
@@ -73,4 +80,5 @@ class Meta:
             "external_computing_service",
             "initiate_oauth",
             "credentials_available",
+            "configured_addons_uris",
         ]
diff --git a/addon_service/authorized_account/serializers.py b/addon_service/authorized_account/serializers.py
index a8aa7110..6367a6d4 100644
--- a/addon_service/authorized_account/serializers.py
+++ b/addon_service/authorized_account/serializers.py
@@ -31,10 +31,16 @@ def __init__(self, *args, **kwargs):
                 f"{self.__class__.__name__} requires {self.REQUIRED_FIELDS} to be instantiated"
             )
 
+        if "context" not in kwargs:
+            return
+
         # Check if it's a POST request and remove the field as it's not in our FE spec
-        if "context" in kwargs and kwargs["context"]["request"].method == "POST":
+        if kwargs["context"]["request"].method == "POST":
             self.fields.pop("configured_storage_addons", None)
 
+        if kwargs["context"]["request"].query_params.get("uris") != "true":
+            self.fields.pop("configured_addons_uris")
+
     display_name = serializers.CharField(
         allow_blank=True, allow_null=True, required=False, max_length=256
     )
diff --git a/addon_service/authorized_account/storage/models.py b/addon_service/authorized_account/storage/models.py
index c9977e72..e13f211d 100644
--- a/addon_service/authorized_account/storage/models.py
+++ b/addon_service/authorized_account/storage/models.py
@@ -24,7 +24,9 @@ class JSONAPIMeta:
 
     @property
     def configured_storage_addons(self):
-        return ConfiguredStorageAddon.objects.filter(base_account=self)
+        return ConfiguredStorageAddon.objects.filter(base_account=self).select_related(
+            "authorized_resource"
+        )
 
     @property
     def config(self) -> StorageConfig:
diff --git a/addon_service/authorized_account/storage/serializers.py b/addon_service/authorized_account/storage/serializers.py
index ea98fdf0..032c796e 100644
--- a/addon_service/authorized_account/storage/serializers.py
+++ b/addon_service/authorized_account/storage/serializers.py
@@ -57,6 +57,13 @@ class AuthorizedStorageAccountSerializer(AuthorizedAccountSerializer):
         "authorized_operations": "addon_service.serializers.AddonOperationSerializer",
     }
 
+    configured_addons_uris = serializers.SerializerMethodField()
+
+    def get_configured_addons_uris(self, obj):
+        return obj.configured_storage_addons.values_list(
+            "authorized_resource__resource_uri", flat=True
+        )
+
     class Meta:
         model = AuthorizedStorageAccount
         fields = [
@@ -75,4 +82,5 @@ class Meta:
             "external_storage_service",
             "initiate_oauth",
             "credentials_available",
+            "configured_addons_uris",
         ]
diff --git a/addon_service/authorized_account/views.py b/addon_service/authorized_account/views.py
index ef65cd8f..22ac765c 100644
--- a/addon_service/authorized_account/views.py
+++ b/addon_service/authorized_account/views.py
@@ -8,6 +8,7 @@
 
 
 class AuthorizedAccountViewSet(RetrieveWriteDeleteViewSet, ABC):
+    allowed_query_params = ["uris"]
 
     def get_permissions(self):
         match self.action:
diff --git a/addon_service/common/queryparams_filter.py b/addon_service/common/queryparams_filter.py
new file mode 100644
index 00000000..ea207e83
--- /dev/null
+++ b/addon_service/common/queryparams_filter.py
@@ -0,0 +1,33 @@
+from rest_framework_json_api.filters import QueryParameterValidationFilter
+
+
+class AllowedQueryParamsFilter(QueryParameterValidationFilter):
+    """
+    JSONAPI supports specific kind of query params.
+    See https://jsonapi.org/format/#query-parameters or check the base class
+    But in some cases we need simple query params like "uris", "title", etc.
+    Basically the purpose of this filter is to bypass JSONAPI query params validation
+    for "allowed_query_params" query params but leave it for others.
+    Note: "allowed_query_params" must be set for all viewsets that are used in request
+    """
+
+    def filter_queryset(self, request, queryset, view):
+        query_params_to_remove = {}
+        # our custom query params just should be listed in allowed_query_params
+        if hasattr(view, "allowed_query_params"):
+            for query_param, value in request.query_params.items():
+                if query_param in view.allowed_query_params:
+                    query_params_to_remove[query_param] = value
+
+        # remove correct query params from request because they'll fail JSONAPI validation
+        request.query_params._mutable = True
+        for allowed_query_param in query_params_to_remove:
+            request.query_params.pop(allowed_query_param)
+
+        # request.query_params contains query params that aren't listed in allowed_query_params.
+        # So it's either incorrect parameter or JSONAPI query parameter
+        self.validate_query_params(request)
+        # return correct query params for usage in other places
+        request.query_params.update(query_params_to_remove)
+        request.query_params._mutable = False
+        return queryset
diff --git a/addon_service/common/view_only_filter.py b/addon_service/common/view_only_filter.py
deleted file mode 100644
index f5b5e42c..00000000
--- a/addon_service/common/view_only_filter.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from rest_framework.filters import BaseFilterBackend
-
-
-class ViewOnlyFilter(BaseFilterBackend):
-    def filter_queryset(self, request, queryset, view):
-        if "view_only" in request.query_params:
-            pass
-        return queryset
diff --git a/addon_service/configured_addon/views.py b/addon_service/configured_addon/views.py
index 8dbc06d0..fe1189c3 100644
--- a/addon_service/configured_addon/views.py
+++ b/addon_service/configured_addon/views.py
@@ -5,12 +5,11 @@
     SessionUserIsOwnerOrResourceAdmin,
     SessionUserMayConnectAddon,
 )
-from addon_service.common.view_only_filter import ViewOnlyFilter
 from addon_service.common.viewsets import RetrieveWriteDeleteViewSet
 
 
 class ConfiguredAddonViewSet(RetrieveWriteDeleteViewSet):
-    filter_backends = [ViewOnlyFilter]  # Ensures `view_only` is always allowed
+    allowed_query_params = ["view_only"]
 
     def get_permissions(self):
         match self.action:
diff --git a/addon_service/resource_reference/views.py b/addon_service/resource_reference/views.py
index 1549f0e6..7d8eeec2 100644
--- a/addon_service/resource_reference/views.py
+++ b/addon_service/resource_reference/views.py
@@ -2,7 +2,6 @@
 from addon_service.common.viewsets import RestrictedReadOnlyViewSet
 from addon_service.serializers import ResourceReferenceSerializer
 
-from ..common.view_only_filter import ViewOnlyFilter
 from .models import ResourceReference
 
 
@@ -10,6 +9,6 @@ class ResourceReferenceViewSet(RestrictedReadOnlyViewSet):
     queryset = ResourceReference.objects.all()
     serializer_class = ResourceReferenceSerializer
     permission_classes = [SessionUserCanViewReferencedResource]
-    filter_backends = [ViewOnlyFilter]
+    allowed_query_params = ["view_only"]
     # Satisfies requirements of `RestrictedReadOnlyViewSet.list`
     required_list_filter_fields = ("resource_uri",)
diff --git a/addon_service/user_reference/views.py b/addon_service/user_reference/views.py
index 944e4f07..f33b53af 100644
--- a/addon_service/user_reference/views.py
+++ b/addon_service/user_reference/views.py
@@ -11,5 +11,6 @@ class UserReferenceViewSet(RestrictedReadOnlyViewSet):
     permission_classes = [
         SessionUserIsOwner,
     ]
+    allowed_query_params = ["uris"]
     # Satisfies requirements of `RestrictedReadOnlyViewSet.list`
     required_list_filter_fields = ("user_uri",)
diff --git a/app/settings.py b/app/settings.py
index bff9b1a7..a8b8d026 100644
--- a/app/settings.py
+++ b/app/settings.py
@@ -193,7 +193,7 @@
         "rest_framework_json_api.renderers.BrowsableAPIRenderer",
     ),
     "DEFAULT_FILTER_BACKENDS": (
-        "rest_framework_json_api.filters.QueryParameterValidationFilter",
+        "addon_service.common.queryparams_filter.AllowedQueryParamsFilter",
         "rest_framework_json_api.filters.OrderingFilter",
         "rest_framework_json_api.django_filters.DjangoFilterBackend",
         "rest_framework.filters.SearchFilter",


Reviews:
user: login: opaduchak
body: Looks great, please also add this attribute to citation and compute accounts
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-03-04T11:45:43Z
user: login: opaduchak
body: Looks great!
I have one small sugestion, please do it if you have the capacity
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-12T16:15:05Z
user: login: opaduchak
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-14T10:03:32Z
user: login: opaduchak
body: LGTM!

state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-17T14:22:20Z

Review comments:
diff_hunk: @@ -57,6 +57,16 @@ class AuthorizedStorageAccountSerializer(AuthorizedAccountSerializer):
         "authorized_operations": "addon_service.serializers.AddonOperationSerializer",
     }
 
+    configured_storage_addons_uris = serializers.SerializerMethodField()
+
+    def get_configured_storage_addons_uris(self, obj):
+        return obj.configured_storage_addons.select_related(
+            'authorized_resource'
+        ).values_list(
+            'authorized_resource__resource_uri',
+            flat=True
+        )
path: addon_service/authorized_account/storage/serializers.py
user: login: opaduchak
body: this should be defined also for `AuthorizedCitationAccount` and `AuthorizedComputingAccount` 
created_at: 2025-03-04T11:44:55Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,9 @@
+from rest_framework.filters import BaseFilterBackend
+
+
+class URIFilter(BaseFilterBackend):
+
+    def filter_queryset(self, request, queryset, view):
+        if 'uris' in request.query_params:
+            pass
+        return queryset
path: addon_service/common/uri_filter.py
user: login: opaduchak
body: I'd try to make it more universal (to be able to handle arbitrary number of any queryparams)
created_at: 2025-03-12T16:14:16Z
author_association: COLLABORATOR
diff_hunk: @@ -11,5 +11,6 @@ class UserReferenceViewSet(RestrictedReadOnlyViewSet):
     permission_classes = [
         SessionUserIsOwner,
     ]
+    allowed_query_params = ['uris']
path: addon_service/user_reference/views.py
user: login: opaduchak
body: may you also do this for `ViewOnlyFilter` as well, please?
created_at: 2025-03-14T10:03:29Z
author_association: COLLABORATOR

PR comments:
user: login: ihorsokhanexoft
created_at: 2025-03-11T13:06:22Z
body: @opaduchak I noticed that `configured_storage_addons` doesn't use .active() method. Is it correct for other addons accounts?
author_association: CONTRIBUTOR
user: login: opaduchak
created_at: 2025-03-11T13:30:06Z
body: > @opaduchak I noticed that `configured_storage_addons` doesn't use .active() method. Is it correct for other addons accounts?

I believe so, because we are already fetching active accounts
author_association: COLLABORATOR
user: login: ihorsokhanexoft
created_at: 2025-03-11T13:33:48Z
body: > > @opaduchak I noticed that `configured_storage_addons` doesn't use .active() method. Is it correct for other addons accounts?
> 
> I believe so, because we are already fetching active accounts

So, no need to use .active(), right?
author_association: CONTRIBUTOR