Repository: intelowlproject/GreedyBear
Issue category: Performance Issue

Issue:
Feed API slow
Querying the API, e.g. `/api/feeds/all/all/recent.json`, takes a while. Requesting the feed on my GreedyBear instancees with `curl -so /dev/null -w '%{time_total}\n' http://HOSTNAME/api/feeds/all/all/recent.json` takes about 6 seconds. This also produces significant CPU load on the server. However the Honeynet instance seems to be much quicker. Is it just an issue of my installations?


Hints:

Patch:
diff --git a/api/serializers.py b/api/serializers.py
index 937ce0f..4d2d676 100644
--- a/api/serializers.py
+++ b/api/serializers.py
@@ -47,14 +47,10 @@ def validate(self, data):
         return data
 
 
-def feed_type_validation(feed_type):
-    feed_choices = ["log4j", "cowrie", "all"]
-    generalHoneypots = GeneralHoneypot.objects.all().filter(active=True)
-    feed_choices.extend([hp.name.lower() for hp in generalHoneypots])  # FEEDS
-
-    if feed_type not in feed_choices:
-        logger.info(f"Feed type {feed_type} not in feed_choices {feed_choices}")
-        raise serializers.ValidationError("Invalid feed_type")
+def feed_type_validation(feed_type: str, valid_feed_types: frozenset) -> str:
+    if feed_type not in valid_feed_types:
+        logger.info(f"Feed type {feed_type} not in feed_choices {valid_feed_types}")
+        raise serializers.ValidationError(f"Invalid feed_type: {feed_type}")
     return feed_type
 
 
@@ -66,7 +62,7 @@ class FeedsSerializer(serializers.Serializer):
 
     def validate_feed_type(self, feed_type):
         logger.debug(f"FeedsSerializer - Validation feed_type: '{feed_type}'")
-        return feed_type_validation(feed_type)
+        return feed_type_validation(feed_type, self.context["valid_feed_types"])
 
 
 class FeedsResponseSerializer(serializers.Serializer):
@@ -80,4 +76,4 @@ class FeedsResponseSerializer(serializers.Serializer):
 
     def validate_feed_type(self, feed_type):
         logger.debug(f"FeedsResponseSerializer - validation feed_type: '{feed_type}'")
-        return feed_type_validation(feed_type)
+        return feed_type_validation(feed_type, self.context["valid_feed_types"])
diff --git a/api/views.py b/api/views.py
index d40c1c2..c929d31 100644
--- a/api/views.py
+++ b/api/views.py
@@ -13,6 +13,7 @@
 from django.http import HttpResponse, HttpResponseServerError, StreamingHttpResponse
 from greedybear.consts import FEEDS_LICENSE, GET, PAYLOAD_REQUEST, SCANNER
 from greedybear.models import IOC, GeneralHoneypot, Statistics, viewType
+from greedybear.settings import SKIP_FEED_VALIDATION
 from rest_framework import status, viewsets
 from rest_framework.decorators import action, api_view, authentication_classes, permission_classes
 from rest_framework.permissions import IsAuthenticated
@@ -56,8 +57,11 @@ def feeds(request, feed_type, attack_type, age, format_):
     """
     logger.info(f"request /api/feeds with params: feed type: {feed_type}, " f"attack_type: {attack_type}, Age: {age}, format: {format_}")
 
-    iocs_queryset = get_queryset(request, feed_type, attack_type, age, format_)
-    return feeds_response(request, iocs_queryset, feed_type, format_)
+    general_honeypots = GeneralHoneypot.objects.all().filter(active=True)
+    valid_feed_types = frozenset(["log4j", "cowrie", "all"] + [hp.name.lower() for hp in general_honeypots])
+
+    iocs_queryset = get_queryset(request, feed_type, valid_feed_types, attack_type, age, format_)
+    return feeds_response(request, iocs_queryset, feed_type, valid_feed_types, format_)
 
 
 @api_view([GET])
@@ -74,26 +78,31 @@ def feeds_pagination(request):
     params = request.query_params
     logger.info(f"request /api/feeds with params: {params}")
 
+    general_honeypots = GeneralHoneypot.objects.all().filter(active=True)
+    valid_feed_types = frozenset(["log4j", "cowrie", "all"] + [hp.name.lower() for hp in general_honeypots])
+
     paginator = CustomPageNumberPagination()
     iocs_queryset = get_queryset(
         request,
         params["feed_type"],
+        valid_feed_types,
         params["attack_type"],
         params["age"],
         "json",
     )
     iocs = paginator.paginate_queryset(iocs_queryset, request)
-    resp_data = feeds_response(request, iocs, params["feed_type"], "json", dict_only=True)
+    resp_data = feeds_response(request, iocs, params["feed_type"], valid_feed_types, "json", dict_only=True)
     return paginator.get_paginated_response(resp_data)
 
 
-def get_queryset(request, feed_type, attack_type, age, format_):
+def get_queryset(request, feed_type, valid_feed_types, attack_type, age, format_):
     """
     Build a queryset to filter IOC data based on the request parameters.
 
     Args:
         request: The incoming request object.
         feed_type (str): Type of feed (e.g., log4j, cowrie, etc.).
+        valid_feed_types (frozenset): The set of all valid feed types.
         attack_type (str): Type of attack (e.g., all, specific attack types).
         age (str): Age of the data to filter (e.g., recent, persistent).
         format_ (str): Desired format of the response (e.g., json, csv, txt).
@@ -110,7 +119,8 @@ def get_queryset(request, feed_type, attack_type, age, format_):
             "attack_type": attack_type,
             "age": age,
             "format": format_,
-        }
+        },
+        context={"valid_feed_types": valid_feed_types},
     )
     serializer.is_valid(raise_exception=True)
 
@@ -141,7 +151,7 @@ def get_queryset(request, feed_type, attack_type, age, format_):
         # ordering by "feed_type" is done in feed_response function
         if ordering is None or ordering == "feed_type" or ordering == "-feed_type":
             ordering = "-last_seen"
-        iocs = IOC.objects.exclude(general_honeypot__active=False).filter(**query_dict).order_by(ordering)[:5000]
+        iocs = IOC.objects.exclude(general_honeypot__active=False).filter(**query_dict).order_by(ordering).prefetch_related("general_honeypot")[:5000]
     elif age == "persistent":
         # scanners detected in the last 14 days
         fourteen_days_ago = datetime.utcnow() - timedelta(days=14)
@@ -153,7 +163,7 @@ def get_queryset(request, feed_type, attack_type, age, format_):
         # ordering by "feed_type" is done in feed_response function
         if ordering is None or ordering == "feed_type" or ordering == "-feed_type":
             ordering = "-times_seen"
-        iocs = IOC.objects.exclude(general_honeypot__active=False).filter(**query_dict).order_by(ordering)[:5000]
+        iocs = IOC.objects.exclude(general_honeypot__active=False).filter(**query_dict).order_by(ordering).prefetch_related("general_honeypot")[:5000]
 
     # save request source for statistics
     source_ip = str(request.META["REMOTE_ADDR"])
@@ -164,7 +174,7 @@ def get_queryset(request, feed_type, attack_type, age, format_):
     return iocs
 
 
-def feeds_response(request, iocs, feed_type, format_, dict_only=False):
+def feeds_response(request, iocs, feed_type, valid_feed_types, format_, dict_only=False):
     """
     Format the IOC data into the requested format (e.g., JSON, CSV, TXT).
 
@@ -172,6 +182,7 @@ def feeds_response(request, iocs, feed_type, format_, dict_only=False):
         request: The incoming request object.
         iocs (QuerySet): The filtered queryset of IOC data.
         feed_type (str): Type of feed (e.g., log4j, cowrie, etc.).
+        valid_feed_types (frozenset): The set of all valid feed types.
         format_ (str): Desired format of the response (e.g., json, csv, txt).
 
     Returns:
@@ -214,8 +225,8 @@ def feeds_response(request, iocs, feed_type, format_, dict_only=False):
                 elif ioc.cowrie:
                     ioc_feed_type = "cowrie"
                 else:
-                    ioc_feed_type = str(ioc.general_honeypot.first()).lower()
-
+                    # first() can not be used here because is does not work with prefetching
+                    ioc_feed_type = str(ioc.general_honeypot.all()[0]).lower()
             data_ = {
                 "value": ioc.name,
                 SCANNER: ioc.scanner,
@@ -226,7 +237,13 @@ def feeds_response(request, iocs, feed_type, format_, dict_only=False):
                 "feed_type": ioc_feed_type,
             }
 
-            serializer_item = FeedsResponseSerializer(data=data_)
+            if SKIP_FEED_VALIDATION:
+                json_list.append(data_)
+                continue
+            serializer_item = FeedsResponseSerializer(
+                data=data_,
+                context={"valid_feed_types": valid_feed_types},
+            )
             serializer_item.is_valid(raise_exception=True)
             json_list.append(serializer_item.data)
 
diff --git a/docker/env_file_template b/docker/env_file_template
index 7fe51ce..8eb3d0a 100644
--- a/docker/env_file_template
+++ b/docker/env_file_template
@@ -41,3 +41,6 @@ MOCK_CONNECTIONS=False
 
 # True for public deployment, False for internal deployment
 PUBLIC_DEPLOYMENT=False
+
+# True for quicker feed generation, may lead to invalid feeds
+SKIP_FEED_VALIDATION=False
diff --git a/greedybear/settings.py b/greedybear/settings.py
index 93f3945..7146a48 100644
--- a/greedybear/settings.py
+++ b/greedybear/settings.py
@@ -405,3 +405,6 @@
         EMAIL_PORT = os.environ.get("EMAIL_PORT")
         EMAIL_USE_TLS = os.environ.get("EMAIL_USE_TLS", False) == "True"
         EMAIL_USE_SSL = os.environ.get("EMAIL_USE_SSL", False) == "True"
+
+
+SKIP_FEED_VALIDATION = os.environ.get("SKIP_FEED_VALIDATION", False) == "True"


Test patch:
diff --git a/tests/test_serializers.py b/tests/test_serializers.py
index b8e2dbc..c538a1b 100644
--- a/tests/test_serializers.py
+++ b/tests/test_serializers.py
@@ -31,13 +31,20 @@ def test_valid_fields(self):
 
         for element in valid_data_choices:
             data_ = {"feed_type": element[0], "attack_type": element[1], "age": element[2], "format": element[3]}
-            serializer = FeedsSerializer(data=data_)
+            serializer = FeedsSerializer(
+                data=data_,
+                context={"valid_feed_types": frozenset(feed_type_choices)},
+            )
             valid = serializer.is_valid(raise_exception=True)
             self.assertEqual(valid, True)
 
     def test_invalid_fields(self):
+        valid_feed_types = frozenset(["all", "log4j", "cowrie", "adbhoney"])
         data_ = {"feed_type": "invalid_feed_type", "attack_type": "invalid_attack_type", "age": "invalid_age", "format": "invalid_format"}
-        serializer = FeedsSerializer(data=data_)
+        serializer = FeedsSerializer(
+            data=data_,
+            context={"valid_feed_types": valid_feed_types},
+        )
         try:
             serializer.is_valid(raise_exception=True)
         except ValidationError:
@@ -78,11 +85,15 @@ def test_valid_fields(self):
                 "times_seen": "5",
                 "feed_type": element[2],
             }
-            serializer = FeedsResponseSerializer(data=data_)
+            serializer = FeedsResponseSerializer(
+                data=data_,
+                context={"valid_feed_types": frozenset(feed_type_choices)},
+            )
             valid = serializer.is_valid(raise_exception=True)
             self.assertEqual(valid, True)
 
     def test_invalid_fields(self):
+        valid_feed_types = frozenset(["all", "log4j", "cowrie", "adbhoney"])
         data_ = {
             "value": True,
             SCANNER: "invalid_scanner",
@@ -92,7 +103,10 @@ def test_invalid_fields(self):
             "times_seen": "invalid_times_seen",
             "feed_type": "invalid_feed_type",
         }
-        serializer = FeedsResponseSerializer(data=data_)
+        serializer = FeedsResponseSerializer(
+            data=data_,
+            context={"valid_feed_types": valid_feed_types},
+        )
         try:
             serializer.is_valid(raise_exception=True)
         except ValidationError: