Repository: apluslms/a-plus

Issue:
Possible performance issue in points cache
(From #880, delayed feedback)

https://github.com/apluslms/a-plus/blob/8a852f917e815876892ffa82eeca3e08d82d4882/exercise/cache/points.py#L178-L181

This could potentially be a performance bottleneck. Querying all exercises one-by-one from the database is typically slow. Though, we can't be certain without performance testing.

If it were faster to fetch all exercises in one query from the database, then this line could pick the right exercise from a dictionary with the id, or something like that. The dictionary would be filled once with all exercises in one query.


Discussion:
@Mankro  I did some quick benchmarking (just simple timing in Python).

In the A+ Manual course, the total time taken by querying exercises one by one is ~200 ms per cache regeneration. After changing this so that all exercises are fetched first, the time dropped to ~20 ms.

---

A much worse bottleneck seems to be the reveal mode DEADLINE_ALL. I tested it by first enabling DEADLINE_ALL for 7 exercises, which increased the total time taken by the following lines from ~50 ms to ~150 ms. After I enabled it for 13 more exercises, the average total time went up to ~350 ms.

https://github.com/apluslms/a-plus/blob/4312c99fbb257d190c37b4589bd28671e32c97cf/exercise/cache/points.py#L182-L185

The reason this mode is so much slower is because it needs to find the maximum deadline deviation received by _any student in the course_. This information cannot be derived from the intermediate cache that is passed to ExerciseRevealState:

https://github.com/apluslms/a-plus/blob/4312c99fbb257d190c37b4589bd28671e32c97cf/exercise/cache/points.py#L183

Instead, it needs to query it from the database:

https://github.com/apluslms/a-plus/blob/4312c99fbb257d190c37b4589bd28671e32c97cf/exercise/reveal_states.py#L88-L90

Seems like this could absolutely kill the CachedPoints regeneration performance, if a course decided to use this reveal mode for all of its exercises. Maybe we should cache the maximum deadline deviations separately, and then we could use that cache when regenerating CachedPoints? Deadline deviations are updated much more rarely than CachedPoints is regenerated.

Resolving patch:
diff --git a/deviations/models.py b/deviations/models.py
--- a/deviations/models.py
+++ b/deviations/models.py
@@ -1,4 +1,5 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
+from typing import Optional
 
 from django.urls import reverse
 from django.db import models
@@ -55,8 +56,17 @@ class Meta(SubmissionRuleDeviation.Meta):
     def get_extra_time(self):
         return timedelta(minutes=self.extra_minutes)
 
-    def get_new_deadline(self):
-        return self.get_normal_deadline() + self.get_extra_time()
+    def get_new_deadline(self, normal_deadline: Optional[datetime] = None) -> datetime:
+        """
+        Returns the new deadline after adding the extra time to the normal
+        deadline.
+
+        The `normal_deadline` argument can be provided if it is known by the
+        caller, to avoid querying it.
+        """
+        if normal_deadline is None:
+            normal_deadline = self.get_normal_deadline()
+        return normal_deadline + self.get_extra_time()
 
     def get_normal_deadline(self):
         return self.exercise.course_module.closing_time
diff --git a/exercise/cache/points.py b/exercise/cache/points.py
--- a/exercise/cache/points.py
+++ b/exercise/cache/points.py
@@ -79,7 +79,6 @@ def _generate_data(
         categories = data['categories']
         total = data['total']
         data['invalidate_time'] = None
-        exercise_revealed = {}
 
         # Augment submission parameters.
         def r_augment(children: List[Dict[str, Any]]) -> None:
@@ -131,80 +130,11 @@ def r_augment(children: List[Dict[str, Any]]) -> None:
         })
 
         if user.is_authenticated:
-            # Augment deviation data.
-            def get_max_deviations(
-                    cls: Type[SubmissionRuleDeviation],
-                    field: str,
-                    ) -> Iterable[Tuple[Dict[str, Any], int]]:
-                # This function gets the maximum deviation of the given type
-                # for the current user, and returns the corresponding cache
-                # item as well as the deviation value.
-                results = (
-                    cls.objects.filter(
-                        Q(exercise__course_module__course_instance=instance)
-                        & (
-                            Q(submitter=user.userprofile)
-                            | Q(exercise__submissions__submitters=user.userprofile)
-                        )
-                    )
-                    .values('exercise')
-                    .annotate(max_deviation=Max(field))
-                )
-
-                for result in results:
-                    try:
-                        tree = self._by_idx(modules, exercise_index[result['exercise']])
-                    except KeyError:
-                        self.dirty = True
-                        continue
-                    entry = tree[-1]
-                    yield entry, result['max_deviation']
-
-            for entry, extra_minutes in (
-                get_max_deviations(DeadlineRuleDeviation, 'extra_minutes')
-            ):
-                entry['personal_deadline'] = (
-                    entry['closing_time'] + datetime.timedelta(minutes=extra_minutes)
-                )
-
-            for entry, extra_submissions in (
-                get_max_deviations(MaxSubmissionsRuleDeviation, 'extra_submissions')
-            ):
-                entry['personal_max_submissions'] = (
-                    entry['max_submissions'] + extra_submissions
-                )
-
-            # Augment exercise reveal rules.
-            def r_reveal_feedback(children: List[Dict[str, Any]]) -> None:
-                for entry in children:
-                    if entry['submittable']:
-                        exercise = BaseExercise.objects.get(id=entry['id'])
-                        rule = exercise.active_submission_feedback_reveal_rule
-                        state = ExerciseRevealState(entry)
-                        is_revealed = rule.is_revealed(state)
-                        reveal_time = rule.get_reveal_time(state)
-                        entry['feedback_revealed'] = is_revealed
-                        entry['feedback_reveal_time'] = reveal_time
-                        exercise_revealed[exercise.id] = is_revealed, reveal_time
-                        if (
-                            reveal_time is not None
-                            and reveal_time > timezone.now()
-                            and (
-                                data['invalidate_time'] is None
-                                or reveal_time < data['invalidate_time']
-                            )
-                        ):
-                            data['invalidate_time'] = reveal_time
-                    r_reveal_feedback(entry.get('children', []))
-            if not self.is_staff:
-                for module in modules:
-                    r_reveal_feedback(module['children'])
-
             # Augment submission data.
             submissions = (
                 user.userprofile.submissions.exclude_errors()
                 .filter(exercise__course_module__course_instance=instance)
-                .prefetch_related('exercise')
+                .prefetch_related('exercise', 'notifications')
                 .only('id', 'exercise', 'submission_time', 'status', 'grade', 'force_exercise_points')
             )
             for submission in submissions:
@@ -214,11 +144,6 @@ def r_reveal_feedback(children: List[Dict[str, Any]]) -> None:
                     self.dirty = True
                     continue
                 entry = tree[-1]
-                if self.is_staff:
-                    is_revealed = True
-                    reveal_time = None
-                else:
-                    is_revealed, reveal_time = exercise_revealed[submission.exercise.id]
                 ready = submission.status == Submission.STATUS.READY
                 unofficial = submission.status == Submission.STATUS.UNOFFICIAL
                 if ready or submission.status in (Submission.STATUS.WAITING, Submission.STATUS.INITIALIZED):
@@ -230,16 +155,14 @@ def r_reveal_feedback(children: List[Dict[str, Any]]) -> None:
                     'points_to_pass': entry['points_to_pass'],
                     'confirm_the_level': entry.get('confirm_the_level', False),
                     'submission_count': 1, # to fool points badge
-                    'points': submission.grade if is_revealed else 0,
-                    'formatted_points': format_points(submission.grade, is_revealed, False),
+                    'points': submission.grade,
+                    'formatted_points': format_points(submission.grade, True, False),
                     'graded': submission.is_graded, # TODO: should this be official (is_graded = ready or unofficial)
-                    'passed': (submission.grade >= entry['points_to_pass']) if is_revealed else False,
+                    'passed': (submission.grade >= entry['points_to_pass']),
                     'submission_status': submission.status if not submission.is_graded else False,
                     'unofficial': unofficial,
                     'date': submission.submission_time,
                     'url': submission.get_url('submission-plain'),
-                    'feedback_revealed': is_revealed,
-                    'feedback_reveal_time': reveal_time,
                 })
                 # TODO: implement way to select algorithm for the best
                 if submission.exercise.grading_mode == BaseExercise.GRADING_MODE.BEST:
@@ -262,9 +185,9 @@ def r_reveal_feedback(children: List[Dict[str, Any]]) -> None:
                     entry.update({
                         'best_submission': submission.id,
                         'submission_date': submission.submission_time,
-                        'points': submission.grade if is_revealed else 0,
-                        'formatted_points': format_points(submission.grade, is_revealed, False),
-                        'passed': (ready and submission.grade >= entry['points_to_pass']) if is_revealed else False,
+                        'points': submission.grade,
+                        'formatted_points': format_points(submission.grade, True, False),
+                        'passed': (ready and submission.grade >= entry['points_to_pass']),
                         'graded': True,
                         'unofficial': False,
                         'forced_points': True,
@@ -283,9 +206,9 @@ def r_reveal_feedback(children: List[Dict[str, Any]]) -> None:
                         entry.update({
                             'best_submission': submission.id,
                             'submission_date': submission.submission_time,
-                            'points': submission.grade if is_revealed else 0,
-                            'formatted_points': format_points(submission.grade, is_revealed, False),
-                            'passed': (ready and submission.grade >= entry['points_to_pass']) if is_revealed else False,
+                            'points': submission.grade,
+                            'formatted_points': format_points(submission.grade, True, False),
+                            'passed': (ready and submission.grade >= entry['points_to_pass']),
                             'graded': ready, # != unofficial
                             'unofficial': unofficial,
                         })
@@ -294,6 +217,95 @@ def r_reveal_feedback(children: List[Dict[str, Any]]) -> None:
                     if submission.notifications.filter(seen=False).count() > 0:
                         entry['unseen'] = True
 
+            # Augment deviation data.
+            def get_max_deviations(
+                    cls: Type[SubmissionRuleDeviation],
+                    field: str,
+                    ) -> Iterable[Tuple[Dict[str, Any], int]]:
+                # This function gets the maximum deviation of the given type
+                # for the current user, and returns the corresponding cache
+                # item as well as the deviation value.
+                results = (
+                    cls.objects.filter(
+                        Q(exercise__course_module__course_instance=instance)
+                        & (
+                            Q(submitter=user.userprofile)
+                            | Q(exercise__submissions__submitters=user.userprofile)
+                        )
+                    )
+                    .values('exercise')
+                    .annotate(max_deviation=Max(field))
+                )
+
+                for result in results:
+                    try:
+                        tree = self._by_idx(modules, exercise_index[result['exercise']])
+                    except KeyError:
+                        self.dirty = True
+                        continue
+                    entry = tree[-1]
+                    yield entry, result['max_deviation']
+
+            for entry, extra_minutes in (
+                get_max_deviations(DeadlineRuleDeviation, 'extra_minutes')
+            ):
+                entry['personal_deadline'] = (
+                    entry['closing_time'] + datetime.timedelta(minutes=extra_minutes)
+                )
+
+            for entry, extra_submissions in (
+                get_max_deviations(MaxSubmissionsRuleDeviation, 'extra_submissions')
+            ):
+                entry['personal_max_submissions'] = (
+                    entry['max_submissions'] + extra_submissions
+                )
+
+            # Augment exercise reveal rules.
+            if not self.is_staff:
+                for exercise in (
+                    BaseExercise.objects
+                    .filter(course_module__course_instance=instance)
+                    .prefetch_related('submission_feedback_reveal_rule')
+                    .only('id', 'submission_feedback_reveal_rule')
+                ):
+                    try:
+                        tree = self._by_idx(modules, exercise_index[exercise.id])
+                    except KeyError:
+                        self.dirty = True
+                        continue
+                    entry = tree[-1]
+                    rule = exercise.active_submission_feedback_reveal_rule
+                    state = ExerciseRevealState(entry)
+                    is_revealed = rule.is_revealed(state)
+                    reveal_time = rule.get_reveal_time(state)
+
+                    entry.update({
+                        'points': entry['points'] if is_revealed else 0,
+                        'formatted_points': format_points(entry['points'], is_revealed, False),
+                        'passed': entry['passed'] if is_revealed else False,
+                        'feedback_revealed': is_revealed,
+                        'feedback_reveal_time': reveal_time,
+                    })
+
+                    for submission in entry['submissions']:
+                        submission.update({
+                            'points': submission['points'] if is_revealed else 0,
+                            'formatted_points': format_points(submission['points'], is_revealed, False),
+                            'passed': submission['passed'] if is_revealed else False,
+                            'feedback_revealed': is_revealed,
+                            'feedback_reveal_time': reveal_time,
+                        })
+
+                    if (
+                        reveal_time is not None
+                        and reveal_time > timezone.now()
+                        and (
+                            data['invalidate_time'] is None
+                            or reveal_time < data['invalidate_time']
+                        )
+                    ):
+                        data['invalidate_time'] = reveal_time
+
         # Confirm points.
         def r_check(parent: Dict[str, Any], children: List[Dict[str, Any]]) -> None:
             for entry in children:
diff --git a/exercise/reveal_states.py b/exercise/reveal_states.py
--- a/exercise/reveal_states.py
+++ b/exercise/reveal_states.py
@@ -3,6 +3,7 @@
 
 from django.contrib.auth.models import User
 
+from deviations.models import DeadlineRuleDeviation
 from .exercise_models import BaseExercise
 
 
@@ -60,6 +61,9 @@ def __init__(
         else:
             self.cache = exercise
 
+        self.max_deviation_fetched: bool = False
+        self.max_deviation: Optional[DeadlineRuleDeviation] = None
+
     def get_points(self) -> Optional[int]:
         return self.cache['points']
 
@@ -84,12 +88,19 @@ def get_deadline(self) -> Optional[datetime.datetime]:
 
     def get_latest_deadline(self) -> Optional[datetime.datetime]:
         deadlines = self._get_common_deadlines()
-        exercise = BaseExercise.objects.get(id=self.cache['id'])
-        # This is the only thing that we don't get from the cache.
-        # Caching this could be considered.
-        deviation = exercise.deadlineruledeviation_set.order_by('-extra_minutes').first()
-        if deviation is not None:
-            deadlines.append(deviation.get_new_deadline())
+        # This is the only thing that we don't get from CachedPoints. It is
+        # cached within this object, though, so it won't have to be fetched
+        # again if RevealRule.is_revealed and RevealRule.get_reveal_time are
+        # called separately.
+        if not self.max_deviation_fetched:
+            self.max_deviation = (
+                DeadlineRuleDeviation.objects
+                .filter(exercise_id=self.cache['id'])
+                .order_by('-extra_minutes').first()
+            )
+            self.max_deviation_fetched = True
+        if self.max_deviation is not None:
+            deadlines.append(self.max_deviation.get_new_deadline(self.cache['closing_time']))
         return max(deadlines)
 
     def _get_common_deadlines(self) -> List[datetime.datetime]:


Test patch:
diff --git a/deviations/tests.py b/deviations/tests.py
--- a/deviations/tests.py
+++ b/deviations/tests.py
@@ -72,5 +72,8 @@ def test_deadline_rule_deviation_extra_time(self):
     def test_deadline_rule_deviation_new_deadline(self):
         self.assertEqual(self.two_days_from_now, self.deadline_rule_deviation.get_new_deadline())
 
+    def test_deadline_rule_deviation_new_deadline_with_normal_deadline(self):
+        self.assertEqual(self.tomorrow, self.deadline_rule_deviation.get_new_deadline(self.today))
+
     def test_deadline_rule_deviation_normal_deadline(self):
         self.assertEqual(self.tomorrow, self.deadline_rule_deviation.get_normal_deadline())