Repository: learningequality/kolibri

Issue:
Unattempted answers in a quiz should not show up as difficult questions?
### Observed behavior
Unattempted answers in a quiz show up as _difficult questions_  immediately upon learner opening the quiz.

![unattempted-diffque](https://user-images.githubusercontent.com/1457929/58970422-b2349200-87b9-11e9-9231-e55c60880406.gif)

In the above example all 5 questions are categorized as difficult even though the second learner has just started the quiz, and haven't even viewed the questions 2-5.


### Expected behavior
Label questions as difficult _only_ after the quiz has been submitted? Or more complex one would be to have the `viewed and skipped more than once` as the threshold for unattempted question labeled as difficult?

### User-facing consequences
Coaches getting overestimates about the number of difficult questions. 

### Errors and logs

…

### Steps to reproduce

…

### Context

 * Kolibri version: 0.12.4b6 upgraded from 0.10
 * Operating system: Ubuntu 16.04
 * Browser: all

cc @rtibbles @indirectlylit (I did not put the milestone, in case the fix is out of the scope for now)


Discussion:

Resolving patch:
diff --git a/kolibri/plugins/coach/api.py b/kolibri/plugins/coach/api.py
--- a/kolibri/plugins/coach/api.py
+++ b/kolibri/plugins/coach/api.py
@@ -2,6 +2,7 @@
 
 from django.db.models import Count
 from django.db.models import F
+from django.db.models import Q
 from django.db.models import Sum
 from rest_framework import pagination
 from rest_framework import permissions
@@ -328,7 +329,12 @@ def retrieve(self, request, pk):
         Get the difficult questions for a particular quiz.
         """
         group_id = request.GET.get("group_id", None)
-        queryset = ExamAttemptLog.objects.filter(examlog__exam=pk)
+        # Only return logs when the learner has submitted the Quiz OR
+        # the coach has deactivated the Quiz. Do not return logs when Quiz is still
+        # in-progress.
+        queryset = ExamAttemptLog.objects.filter(
+            Q(examlog__closed=True) | Q(examlog__exam__active=False), examlog__exam=pk
+        )
         if group_id is not None:
             queryset = HierarchyRelationsFilter(queryset).filter_by_hierarchy(
                 ancestor_collection=group_id, target_user=F("user")
@@ -339,10 +345,15 @@ def retrieve(self, request, pk):
         data = queryset.values("item", "content_id").annotate(correct=Sum("correct"))
 
         # Instead of inferring the totals from the number of logs, use the total
-        # number of people who took the exam as our guide, as people who started the exam
+        # number of people who submitted (if quiz is active) or started the exam
+        # (if quiz is inactive) as our guide, as people who started the exam
         # but did not attempt the question are still important.
         total = (
-            HierarchyRelationsFilter(ExamLog.objects.filter(exam_id=pk))
+            HierarchyRelationsFilter(
+                ExamLog.objects.filter(
+                    Q(closed=True) | Q(exam__active=False), exam_id=pk
+                )
+            )
             .filter_by_hierarchy(
                 ancestor_collection=collection_id, target_user=F("user")
             )
diff --git a/kolibri/plugins/coach/class_summary_api.py b/kolibri/plugins/coach/class_summary_api.py
--- a/kolibri/plugins/coach/class_summary_api.py
+++ b/kolibri/plugins/coach/class_summary_api.py
@@ -239,6 +239,7 @@ class Meta:
             "data_model_version",
             "question_count",
             "learners_see_fixed_order",
+            "seed",
         )
 
 


Test patch:
diff --git a/kolibri/plugins/coach/test/test_difficult_questions.py b/kolibri/plugins/coach/test/test_difficult_questions.py
--- a/kolibri/plugins/coach/test/test_difficult_questions.py
+++ b/kolibri/plugins/coach/test/test_difficult_questions.py
@@ -486,11 +486,20 @@ def setUp(self):
             learner_group=self.group,
         )
 
+        self.classroom_group_learner_2 = helpers.create_learner(
+            username="classroom_group_learner_2",
+            password=DUMMY_PASSWORD,
+            facility=self.facility,
+            classroom=self.classroom,
+            learner_group=self.group,
+        )
+
         self.quiz = Exam.objects.create(
             title="My Lesson",
             creator=self.facility_and_classroom_coach,
             collection=self.classroom,
             question_count=5,
+            active=False,
         )
         self.assignment_1 = ExamAssignment.objects.create(
             exam=self.quiz,
@@ -500,46 +509,38 @@ def setUp(self):
         self.quiz_difficulties_basename = "kolibri:coach:quizdifficulties"
         self.content_id = "25f32edcec565396a1840c5413c92451"
 
-    def test_learner_cannot_access(self):
-        self.client.login(username=self.learner.username, password=DUMMY_PASSWORD)
-        response = self.client.get(
+    def _get_quiz_difficulties(self, for_group=False):
+        data = {"group_id": self.group.id} if for_group else {}
+        return self.client.get(
             reverse(
                 self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
+            ),
+            data=data,
+        )
+
+    def _login_as_coach(self):
+        self.client.login(
+            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
         )
+
+    def test_learner_cannot_access(self):
+        self.client.login(username=self.learner.username, password=DUMMY_PASSWORD)
+        response = self._get_quiz_difficulties()
         self.assertEqual(response.status_code, 403)
 
     def test_learner_cannot_access_by_group_id(self):
         self.client.login(username=self.learner.username, password=DUMMY_PASSWORD)
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            ),
-            data={"group_id": self.group.id},
-        )
+        response = self._get_quiz_difficulties(for_group=True)
         self.assertEqual(response.status_code, 403)
 
     def test_coach_no_progress(self):
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
         self.assertEqual(len(response.data), 0)
 
     def test_coach_no_progress_by_group_id(self):
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            ),
-            data={"group_id": self.group.id},
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties(for_group=True)
         self.assertEqual(len(response.data), 0)
 
     def _set_one_difficult(self, user):
@@ -558,28 +559,63 @@ def _set_one_difficult(self, user):
 
     def test_coach_one_difficult(self):
         self._set_one_difficult(self.classroom_group_learner)
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
         self.assertEqual(len(response.data), 1)
         self.assertEqual(response.data[0]["total"], 1)
         self.assertEqual(response.data[0]["correct"], 0)
 
-    def test_coach_one_two_started_difficult(self):
+    def test_active_and_unsubmitted_quizzes_are_not_returned(self):
         self._set_one_difficult(self.classroom_group_learner)
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
+
+        # Reactivate exam, but flag learner as not having submitted it
+        self.quiz.active = True
+        self.quiz.save()
+        self.examlog.closed = False
+        self.examlog.save()
+
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
+        self.assertEqual(len(response.data), 0)
+
+    def test_active_and_submtted_quizzes_are_returned(self):
+        self._set_one_difficult(self.classroom_group_learner)
+
+        # Reactivate exam, and flag learner as having submitted it
+        self.quiz.active = True
+        self.quiz.save()
+        self.examlog.closed = True
+        self.examlog.save()
+
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
+        self.assertEqual(len(response.data), 1)
+        self.assertEqual(response.data[0]["total"], 1)
+        self.assertEqual(response.data[0]["correct"], 0)
+
+    def test_submitted_quizzes_are_in_total(self):
+        self._set_one_difficult(self.classroom_group_learner)
+
+        self._login_as_coach()
+
+        # Reactivate quiz and simulate 2 quiz submissions.
+        self.quiz.active = True
+        self.quiz.save()
+        self.examlog.closed = True
+        self.examlog.save()
+        ExamLog.objects.create(
+            user=self.classroom_group_learner_2, exam=self.quiz, closed=True
         )
+
+        response = self._get_quiz_difficulties()
+        self.assertEqual(len(response.data), 1)
+        self.assertEqual(response.data[0]["total"], 2)
+        self.assertEqual(response.data[0]["correct"], 0)
+
+    def test_coach_one_two_started_difficult(self):
+        self._set_one_difficult(self.classroom_group_learner)
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
         self.assertEqual(len(response.data), 1)
         self.assertEqual(response.data[0]["total"], 1)
         self.assertEqual(response.data[0]["correct"], 0)
@@ -591,29 +627,16 @@ def test_coach_one_difficult_repeated_assignment(self):
             collection=self.group,
         )
         self._set_one_difficult(self.classroom_group_learner)
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
         self.assertEqual(len(response.data), 1)
         self.assertEqual(response.data[0]["total"], 1)
         self.assertEqual(response.data[0]["correct"], 0)
 
     def test_coach_one_difficult_by_group_id(self):
         self._set_one_difficult(self.classroom_group_learner)
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            ),
-            data={"group_id": self.group.id},
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties(for_group=True)
         self.assertEqual(len(response.data), 1)
         self.assertEqual(response.data[0]["total"], 1)
         self.assertEqual(response.data[0]["correct"], 0)
@@ -630,14 +653,8 @@ def test_coach_two_difficult(self):
             item="notatest",
             content_id=self.content_id,
         )
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
         self.assertEqual(len(response.data), 2)
         self.assertEqual(response.data[0]["total"], 1)
         self.assertEqual(response.data[0]["correct"], 0)
@@ -656,14 +673,8 @@ def test_coach_one_difficult_one_not(self):
             item="notatest",
             content_id=self.content_id,
         )
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            )
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties()
         self.assertEqual(len(response.data), 2)
         self.assertTrue(
             any(map(lambda x: x["total"] == 1 and x["correct"] == 0, response.data))
@@ -684,15 +695,8 @@ def test_coach_difficult_by_group_id(self):
             item="notatest",
             content_id=self.content_id,
         )
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            ),
-            data={"group_id": self.group.id},
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties(for_group=True)
         self.assertEqual(len(response.data), 2)
         self.assertTrue(
             any(map(lambda x: x["total"] == 1 and x["correct"] == 0, response.data))
@@ -708,15 +712,8 @@ def test_coach_difficult_both_assigned_by_group_id(self):
         )
         self.classroom.add_member(learner2)
         self._set_one_difficult(learner2)
-        self.client.login(
-            username=self.facility_and_classroom_coach.username, password=DUMMY_PASSWORD
-        )
-        response = self.client.get(
-            reverse(
-                self.quiz_difficulties_basename + "-detail", kwargs={"pk": self.quiz.id}
-            ),
-            data={"group_id": self.group.id},
-        )
+        self._login_as_coach()
+        response = self._get_quiz_difficulties(for_group=True)
         self.assertEqual(len(response.data), 1)
         self.assertEqual(response.data[0]["total"], 1)
         self.assertEqual(response.data[0]["correct"], 0)