Repository: e-valuation/EvaP

Issue:
Weigh results of questions by number of answers
Currently, we calculate a grade distribution for each question, and then calculate an average distribution for groups of questions. If one question has 1 vote, and another question has 10 votes, that one vote will have roughly 10 times the weight of the other votes in that average distribution, which doesn't seem right.


Discussion:

Resolving patch:
diff --git a/evap/results/exporters.py b/evap/results/exporters.py
--- a/evap/results/exporters.py
+++ b/evap/results/exporters.py
@@ -133,21 +133,21 @@ def export(self, response, course_types_list, include_not_enough_voters=False, i
                             continue
                         qn_results = results[questionnaire.id]
                         values = []
-                        total_count = 0
+                        count_sum = 0
                         approval_count = 0
 
                         for grade_result in qn_results:
                             if grade_result.question.id == question.id:
                                 if grade_result.has_answers:
-                                    values.append(grade_result.average * grade_result.total_count)
-                                    total_count += grade_result.total_count
+                                    values.append(grade_result.average * grade_result.count_sum)
+                                    count_sum += grade_result.count_sum
                                     if grade_result.question.is_yes_no_question:
                                         approval_count += grade_result.approval_count
                         if values:
-                            avg = sum(values) / total_count
+                            avg = sum(values) / count_sum
 
                             if question.is_yes_no_question:
-                                percent_approval = approval_count / total_count if total_count > 0 else 0
+                                percent_approval = approval_count / count_sum if count_sum > 0 else 0
                                 writec(self, "{:.0%}".format(percent_approval), self.grade_to_style(avg))
                             else:
                                 writec(self, avg, self.grade_to_style(avg))
diff --git a/evap/results/tools.py b/evap/results/tools.py
--- a/evap/results/tools.py
+++ b/evap/results/tools.py
@@ -56,7 +56,7 @@ def __init__(self, question, counts):
         self.counts = counts
 
     @property
-    def total_count(self):
+    def count_sum(self):
         if not self.is_published:
             return None
         return sum(self.counts)
@@ -72,7 +72,7 @@ def approval_count(self):
     def average(self):
         if not self.has_answers:
             return None
-        return sum(answer * count for answer, count in enumerate(self.counts, start=1)) / self.total_count
+        return sum(answer * count for answer, count in enumerate(self.counts, start=1)) / self.count_sum
 
     @property
     def has_answers(self):
@@ -147,12 +147,12 @@ def normalized_distribution(distribution):
     return tuple((value / distribution_sum) for value in distribution)
 
 
-def avg_distribution(distributions, weights=itertools.repeat(1)):
-    if all(distribution is None for distribution in distributions):
+def avg_distribution(weighted_distributions):
+    if all(distribution is None for distribution, __ in weighted_distributions):
         return None
 
     summed_distribution = [0, 0, 0, 0, 0]
-    for distribution, weight in zip(distributions, weights):
+    for distribution, weight in weighted_distributions:
         if distribution:
             for index, value in enumerate(distribution):
                 summed_distribution[index] += weight * value
@@ -160,11 +160,15 @@ def avg_distribution(distributions, weights=itertools.repeat(1)):
 
 
 def average_grade_questions_distribution(results):
-    return avg_distribution([normalized_distribution(result.counts) for result in results if result.question.is_grade_question])
+    return avg_distribution(
+        [(normalized_distribution(result.counts), result.count_sum) for result in results if result.question.is_grade_question]
+    )
 
 
 def average_non_grade_rating_questions_distribution(results):
-    return avg_distribution([normalized_distribution(result.counts) for result in results if result.question.is_non_grade_rating_question])
+    return avg_distribution(
+        [(normalized_distribution(result.counts), result.count_sum) for result in results if result.question.is_non_grade_rating_question],
+    )
 
 
 def calculate_average_distribution(course):
@@ -180,16 +184,21 @@ def calculate_average_distribution(course):
     course_results = grouped_results.pop(None, [])
 
     average_contributor_distribution = avg_distribution([
-        avg_distribution(
-            [average_grade_questions_distribution(results), average_non_grade_rating_questions_distribution(results)],
-            [settings.CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT, settings.CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT]
-        ) for results in grouped_results.values()
+        (
+            avg_distribution([
+                (average_grade_questions_distribution(contributor_results), settings.CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT),
+                (average_non_grade_rating_questions_distribution(contributor_results), settings.CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT)
+            ]),
+            sum(result.count_sum for result in contributor_results if result.question.is_rating_question)
+        )
+        for contributor_results in grouped_results.values()
     ])
 
-    return avg_distribution(
-        [average_grade_questions_distribution(course_results), average_non_grade_rating_questions_distribution(course_results), average_contributor_distribution],
-        [settings.COURSE_GRADE_QUESTIONS_WEIGHT, settings.COURSE_NON_GRADE_QUESTIONS_WEIGHT, settings.CONTRIBUTIONS_WEIGHT]
-    )
+    return avg_distribution([
+        (average_grade_questions_distribution(course_results), settings.COURSE_GRADE_QUESTIONS_WEIGHT),
+        (average_non_grade_rating_questions_distribution(course_results), settings.COURSE_NON_GRADE_QUESTIONS_WEIGHT),
+        (average_contributor_distribution, settings.CONTRIBUTIONS_WEIGHT)
+    ])
 
 
 def distribution_to_grade(distribution):
diff --git a/evap/results/views.py b/evap/results/views.py
--- a/evap/results/views.py
+++ b/evap/results/views.py
@@ -143,7 +143,7 @@ def add_warnings(course, course_result):
     # calculate the median values of how many people answered a questionnaire across all contributions
     questionnaire_max_answers = defaultdict(list)
     for questionnaire_result in course_result.questionnaire_results:
-        max_answers = max((question_result.total_count for question_result in questionnaire_result.question_results if question_result.question.is_rating_question), default=0)
+        max_answers = max((question_result.count_sum for question_result in questionnaire_result.question_results if question_result.question.is_rating_question), default=0)
         questionnaire_max_answers[questionnaire_result.questionnaire].append(max_answers)
 
     questionnaire_warning_thresholds = {}
@@ -152,11 +152,11 @@ def add_warnings(course, course_result):
 
     for questionnaire_result in course_result.questionnaire_results:
         rating_results = [question_result for question_result in questionnaire_result.question_results if question_result.question.is_rating_question]
-        max_answers = max((rating_result.total_count for rating_result in rating_results), default=0)
+        max_answers = max((rating_result.count_sum for rating_result in rating_results), default=0)
         questionnaire_result.warning = 0 < max_answers < questionnaire_warning_thresholds[questionnaire_result.questionnaire]
 
         for rating_result in rating_results:
-            rating_result.warning = questionnaire_result.warning or rating_result.has_answers and rating_result.total_count < questionnaire_warning_thresholds[questionnaire_result.questionnaire]
+            rating_result.warning = questionnaire_result.warning or rating_result.has_answers and rating_result.count_sum < questionnaire_warning_thresholds[questionnaire_result.questionnaire]
 
 
 def user_can_see_text_answer(user, represented_users, text_answer, public_view=False):


Test patch:
diff --git a/evap/results/tests/test_tools.py b/evap/results/tests/test_tools.py
--- a/evap/results/tests/test_tools.py
+++ b/evap/results/tests/test_tools.py
@@ -50,7 +50,7 @@ def test_calculation_results(self):
         self.assertEqual(len(questionnaire_result.question_results), 1)
         question_result = questionnaire_result.question_results[0]
 
-        self.assertEqual(question_result.total_count, 150)
+        self.assertEqual(question_result.count_sum, 150)
         self.assertAlmostEqual(question_result.average, float(109) / 30)
         self.assertEqual(question_result.counts, (5, 15, 40, 60, 30))
 
@@ -85,6 +85,7 @@ def setUpTestData(cls):
         cls.questionnaire = mommy.make(Questionnaire)
         cls.question_grade = mommy.make(Question, questionnaire=cls.questionnaire, type="G")
         cls.question_likert = mommy.make(Question, questionnaire=cls.questionnaire, type="L")
+        cls.question_likert_2 = mommy.make(Question, questionnaire=cls.questionnaire, type="L")
         cls.general_contribution = cls.course.general_contribution
         cls.general_contribution.questionnaires.set([cls.questionnaire])
         cls.contribution1 = mommy.make(Contribution, contributor=mommy.make(UserProfile), course=cls.course, questionnaires=[cls.questionnaire])
@@ -98,71 +99,74 @@ def test_average_grade(self):
         mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=4, count=2)
         mommy.make(RatingAnswerCounter, question=question_grade2, contribution=self.contribution1, answer=1, count=1)
         mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=3, count=4)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.general_contribution, answer=5, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.general_contribution, answer=5, count=5)
+        mommy.make(RatingAnswerCounter, question=self.question_likert_2, contribution=self.general_contribution, answer=3, count=3)
 
         contributor_weights_sum = settings.CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT + settings.CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT
-        contributor1_average = (settings.CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT * (2 + 1) / 2 + (settings.CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT) * 3) / contributor_weights_sum  # 2.4
+        contributor1_average = ((settings.CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT * ((2 * 1) + (1 * 1)) / (1 + 1)) + (settings.CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT * 3)) / contributor_weights_sum  # 2.4
         contributor2_average = 4
-        contributors_average = (contributor1_average + contributor2_average) / 2  # 3.2
+        contributors_average = (((1 + 1 + 4) * contributor1_average) + (2 * contributor2_average)) / (1 + 1 + 4 + 2)  # 2.8
 
-        course_non_grade_average = 5
+        course_non_grade_average = ((5 * 5) + (3 * 3)) / (5 + 3)  # 4.25
 
         contributors_percentage = settings.CONTRIBUTIONS_WEIGHT / (settings.CONTRIBUTIONS_WEIGHT + settings.COURSE_NON_GRADE_QUESTIONS_WEIGHT)  # 0.375
         course_non_grade_percentage = settings.COURSE_NON_GRADE_QUESTIONS_WEIGHT / (settings.CONTRIBUTIONS_WEIGHT + settings.COURSE_NON_GRADE_QUESTIONS_WEIGHT)  # 0.625
 
-        total_grade = contributors_percentage * contributors_average + course_non_grade_percentage * course_non_grade_average  # 1.2 + 3.125 = 4.325
+        total_grade = contributors_percentage * contributors_average + course_non_grade_percentage * course_non_grade_average  # 1.05 + 2.65625 = 3.70625
 
         average_grade = distribution_to_grade(calculate_average_distribution(self.course))
         self.assertAlmostEqual(average_grade, total_grade)
-        self.assertAlmostEqual(average_grade, 4.325)
+        self.assertAlmostEqual(average_grade, 3.70625)
 
     @override_settings(CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT=4, CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT=6, CONTRIBUTIONS_WEIGHT=3, COURSE_GRADE_QUESTIONS_WEIGHT=2, COURSE_NON_GRADE_QUESTIONS_WEIGHT=5)
     def test_distribution_without_course_grade_question(self):
         mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution1, answer=1, count=1)
         mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution1, answer=3, count=1)
-        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=4, count=2)
-        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=2, count=2)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=3, count=4)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=5, count=4)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.general_contribution, answer=5, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=4, count=1)
+        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=2, count=1)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=3, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=5, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.general_contribution, answer=5, count=5)
+        mommy.make(RatingAnswerCounter, question=self.question_likert_2, contribution=self.general_contribution, answer=3, count=3)
 
         # contribution1: 0.4 * (0.5, 0, 0.5, 0, 0) + 0.6 * (0, 0, 0.5, 0, 0.5) = (0.2, 0, 0.5, 0, 0.3)
         # contribution2: (0, 0.5, 0, 0.5, 0)
-        # contributions: (0.1, 0.25, 0.25, 0.25, 0.15)
+        # contributions: (8 / 10) * (0.2, 0, 0.5, 0, 0.3) + (2 / 10) * (0, 0.5, 0, 0.5, 0) = (0.16, 0.1, 0.4, 0.1, 0.24)
 
-        # course_non_grade: (0, 0, 0, 0, 1)
+        # course_non_grade: (0, 0, 0.375, 0, 0.625)
 
-        # total: 0.375 * (0.1, 0.25, 0.25, 0.25, 0.15) + 0.625 * (0, 0, 0, 0, 1) = (0.0375, 0.09375, 0.09375, 0.09375, 0.68125)
+        # total: 0.375 * (0.16, 0.1, 0.4, 0.1, 0.24) + 0.625 * (0, 0, 0.375, 0, 0.625) = (0.06, 0.0375, 0.384375, 0.0375, 0.480625)
 
         distribution = calculate_average_distribution(self.course)
-        self.assertAlmostEqual(distribution[0], 0.0375)
-        self.assertAlmostEqual(distribution[1], 0.09375)
-        self.assertAlmostEqual(distribution[2], 0.09375)
-        self.assertAlmostEqual(distribution[3], 0.09375)
-        self.assertAlmostEqual(distribution[4], 0.68125)
+        self.assertAlmostEqual(distribution[0], 0.06)
+        self.assertAlmostEqual(distribution[1], 0.0375)
+        self.assertAlmostEqual(distribution[2], 0.384375)
+        self.assertAlmostEqual(distribution[3], 0.0375)
+        self.assertAlmostEqual(distribution[4], 0.480625)
 
     @override_settings(CONTRIBUTOR_GRADE_QUESTIONS_WEIGHT=4, CONTRIBUTOR_NON_GRADE_RATING_QUESTIONS_WEIGHT=6, CONTRIBUTIONS_WEIGHT=3, COURSE_GRADE_QUESTIONS_WEIGHT=2, COURSE_NON_GRADE_QUESTIONS_WEIGHT=5)
     def test_distribution_with_course_grade_question(self):
         mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution1, answer=1, count=1)
         mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution1, answer=3, count=1)
-        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=4, count=2)
-        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=2, count=2)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=3, count=4)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=5, count=4)
-        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.general_contribution, answer=5, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=4, count=1)
+        mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.contribution2, answer=2, count=1)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=3, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.contribution1, answer=5, count=3)
+        mommy.make(RatingAnswerCounter, question=self.question_likert, contribution=self.general_contribution, answer=5, count=5)
+        mommy.make(RatingAnswerCounter, question=self.question_likert_2, contribution=self.general_contribution, answer=3, count=3)
         mommy.make(RatingAnswerCounter, question=self.question_grade, contribution=self.general_contribution, answer=2, count=10)
 
         # contributions and course_non_grade are as above
         # course_grade: (0, 1, 0, 0, 0)
 
-        # total: 0.3 * (0.1, 0.25, 0.25, 0.25, 0.15) + 0.2 * (0, 1, 0, 0, 0) + 0.5 * (0, 0, 0, 0, 1) = (0.03, 0.275, 0.075, 0.075, 0.545)
+        # total: 0.3 * (0.16, 0.1, 0.4, 0.1, 0.24) + 0.2 * (0, 1, 0, 0, 0) + 0.5 * (0, 0, 0.375, 0, 0.625) = (0.048, 0.23, 0.3075, 0.03, 0.3845)
 
         distribution = calculate_average_distribution(self.course)
-        self.assertAlmostEqual(distribution[0], 0.03)
-        self.assertAlmostEqual(distribution[1], 0.275)
-        self.assertAlmostEqual(distribution[2], 0.075)
-        self.assertAlmostEqual(distribution[3], 0.075)
-        self.assertAlmostEqual(distribution[4], 0.545)
+        self.assertAlmostEqual(distribution[0], 0.048)
+        self.assertAlmostEqual(distribution[1], 0.23)
+        self.assertAlmostEqual(distribution[2], 0.3075)
+        self.assertAlmostEqual(distribution[3], 0.03)
+        self.assertAlmostEqual(distribution[4], 0.3845)
 
     def test_single_result_calculate_average_distribution(self):
         single_result_course = mommy.make(Course, state='published')