Repository: e-valuation/EvaP

Issue:
Anonymization script: scramble evaluation results
This is a follow up to #1287. We came up with the idea of scrambling evaluation results as a part of the anonymization. Quote from @janno42:

"then @karyon and i discussed that it is better to randomize answers because there might be the following case: users who have access to production might find unpublished courses in the anonymized data and can find results there based on answer distribution of other courses and de-anonymizing contributor's names. this is a very special case and there wouldn't be much data available as a course is only then not published if less than two participants voted for it - but nevertheless there might be data leaked.
so we decided that answer counters should be set randomly on anonymization. the answer count per question should (roughly) stay the same during the process and it must be made sure that the answer count never exceeds the number of voters in the course."

We had two implementations flying around in that debate:
- One summed up the answer count, iterated over the RatingAnswerCounters and then took a random number between 0 and the remaining answers and assigned it to the current counter. This can be found [here](https://github.com/fsr-de/EvaP/pull/1287/commits/2c20342a61409f560aeea93a0d77e2bec9a5ee42#diff-8966c64d41a29b22e8ee43d34675dc33R220) but is biased towards giving higher vote counts to the answer counters we are looking at earlier
- The other one was given [here](https://github.com/fsr-de/EvaP/pull/1287#pullrequestreview-218545677). Basically, give every counter a random number between 0 and the count of votes, then divide by a factor so that the vote count matches the earlier result again. In order to keep the count of votes equal, one would have to then distribute the error evenly to the counters. This will give more uniform results but has more clean up to do.


Discussion:

Resolving patch:
diff --git a/evap/evaluation/management/commands/anonymize.py b/evap/evaluation/management/commands/anonymize.py
--- a/evap/evaluation/management/commands/anonymize.py
+++ b/evap/evaluation/management/commands/anonymize.py
@@ -1,14 +1,18 @@
 from datetime import date, timedelta
 import os
 import itertools
+from math import floor
 import random
 
+from collections import defaultdict
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.core.serializers.base import ProgressBar
 from django.db import transaction
 
-from evap.evaluation.models import (Contribution, Course, CourseType, Degree, RatingAnswerCounter, Semester, TextAnswer,
-                                    UserProfile)
+from evap.evaluation.models import (CHOICES, Contribution, Course, CourseType, Degree,
+        NO_ANSWER, RatingAnswerCounter, Semester, TextAnswer, UserProfile)
+
 
 class Command(BaseCommand):
     args = ''
@@ -60,8 +64,10 @@ def anonymize_data(self):
                 self.anonymize_users(first_names, last_names)
                 self.anonymize_courses()
                 self.anonymize_evaluations()
-                self.anonymize_questionnaires(lorem_ipsum)
+                self.anonymize_questionnaires()
+                self.anonymize_answers(lorem_ipsum)
 
+                self.stdout.write("")
                 self.stdout.write("Done.")
 
         except Exception:
@@ -95,7 +101,7 @@ def anonymize_users(self, first_names, last_names):
             # Give users unique temporary names to counter identity errors due to the names being unique
             if user.username in Command.ignore_usernames:
                 continue
-            user.username = "<User #" + str(i) + ">"
+            user.username = f"<User #{i}>"
             user.save()
 
         # Actually replace all the real user data
@@ -145,7 +151,7 @@ def anonymize_courses(self):
             random.shuffle(courses)
             public_courses = [course for course in courses if not course.is_private]
 
-            self.stdout.write("Anonymizing " + str(len(courses)) + " courses of semester " + str(semester) + "...")
+            self.stdout.write(f"Anonymizing {len(courses)} courses of semester {semester}...")
 
             # Shuffle public courses' names in order to decouple them from the results.
             # Also, assign public courses' names to private ones as their names may be confidential.
@@ -155,8 +161,8 @@ def anonymize_courses(self):
 
             for i, course in enumerate(courses):
                 # Give courses unique temporary names to counter identity errors due to the names being unique
-                course.name_de = "<Veranstaltung #" + str(i) + ">"
-                course.name_en = "<Course #" + str(i) + ">"
+                course.name_de = f"<Veranstaltung #{i}>"
+                course.name_en = f"<Course #{i}>"
                 course.save()
 
             for i, course in enumerate(courses):
@@ -165,16 +171,15 @@ def anonymize_courses(self):
                     course.name_de = name[0]
                     course.name_en = name[1]
                 else:
-                    course.name_de = "Veranstaltung #" + str(i + 1)
-                    course.name_en = "Course #" + str(i + 1)
+                    course.name_de = f"Veranstaltung #{i + 1}"
+                    course.name_en = f"Course #{i + 1}"
                 course.save()
 
     def anonymize_evaluations(self):
         for semester in Semester.objects.all():
             evaluations = list(semester.evaluations.all())
             random.shuffle(evaluations)
-            self.stdout.write("Anonymizing " + str(len(evaluations)) + " evaluations of semester " + str(semester) +
-                "...")
+            self.stdout.write(f"Anonymizing {len(evaluations)} evaluations of semester {semester}...")
 
             self.stdout.write("Shuffling evaluation names...")
             named_evaluations = (evaluation for evaluation in evaluations if evaluation.name_de and evaluation.name_en)
@@ -184,9 +189,9 @@ def anonymize_evaluations(self):
             for i, evaluation in enumerate(evaluations):
                 # Give evaluations unique temporary names to counter identity errors due to the names being unique
                 if evaluation.name_de:
-                    evaluation.name_de = "<Evaluierung #" + str(i) + ">"
+                    evaluation.name_de = f"<Evaluierung #{i}>"
                 if evaluation.name_en:
-                    evaluation.name_en = "<Evaluation #" + str(i) + ">"
+                    evaluation.name_en = f"<Evaluation #{i}>"
                 evaluation.save()
 
             for i, evaluation in enumerate(evaluations):
@@ -197,16 +202,15 @@ def anonymize_evaluations(self):
                     evaluation.name_de = name[0]
                     evaluation.name_en = name[1]
                 else:
-                    evaluation.name_de = "Evaluierung #" + str(i + 1)
-                    evaluation.name_en = "Evaluation #" + str(i + 1)
+                    evaluation.name_de = f"Evaluierung #{i + 1}"
+                    evaluation.name_en = f"Evaluation #{i + 1}"
                 evaluation.save()
 
-    def anonymize_questionnaires(self, lorem_ipsum):
-        # questionnaires = Questionnaire.objects.all()
-
+    def anonymize_questionnaires(self):
         self.stdout.write("REMINDER: You still need to randomize the questionnaire names...")
         self.stdout.write("REMINDER: You still need to randomize the questionnaire questions...")
 
+    def anonymize_answers(self, lorem_ipsum):
         self.stdout.write("Replacing text answers with fake ones...")
         for text_answer in TextAnswer.objects.all():
             text_answer.answer = self.lorem(text_answer.answer, lorem_ipsum)
@@ -214,6 +218,48 @@ def anonymize_questionnaires(self, lorem_ipsum):
                 text_answer.original_answer = self.lorem(text_answer.original_answer, lorem_ipsum)
             text_answer.save()
 
+        self.stdout.write("Shuffling rating answer counter counts...")
+
+        contributions = Contribution.objects.all().prefetch_related("ratinganswercounter_set__question")
+        try:
+            self.stdout.ending = ""
+            progress_bar = ProgressBar(self.stdout, contributions.count())
+            for contribution_counter, contribution in enumerate(contributions):
+                progress_bar.update(contribution_counter + 1)
+
+                counters_per_question = defaultdict(list)
+                for counter in contribution.ratinganswercounter_set.all():
+                    counters_per_question[counter.question].append(counter)
+
+                for question, counters in counters_per_question.items():
+                    original_sum = sum(counter.count for counter in counters)
+
+                    missing_values = set(CHOICES[question.type].values).difference(set(c.answer for c in counters))
+                    missing_values.discard(NO_ANSWER)  # don't add NO_ANSWER counter if it didn't exist before
+                    for value in missing_values:
+                        counters.append(RatingAnswerCounter(question=question, contribution=contribution, answer=value, count=0))
+
+                    generated_counts = [random.random() for c in counters]
+                    generated_sum = sum(generated_counts)
+                    generated_counts = [floor(count / generated_sum * original_sum) for count in generated_counts]
+
+                    to_add = original_sum - sum(generated_counts)
+                    index = random.randint(0, len(generated_counts) - 1)
+                    generated_counts[index] += to_add
+
+                    for counter, generated_count in zip(counters, generated_counts):
+                        assert generated_count >= 0
+                        counter.count = generated_count
+
+                        if counter.count:
+                            counter.save()
+                        elif counter.id:
+                            counter.delete()
+
+                    assert original_sum == sum(counter.count for counter in counters)
+        finally:
+            self.stdout.ending = "\n"
+
     # Returns a string with the same number of lorem ipsum words as the given text
     @staticmethod
     def lorem(text, lorem_ipsum):


Test patch:
diff --git a/evap/evaluation/tests/test_commands.py b/evap/evaluation/tests/test_commands.py
--- a/evap/evaluation/tests/test_commands.py
+++ b/evap/evaluation/tests/test_commands.py
@@ -1,21 +1,26 @@
+from collections import defaultdict
 from datetime import datetime, date, timedelta
 from io import StringIO
+from itertools import chain, cycle
 import os
+import random
 from unittest.mock import patch
 
 from django.conf import settings
 from django.core import management, mail
+from django.db.models import Sum
 from django.test import TestCase
 from django.test.utils import override_settings
 
 from model_mommy import mommy
 
-from evap.evaluation.models import Course, Evaluation, EmailTemplate, Semester, UserProfile
+from evap.evaluation.models import (CHOICES, Contribution, Course, Evaluation, EmailTemplate, NO_ANSWER,
+    Question, Questionnaire, RatingAnswerCounter, Semester, UserProfile)
 
 
 class TestAnonymizeCommand(TestCase):
-    @patch('builtins.input')
-    def test_anonymize_does_not_crash(self, mock_input):
+    @classmethod
+    def setUpTestData(cls):
         mommy.make(EmailTemplate, name="name", subject="Subject", body="Body.")
         mommy.make(UserProfile,
           username="secret.username",
@@ -26,38 +31,112 @@ def test_anonymize_does_not_crash(self, mock_input):
           login_key=1234567890,
           login_key_valid_until=date.today())
         semester1 = mommy.make(Semester, name_de="S1", name_en="S1")
-        semester2 = mommy.make(Semester, name_de="S2", name_en="S2")
-        course1 = mommy.make(Course,
+        mommy.make(Semester, name_de="S2", name_en="S2")
+        cls.course = mommy.make(
+            Course,
             semester=semester1,
             name_de="Eine private Veranstaltung",
             name_en="A private course",
             is_private=True,
         )
-        course2 = mommy.make(Course,
+        course2 = mommy.make(
+            Course,
             semester=semester1,
             name_de="Veranstaltungsexperimente",
             name_en="Course experiments",
         )
-        mommy.make(Evaluation,
-            course=course1,
+        cls.evaluation = mommy.make(
+            Evaluation,
+            course=cls.course,
             name_de="Wie man Software testet",
             name_en="Testing your software",
         )
-        mommy.make(Evaluation,
-            course=course2,
-            name_de="Einführung in Python",
-            name_en="Introduction to Python",
-        )
-        mommy.make(Evaluation,
+        mommy.make(
+            Evaluation,
             course=course2,
             name_de="Die Entstehung von Unicode 😄",
             name_en="History of Unicode 😄",
         )
 
-        mock_input.return_value = 'yes'
+        cls.contributor_questionnaire = mommy.make(Questionnaire, type=Questionnaire.CONTRIBUTOR)
+        cls.general_questionnaire = mommy.make(Questionnaire, type=Questionnaire.TOP)
+
+        cls.contributor_questions = mommy.make(Question, _quantity=10,
+                questionnaire=cls.contributor_questionnaire, type=cycle(iter(CHOICES.keys())))
+        cls.general_questions = mommy.make(Question, _quantity=10,
+                questionnaire=cls.contributor_questionnaire, type=cycle(iter(CHOICES.keys())))
+
+        cls.contributor = mommy.make(UserProfile)
+
+        cls.contribution = mommy.make(Contribution, contributor=cls.contributor, evaluation=cls.evaluation,
+            questionnaires=[cls.contributor_questionnaire, cls.contributor_questionnaire])
+
+        cls.general_contribution = cls.evaluation.general_contribution
+        cls.general_contribution.questionnaires.set([cls.general_questionnaire])
+        cls.general_contribution.save()
+
+    def setUp(self):
+        self.input_patch = patch('builtins.input')
+        self.input_mock = self.input_patch.start()
+        self.input_mock.return_value = 'yes'
+        self.addCleanup(self.input_patch.stop)
+
+    def test_no_empty_rating_answer_counters_left(self):
+        for question in chain(self.contributor_questions, self.general_questions):
+            choices = [choice for choice in CHOICES[question.type].values if choice != NO_ANSWER]
+            for answer in choices:
+                mommy.make(RatingAnswerCounter, question=question, contribution=self.contribution, count=1, answer=answer)
+
+        old_count = RatingAnswerCounter.objects.count()
+
+        random.seed(0)
+        management.call_command('anonymize', stdout=StringIO())
+
+        new_count = RatingAnswerCounter.objects.count()
+        self.assertLess(new_count, old_count)
+
+        for counter in RatingAnswerCounter.objects.all():
+            self.assertGreater(counter.count, 0)
+
+    def test_question_with_no_answers(self):
+        management.call_command('anonymize', stdout=StringIO())
+        self.assertEqual(RatingAnswerCounter.objects.count(), 0)
+
+    def test_answer_count_unchanged(self):
+        answers_per_question = defaultdict(int)
+        random.seed(0)
+        for question in chain(self.contributor_questions, self.general_questions):
+            choices = [choice for choice in CHOICES[question.type].values if choice != NO_ANSWER]
+            for answer in choices:
+                count = random.randint(10, 100)
+                mommy.make(RatingAnswerCounter, question=question, contribution=self.contribution, count=count, answer=answer)
+                answers_per_question[question] += count
 
         management.call_command('anonymize', stdout=StringIO())
 
+        for question in chain(self.contributor_questions, self.general_questions):
+            answer_count = RatingAnswerCounter.objects.filter(question=question).aggregate(Sum('count'))["count__sum"]
+            self.assertEqual(answers_per_question[question], answer_count)
+
+    def test_single_result_anonymization(self):
+        questionnaire = Questionnaire.single_result_questionnaire()
+        single_result = mommy.make(Evaluation, is_single_result=True, course=self.course)
+        single_result.general_contribution.questionnaires.set([questionnaire])
+        question = Question.objects.get(questionnaire=questionnaire)
+
+        answer_count_before = 0
+        choices = [choice for choice in CHOICES[question.type].values if choice != NO_ANSWER]
+        random.seed(0)
+        for answer in choices:
+            count = random.randint(50, 100)
+            mommy.make(RatingAnswerCounter, question=question, contribution=single_result.general_contribution, count=count, answer=answer)
+            answer_count_before += count
+
+        management.call_command('anonymize', stdout=StringIO())
+
+        self.assertLessEqual(RatingAnswerCounter.objects.count(), len(choices))
+        self.assertEqual(RatingAnswerCounter.objects.aggregate(Sum('count'))["count__sum"], answer_count_before)
+
 
 class TestRunCommand(TestCase):
     def test_calls_runserver(self):