Repository: e-valuation/EvaP

Issue:
Swapping contributors in course edit form fails
When you exchange two contributors in the course edit form, it says "a contribution with this contributor already exists"

you can also e.g. replace contributor A by someone who is not yet a contributor, and then replace contributor B by contributor A.

reordering the contribution works, the problem occurs when changing the contributors of the contributions.

Swapping contributors in course edit form fails
When you exchange two contributors in the course edit form, it says "a contribution with this contributor already exists"

you can also e.g. replace contributor A by someone who is not yet a contributor, and then replace contributor B by contributor A.

reordering the contribution works, the problem occurs when changing the contributors of the contributions.



Discussion:

Resolving patch:
diff --git a/evap/staff/forms.py b/evap/staff/forms.py
--- a/evap/staff/forms.py
+++ b/evap/staff/forms.py
@@ -4,6 +4,7 @@
 from django.forms.models import BaseInlineFormSet
 from django.utils.translation import ugettext_lazy as _
 from django.utils.text import normalize_newlines
+from django.http.request import QueryDict
 from django.core.exceptions import ValidationError
 from django.contrib.auth.models import Group
 
@@ -316,8 +317,9 @@ def clean(self):
 
 
 class ContributionFormSet(AtLeastOneFormSet):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    def __init__(self, data=None, *args, **kwargs):
+        data = self.handle_moved_contributors(data, *args, **kwargs)
+        super().__init__(data, *args, **kwargs)
         self.queryset = self.instance.contributions.exclude(contributor=None)
 
     def handle_deleted_and_added_contributions(self):
@@ -334,11 +336,51 @@ def handle_deleted_and_added_contributions(self):
                     continue
                 if not deleted_form.cleaned_data['contributor'] == form_with_errors.cleaned_data['contributor']:
                     continue
-                form_with_errors.cleaned_data['id'] = deleted_form.cleaned_data['id']
                 form_with_errors.instance = deleted_form.instance
                 # we modified the form, so we have to force re-validation
                 form_with_errors.full_clean()
 
+    def handle_moved_contributors(self, data, *args, **kwargs):
+        """
+            Work around https://code.djangoproject.com/ticket/25139
+            Basically, if the user assigns a contributor who already has a contribution to a new contribution,
+            this moves the contributor (and all the data of the new form they got assigned to) back to the original contribution.
+        """
+        if data is None or 'instance' not in kwargs:
+            return data
+
+        course = kwargs['instance']
+        total_forms = int(data['contributions-TOTAL_FORMS'])
+        for i in range(0, total_forms):
+            prefix = "contributions-" + str(i) + "-"
+            current_id = data.get(prefix + 'id', '')
+            contributor = data.get(prefix + 'contributor', '')
+            if contributor == '':
+                continue
+            # find the contribution that the contributor had before the user messed with it
+            try:
+                previous_id = str(Contribution.objects.get(contributor=contributor, course=course).id)
+            except Contribution.DoesNotExist:
+                continue
+
+            if current_id == previous_id:
+                continue
+
+            # find the form with that previous contribution and then swap the contributions
+            for j in range(0, total_forms):
+                other_prefix = "contributions-" + str(j) + "-"
+                other_id = data[other_prefix + 'id']
+                if other_id == previous_id:
+                    # swap all the data. the contribution's ids stay in place.
+                    data2 = data.copy()
+                    data = QueryDict(mutable=True)
+                    for key, value in data2.lists():
+                        if not key.endswith('-id'):
+                            key = key.replace(prefix, '%temp%').replace(other_prefix, prefix).replace('%temp%', other_prefix)
+                        data.setlist(key, value)
+                    break
+        return data
+
     def clean(self):
         self.handle_deleted_and_added_contributions()
 


Test patch:
diff --git a/deployment/testing_environment/modules/evap/manifests/init.pp b/deployment/testing_environment/modules/evap/manifests/init.pp
--- a/deployment/testing_environment/modules/evap/manifests/init.pp
+++ b/deployment/testing_environment/modules/evap/manifests/init.pp
@@ -23,5 +23,9 @@
         provider    => shell,
         command     => 'python3 manage.py createcachetable',
         cwd         => '/vagrant'
+    } -> exec { 'evap-cache-warmup':
+        provider    => shell,
+        command     => 'python3 manage.py refresh_results_cache',
+        cwd         => '/vagrant'
     }
 }
diff --git a/evap/contributor/tests/test_forms.py b/evap/contributor/tests/test_forms.py
--- a/evap/contributor/tests/test_forms.py
+++ b/evap/contributor/tests/test_forms.py
@@ -4,7 +4,7 @@
 
 from evap.evaluation.models import UserProfile, Course, Questionnaire, Contribution
 from evap.contributor.forms import DelegatesForm, EditorContributionForm
-from evap.evaluation.tests.test_utils import WebTest, get_form_data_from_instance
+from evap.evaluation.tests.test_utils import WebTest, get_form_data_from_instance, to_querydict
 from evap.staff.forms import ContributionFormSet
 
 
@@ -45,18 +45,18 @@ def test_editors_cannot_change_responsible(self):
 
         InlineContributionFormset = inlineformset_factory(Course, Contribution, formset=ContributionFormSet, form=EditorContributionForm, extra=0)
 
-        data = {
+        data = to_querydict({
             'contributions-TOTAL_FORMS': 1,
             'contributions-INITIAL_FORMS': 1,
             'contributions-MAX_NUM_FORMS': 5,
             'contributions-0-id': contribution1.pk,
             'contributions-0-course': course.pk,
-            'contributions-0-questionnaires': [questionnaire.pk],
+            'contributions-0-questionnaires': questionnaire.pk,
             'contributions-0-order': 1,
             'contributions-0-responsibility': "RESPONSIBLE",
             'contributions-0-comment_visibility': "ALL",
             'contributions-0-contributor': user1.pk,
-        }
+        })
 
         formset = InlineContributionFormset(instance=course, data=data.copy())
         self.assertTrue(formset.is_valid())
diff --git a/evap/evaluation/tests/test_coverage.py b/evap/evaluation/tests/test_coverage.py
--- a/evap/evaluation/tests/test_coverage.py
+++ b/evap/evaluation/tests/test_coverage.py
@@ -4,7 +4,7 @@
 
 from evap.evaluation.models import Semester, Questionnaire, UserProfile, Course, \
                             EmailTemplate, Degree, CourseType, Contribution
-from evap.evaluation.tests.test_utils import WebTest
+from evap.evaluation.tests.test_utils import WebTest, to_querydict
 from evap.staff.forms import ContributionFormSet, ContributionForm
 
 from model_mommy import mommy
@@ -181,16 +181,16 @@ def test_contributor_form_set(self):
 
         ContributionFormset = inlineformset_factory(Course, Contribution, formset=ContributionFormSet, form=ContributionForm, extra=0)
 
-        data = {
+        data = to_querydict({
             'contributions-TOTAL_FORMS': 1,
             'contributions-INITIAL_FORMS': 0,
             'contributions-MAX_NUM_FORMS': 5,
             'contributions-0-course': course.pk,
-            'contributions-0-questionnaires': [1],
+            'contributions-0-questionnaires': 1,
             'contributions-0-order': 0,
             'contributions-0-responsibility': "RESPONSIBLE",
             'contributions-0-comment_visibility': "ALL",
-        }
+        })
         # no contributor and no responsible
         self.assertFalse(ContributionFormset(instance=course, form_kwargs={'course': course}, data=data.copy()).is_valid())
         # valid
@@ -200,7 +200,7 @@ def test_contributor_form_set(self):
         data['contributions-TOTAL_FORMS'] = 2
         data['contributions-1-contributor'] = 1
         data['contributions-1-course'] = course.pk
-        data['contributions-1-questionnaires'] = [1]
+        data['contributions-1-questionnaires'] = 1
         data['contributions-1-order'] = 1
         self.assertFalse(ContributionFormset(instance=course, form_kwargs={'course': course}, data=data).is_valid())
         # two responsibles
diff --git a/evap/evaluation/tests/test_utils.py b/evap/evaluation/tests/test_utils.py
--- a/evap/evaluation/tests/test_utils.py
+++ b/evap/evaluation/tests/test_utils.py
@@ -1,9 +1,18 @@
+from django.http.request import QueryDict
+
 from django_webtest import WebTest as DjangoWebTest
 from model_mommy import mommy
 
 from evap.evaluation.models import Contribution, Course, UserProfile, Questionnaire, Degree
 
 
+def to_querydict(dict):
+    querydict = QueryDict(mutable=True)
+    for key, value in dict.items():
+        querydict[key] = value
+    return querydict
+
+
 # taken from http://lukeplant.me.uk/blog/posts/fuzzy-testing-with-assertnumqueries/
 class FuzzyInt(int):
     def __new__(cls, lowest, highest):
diff --git a/evap/staff/tests/test_forms.py b/evap/staff/tests/test_forms.py
--- a/evap/staff/tests/test_forms.py
+++ b/evap/staff/tests/test_forms.py
@@ -3,7 +3,7 @@
 from model_mommy import mommy
 
 from evap.evaluation.models import UserProfile, CourseType, Course, Questionnaire, Contribution, Semester, Degree
-from evap.evaluation.tests.test_utils import get_form_data_from_instance, course_with_responsible_and_editor
+from evap.evaluation.tests.test_utils import get_form_data_from_instance, course_with_responsible_and_editor, to_querydict
 from evap.staff.forms import UserForm, SingleResultForm, ContributionFormSet, ContributionForm, CourseForm, \
     CourseEmailForm
 from evap.contributor.forms import CourseForm as ContributorCourseForm
@@ -125,31 +125,31 @@ def test_dont_validate_deleted_contributions(self):
         contribution_formset = inlineformset_factory(Course, Contribution, formset=ContributionFormSet, form=ContributionForm, extra=0)
 
         # Here we have two responsibles (one of them deleted), and a deleted contributor with no questionnaires.
-        data = {
+        data = to_querydict({
             'contributions-TOTAL_FORMS': 3,
             'contributions-INITIAL_FORMS': 0,
             'contributions-MAX_NUM_FORMS': 5,
             'contributions-0-course': course.pk,
-            'contributions-0-questionnaires': [questionnaire.pk],
+            'contributions-0-questionnaires': questionnaire.pk,
             'contributions-0-order': 0,
             'contributions-0-responsibility': "RESPONSIBLE",
             'contributions-0-comment_visibility': "ALL",
             'contributions-0-contributor': user1.pk,
             'contributions-0-DELETE': 'on',
             'contributions-1-course': course.pk,
-            'contributions-1-questionnaires': [questionnaire.pk],
+            'contributions-1-questionnaires': questionnaire.pk,
             'contributions-1-order': 0,
             'contributions-1-responsibility': "RESPONSIBLE",
             'contributions-1-comment_visibility': "ALL",
             'contributions-1-contributor': user2.pk,
             'contributions-2-course': course.pk,
-            'contributions-2-questionnaires': [],
+            'contributions-2-questionnaires': "",
             'contributions-2-order': 1,
-            'contributions-2-responsibility': "NONE",
+            'contributions-2-responsibility': "CONTRIBUTOR",
             'contributions-2-comment_visibility': "OWN",
             'contributions-2-contributor': user2.pk,
             'contributions-2-DELETE': 'on',
-        }
+        })
 
         formset = contribution_formset(instance=course, form_kwargs={'course': course}, data=data)
         self.assertTrue(formset.is_valid())
@@ -168,25 +168,26 @@ def test_take_deleted_contributions_into_account(self):
 
         contribution_formset = inlineformset_factory(Course, Contribution, formset=ContributionFormSet, form=ContributionForm, extra=0)
 
-        data = {
+        data = to_querydict({
             'contributions-TOTAL_FORMS': 2,
             'contributions-INITIAL_FORMS': 1,
             'contributions-MAX_NUM_FORMS': 5,
             'contributions-0-id': contribution1.pk,
             'contributions-0-course': course.pk,
-            'contributions-0-questionnaires': [questionnaire.pk],
+            'contributions-0-questionnaires': questionnaire.pk,
             'contributions-0-order': 0,
             'contributions-0-responsibility': "RESPONSIBLE",
             'contributions-0-comment_visibility': "ALL",
             'contributions-0-contributor': user1.pk,
             'contributions-0-DELETE': 'on',
             'contributions-1-course': course.pk,
-            'contributions-1-questionnaires': [questionnaire.pk],
+            'contributions-1-questionnaires': questionnaire.pk,
             'contributions-1-order': 0,
+            'contributions-1-id': '',
             'contributions-1-responsibility': "RESPONSIBLE",
             'contributions-1-comment_visibility': "ALL",
             'contributions-1-contributor': user1.pk,
-        }
+        })
 
         formset = contribution_formset(instance=course, form_kwargs={'course': course}, data=data)
         self.assertTrue(formset.is_valid())
@@ -223,6 +224,105 @@ def test_obsolete_staff_only(self):
         self.assertEqual(expected, set(formset.forms[0].fields['questionnaires'].queryset.all()))
         self.assertEqual(expected, set(formset.forms[1].fields['questionnaires'].queryset.all()))
 
+class ContributionFormset775RegressionTests(TestCase):
+    """
+        Various regression tests for #775
+    """
+    @classmethod
+    def setUpTestData(cls):
+        cls.course = mommy.make(Course, name_en="some course")
+        cls.user1 = mommy.make(UserProfile)
+        cls.user2 = mommy.make(UserProfile)
+        mommy.make(UserProfile)
+        cls.questionnaire = mommy.make(Questionnaire, is_for_contributors=True)
+        cls.contribution1 = mommy.make(Contribution, responsible=True, contributor=cls.user1, course=cls.course)
+        cls.contribution2 = mommy.make(Contribution, contributor=cls.user2, course=cls.course)
+
+        cls.contribution_formset = inlineformset_factory(Course, Contribution, formset=ContributionFormSet, form=ContributionForm, extra=0)
+
+    def setUp(self):
+        self.data = to_querydict({
+            'contributions-TOTAL_FORMS': 2,
+            'contributions-INITIAL_FORMS': 2,
+            'contributions-MAX_NUM_FORMS': 5,
+            'contributions-0-id': str(self.contribution1.pk), # browsers send strings so we should too
+            'contributions-0-course': self.course.pk,
+            'contributions-0-questionnaires': self.questionnaire.pk,
+            'contributions-0-order': 0,
+            'contributions-0-responsibility': "RESPONSIBLE",
+            'contributions-0-comment_visibility': "ALL",
+            'contributions-0-contributor': self.user1.pk,
+            'contributions-1-id': str(self.contribution2.pk),
+            'contributions-1-course': self.course.pk,
+            'contributions-1-questionnaires': self.questionnaire.pk,
+            'contributions-1-order': 0,
+            'contributions-1-responsibility': "CONTRIBUTOR",
+            'contributions-1-comment_visibility': "OWN",
+            'contributions-1-contributor': self.user2.pk,
+        })
+
+    def test_swap_contributors(self):
+        formset = self.contribution_formset(instance=self.course, form_kwargs={'course': self.course}, data=self.data)
+        self.assertTrue(formset.is_valid())
+
+        # swap contributors, should still be valid
+        self.data['contributions-0-contributor'] = self.user2.pk
+        self.data['contributions-1-contributor'] = self.user1.pk
+        formset = self.contribution_formset(instance=self.course, form_kwargs={'course': self.course}, data=self.data)
+        self.assertTrue(formset.is_valid())
+
+    def test_move_and_delete(self):
+        # move contributor2 to the first contribution and delete the second one
+        # after saving, only one contribution should exist and have the contributor2
+        self.data['contributions-0-contributor'] = self.user2.pk
+        self.data['contributions-1-contributor'] = self.user2.pk
+        self.data['contributions-1-DELETE'] = 'on'
+        formset = self.contribution_formset(instance=self.course, form_kwargs={'course': self.course}, data=self.data)
+        self.assertTrue(formset.is_valid())
+        formset.save()
+        self.assertTrue(Contribution.objects.filter(contributor=self.user2, course=self.course).exists())
+        self.assertFalse(Contribution.objects.filter(contributor=self.user1, course=self.course).exists())
+
+    def test_extra_form(self):
+        # make sure nothing crashes when an extra form is present.
+        self.data['contributions-0-contributor'] = self.user2.pk
+        self.data['contributions-1-contributor'] = self.user1.pk
+        self.data['contributions-2-TOTAL_FORMS'] = 3
+        self.data['contributions-2-id'] = ""
+        self.data['contributions-2-order'] = -1
+        self.data['contributions-2-responsibility'] = "CONTRIBUTOR"
+        self.data['contributions-2-comment_visibility'] = "OWN"
+        formset = self.contribution_formset(instance=self.course, form_kwargs={'course': self.course}, data=self.data)
+        self.assertTrue(formset.is_valid())
+
+    def test_swap_contributors_with_extra_form(self):
+        # moving a contributor to an extra form should work.
+        # first, the second contributor is deleted and removed from self.data
+        self.contribution2.delete()
+        self.data['contributions-TOTAL_FORMS'] = 2
+        self.data['contributions-INITIAL_FORMS'] = 1
+        self.data['contributions-0-contributor'] = self.user2.pk
+        self.data['contributions-1-contributor'] = self.user1.pk
+        self.data['contributions-1-id'] = ""
+        self.data['contributions-1-order'] = -1
+        self.data['contributions-1-responsibility'] = "CONTRIBUTOR"
+        self.data['contributions-1-comment_visibility'] = "OWN"
+
+        formset = self.contribution_formset(instance=self.course, form_kwargs={'course': self.course}, data=self.data)
+        self.assertTrue(formset.is_valid())
+
+    def test_handle_multivaluedicts(self):
+        # make sure the workaround is not eating questionnaires
+        # first, swap contributors to trigger the workaround
+        self.data['contributions-0-contributor'] = self.user2.pk
+        self.data['contributions-1-contributor'] = self.user1.pk
+
+        questionnaire = mommy.make(Questionnaire, is_for_contributors=True)
+        self.data.appendlist('contributions-0-questionnaires', questionnaire.pk)
+        formset = self.contribution_formset(instance=self.course, form_kwargs={'course': self.course}, data=self.data)
+        formset.save()
+        self.assertEquals(Questionnaire.objects.filter(contributions=self.contribution2).count(), 2)
+
 
 class CourseFormTests(TestCase):
     def helper_test_course_form_same_name(self, CourseFormClass):