Repository: comic/grand-challenge.org

Issue:
Reader study order randomization for images
**Describe the solution you'd like**
There should be a flag on the reader-studies page that randomizes the order of scans presented to a reader.

**Additional context**
Randomized trials are standard scientific method to counteract biases in answers. We should have that feature.


Discussion:
(How do I add these to our project again? I do not remember how we do this)
Not too sure where this responsibility should lie. The front end could also do a `random.choice` from the unanswered hangings, with maybe a boolean to turn this functionality on and off on the backend. 
Maybe use the username for the seed of the shuffler? Use a fixed algorithm for generating the shuffle?
Possibly use numpy for this, which guarantees compatability across versions: https://docs.scipy.org/doc/numpy-1.16.0/reference/generated/numpy.random.RandomState.html#numpy.random.RandomState

```
np.random.RandomState(user.pk % 2**32).shuffle(hanging)
```

We already use numpy in the project, so no new dependency.

Resolving patch:
diff --git a/app/grandchallenge/reader_studies/forms.py b/app/grandchallenge/reader_studies/forms.py
--- a/app/grandchallenge/reader_studies/forms.py
+++ b/app/grandchallenge/reader_studies/forms.py
@@ -58,6 +58,7 @@ class Meta(ReaderStudyCreateForm.Meta):
             "logo",
             "description",
             "workstation",
+            "shuffle_hanging_list",
             "hanging_list",
         )
         widgets = {
@@ -65,6 +66,12 @@ class Meta(ReaderStudyCreateForm.Meta):
         }
         help_texts = {
             **READER_STUDY_HELP_TEXTS,
+            "shuffle_hanging_list": (
+                "If true, each reader will read the images in a unique "
+                "order. The ordering for each user will be consistent over "
+                "time. If false, the readers will all read the images in the "
+                "order that you define in the hanging_list field."
+            ),
             "hanging_list": (
                 "A list of hangings. "
                 "The hanging defines which image (the hanging value) "
diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py
--- a/app/grandchallenge/reader_studies/models.py
+++ b/app/grandchallenge/reader_studies/models.py
@@ -7,6 +7,7 @@
 from django.db import models
 from django_extensions.db.models import TitleSlugDescriptionModel
 from guardian.shortcuts import assign_perm, get_objects_for_group, remove_perm
+from numpy.random.mtrand import RandomState
 
 from grandchallenge.challenges.models import get_logo_path
 from grandchallenge.core.models import UUIDModel
@@ -216,6 +217,22 @@ def hanging_list_images(self):
 
         return hanging_list_images
 
+    def get_hanging_list_images_for_user(self, *, user):
+        """
+        Returns a shuffled list of the hanging list images for a particular
+        user. The shuffle is seeded with the users pk, and using RandomState
+        from numpy guarantees that the ordering will be consistent across
+        python/library versions. Returns the normal list if
+        shuffle_hanging_list is false.
+        """
+        hanging_list = self.hanging_list_images
+
+        if self.shuffle_hanging_list and hanging_list is not None:
+            # In place shuffle
+            RandomState(seed=int(user.pk)).shuffle(hanging_list)
+
+        return hanging_list
+
 
 ANSWER_TYPE_2D_BOUNDING_BOX_SCHEMA = {
     "$schema": "http://json-schema.org/draft-07/schema#",
diff --git a/app/grandchallenge/reader_studies/serializers.py b/app/grandchallenge/reader_studies/serializers.py
--- a/app/grandchallenge/reader_studies/serializers.py
+++ b/app/grandchallenge/reader_studies/serializers.py
@@ -1,7 +1,10 @@
 from rest_framework.exceptions import ValidationError
 from rest_framework.fields import CharField
 from rest_framework.relations import SlugRelatedField, HyperlinkedRelatedField
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import (
+    HyperlinkedModelSerializer,
+    SerializerMethodField,
+)
 
 from grandchallenge.cases.models import Image
 from grandchallenge.reader_studies.models import ReaderStudy, Question, Answer
@@ -32,6 +35,7 @@ class Meta:
 
 class ReaderStudySerializer(HyperlinkedModelSerializer):
     questions = QuestionSerializer(many=True, read_only=True)
+    hanging_list_images = SerializerMethodField()
 
     class Meta:
         model = ReaderStudy
@@ -45,6 +49,12 @@ class Meta:
             "title",
         )
 
+    def get_hanging_list_images(self, obj: ReaderStudy):
+        """ Used by hanging_list_images serializer field """
+        return obj.get_hanging_list_images_for_user(
+            user=self.context["request"].user
+        )
+
 
 class AnswerSerializer(HyperlinkedModelSerializer):
     creator = SlugRelatedField(read_only=True, slug_field="username")


Test patch:
diff --git a/app/tests/reader_studies_tests/validation_tests.py b/app/tests/reader_studies_tests/validation_tests.py
--- a/app/tests/reader_studies_tests/validation_tests.py
+++ b/app/tests/reader_studies_tests/validation_tests.py
@@ -2,8 +2,9 @@
 
 from grandchallenge.core.validators import JSONSchemaValidator
 from grandchallenge.reader_studies.models import HANGING_LIST_SCHEMA
-from tests.factories import ImageFactory
+from tests.factories import ImageFactory, UserFactory
 from tests.reader_studies_tests.factories import ReaderStudyFactory
+from tests.utils import get_view_for_user
 
 
 @pytest.mark.django_db
@@ -71,3 +72,68 @@ def test_non_unique_images(image_names, expected):
     rs.images.set(images)
     rs.save()
     assert rs.non_unique_study_image_names == expected
+
+
+@pytest.mark.django_db
+def test_hanging_list_shuffle_per_user(client):
+    hanging_list = [{"main": f"image_{n}"} for n in range(10)]
+
+    rs = ReaderStudyFactory(hanging_list=hanging_list)
+    images = [ImageFactory(name=f"image_{n}") for n in range(10)]
+    rs.images.set(images)
+    rs.save()
+
+    # The shuffling is seeded with the users pk, so needs to stay constant
+    u1, u2 = UserFactory(pk=1_000_000), UserFactory(pk=1_000_001)
+
+    rs.add_reader(user=u1)
+    rs.add_reader(user=u2)
+
+    assert rs.get_hanging_list_images_for_user(
+        user=u1
+    ) == rs.get_hanging_list_images_for_user(user=u2)
+
+    rs.shuffle_hanging_list = True
+    rs.save()
+
+    u1_list = rs.get_hanging_list_images_for_user(user=u1)
+    u2_list = rs.get_hanging_list_images_for_user(user=u2)
+
+    # Check that the list is different per user and contains all of the images
+    assert u1_list != u2_list
+    assert (
+        {u["main"] for u in u1_list}
+        == {u["main"] for u in u2_list}
+        == {im.api_url for im in images}
+    )
+
+    # Check that repeat requests return the same list
+    assert rs.get_hanging_list_images_for_user(
+        user=u1
+    ) == rs.get_hanging_list_images_for_user(user=u1)
+
+    # Check that the list is consistent over time, if not, maybe numpy has
+    # changed their implementation
+    api_to_image = {im.api_url: im.name for im in images}
+    assert [api_to_image[h["main"]] for h in u1_list] == [
+        "image_8",
+        "image_3",
+        "image_7",
+        "image_1",
+        "image_4",
+        "image_9",
+        "image_0",
+        "image_5",
+        "image_6",
+        "image_2",
+    ]
+
+    # Check that the api is hooked up
+    response = get_view_for_user(
+        client=client,
+        viewname="api:reader-study-detail",
+        reverse_kwargs={"pk": rs.pk},
+        user=u1,
+    )
+    assert response.status_code == 200
+    assert response.json()["hanging_list_images"] == u1_list