Repository: comic/grand-challenge.org

Issue:
Add a button to download answers / questions for reader studies in csv
The button should only be available for challenge editors.



Discussion:

Resolving patch:
diff --git a/app/grandchallenge/core/management/commands/init_gc_demo.py b/app/grandchallenge/core/management/commands/init_gc_demo.py
--- a/app/grandchallenge/core/management/commands/init_gc_demo.py
+++ b/app/grandchallenge/core/management/commands/init_gc_demo.py
@@ -25,6 +25,7 @@
 )
 from grandchallenge.evaluation.models import Job, Method, Result, Submission
 from grandchallenge.pages.models import Page
+from grandchallenge.reader_studies.models import Answer, Question, ReaderStudy
 from grandchallenge.workstations.models import Workstation
 
 logger = logging.getLogger(__name__)
@@ -55,7 +56,6 @@ def handle(self, *args, **options):
 
         # Set the default domain that is used in RequestFactory
         site = Site.objects.get(pk=settings.SITE_ID)
-
         if site.domain == "gc.localhost":
             # Already initialised
             return
@@ -84,6 +84,7 @@ def handle(self, *args, **options):
         self._create_external_challenge()
         self._create_workstation()
         self._create_algorithm_demo()
+        self._create_reader_studies()
         self._log_tokens()
 
     def _create_flatpages(self, site):
@@ -285,6 +286,27 @@ def _create_workstation(self):
         w.add_editor(user=self.users["workstation"])
         w.add_user(user=self.users["algorithm"])
 
+    def _create_reader_studies(self):
+        reader_study = ReaderStudy.objects.create(
+            title="Reader Study",
+            workstation=Workstation.objects.first(),
+            logo=get_temporary_image(),
+        )
+        reader_study.editors_group.user_set.add(self.users["readerstudy"])
+        reader_study.readers_group.user_set.add(self.users["demo"])
+
+        question = Question.objects.create(
+            reader_study=reader_study,
+            question_text="foo",
+            answer_type=Question.ANSWER_TYPE_SINGLE_LINE_TEXT,
+        )
+
+        answer = Answer.objects.create(
+            creator=self.users["readerstudy"], question=question, answer="foo"
+        )
+        answer.images.add(grandchallenge.cases.models.Image.objects.first())
+        answer.save()
+
     @staticmethod
     def _log_tokens():
         out = [f"\t{t.user} token is: {t}\n" for t in Token.objects.all()]
diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py
--- a/app/grandchallenge/reader_studies/models.py
+++ b/app/grandchallenge/reader_studies/models.py
@@ -455,6 +455,8 @@ class Question(UUIDModel):
     )
     order = models.PositiveSmallIntegerField(default=100)
 
+    csv_headers = ["Question text", "Answer type", "Required", "Image port"]
+
     class Meta:
         ordering = ("order", "created")
 
@@ -469,6 +471,15 @@ def __str__(self):
             ")"
         )
 
+    @property
+    def csv_values(self):
+        return [
+            self.question_text,
+            self.get_answer_type_display(),
+            self.required,
+            f"{self.get_image_port_display() + ' port,' if self.image_port else ''}",
+        ]
+
     @property
     def api_url(self):
         return reverse(
@@ -527,6 +538,8 @@ class Answer(UUIDModel):
     images = models.ManyToManyField("cases.Image", related_name="answers")
     answer = JSONField()
 
+    csv_headers = Question.csv_headers + ["Answer", "Images", "Creator"]
+
     class Meta:
         ordering = ("creator", "created")
 
@@ -539,6 +552,14 @@ def api_url(self):
             "api:reader-studies-answer-detail", kwargs={"pk": self.pk}
         )
 
+    @property
+    def csv_values(self):
+        return self.question.csv_values + [
+            self.answer,
+            "; ".join(self.images.values_list("name", flat=True)),
+            self.creator.username,
+        ]
+
     def save(self, *args, **kwargs):
         adding = self._state.adding
 
diff --git a/app/grandchallenge/reader_studies/views.py b/app/grandchallenge/reader_studies/views.py
--- a/app/grandchallenge/reader_studies/views.py
+++ b/app/grandchallenge/reader_studies/views.py
@@ -1,3 +1,6 @@
+import csv
+import re
+
 from dal import autocomplete
 from django.conf import settings
 from django.contrib.auth import get_user_model
@@ -6,6 +9,7 @@
     UserPassesTestMixin,
 )
 from django.contrib.messages.views import SuccessMessageMixin
+from django.http import Http404, HttpResponse
 from django.views.generic import (
     CreateView,
     DetailView,
@@ -236,13 +240,69 @@ class ReadersUpdate(ReaderStudyUserGroupUpdateMixin):
     success_message = "Readers successfully updated"
 
 
-class ReaderStudyViewSet(ReadOnlyModelViewSet):
+class ExportCSVMixin(object):
+    def _check_export_perms(self, user, obj):
+        if not (user and user.has_perm(self.export_permission, obj)):
+            raise Http404()
+
+    def _create_dicts(self, headers, data):
+        return map(lambda x: dict(zip(headers, x)), data)
+
+    def _preprocess_data(self, data):
+        processed = []
+        for entry in data:
+            processed.append(
+                map(lambda x: re.sub(r"[\n\r\t]", " ", str(x)), entry)
+            )
+        return processed
+
+    def _create_csv_response(self, data, headers, filename="export.csv"):
+        response = HttpResponse(content_type="text/csv")
+        response["Content-Disposition"] = f'attachment; filename="{filename}"'
+        writer = csv.DictWriter(
+            response,
+            quoting=csv.QUOTE_ALL,
+            escapechar="\\",
+            fieldnames=headers,
+        )
+        writer.writeheader()
+        csv_dict = self._create_dicts(headers, self._preprocess_data(data))
+        writer.writerows(csv_dict)
+
+        return response
+
+
+class ReaderStudyViewSet(ExportCSVMixin, ReadOnlyModelViewSet):
     serializer_class = ReaderStudySerializer
     queryset = ReaderStudy.objects.all().prefetch_related(
         "images", "questions"
     )
     permission_classes = [DjangoObjectPermissions]
     filter_backends = [ObjectPermissionsFilter]
+    export_permission = (
+        f"{ReaderStudy._meta.app_label}.change_{ReaderStudy._meta.model_name}"
+    )
+
+    @action(detail=True)
+    def export_answers(self, request, pk=None):
+        reader_study = self.get_object()
+        self._check_export_perms(request.user, reader_study)
+
+        data = [
+            answer.csv_values
+            for answer in Answer.objects.select_related(
+                "question__reader_study"
+            )
+            .select_related("creator")
+            .prefetch_related("images")
+            .filter(question__reader_study=reader_study)
+        ]
+
+        return self._create_csv_response(
+            data,
+            Answer.csv_headers,
+            filename=f"{reader_study.slug}-answers.csv",
+        )
 
 
 class QuestionViewSet(ReadOnlyModelViewSet):


Test patch:
diff --git a/app/tests/reader_studies_tests/test_api.py b/app/tests/reader_studies_tests/test_api.py
--- a/app/tests/reader_studies_tests/test_api.py
+++ b/app/tests/reader_studies_tests/test_api.py
@@ -1,6 +1,10 @@
+import csv
+import re
+
 import pytest
 
 from grandchallenge.reader_studies.models import Answer, Question
+from grandchallenge.reader_studies.views import ExportCSVMixin
 from tests.factories import ImageFactory, UserFactory
 from tests.reader_studies_tests.factories import (
     AnswerFactory,
@@ -294,3 +298,149 @@ def test_answer_is_correct_type(client, answer_type, answer, expected):
         content_type="application/json",
     )
     assert response.status_code == expected
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "answer_type,answer",
+    (
+        (Question.ANSWER_TYPE_BOOL, True),
+        (Question.ANSWER_TYPE_SINGLE_LINE_TEXT, "dgfsgfds"),
+        (Question.ANSWER_TYPE_MULTI_LINE_TEXT, "dgfsgfds\ndgfsgfds"),
+        (
+            Question.ANSWER_TYPE_2D_BOUNDING_BOX,
+            {
+                "version": {"major": 1, "minor": 0},
+                "type": "2D bounding box",
+                "name": "test_name",
+                "corners": [[0, 0, 0], [10, 0, 0], [10, 10, 0], [0, 0, 0]],
+            },
+        ),
+        (
+            Question.ANSWER_TYPE_DISTANCE_MEASUREMENT,
+            {
+                "version": {"major": 1, "minor": 0},
+                "type": "Distance measurement",
+                "name": "test",
+                "start": (1, 2, 3),
+                "end": (4, 5, 6),
+            },
+        ),
+        (
+            Question.ANSWER_TYPE_MULTIPLE_DISTANCE_MEASUREMENTS,
+            {
+                "version": {"major": 1, "minor": 0},
+                "type": "Multiple distance measurements",
+                "name": "test",
+                "lines": [
+                    {"start": (1, 2, 3), "end": (4, 5, 6)},
+                    {"start": (1, 2, 3), "end": (4, 5, 6)},
+                ],
+            },
+        ),
+    ),
+)
+def test_csv_export(client, answer_type, answer):
+    im = ImageFactory()
+
+    rs = ReaderStudyFactory()
+    rs.images.add(im)
+    rs.save()
+
+    editor = UserFactory()
+    rs.add_editor(editor)
+
+    reader = UserFactory()
+    rs.add_reader(reader)
+
+    q = QuestionFactory(
+        question_text="foo", reader_study=rs, answer_type=answer_type
+    )
+
+    a = AnswerFactory(question=q, answer=answer)
+    a.images.add(im)
+    a.save()
+
+    response = get_view_for_user(
+        viewname="api:reader-study-export-answers",
+        reverse_kwargs={"pk": rs.pk},
+        user=editor,
+        client=client,
+        method=client.get,
+        content_type="application/json",
+    )
+
+    headers = str(response.serialize_headers())
+    content = str(response.content)
+
+    assert response.status_code == 200
+    assert "Content-Type: text/csv" in headers
+    assert f'filename="{rs.slug}-answers.csv"' in headers
+    assert a.question.question_text in content
+    assert a.question.get_answer_type_display() in content
+    assert str(a.question.required) in content
+    assert a.question.get_image_port_display() in content
+    if isinstance(answer, dict):
+        for key in answer:
+            assert key in content
+    else:
+        assert re.sub(r"[\n\r\t]", " ", str(a.answer)) in content
+    assert im.name in content
+    assert a.creator.username in content
+
+    response = get_view_for_user(
+        viewname="api:reader-study-export-answers",
+        reverse_kwargs={"pk": rs.pk},
+        user=reader,
+        client=client,
+        method=client.get,
+        content_type="application/json",
+    )
+    assert response.status_code == 404
+
+
+@pytest.mark.parametrize(
+    "data,elements,lines",
+    (
+        ([["a"], ["b"], ["c"]], 1, 3),
+        ([["a"], ["b,c"], ["c"]], 1, 3),
+        ([["a\nb"], ["b"], ["c"]], 1, 3),
+        ([["a\rb\nc", "\nb", "\rc\r\r"]], 3, 1),
+        ([["a", "a", "\na"], ["b", "b", "b"], ["c", "c", "c"]], 3, 3),
+        (
+            [["a", '{"a":\n{"b": "c\nd"}\n}'], ["b", "b,c,d"], ["c", "d\r"]],
+            2,
+            3,
+        ),
+    ),
+)
+def test_csv_export_preprocessing(tmp_path, data, elements, lines):
+    exporter = ExportCSVMixin()
+    processed = exporter._preprocess_data(data)
+    assert len(processed) == lines
+
+    # Unfortunately, we have to create an actual file here, as both tempfile
+    # and StringIO seem to cause issues with line endings
+    with open(tmp_path / "csv.csv", "w+", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerows(processed)
+
+    with open(tmp_path / "csv.csv", "r", newline="") as f:
+        reader = csv.reader(f)
+        for line in reader:
+            assert len(line) == elements
+        assert reader.line_num == lines
+
+
+def test_csv_export_create_dicts():
+    exporter = ExportCSVMixin()
+    headers = ["foo", "bar"]
+    data = []
+
+    for x in range(10):
+        data.append([f"foo{x}", f"bar{x}"])
+
+    csv_dicts = exporter._create_dicts(headers, data)
+
+    for index, dct in enumerate(csv_dicts):
+        assert dct == {"foo": f"foo{index}", "bar": f"bar{index}"}
diff --git a/app/tests/studies_tests/factories.py b/app/tests/studies_tests/factories.py
--- a/app/tests/studies_tests/factories.py
+++ b/app/tests/studies_tests/factories.py
@@ -11,7 +11,7 @@ class StudyFactory(factory.DjangoModelFactory):
     class Meta:
         model = Study
 
-    name = factory.Sequence(lambda n: "Study {}".format(n))
+    name = factory.Sequence(lambda n: f"Study {n}")
     patient = factory.SubFactory(PatientFactory)
     datetime = factory.fuzzy.FuzzyDateTime(
         datetime.datetime(1950, 1, 1, 0, 0, 0, 0, pytz.UTC)