Repository: comic/grand-challenge.org

Issue:
Make ground truth error message more descriptive
When uploading ground truth to a reader study the validation can fail on many points. A couple of important ones are the images not matching the hanging list, or the formatting being incorrect. When validation fails, an error message is shown, but it is not very descriptive. E.g. when the images do not match the hanging list, the error displayed is `Images provided do not match hanging protocol`. It would be a lot more useful to indicate where the file and the hanging list differ.

We should check if we can make the error more descriptive for all errors that might be raised.


Discussion:

Resolving patch:
diff --git a/app/grandchallenge/reader_studies/forms.py b/app/grandchallenge/reader_studies/forms.py
--- a/app/grandchallenge/reader_studies/forms.py
+++ b/app/grandchallenge/reader_studies/forms.py
@@ -1,5 +1,6 @@
 import csv
 import io
+import itertools
 
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import (
@@ -302,16 +303,23 @@ def clean_ground_truth(self):
             )
         ):
             raise ValidationError(
-                "Fields provided do not match with reader study"
+                f"Fields provided do not match with reader study. Fields should "
+                f"be: {self.reader_study.ground_truth_file_headers}"
             )
 
         values = [x for x in rdr]
 
-        if sorted([sorted(x["images"].split(";")) for x in values]) != sorted(
-            self.reader_study.image_groups
-        ):
+        images = sorted([sorted(x["images"].split(";")) for x in values])
+        if images != sorted(self.reader_study.image_groups):
+            diff = self.reader_study.hanging_list_diff(
+                provided=list(itertools.chain(*images))
+            )
             raise ValidationError(
-                "Images provided do not match hanging protocol"
+                f"Images provided do not match hanging protocol. The following "
+                f"images appear in the file, but not in the hanging list: "
+                f"{', '.join(diff['in_provided_list'])}. These images appear "
+                f"in the hanging list, but not in the file: "
+                f"{', '.join(diff['in_hanging_list'])}."
             )
 
         return values
diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py
--- a/app/grandchallenge/reader_studies/models.py
+++ b/app/grandchallenge/reader_studies/models.py
@@ -235,12 +235,16 @@ class Meta(UUIDModel.Meta, TitleSlugDescriptionModel.Meta):
     def __str__(self):
         return f"{self.title}"
 
+    @property
+    def ground_truth_file_headers(self):
+        return f"images,{','.join([q.question_text for q in self.questions.all()])}"
+
     def get_example_ground_truth_csv(self):
         if len(self.hanging_list) == 0:
             return "No cases in this reader study"
         questions = self.questions.all()
         return (
-            f"images,{','.join([q.question_text for q in questions])}\n"
+            f"{self.ground_truth_file_headers}\n"
             f"{';'.join(self.image_groups[0])},{','.join([q.example_answer for q in questions])}"
         )
 
@@ -362,17 +366,16 @@ def hanging_list_valid(self):
             self.hanging_image_names
         )
 
-    @property
-    def hanging_list_diff(self):
+    def hanging_list_diff(self, provided=None):
         """
         Returns the diff between the images added to the study and the images
         in the hanging list.
         """
+        comparison = provided or self.study_image_names
         return {
-            "in_study_list": set(self.study_image_names)
+            "in_provided_list": set(comparison)
             - set(self.hanging_image_names),
-            "in_hanging_list": set(self.hanging_image_names)
-            - set(self.study_image_names),
+            "in_hanging_list": set(self.hanging_image_names) - set(comparison),
         }
 
     @property


Test patch:
diff --git a/app/tests/reader_studies_tests/resources/ground_truth_wrong_images.csv b/app/tests/reader_studies_tests/resources/ground_truth_wrong_images.csv
new file mode 100644
--- /dev/null
+++ b/app/tests/reader_studies_tests/resources/ground_truth_wrong_images.csv
@@ -0,0 +1,5 @@
+images,foo,bool,choice,mchoice
+im1,'"yes'",'true','"option1"','["option1", "option2"]'
+im2,'"no"','false','"option1"','["option1", "option2"]'
+im3,'"no"','false','"option1"','["option1", "option2"]'
+im5,'"yes"','true','"option1"','["option1", "option2"]'
diff --git a/app/tests/reader_studies_tests/test_forms.py b/app/tests/reader_studies_tests/test_forms.py
--- a/app/tests/reader_studies_tests/test_forms.py
+++ b/app/tests/reader_studies_tests/test_forms.py
@@ -465,8 +465,8 @@ def test_reader_study_add_ground_truth(client, settings):
         )
     assert response.status_code == 200
     assert (
-        "Fields provided do not match with reader study"
-        in response.rendered_content
+        f"Fields provided do not match with reader study. Fields should "
+        f"be: {rs.ground_truth_file_headers}" in response.rendered_content
     )
 
     q.question_text = "foo"
@@ -488,6 +488,30 @@ def test_reader_study_add_ground_truth(client, settings):
         in response.rendered_content
     )
 
+    with open(RESOURCE_PATH / "ground_truth_wrong_images.csv") as gt:
+        response = get_view_for_user(
+            viewname="reader-studies:add-ground-truth",
+            client=client,
+            method=client.post,
+            reverse_kwargs={"slug": rs.slug},
+            data={"ground_truth": gt},
+            follow=True,
+            user=editor,
+        )
+    assert response.status_code == 200
+    assert (
+        "Images provided do not match hanging protocol"
+        in response.rendered_content
+    )
+    assert (
+        f"The following images appear in the file, but not in the hanging "
+        f"list: im5." in response.rendered_content
+    )
+    assert (
+        "These images appear in the hanging list, but not in the file: im4."
+        in response.rendered_content.replace(r"[\n']", "")
+    )
+
     rs.hanging_list = [
         {"primary": "im1"},
         {"primary": "im2"},