Repository: comic/grand-challenge.org

Issue:
It is possible to assign other annotations to annotation answers
The validation for annotation answers will accept any annotation, so it is possible to set say, a bounding box for a distance line answer. Discovered when reading code with @silvandeleemput yesterday.


Discussion:

Resolving patch:
diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py
--- a/app/grandchallenge/reader_studies/models.py
+++ b/app/grandchallenge/reader_studies/models.py
@@ -10,6 +10,7 @@
 from django.dispatch import receiver
 from django_extensions.db.models import TitleSlugDescriptionModel
 from guardian.shortcuts import assign_perm, get_objects_for_group, remove_perm
+from jsonschema import RefResolutionError
 from numpy.random.mtrand import RandomState
 
 from grandchallenge.cases.models import Image
@@ -282,11 +283,7 @@ def get_progress_for_user(self, user):
         hanging_list_count = len(self.hanging_list)
 
         if answerable_question_count == 0 or hanging_list_count == 0:
-            return {
-                "questions": 0.0,
-                "hangings": 0.0,
-                "diff": 0.0,
-            }
+            return {"questions": 0.0, "hangings": 0.0, "diff": 0.0}
 
         expected = hanging_list_count * answerable_question_count
         answers = Answer.objects.filter(
@@ -352,9 +349,13 @@ def delete_reader_study_groups_hook(*_, instance: ReaderStudy, using, **__):
         pass
 
 
-ANSWER_TYPE_ANNOTATIONS_SCHEMA = {
+ANSWER_TYPE_SCHEMA = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+        "STXT": {"type": "string"},
+        "MTXT": {"type": "string"},
+        "BOOL": {"type": "boolean"},
+        "HEAD": {"type": "null"},
         "2DBB": {
             "type": "object",
             "properties": {
@@ -433,7 +434,6 @@ def delete_reader_study_groups_hook(*_, instance: ReaderStudy, using, **__):
             "required": ["version", "type", "lines"],
         },
     },
-    "type": "object",
     "properties": {
         "version": {
             "type": "object",
@@ -441,7 +441,12 @@ def delete_reader_study_groups_hook(*_, instance: ReaderStudy, using, **__):
             "required": ["major", "minor"],
         }
     },
+    # anyOf should exist, check Question.is_answer_valid
     "anyOf": [
+        {"$ref": "#/definitions/STXT"},
+        {"$ref": "#/definitions/MTXT"},
+        {"$ref": "#/definitions/BOOL"},
+        {"$ref": "#/definitions/HEAD"},
         {"$ref": "#/definitions/2DBB"},
         {"$ref": "#/definitions/DIST"},
         {"$ref": "#/definitions/MDIS"},
@@ -449,15 +454,6 @@ def delete_reader_study_groups_hook(*_, instance: ReaderStudy, using, **__):
 }
 
 
-def validate_answer_json(schema: dict, obj: object) -> bool:
-    """The answer type validators must return true or false."""
-    try:
-        JSONSchemaValidator(schema=schema)(obj)
-        return True
-    except ValidationError:
-        return False
-
-
 class Question(UUIDModel):
     ANSWER_TYPE_SINGLE_LINE_TEXT = "STXT"
     ANSWER_TYPE_MULTI_LINE_TEXT = "MTXT"
@@ -480,23 +476,6 @@ class Question(UUIDModel):
         ),
     )
 
-    # A callable for every answer type that would validate the given answer
-    ANSWER_TYPE_VALIDATOR = {
-        ANSWER_TYPE_SINGLE_LINE_TEXT: lambda o: isinstance(o, str),
-        ANSWER_TYPE_MULTI_LINE_TEXT: lambda o: isinstance(o, str),
-        ANSWER_TYPE_BOOL: lambda o: isinstance(o, bool),
-        ANSWER_TYPE_HEADING: lambda o: False,  # Headings are not answerable
-        ANSWER_TYPE_2D_BOUNDING_BOX: lambda o: validate_answer_json(
-            ANSWER_TYPE_ANNOTATIONS_SCHEMA, o
-        ),
-        ANSWER_TYPE_DISTANCE_MEASUREMENT: lambda o: validate_answer_json(
-            ANSWER_TYPE_ANNOTATIONS_SCHEMA, o
-        ),
-        ANSWER_TYPE_MULTIPLE_DISTANCE_MEASUREMENTS: lambda o: validate_answer_json(
-            ANSWER_TYPE_ANNOTATIONS_SCHEMA, o
-        ),
-    }
-
     # What is the orientation of the question form when presented on the
     # front end?
     DIRECTION_HORIZONTAL = "H"
@@ -617,13 +596,32 @@ def clean(self):
             )
 
     def is_answer_valid(self, *, answer):
-        return self.ANSWER_TYPE_VALIDATOR[self.answer_type](answer)
+        try:
+            return (
+                JSONSchemaValidator(
+                    schema={
+                        **ANSWER_TYPE_SCHEMA,
+                        "anyOf": [
+                            {"$ref": f"#/definitions/{self.answer_type}"}
+                        ],
+                    }
+                )(answer)
+                is None
+            )
+        except ValidationError:
+            return False
+        except RefResolutionError:
+            raise RuntimeError(
+                f"#/definitions/{self.answer_type} needs to be defined in "
+                "ANSWER_TYPE_SCHEMA."
+            )
 
 
 class Answer(UUIDModel):
     creator = models.ForeignKey(get_user_model(), on_delete=models.CASCADE)
     question = models.ForeignKey(Question, on_delete=models.CASCADE)
     images = models.ManyToManyField("cases.Image", related_name="answers")
+    # TODO: add validators=[JSONSchemaValidator(schema=ANSWER_TYPE_SCHEMA)],
     answer = JSONField()
 
     csv_headers = Question.csv_headers + [


Test patch:
diff --git a/app/tests/reader_studies_tests/test_api.py b/app/tests/reader_studies_tests/test_api.py
--- a/app/tests/reader_studies_tests/test_api.py
+++ b/app/tests/reader_studies_tests/test_api.py
@@ -142,7 +142,8 @@ def test_answer_creator_is_reader(client):
         (Question.ANSWER_TYPE_MULTI_LINE_TEXT, "dgfsgfds", 201),
         (Question.ANSWER_TYPE_MULTI_LINE_TEXT, True, 400),
         (Question.ANSWER_TYPE_HEADING, True, 400),
-        (Question.ANSWER_TYPE_HEADING, "fdsa", 400),
+        (Question.ANSWER_TYPE_HEADING, "null", 400),
+        (Question.ANSWER_TYPE_HEADING, None, 400),
         (Question.ANSWER_TYPE_2D_BOUNDING_BOX, "", 400),
         (Question.ANSWER_TYPE_2D_BOUNDING_BOX, True, 400),
         (Question.ANSWER_TYPE_2D_BOUNDING_BOX, False, 400),
diff --git a/app/tests/reader_studies_tests/validation_tests.py b/app/tests/reader_studies_tests/validation_tests.py
--- a/app/tests/reader_studies_tests/validation_tests.py
+++ b/app/tests/reader_studies_tests/validation_tests.py
@@ -1,7 +1,10 @@
 import pytest
 
 from grandchallenge.core.validators import JSONSchemaValidator
-from grandchallenge.reader_studies.models import HANGING_LIST_SCHEMA
+from grandchallenge.reader_studies.models import (
+    HANGING_LIST_SCHEMA,
+    Question,
+)
 from tests.factories import ImageFactory, UserFactory
 from tests.reader_studies_tests.factories import ReaderStudyFactory
 from tests.utils import get_view_for_user
@@ -137,3 +140,86 @@ def test_hanging_list_shuffle_per_user(client):
     )
     assert response.status_code == 200
     assert response.json()["hanging_list_images"] == u1_list
+
+
+ANSWER_TYPE_NAMES_AND_ANSWERS = {
+    "STXT": "string test",
+    "MTXT": "multiline string\ntest",
+    "BOOL": True,
+    "2DBB": {
+        "version": {"major": 1, "minor": 0},
+        "type": "2D bounding box",
+        "name": "test_name",
+        "corners": [[0, 0, 0], [10, 0, 0], [10, 10, 0], [0, 0, 0]],
+    },
+    "DIST": {
+        "version": {"major": 1, "minor": 0},
+        "type": "Distance measurement",
+        "name": "test_name",
+        "start": [0, 0, 0],
+        "end": [10, 0, 0],
+    },
+    "MDIS": {
+        "version": {"major": 1, "minor": 0},
+        "type": "Multiple distance measurements",
+        "name": "test_name",
+        "lines": [
+            {
+                "type": "object",
+                "name": "segment1",
+                "start": [0, 0, 0],
+                "end": [10, 0, 0],
+            },
+            {"start": [0, 0, 0], "end": [10, 0, 0]},
+        ],
+    },
+}
+
+
+@pytest.mark.parametrize(
+    "answer_type, answer", ANSWER_TYPE_NAMES_AND_ANSWERS.items()
+)
+def test_answer_type_annotation_schema(answer, answer_type):
+    q = Question(answer_type=answer_type)
+    assert q.is_answer_valid(answer=answer) is True
+
+
+@pytest.mark.parametrize("answer", ANSWER_TYPE_NAMES_AND_ANSWERS.values())
+def test_answer_type_annotation_header_schema_fails(
+    answer, answer_type: str = "HEAD"
+):
+    q = Question(answer_type=answer_type)
+    assert not q.is_answer_valid(answer=answer)
+
+
+@pytest.mark.parametrize(
+    "answer_type, answer_type_check",
+    [
+        ("STXT", "2DBB"),
+        ("STXT", "DIST"),
+        ("MTXT", "MDIS"),
+        ("BOOL", "2DBB"),
+        ("BOOL", "STXT"),
+        ("2DBB", "DIST"),
+        ("2DBB", "STXT"),
+        ("DIST", "MDIS"),
+        ("DIST", "2DBB"),
+        ("MDIS", "2DBB"),
+        ("MDIS", "DIST"),
+    ],
+)
+def test_answer_type_annotation_schema_mismatch(
+    answer_type, answer_type_check
+):
+    a = ANSWER_TYPE_NAMES_AND_ANSWERS[answer_type]
+    assert Question(answer_type=answer_type).is_answer_valid(answer=a) is True
+    assert (
+        Question(answer_type=answer_type_check).is_answer_valid(answer=a)
+        is False
+    )
+
+
+def test_new_answer_type_listed():
+    q = Question(answer_type="TEST")
+    with pytest.raises(RuntimeError):
+        q.is_answer_valid(answer="foo")