Repository: comic/grand-challenge.org

Issue:
Allow null in annotation answers
For string answers we allow empty strings to represent that the question was skipped.

For annotations and choices the front end skips submitting an answer if skipped, causing this issue: https://github.com/DIAGNijmegen/rse-cirrus-core/issues/635

So, we need to represent a question being intentionally left blank, and the way to do this is to add optional `null` (JSON/in the db)/`None` (python) to the schema for the non-boolean & string answer types.


Discussion:
@pkcakeout let me know if you agree with this approach
Yes, storing an explicit `null`/`None` would be fine and a great solution. Alternatively (just in case that is easier to implement) and empty list or dictionary would also work from the cirrus side (for annotations).
No difference in complexity I think - I will first go with explicit `null`/`None`. 

Resolving patch:
diff --git a/app/grandchallenge/reader_studies/migrations/0017_auto_20200421_1348.py b/app/grandchallenge/reader_studies/migrations/0017_auto_20200421_1348.py
new file mode 100644
--- /dev/null
+++ b/app/grandchallenge/reader_studies/migrations/0017_auto_20200421_1348.py
@@ -0,0 +1,561 @@
+# Generated by Django 3.0.5 on 2020-04-21 13:48
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+import grandchallenge.core.validators
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("reader_studies", "0016_auto_20200406_1334"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="answer",
+            name="answer",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                null=True,
+                validators=[
+                    grandchallenge.core.validators.JSONSchemaValidator(
+                        schema={
+                            "$schema": "http://json-schema.org/draft-07/schema#",
+                            "anyOf": [
+                                {"$ref": "#/definitions/null"},
+                                {"$ref": "#/definitions/STXT"},
+                                {"$ref": "#/definitions/MTXT"},
+                                {"$ref": "#/definitions/BOOL"},
+                                {"$ref": "#/definitions/HEAD"},
+                                {"$ref": "#/definitions/2DBB"},
+                                {"$ref": "#/definitions/DIST"},
+                                {"$ref": "#/definitions/MDIS"},
+                                {"$ref": "#/definitions/POIN"},
+                                {"$ref": "#/definitions/MPOI"},
+                                {"$ref": "#/definitions/POLY"},
+                                {"$ref": "#/definitions/MPOL"},
+                                {"$ref": "#/definitions/CHOI"},
+                                {"$ref": "#/definitions/MCHO"},
+                                {"$ref": "#/definitions/MCHD"},
+                            ],
+                            "definitions": {
+                                "2DBB": {
+                                    "properties": {
+                                        "corners": {
+                                            "items": {
+                                                "items": {"type": "number"},
+                                                "maxItems": 3,
+                                                "minItems": 3,
+                                                "type": "array",
+                                            },
+                                            "maxItems": 4,
+                                            "minItems": 4,
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "type": {"enum": ["2D bounding box"]},
+                                    },
+                                    "required": ["version", "type", "corners"],
+                                    "type": "object",
+                                },
+                                "BOOL": {"type": "boolean"},
+                                "CHOI": {"type": "number"},
+                                "DIST": {
+                                    "properties": {
+                                        "end": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "start": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "type": {
+                                            "enum": ["Distance measurement"]
+                                        },
+                                    },
+                                    "required": [
+                                        "version",
+                                        "type",
+                                        "start",
+                                        "end",
+                                    ],
+                                    "type": "object",
+                                },
+                                "HEAD": {"type": "null"},
+                                "MCHD": {
+                                    "items": {"type": "number"},
+                                    "type": "array",
+                                },
+                                "MCHO": {
+                                    "items": {"type": "number"},
+                                    "type": "array",
+                                },
+                                "MDIS": {
+                                    "properties": {
+                                        "lines": {
+                                            "items": {
+                                                "allOf": [
+                                                    {
+                                                        "$ref": "#/definitions/line-object"
+                                                    }
+                                                ]
+                                            },
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "type": {
+                                            "enum": [
+                                                "Multiple distance measurements"
+                                            ]
+                                        },
+                                    },
+                                    "required": ["version", "type", "lines"],
+                                    "type": "object",
+                                },
+                                "MPOI": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "points": {
+                                            "items": {
+                                                "allOf": [
+                                                    {
+                                                        "$ref": "#/definitions/point-object"
+                                                    }
+                                                ]
+                                            },
+                                            "type": "array",
+                                        },
+                                        "type": {"enum": ["Multiple points"]},
+                                    },
+                                    "required": ["version", "type", "points"],
+                                    "type": "object",
+                                },
+                                "MPOL": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "polygons": {
+                                            "items": {
+                                                "$ref": "#/definitions/polygon-object"
+                                            },
+                                            "type": "array",
+                                        },
+                                        "type": {
+                                            "enum": ["Multiple polygons"]
+                                        },
+                                    },
+                                    "required": [
+                                        "type",
+                                        "version",
+                                        "polygons",
+                                    ],
+                                    "type": "object",
+                                },
+                                "MTXT": {"type": "string"},
+                                "POIN": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "type": {"enum": ["Point"]},
+                                    },
+                                    "required": ["version", "type", "point"],
+                                    "type": "object",
+                                },
+                                "POLY": {
+                                    "properties": {
+                                        "groups": {
+                                            "items": {"type": "string"},
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "path_points": {
+                                            "items": {
+                                                "items": {"type": "number"},
+                                                "maxItems": 3,
+                                                "minItems": 3,
+                                                "type": "array",
+                                            },
+                                            "type": "array",
+                                        },
+                                        "seed_point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "sub_type": {"type": "string"},
+                                    },
+                                    "required": [
+                                        "name",
+                                        "seed_point",
+                                        "path_points",
+                                        "sub_type",
+                                        "groups",
+                                        "version",
+                                    ],
+                                    "type": "object",
+                                },
+                                "STXT": {"type": "string"},
+                                "line-object": {
+                                    "properties": {
+                                        "end": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "start": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                    },
+                                    "required": ["start", "end"],
+                                    "type": "object",
+                                },
+                                "null": {"type": "null"},
+                                "point-object": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                    },
+                                    "required": ["point"],
+                                    "type": "object",
+                                },
+                                "polygon-object": {
+                                    "properties": {
+                                        "groups": {
+                                            "items": {"type": "string"},
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "path_points": {
+                                            "items": {
+                                                "items": {"type": "number"},
+                                                "maxItems": 3,
+                                                "minItems": 3,
+                                                "type": "array",
+                                            },
+                                            "type": "array",
+                                        },
+                                        "seed_point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "sub_type": {"type": "string"},
+                                    },
+                                    "required": [
+                                        "name",
+                                        "seed_point",
+                                        "path_points",
+                                        "sub_type",
+                                        "groups",
+                                    ],
+                                    "type": "object",
+                                },
+                            },
+                            "properties": {
+                                "version": {
+                                    "additionalProperties": {"type": "number"},
+                                    "required": ["major", "minor"],
+                                    "type": "object",
+                                }
+                            },
+                        }
+                    )
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="historicalanswer",
+            name="answer",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                null=True,
+                validators=[
+                    grandchallenge.core.validators.JSONSchemaValidator(
+                        schema={
+                            "$schema": "http://json-schema.org/draft-07/schema#",
+                            "anyOf": [
+                                {"$ref": "#/definitions/null"},
+                                {"$ref": "#/definitions/STXT"},
+                                {"$ref": "#/definitions/MTXT"},
+                                {"$ref": "#/definitions/BOOL"},
+                                {"$ref": "#/definitions/HEAD"},
+                                {"$ref": "#/definitions/2DBB"},
+                                {"$ref": "#/definitions/DIST"},
+                                {"$ref": "#/definitions/MDIS"},
+                                {"$ref": "#/definitions/POIN"},
+                                {"$ref": "#/definitions/MPOI"},
+                                {"$ref": "#/definitions/POLY"},
+                                {"$ref": "#/definitions/MPOL"},
+                                {"$ref": "#/definitions/CHOI"},
+                                {"$ref": "#/definitions/MCHO"},
+                                {"$ref": "#/definitions/MCHD"},
+                            ],
+                            "definitions": {
+                                "2DBB": {
+                                    "properties": {
+                                        "corners": {
+                                            "items": {
+                                                "items": {"type": "number"},
+                                                "maxItems": 3,
+                                                "minItems": 3,
+                                                "type": "array",
+                                            },
+                                            "maxItems": 4,
+                                            "minItems": 4,
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "type": {"enum": ["2D bounding box"]},
+                                    },
+                                    "required": ["version", "type", "corners"],
+                                    "type": "object",
+                                },
+                                "BOOL": {"type": "boolean"},
+                                "CHOI": {"type": "number"},
+                                "DIST": {
+                                    "properties": {
+                                        "end": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "start": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "type": {
+                                            "enum": ["Distance measurement"]
+                                        },
+                                    },
+                                    "required": [
+                                        "version",
+                                        "type",
+                                        "start",
+                                        "end",
+                                    ],
+                                    "type": "object",
+                                },
+                                "HEAD": {"type": "null"},
+                                "MCHD": {
+                                    "items": {"type": "number"},
+                                    "type": "array",
+                                },
+                                "MCHO": {
+                                    "items": {"type": "number"},
+                                    "type": "array",
+                                },
+                                "MDIS": {
+                                    "properties": {
+                                        "lines": {
+                                            "items": {
+                                                "allOf": [
+                                                    {
+                                                        "$ref": "#/definitions/line-object"
+                                                    }
+                                                ]
+                                            },
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "type": {
+                                            "enum": [
+                                                "Multiple distance measurements"
+                                            ]
+                                        },
+                                    },
+                                    "required": ["version", "type", "lines"],
+                                    "type": "object",
+                                },
+                                "MPOI": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "points": {
+                                            "items": {
+                                                "allOf": [
+                                                    {
+                                                        "$ref": "#/definitions/point-object"
+                                                    }
+                                                ]
+                                            },
+                                            "type": "array",
+                                        },
+                                        "type": {"enum": ["Multiple points"]},
+                                    },
+                                    "required": ["version", "type", "points"],
+                                    "type": "object",
+                                },
+                                "MPOL": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "polygons": {
+                                            "items": {
+                                                "$ref": "#/definitions/polygon-object"
+                                            },
+                                            "type": "array",
+                                        },
+                                        "type": {
+                                            "enum": ["Multiple polygons"]
+                                        },
+                                    },
+                                    "required": [
+                                        "type",
+                                        "version",
+                                        "polygons",
+                                    ],
+                                    "type": "object",
+                                },
+                                "MTXT": {"type": "string"},
+                                "POIN": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "type": {"enum": ["Point"]},
+                                    },
+                                    "required": ["version", "type", "point"],
+                                    "type": "object",
+                                },
+                                "POLY": {
+                                    "properties": {
+                                        "groups": {
+                                            "items": {"type": "string"},
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "path_points": {
+                                            "items": {
+                                                "items": {"type": "number"},
+                                                "maxItems": 3,
+                                                "minItems": 3,
+                                                "type": "array",
+                                            },
+                                            "type": "array",
+                                        },
+                                        "seed_point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "sub_type": {"type": "string"},
+                                    },
+                                    "required": [
+                                        "name",
+                                        "seed_point",
+                                        "path_points",
+                                        "sub_type",
+                                        "groups",
+                                        "version",
+                                    ],
+                                    "type": "object",
+                                },
+                                "STXT": {"type": "string"},
+                                "line-object": {
+                                    "properties": {
+                                        "end": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "start": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                    },
+                                    "required": ["start", "end"],
+                                    "type": "object",
+                                },
+                                "null": {"type": "null"},
+                                "point-object": {
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                    },
+                                    "required": ["point"],
+                                    "type": "object",
+                                },
+                                "polygon-object": {
+                                    "properties": {
+                                        "groups": {
+                                            "items": {"type": "string"},
+                                            "type": "array",
+                                        },
+                                        "name": {"type": "string"},
+                                        "path_points": {
+                                            "items": {
+                                                "items": {"type": "number"},
+                                                "maxItems": 3,
+                                                "minItems": 3,
+                                                "type": "array",
+                                            },
+                                            "type": "array",
+                                        },
+                                        "seed_point": {
+                                            "items": {"type": "number"},
+                                            "maxItems": 3,
+                                            "minItems": 3,
+                                            "type": "array",
+                                        },
+                                        "sub_type": {"type": "string"},
+                                    },
+                                    "required": [
+                                        "name",
+                                        "seed_point",
+                                        "path_points",
+                                        "sub_type",
+                                        "groups",
+                                    ],
+                                    "type": "object",
+                                },
+                            },
+                            "properties": {
+                                "version": {
+                                    "additionalProperties": {"type": "number"},
+                                    "required": ["major", "minor"],
+                                    "type": "object",
+                                }
+                            },
+                        }
+                    )
+                ],
+            ),
+        ),
+    ]
diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py
--- a/app/grandchallenge/reader_studies/models.py
+++ b/app/grandchallenge/reader_studies/models.py
@@ -743,6 +743,7 @@ def delete_reader_study_groups_hook(*_, instance: ReaderStudy, using, **__):
 ANSWER_TYPE_SCHEMA = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+        "null": {"type": "null"},
         "STXT": {"type": "string"},
         "MTXT": {"type": "string"},
         "BOOL": {"type": "boolean"},
@@ -946,6 +947,7 @@ def delete_reader_study_groups_hook(*_, instance: ReaderStudy, using, **__):
     },
     # anyOf should exist, check Question.is_answer_valid
     "anyOf": [
+        {"$ref": "#/definitions/null"},
         {"$ref": "#/definitions/STXT"},
         {"$ref": "#/definitions/MTXT"},
         {"$ref": "#/definitions/BOOL"},
@@ -1155,18 +1157,9 @@ def assign_permissions(self):
     def clean(self):
         # Make sure that the image port is only set when using drawn
         # annotations.
-        if (
-            self.answer_type
-            in [
-                self.ANSWER_TYPE_2D_BOUNDING_BOX,
-                self.ANSWER_TYPE_DISTANCE_MEASUREMENT,
-                self.ANSWER_TYPE_MULTIPLE_DISTANCE_MEASUREMENTS,
-                self.ANSWER_TYPE_POINT,
-                self.ANSWER_TYPE_MULTIPLE_POINTS,
-                self.ANSWER_TYPE_POLYGON,
-                self.ANSWER_TYPE_MULTIPLE_POLYGONS,
-            ]
-        ) != bool(self.image_port):
+        if (self.answer_type in self.annotation_types) != bool(
+            self.image_port
+        ):
             raise ValidationError(
                 "The image port must (only) be set for annotation questions."
             )
@@ -1181,17 +1174,32 @@ def clean(self):
                 "(otherwise the user will need to tick a box for each image!)"
             )
 
+    @property
+    def annotation_types(self):
+        return [
+            self.ANSWER_TYPE_2D_BOUNDING_BOX,
+            self.ANSWER_TYPE_DISTANCE_MEASUREMENT,
+            self.ANSWER_TYPE_MULTIPLE_DISTANCE_MEASUREMENTS,
+            self.ANSWER_TYPE_POINT,
+            self.ANSWER_TYPE_MULTIPLE_POINTS,
+            self.ANSWER_TYPE_POLYGON,
+            self.ANSWER_TYPE_MULTIPLE_POLYGONS,
+        ]
+
     def is_answer_valid(self, *, answer):
         """Validates ``answer`` against ``ANSWER_TYPE_SCHEMA``."""
+        allowed_types = [
+            {"$ref": f"#/definitions/{self.answer_type}"},
+        ]
+
+        allow_null = self.answer_type in self.annotation_types
+        if allow_null:
+            allowed_types.append({"$ref": "#/definitions/null"})
+
         try:
             return (
                 JSONSchemaValidator(
-                    schema={
-                        **ANSWER_TYPE_SCHEMA,
-                        "anyOf": [
-                            {"$ref": f"#/definitions/{self.answer_type}"}
-                        ],
-                    }
+                    schema={**ANSWER_TYPE_SCHEMA, "anyOf": allowed_types}
                 )(answer)
                 is None
             )
@@ -1224,8 +1232,9 @@ class Answer(UUIDModel):
     creator = models.ForeignKey(get_user_model(), on_delete=models.CASCADE)
     question = models.ForeignKey(Question, on_delete=models.CASCADE)
     images = models.ManyToManyField("cases.Image", related_name="answers")
-    # TODO: add validators=[JSONSchemaValidator(schema=ANSWER_TYPE_SCHEMA)],
-    answer = JSONField()
+    answer = JSONField(
+        null=True, validators=[JSONSchemaValidator(schema=ANSWER_TYPE_SCHEMA)],
+    )
     is_ground_truth = models.BooleanField(default=False)
     score = models.FloatField(null=True)
     history = HistoricalRecords(
@@ -1300,6 +1309,17 @@ def validate(  # noqa: C901
         instance=None,
     ):
         """Validates all fields provided for ``answer``."""
+        if question.answer_type == question.ANSWER_TYPE_HEADING:
+            # Maintained for historical consistency
+            raise ValidationError("Headings are not answerable.")
+
+        if not question.is_answer_valid(answer=answer):
+            raise ValidationError(
+                f"Your answer is not the correct type. "
+                f"{question.get_answer_type_display()} expected, "
+                f"{type(answer)} found."
+            )
+
         if len(images) == 0:
             raise ValidationError(
                 "You must specify the images that this answer corresponds to."
@@ -1331,6 +1351,7 @@ def validate(  # noqa: C901
                 raise ValidationError(
                     "This user is not a reader for this study."
                 )
+
         if (
             question.answer_type == Question.ANSWER_TYPE_CHOICE
             and answer not in question.options.values_list("id", flat=True)
@@ -1338,6 +1359,7 @@ def validate(  # noqa: C901
             raise ValidationError(
                 "Provided option is not valid for this question"
             )
+
         if question.answer_type in (
             Question.ANSWER_TYPE_MULTIPLE_CHOICE,
             Question.ANSWER_TYPE_MULTIPLE_CHOICE_DROPDOWN,
@@ -1347,12 +1369,6 @@ def validate(  # noqa: C901
                 raise ValidationError(
                     "Provided options are not valid for this question"
                 )
-        if not question.is_answer_valid(answer=answer):
-            raise ValidationError(
-                f"Your answer is not the correct type. "
-                f"{question.get_answer_type_display()} expected, "
-                f"{type(answer)} found."
-            )
 
     @property
     def answer_text(self):


Test patch:
diff --git a/app/tests/reader_studies_tests/test_api.py b/app/tests/reader_studies_tests/test_api.py
--- a/app/tests/reader_studies_tests/test_api.py
+++ b/app/tests/reader_studies_tests/test_api.py
@@ -521,6 +521,20 @@ def test_answer_creator_is_reader(client):
             },
             400,
         ),
+        (Question.ANSWER_TYPE_SINGLE_LINE_TEXT, None, 400),
+        (Question.ANSWER_TYPE_MULTI_LINE_TEXT, None, 400),
+        (Question.ANSWER_TYPE_BOOL, None, 400),
+        (Question.ANSWER_TYPE_HEADING, None, 400),
+        (Question.ANSWER_TYPE_2D_BOUNDING_BOX, None, 201),
+        (Question.ANSWER_TYPE_DISTANCE_MEASUREMENT, None, 201),
+        (Question.ANSWER_TYPE_MULTIPLE_DISTANCE_MEASUREMENTS, None, 201),
+        (Question.ANSWER_TYPE_POINT, None, 201),
+        (Question.ANSWER_TYPE_MULTIPLE_POINTS, None, 201),
+        (Question.ANSWER_TYPE_POLYGON, None, 201),
+        (Question.ANSWER_TYPE_MULTIPLE_POLYGONS, None, 201),
+        (Question.ANSWER_TYPE_CHOICE, None, 400),
+        (Question.ANSWER_TYPE_MULTIPLE_CHOICE, None, 400),
+        (Question.ANSWER_TYPE_MULTIPLE_CHOICE_DROPDOWN, None, 400),
     ),
 )
 def test_answer_is_correct_type(client, answer_type, answer, expected):