Repository: comic/grand-challenge.org

Issue:
Non required questions with null answers are being validated
Questions that are not required and to which an empty answer is being posted are being validated incorrectly.

See this sentry issue, as a result of a number type question with a minimum limit set: 
- https://grand-challenge.sentry.io/issues/4295894591/?project=303639&query=is%3Aunresolved&referrer=issue-stream&stream_index=0




Discussion:
Also likely cause of:
- https://github.com/DIAGNijmegen/rse-cirrus-core/issues/2582

Resolving patch:
diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py
--- a/app/grandchallenge/reader_studies/models.py
+++ b/app/grandchallenge/reader_studies/models.py
@@ -972,6 +972,12 @@ def get_annotation_types():
             AnswerType.MULTIPLE_ELLIPSES,
         ]
 
+    @staticmethod
+    def get_image_types():
+        return [
+            AnswerType.MASK,
+        ]
+
     @staticmethod
     def get_widget_required_types():
         return [
@@ -1084,6 +1090,32 @@ class QuestionWidgetKindChoices(models.TextChoices):
     for answer_type, options in ANSWER_TYPE_TO_QUESTION_WIDGET.items()
 }
 
+EMPTY_ANSWER_VALUES = {
+    AnswerType.SINGLE_LINE_TEXT: "",
+    AnswerType.MULTI_LINE_TEXT: "",
+    AnswerType.TEXT: "",
+    AnswerType.NUMBER: None,
+    AnswerType.CHOICE: None,
+    AnswerType.BOOL: None,
+    AnswerType.BOUNDING_BOX_2D: None,
+    AnswerType.MULTIPLE_2D_BOUNDING_BOXES: None,
+    AnswerType.DISTANCE_MEASUREMENT: None,
+    AnswerType.MULTIPLE_DISTANCE_MEASUREMENTS: None,
+    AnswerType.POINT: None,
+    AnswerType.MULTIPLE_POINTS: None,
+    AnswerType.POLYGON: None,
+    AnswerType.MULTIPLE_POLYGONS: None,
+    AnswerType.LINE: None,
+    AnswerType.MULTIPLE_LINES: None,
+    AnswerType.MASK: None,
+    AnswerType.ANGLE: None,
+    AnswerType.MULTIPLE_ANGLES: None,
+    AnswerType.ELLIPSE: None,
+    AnswerType.MULTIPLE_ELLIPSES: None,
+    AnswerType.MULTIPLE_CHOICE: [],
+    AnswerType.MULTIPLE_CHOICE_DROPDOWN: [],
+}
+
 
 class Question(UUIDModel, OverlaySegmentsMixin):
     AnswerType = AnswerType
@@ -1434,18 +1466,22 @@ def _clean_text_options(self):
             )
 
     @property
-    def allow_null_types(self):
-        return [
-            *self.AnswerType.get_annotation_types(),
-            self.AnswerType.CHOICE,
-            self.AnswerType.NUMBER,
-        ]
+    def empty_answer_value(self):
+        """Returns the answer value which is considered to be empty"""
+        if self.answer_type not in EMPTY_ANSWER_VALUES:
+            raise RuntimeError(
+                f"{self.answer_type} has no representation of an empty value."
+            )
+        return EMPTY_ANSWER_VALUES[self.answer_type]
 
     def is_answer_valid(self, *, answer):
         """Validates ``answer`` against ``ANSWER_TYPE_SCHEMA``."""
+        if self.answer_type == Question.AnswerType.HEADING:  # Never valid
+            return False
+
         allowed_types = [{"$ref": f"#/definitions/{self.answer_type}"}]
 
-        if self.answer_type in self.allow_null_types:
+        if self.empty_answer_value is None:
             allowed_types.append({"$ref": "#/definitions/null"})
 
         try:
@@ -1463,9 +1499,35 @@ def is_answer_valid(self, *, answer):
                 "ANSWER_TYPE_SCHEMA."
             )
 
+    @property
+    def validators(self):
+        if self.answer_min_value is not None:
+            yield MinValueValidator(self.answer_min_value)
+
+        if self.answer_max_value is not None:
+            yield MaxValueValidator(self.answer_max_value)
+
+        if self.answer_step_size:
+            yield StepValueValidator(
+                limit_value=self.answer_step_size,
+                offset=self.answer_min_value,
+            )
+
+        if self.answer_min_length is not None:
+            yield MinLengthValidator(self.answer_min_length)
+
+        if self.answer_max_length is not None:
+            yield MaxLengthValidator(self.answer_max_length)
+
+        if self.answer_match_pattern:
+            yield RegexValidator(
+                regex=self.answer_match_pattern,
+                message="Enter a valid answer that matches with the requested format",
+            )
+
     @property
     def is_image_type(self):
-        return self.answer_type in [self.AnswerType.MASK]
+        return self.answer_type in self.AnswerType.get_image_types()
 
     def get_absolute_url(self):
         return self.reader_study.get_absolute_url() + "#questions"
@@ -1612,39 +1674,20 @@ def validate(  # noqa: C901
                     "Provided options are not valid for this question"
                 )
 
-        if (
-            question.answer_type
-            in (Question.AnswerType.NUMBER, Question.AnswerType.TEXT)
-            and question.required
-            and answer is None
-        ):
+        # Image types can have empty answers while an image is being uploaded,
+        # as such we never consider it empty.
+        answer_is_empty = (
+            answer == question.empty_answer_value
+            and not question.is_image_type
+        )
+        if question.required and answer_is_empty:
             raise ValidationError(
-                "Answer for required question cannot be None"
+                "Answer for required question cannot be empty"
             )
 
-        if question.answer_min_value is not None:
-            MinValueValidator(question.answer_min_value)(answer)
-
-        if question.answer_max_value is not None:
-            MaxValueValidator(question.answer_max_value)(answer)
-
-        if question.answer_step_size:
-            StepValueValidator(
-                limit_value=question.answer_step_size,
-                offset=question.answer_min_value,
-            )(answer)
-
-        if question.answer_min_length is not None:
-            MinLengthValidator(question.answer_min_length)(answer)
-
-        if question.answer_max_length is not None:
-            MaxLengthValidator(question.answer_max_length)(answer)
-
-        if question.answer_match_pattern:
-            RegexValidator(
-                regex=question.answer_match_pattern,
-                message="Enter a valid answer that matches with the requested format",
-            )(answer)
+        if not answer_is_empty:
+            for validator in question.validators:
+                validator(answer)
 
     @property
     def answer_text(self):


Test patch:
diff --git a/app/tests/reader_studies_tests/test_api.py b/app/tests/reader_studies_tests/test_api.py
--- a/app/tests/reader_studies_tests/test_api.py
+++ b/app/tests/reader_studies_tests/test_api.py
@@ -281,14 +281,19 @@ def test_answer_creator_is_reader(client):
         (Question.AnswerType.NUMBER, "12", 400),
         (Question.AnswerType.NUMBER, True, 400),
         (Question.AnswerType.SINGLE_LINE_TEXT, "dgfsgfds", 201),
+        (Question.AnswerType.SINGLE_LINE_TEXT, None, 400),
         (Question.AnswerType.SINGLE_LINE_TEXT, True, 400),
         (Question.AnswerType.SINGLE_LINE_TEXT, 12, 400),
         (Question.AnswerType.MULTI_LINE_TEXT, "dgfsgfds", 201),
+        (Question.AnswerType.MULTI_LINE_TEXT, None, 400),
         (Question.AnswerType.MULTI_LINE_TEXT, True, 400),
         (Question.AnswerType.MULTI_LINE_TEXT, 12, 400),
+        (Question.AnswerType.MULTIPLE_CHOICE, None, 400),
+        (Question.AnswerType.MULTIPLE_CHOICE_DROPDOWN, None, 400),
+        # Headings are always incorrect when answering
         (Question.AnswerType.HEADING, True, 400),
         (Question.AnswerType.HEADING, "null", 400),
-        (Question.AnswerType.HEADING, None, 400),
+        # Annotations
         (Question.AnswerType.BOUNDING_BOX_2D, "", 400),
         (Question.AnswerType.BOUNDING_BOX_2D, True, 400),
         (Question.AnswerType.BOUNDING_BOX_2D, False, 400),
@@ -782,24 +787,6 @@ def test_answer_creator_is_reader(client):
             },
             400,
         ),
-        (Question.AnswerType.SINGLE_LINE_TEXT, None, 400),
-        (Question.AnswerType.MULTI_LINE_TEXT, None, 400),
-        (Question.AnswerType.BOOL, None, 400),
-        (Question.AnswerType.NUMBER, None, 400),
-        (Question.AnswerType.HEADING, None, 400),
-        (Question.AnswerType.BOUNDING_BOX_2D, None, 201),
-        (Question.AnswerType.MULTIPLE_2D_BOUNDING_BOXES, None, 201),
-        (Question.AnswerType.DISTANCE_MEASUREMENT, None, 201),
-        (Question.AnswerType.MULTIPLE_DISTANCE_MEASUREMENTS, None, 201),
-        (Question.AnswerType.POINT, None, 201),
-        (Question.AnswerType.MULTIPLE_POINTS, None, 201),
-        (Question.AnswerType.POLYGON, None, 201),
-        (Question.AnswerType.MULTIPLE_POLYGONS, None, 201),
-        (Question.AnswerType.ANGLE, None, 201),
-        (Question.AnswerType.MULTIPLE_ANGLES, None, 201),
-        (Question.AnswerType.CHOICE, None, 400),
-        (Question.AnswerType.MULTIPLE_CHOICE, None, 400),
-        (Question.AnswerType.MULTIPLE_CHOICE_DROPDOWN, None, 400),
         (Question.AnswerType.ELLIPSE, "wwoljg", 400),
         (Question.AnswerType.ELLIPSE, True, 400),
         (Question.AnswerType.ELLIPSE, 42, 400),
@@ -935,34 +922,53 @@ def test_answer_is_correct_type(client, answer_type, answer, expected):
 
 @pytest.mark.django_db
 @pytest.mark.parametrize(
-    "answer_type", (Question.AnswerType.CHOICE, Question.AnswerType.NUMBER)
+    "answer_type,answer",
+    (
+        # Blank answers
+        (Question.AnswerType.SINGLE_LINE_TEXT, ""),
+        (Question.AnswerType.MULTI_LINE_TEXT, ""),
+        (Question.AnswerType.TEXT, ""),
+        # Null answers
+        (Question.AnswerType.NUMBER, None),
+        (Question.AnswerType.CHOICE, None),
+        (Question.AnswerType.BOUNDING_BOX_2D, None),
+        (Question.AnswerType.MULTIPLE_2D_BOUNDING_BOXES, None),
+        (Question.AnswerType.DISTANCE_MEASUREMENT, None),
+        (Question.AnswerType.MULTIPLE_DISTANCE_MEASUREMENTS, None),
+        (Question.AnswerType.POINT, None),
+        (Question.AnswerType.MULTIPLE_POINTS, None),
+        (Question.AnswerType.POLYGON, None),
+        (Question.AnswerType.MULTIPLE_POLYGONS, None),
+        (Question.AnswerType.LINE, None),
+        (Question.AnswerType.MULTIPLE_LINES, None),
+        (Question.AnswerType.ANGLE, None),
+        (Question.AnswerType.MULTIPLE_ANGLES, None),
+        (Question.AnswerType.ELLIPSE, None),
+        (Question.AnswerType.MULTIPLE_ELLIPSES, None),
+        # Empty-collection answers
+        (Question.AnswerType.MULTIPLE_CHOICE, []),
+        (Question.AnswerType.MULTIPLE_CHOICE_DROPDOWN, []),
+    ),
+)
+@pytest.mark.parametrize(
+    "is_required,expected_status_code",
+    (
+        (False, 201),
+        (True, 400),
+    ),
 )
-def test_only_non_required_can_be_null(client, answer_type):
+def test_empty_answers(
+    client, answer_type, answer, is_required, expected_status_code
+):
     rs = ReaderStudyFactory()
     ds = DisplaySetFactory(reader_study=rs)
     reader = UserFactory()
     rs.add_reader(reader)
 
     q = QuestionFactory(
-        reader_study=rs, answer_type=answer_type, required=True
-    )
-
-    response = get_view_for_user(
-        viewname="api:reader-studies-answer-list",
-        user=reader,
-        client=client,
-        method=client.post,
-        data={
-            "answer": None,
-            "display_set": ds.api_url,
-            "question": q.api_url,
-        },
-        content_type="application/json",
-    )
-    assert response.status_code == 400
-
-    q = QuestionFactory(
-        reader_study=rs, answer_type=answer_type, required=False
+        reader_study=rs,
+        answer_type=answer_type,
+        required=is_required,
     )
 
     response = get_view_for_user(
@@ -971,13 +977,13 @@ def test_only_non_required_can_be_null(client, answer_type):
         client=client,
         method=client.post,
         data={
-            "answer": None,
+            "answer": answer,
             "display_set": ds.api_url,
             "question": q.api_url,
         },
         content_type="application/json",
     )
-    assert response.status_code == 201
+    assert response.status_code == expected_status_code
 
 
 @pytest.mark.django_db
diff --git a/app/tests/reader_studies_tests/test_models.py b/app/tests/reader_studies_tests/test_models.py
--- a/app/tests/reader_studies_tests/test_models.py
+++ b/app/tests/reader_studies_tests/test_models.py
@@ -429,6 +429,22 @@ def test_validate_answer():
 
 
 NUMBER_ANSWER_VALIDATION_INPUT = (
+    (
+        AnswerType.NUMBER,
+        None,
+        {
+            "required": False,
+        },
+        nullcontext(),
+    ),
+    (
+        AnswerType.NUMBER,
+        None,
+        {
+            "required": True,
+        },
+        pytest.raises(ValidationError),
+    ),
     (
         AnswerType.NUMBER,
         2,
@@ -512,9 +528,19 @@ def test_validate_answer():
     (
         AnswerType.TEXT,
         "",
-        {},
+        {
+            "required": False,
+        },
         nullcontext(),
     ),
+    (
+        AnswerType.TEXT,
+        "",
+        {
+            "required": True,
+        },
+        pytest.raises(ValidationError),
+    ),
     (
         AnswerType.TEXT,
         "",
@@ -566,12 +592,21 @@ def test_validate_answer():
     ),
     (
         AnswerType.TEXT,
-        "",
+        "not hello world",
         {
             "answer_match_pattern": r"^hello world$",
         },
         pytest.raises(ValidationError),
     ),
+    (
+        AnswerType.TEXT,
+        "",
+        {
+            "answer_min_size": 1,
+            "required": False,
+        },
+        nullcontext(),
+    ),
 )
 
 
diff --git a/app/tests/reader_studies_tests/validation_tests.py b/app/tests/reader_studies_tests/validation_tests.py
--- a/app/tests/reader_studies_tests/validation_tests.py
+++ b/app/tests/reader_studies_tests/validation_tests.py
@@ -232,7 +232,7 @@ def test_new_answer_type_listed():
     [
         ["STXT", False],
         ["MTXT", False],
-        ["BOOL", False],
+        ["BOOL", True],
         ["NUMB", True],
         ["2DBB", True],
         ["M2DB", True],