GitHub PullRequestReviewCommentEvent at 2014-05-14 17:58:17
Repository: edx-ora2
Actor: stephensanchez
Payload: {"comment":{"url":"https://api.github.com/repos/edx/edx-ora2/pulls/comments/12650716","id":12650716,"diff_hunk":"@@ -0,0 +1,504 @@\n+\"\"\"\n+Public interface for student training:\n+\n+* Staff can create assessments for example responses.\n+* Students assess an example response, then compare the scores\n+    they gave to to the instructor's assessment.\n+\n+\"\"\"\n+import logging\n+from django.db import DatabaseError\n+from submissions import api as sub_api\n+from openassessment.assessment.models import StudentTrainingWorkflow\n+from openassessment.assessment.serializers import (\n+    deserialize_training_examples, serialize_training_example,\n+    validate_training_example_format,\n+    InvalidTrainingExample, InvalidRubric\n+)\n+from openassessment.assessment.errors import (\n+    StudentTrainingRequestError, StudentTrainingInternalError\n+)\n+\n+\n+logger = logging.getLogger(__name__)\n+\n+\n+def submitter_is_finished(submission_uuid, requirements):   # pylint:disable=W0613\n+    \"\"\"\n+    Check whether the student has correctly assessed\n+    all the training example responses.\n+\n+    Args:\n+        submission_uuid (str): The UUID of the student's submission.\n+        requirements (dict): Not used.\n+\n+    Returns:\n+        bool\n+\n+    \"\"\"\n+    try:\n+        workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)\n+    except StudentTrainingWorkflow.DoesNotExist:\n+        return False\n+    else:\n+        return workflow.is_complete\n+\n+\n+def assessment_is_finished(submission_uuid, requirements):\n+    \"\"\"\n+    When the submitter has finished assessing the training\n+    examples, then this step is finished.\n+\n+    Args:\n+        Same as `submitter_is_finished()`\n+\n+    Returns:\n+        bool\n+\n+    \"\"\"\n+    return submitter_is_finished(submission_uuid, requirements)\n+\n+\n+def get_score(submission_uuid, requirements):   # pylint:disable=W0613\n+    \"\"\"\n+    Training is either complete or incomplete; there is no score.\n+    \"\"\"\n+    return None\n+\n+\n+def validate_training_examples(rubric, examples):\n+    \"\"\"\n+    Validate that the training examples match the rubric.\n+\n+    Args:\n+        rubric (dict): Serialized rubric model.\n+        examples (list): List of serialized training examples.\n+\n+    Returns:\n+        tuple of `(is_valid, errors)` where `is_valid` is a boolean indicating\n+        whether the examples are valid, and `errors` is a list of unicode strings\n+        describing validation errors.\n+\n+    Raises:\n+        StudentTrainingRequestError\n+        StudentTrainingInternalError\n+\n+    Example usage:\n+\n+        >>> options = [\n+        >>>     {\n+        >>>         \"order_num\": 0,\n+        >>>         \"name\": \"poor\",\n+        >>>         \"explanation\": \"Poor job!\",\n+        >>>         \"points\": 0,\n+        >>>     },\n+        >>>     {\n+        >>>         \"order_num\": 1,\n+        >>>         \"name\": \"good\",\n+        >>>         \"explanation\": \"Good job!\",\n+        >>>         \"points\": 1,\n+        >>>     },\n+        >>>     {\n+        >>>         \"order_num\": 2,\n+        >>>         \"name\": \"excellent\",\n+        >>>         \"explanation\": \"Excellent job!\",\n+        >>>         \"points\": 2,\n+        >>>     },\n+        >>> ]\n+        >>>\n+        >>> rubric = {\n+        >>>     \"prompt\": \"Write an essay!\",\n+        >>>     \"criteria\": [\n+        >>>         {\n+        >>>             \"order_num\": 0,\n+        >>>             \"name\": \"vocabulary\",\n+        >>>             \"prompt\": \"How varied is the vocabulary?\",\n+        >>>             \"options\": options\n+        >>>         },\n+        >>>         {\n+        >>>             \"order_num\": 1,\n+        >>>             \"name\": \"grammar\",\n+        >>>             \"prompt\": \"How correct is the grammar?\",\n+        >>>             \"options\": options\n+        >>>         }\n+        >>>     ]\n+        >>> }\n+        >>>\n+        >>> examples = [\n+        >>>     {\n+        >>>         'answer': u'Lorem ipsum',\n+        >>>         'options_selected': {\n+        >>>             'vocabulary': 'good',\n+        >>>             'grammar': 'excellent'\n+        >>>         }\n+        >>>     },\n+        >>>     {\n+        >>>         'answer': u'Doler',\n+        >>>         'options_selected': {\n+        >>>             'vocabulary': 'good',\n+        >>>             'grammar': 'poor'\n+        >>>         }\n+        >>>     }\n+        >>> ]\n+        >>>\n+        >>> is_valid, errors = validate_training_examples(rubric, examples)\n+\n+    \"\"\"\n+    errors = []\n+\n+    # Construct a list of valid options for each criterion\n+    try:\n+        criteria_options = {\n+            unicode(criterion['name']): [\n+                unicode(option['name'])\n+                for option in criterion['options']\n+            ]\n+            for criterion in rubric['criteria']\n+        }\n+    except (ValueError, KeyError):\n+        msg = u\"Could not parse serialized rubric\"\n+        return False, [msg]\n+\n+    # Check each example\n+    for order_num, example_dict in enumerate(examples, start=1):\n+\n+        # Check the structure of the example dict\n+        is_format_valid, format_errors = validate_training_example_format(example_dict)\n+        if not is_format_valid:\n+            format_errors = [\n+                \"Example {} has a validation error: {}\".format(order_num, error)\n+                for error in format_errors\n+            ]\n+            errors.extend(format_errors)\n+        else:\n+            # Check each selected option in the example (one per criterion)\n+            options_selected = example_dict['options_selected']\n+            for criterion_name, option_name in options_selected.iteritems():\n+                if criterion_name in criteria_options:\n+                    valid_options = criteria_options[criterion_name]\n+                    if option_name not in valid_options:\n+                        msg = u\"Example {} has an invalid option for \\\"{}\\\": \\\"{}\\\"\".format(\n+                            order_num, criterion_name, option_name\n+                        )\n+                        errors.append(msg)\n+                else:\n+                    msg = u\"Example {} has an extra option for \\\"{}\\\"\".format(\n+                        order_num, criterion_name\n+                    )\n+                    errors.append(msg)\n+\n+            # Check for missing criteria\n+            for missing_criterion in set(criteria_options.keys()) - set(options_selected.keys()):\n+                msg = u\"Example {} is missing an option for \\\"{}\\\"\".format(\n+                    order_num, missing_criterion\n+                )\n+                errors.append(msg)\n+\n+    is_valid = (len(errors) == 0)\n+    return is_valid, errors\n+\n+\n+def create_training_workflow(submission_uuid, rubric, examples):\n+    \"\"\"\n+    Start the training workflow.\n+\n+    Args:\n+        submission_uuid (str): The UUID of the student's submission.\n+        rubric (dict): Serialized rubric model.\n+        examples (list): The serialized training examples the student will need to assess.\n+\n+    Returns:\n+        None\n+\n+    Raises:\n+        StudentTrainingRequestError\n+        StudentTrainingInternalError\n+\n+    Example usage:\n+\n+        >>> options = [\n+        >>>     {\n+        >>>         \"order_num\": 0,\n+        >>>         \"name\": \"poor\",\n+        >>>         \"explanation\": \"Poor job!\",\n+        >>>         \"points\": 0,\n+        >>>     },\n+        >>>     {\n+        >>>         \"order_num\": 1,\n+        >>>         \"name\": \"good\",\n+        >>>         \"explanation\": \"Good job!\",\n+        >>>         \"points\": 1,\n+        >>>     },\n+        >>>     {\n+        >>>         \"order_num\": 2,\n+        >>>         \"name\": \"excellent\",\n+        >>>         \"explanation\": \"Excellent job!\",\n+        >>>         \"points\": 2,\n+        >>>     },\n+        >>> ]\n+        >>>\n+        >>> rubric = {\n+        >>>     \"prompt\": \"Write an essay!\",\n+        >>>     \"criteria\": [\n+        >>>         {\n+        >>>             \"order_num\": 0,\n+        >>>             \"name\": \"vocabulary\",\n+        >>>             \"prompt\": \"How varied is the vocabulary?\",\n+        >>>             \"options\": options\n+        >>>         },\n+        >>>         {\n+        >>>             \"order_num\": 1,\n+        >>>             \"name\": \"grammar\",\n+        >>>             \"prompt\": \"How correct is the grammar?\",\n+        >>>             \"options\": options\n+        >>>         }\n+        >>>     ]\n+        >>> }\n+        >>>\n+        >>> examples = [\n+        >>>     {\n+        >>>         'answer': u'Lorem ipsum',\n+        >>>         'options_selected': {\n+        >>>             'vocabulary': 'good',\n+        >>>             'grammar': 'excellent'\n+        >>>         }\n+        >>>     },\n+        >>>     {\n+        >>>         'answer': u'Doler',\n+        >>>         'options_selected': {\n+        >>>             'vocabulary': 'good',\n+        >>>             'grammar': 'poor'\n+        >>>         }\n+        >>>     }\n+        >>> ]\n+        >>>\n+        >>> create_training_workflow(\"5443ebbbe2297b30f503736e26be84f6c7303c57\", rubric, examples)\n+\n+    \"\"\"\n+    try:\n+        # Check that examples were provided\n+        if len(examples) == 0:\n+            msg = (\n+                u\"No examples provided for student training workflow \"\n+                u\"(attempted to create workflow for student with submission UUID {})\"\n+            ).format(submission_uuid)\n+            raise StudentTrainingRequestError(msg)\n+\n+        # Ensure that a workflow doesn't already exist for this submission\n+        already_exists = StudentTrainingWorkflow.objects.filter(\n+            submission_uuid=submission_uuid\n+        ).exists()\n+\n+        if already_exists:\n+            msg = (\n+                u\"Student training workflow already exists for the student \"\n+                u\"associated with submission UUID {}\"\n+            ).format(submission_uuid)\n+            raise StudentTrainingRequestError(msg)\n+\n+        # Create the training examples\n+        try:\n+            examples = deserialize_training_examples(examples, rubric)\n+        except (InvalidRubric, InvalidTrainingExample) as ex:\n+            raise StudentTrainingRequestError(ex.message)","path":"apps/openassessment/assessment/api/student_training.py","position":303,"original_position":303,"commit_id":"f034b1dd1f9a30d345bf7c0fae5fce7f4e420297","original_commit_id":"f034b1dd1f9a30d345bf7c0fae5fce7f4e420297","user":{"login":"stephensanchez","id":235752,"avatar_url":"https://avatars.githubusercontent.com/u/235752?","gravatar_id":"1bb01f02977bfc7753dc9db8ab03116d","url":"https://api.github.com/users/stephensanchez","html_url":"https://github.com/stephensanchez","followers_url":"https://api.github.com/users/stephensanchez/followers","following_url":"https://api.github.com/users/stephensanchez/following{/other_user}","gists_url":"https://api.github.com/users/stephensanchez/gists{/gist_id}","starred_url":"https://api.github.com/users/stephensanchez/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stephensanchez/subscriptions","organizations_url":"https://api.github.com/users/stephensanchez/orgs","repos_url":"https://api.github.com/users/stephensanchez/repos","events_url":"https://api.github.com/users/stephensanchez/events{/privacy}","received_events_url":"https://api.github.com/users/stephensanchez/received_events","type":"User","site_admin":false},"body":"Worth logging?","created_at":"2014-05-14T17:58:17Z","updated_at":"2014-05-14T17:58:17Z","html_url":"https://github.com/edx/edx-ora2/pull/323#discussion_r12650716","pull_request_url":"https://api.github.com/repos/edx/edx-ora2/pulls/323","_links":{"self":{"href":"https://api.github.com/repos/edx/edx-ora2/pulls/comments/12650716"},"html":{"href":"https://github.com/edx/edx-ora2/pull/323#discussion_r12650716"},"pull_request":{"href":"https://api.github.com/repos/edx/edx-ora2/pulls/323"}}}}