rubrics + essay prompt field + templates

app.py

@@ -1,7 +1,12 @@
+from typing import Dict
 import gradio as gr
 import openai
 import os
+import json
 
+from langchain import PromptTemplate
+from langchain.output_parsers import PydanticOutputParser
+from pydantic import BaseModel, Field
 
 MODEL = "gpt-3.5-turbo"
 
@@ -10,30 +15,63 @@ try:
 except:
     print("Set the OPENAI_API_KEY environment variable")
     exit()
+
+with open('./rubrics/act_rubric.json', 'r') as j:
+     act_rubric = json.loads(j.read())
+
+class ScoreDescription(BaseModel):
+    score: int = Field(description="The score given")
+    description: str = Field(description="Why the score was given")
+
+class ACTScore(BaseModel):
+    sub_scores: Dict[str, ScoreDescription] = Field(description="The sub-scores of the essay for each category in the rubric")
+    overall_feedback: str = Field(description="Overall feedback for the essay")
+
+parser = PydanticOutputParser(pydantic_object=ACTScore)
+
+grader_template = PromptTemplate(
+    input_variables=['rubric', 'essay_prompt', 'essay'],
+    template= """
+        You are an essay grader provided with the following grading rubric:\n
+        {rubric}
+        \n
+        The essay writer was given the following instructions to write the essay: \n
+        {essay_prompt}
+        \n
+        Grade the following essay. Provide sub-scores and rationale for each sub-score. \n
+        {essay}
+        \n
+
+        Format description:
+        {format_description}
+    """,
+    partial_variables={
+        'format_description': parser.get_format_instructions()
+    }
+
+)
     
-def get_prompt(essay):
-    return f"""
-    Provide an estimate score and sub-scores for the following ACT writing sample using the ACT scoring rubric.
-    For each sub-score also provide one example of support and one example of what could be improved.
-    Each score should be a single number with one decimal point of accuracy and not a range.
-    This is the provided writing sample:
-    "{essay}"
-    """
+def get_prompt(essay, essay_prompt):
+    return grader_template.format(
+        rubric=act_rubric,
+        essay=essay,
+        essay_prompt=essay_prompt
+    )
 
 
 
-def grade_essay(essay):
+def grade_essay(essay, essay_prompt):
     response = openai.ChatCompletion.create(
         model=MODEL,
-        messages=[{"role":"user", "content":get_prompt(essay)}],
-        temperature=1,
-        max_tokens=1000
+        messages=[{"role":"user", "content":get_prompt(essay, essay_prompt)}],
+        temperature=0.0,
+        max_tokens=1000,
     )
     
-    return response['choices'][0]['message']['content']
-
+    result = response['choices'][0]['message']['content']
+    return result
 
-demo = gr.Interface(fn=grade_essay, inputs="text", outputs="text")
+demo = gr.Interface(fn=grade_essay, inputs=[gr.Textbox(lines=10, placeholder='Essay'), gr.Textbox(lines=10, placeholder='Essay Prompt')], outputs="text")
 
 demo.launch()
 

requirements.txt

@@ -1,2 +1,4 @@
 gradio==3.27.0
-openai==0.27.4
+openai==0.27.4
+langchain
+pydantic

rubrics/act_rubric.json

@@ -0,0 +1,150 @@
+{
+    "overall_scores": [
+        {
+            "score": "6",
+            "description": "Responses at this scorepoint demonstrate effective skill in writing an argumentative essay."
+        },
+        {
+            "score": "5",
+            "description": "Responses at this scorepoint demonstrate well-developed skill in writing an argumentative essay."
+        },
+        {
+            "score": "4",
+            "description": "Responses at this scorepoint demonstrate adequate skill in writing an argumentative essay."
+        },
+        {
+            "score": "3",
+            "description": "Responses at this scorepoint demonstrate some developing skill in writing an argumentative essay."
+        },
+        {
+            "score": "2",
+            "description": "Responses at this scorepoint demonstrate weak or inconsistent skill in writing an argumentative essay."
+        },
+        {
+            "score": "1",
+            "description": "Responses at this scorepoint demonstrate little or no skill in writing an argumentative essay."
+        }
+    ],
+    "categories": [
+        {
+            "name": "Ideas and Analysis",
+            "description": "Scores in this domain reflect the ability to generate productive ideas and engage critically with multiple perspectives on the given issue.",
+            "levels": [
+                {
+                    "score": "6",
+                    "description": "The writer generates an argument that critically engages with multiple perspectives on the given issue. The argument’s thesis reflects nuance and precision in thought and purpose. The argument establishes and employs an insightful context for analysis of the issue and its perspectives. The analysis examines implications, complexities and tensions, and/or underlying values and assumptions."
+                },
+                {
+                    "score": "5",
+                    "description": "The writer generates an argument that productively engages with multiple perspectives on the given issue. The argument’s thesis reflects precision in thought and purpose. The argument establishes and employs a thoughtful context for analysis of the issue and its perspectives. The analysis addresses implications, complexities and tensions, and/or underlying values and assumptions."
+                },
+                {
+                    "score": "4",
+                    "description": "The writer generates an argument that engages with multiple perspectives on the given issue. The argument’s thesis reflects clarity in thought and purpose. The argument establishes and employs a relevant context for analysis of the issue and its perspectives. The analysis recognizes implications, complexities and tensions, and/or underlying values and assumptions."
+                },
+                {
+                    "score": "3",
+                    "description": "The writer generates an argument that responds to multiple perspectives on the given issue. The argument’s thesis reflects some clarity in thought and purpose. The argument establishes a limited or tangential context for analysis of the issue and its perspectives. Analysis is simplistic or somewhat unclear."
+                },
+                {
+                    "score": "2",
+                    "description": "The writer generates an argument that weakly responds to multiple perspectives on the given issue. The argument’s thesis, if evident, reflects little clarity in thought and purpose. Attempts at analysis are incomplete, largely irrelevant, or consist primarily of restatement of the issue and its perspectives."
+                },
+                {
+                    "score": "1",
+                    "description": "The writer fails to generate an argument that responds intelligibly to the task. The writer’s intentions are difficult to discern. Attempts at analysis are unclear or irrelevant."
+                }
+            ]
+        },
+        {
+            "name": "Development and Support",
+            "description": "Scores in this domain reflect the ability to discuss ideas, offer rationale, and bolster an argument.",
+            "levels": [
+                {
+                    "score": "6",
+                    "description": "Development of ideas and support for claims deepen insight and broaden context. An integrated line of skillful reasoning and illustration effectively conveys the significance of the argument. Qualifications and complications enrich and bolster ideas and analysis."
+                },
+                {
+                    "score": "5",
+                    "description": "Development of ideas and support for claims deepen understanding. A mostly integrated line of purposeful reasoning and illustration capably conveys the significance of the argument. Qualifications and complications enrich ideas and analysis."
+                },
+                {
+                    "score": "4",
+                    "description": "Development of ideas and support for claims clarify meaning and purpose. Lines of clear reasoning and illustration adequately convey the significance of the argument. Qualifications and complications extend ideas and analysis."
+                },
+                {
+                    "score": "3",
+                    "description": "Development of ideas and support for claims are mostly relevant but are overly general or simplistic. Reasoning and illustration largely clarify the argument but may be somewhat repetitious or imprecise."
+                },
+                {
+                    "score": "2",
+                    "description": "Development of ideas and support for claims are weak, confused, or disjointed. Reasoning and illustration are inadequate, illogical, or circular, and fail to fully clarify the argument."
+                },
+                {
+                    "score": "1",
+                    "description": "Ideas lack development, and claims lack support. Reasoning and illustration are unclear, incoherent, or largely absent."
+                }
+            ]
+        },
+        {
+            "name": "Organization",
+            "description": "Scores in this domain reflect the ability to organize ideas with clarity and purpose.",
+            "levels": [
+                {
+                    "score": "6",
+                    "description": "The response exhibits a skillful organizational strategy. The response is unified by a controlling idea or purpose, and a logical progression of ideas increases the effectiveness of the writer’s argument. Transitions between and within paragraphs strengthen the relationships among ideas."
+                },
+                {
+                    "score": "5",
+                    "description": "The response exhibits a productive organizational strategy. The response is mostly unified by a controlling idea or purpose, and a logical sequencing of ideas contributes to the effectiveness of the argument. Transitions between and within paragraphs consistently clarify the relationships among ideas."
+                },
+                {
+                    "score": "4",
+                    "description": "The response exhibits a clear organizational strategy. The overall shape of the response reflects an emergent controlling idea or purpose. Ideas are logically grouped and sequenced. Transitions between and within paragraphs clarify the relationships among ideas."
+                },
+                {
+                    "score": "3",
+                    "description": "The response exhibits a basic organizational structure. The response largely coheres, with most ideas logically grouped. Transitions between and within paragraphs sometimes clarify the relationships among ideas."
+                },
+                {
+                    "score": "2",
+                    "description": "The response exhibits a rudimentary organizational structure. Grouping of ideas is inconsistent and often unclear. Transitions between and within paragraphs are misleading or poorly formed."
+                },
+                {
+                    "score": "1",
+                    "description": "The response does not exhibit an organizational structure. There is little grouping of ideas. When present, transitional devices fail to connect ideas."
+                }
+            ]
+        },
+        {
+            "name": "Language Use and Conventions",
+            "description": "Scores in this domain reflect the ability to use written language to convey arguments with clarity.",
+            "levels": [
+                {
+                    "score": "6",
+                    "description": "The use of language enhances the argument. Word choice is skillful and precise. Sentence structures are consistently varied and clear. Stylistic and register choices, including voice and tone, are strategic and effective. While a few minor errors in grammar, usage, and mechanics may be present, they do not impede understanding."
+                },
+                {
+                    "score": "5",
+                    "description": "The use of language works in service of the argument. Word choice is precise. Sentence structures are clear and varied often. Stylistic and register choices, including voice and tone, are purposeful and productive. While minor errors in grammar, usage, and mechanics may be present, they do not impede understanding."
+                },
+                {
+                    "score": "4",
+                    "description": "The use of language conveys the argument with clarity. Word choice is adequate and sometimes precise. Sentence structures are clear and demonstrate some variety. Stylistic and register choices, including voice and tone, are appropriate for the rhetorical purpose. While errors in grammar, usage, and mechanics are present, they rarely impede understanding."
+                },
+                {
+                    "score": "3",
+                    "description": "The use of language is basic and only somewhat clear. Word choice is general and occasionally imprecise. Sentence structures are usually clear but show little variety. Stylistic and register choices, including voice and tone, are not always appropriate for the rhetorical purpose. Distracting errors in grammar, usage, and mechanics may be present, but they generally do not impede understanding."
+                },
+                {
+                    "score": "2",
+                    "description": "The use of language is inconsistent and often unclear. Word choice is rudimentary and frequently imprecise. Sentence structures are sometimes unclear. Stylistic and register choices, including voice and tone, are inconsistent and are not always appropriate for the rhetorical purpose. Distracting errors in grammar, usage, and mechanics are present, and they sometimes impede understanding."
+                },
+                {
+                    "score": "1",
+                    "description": "The use of language fails to demonstrate skill in responding to the task. Word choice is imprecise and often difficult to comprehend. Sentence structures are often unclear. Stylistic and register choices are difficult to identify. Errors in grammar, usage, and mechanics are pervasive and often impede understanding."
+                }
+            ]
+        }
+    ]
+}