Add logging and error validation to nlu endpoint

app.py

@@ -1,33 +1,42 @@
 """FastAPI endpoint
 To run locally use 'uvicorn app:app --host localhost --port 7860'
+or
+`python -m uvicorn app:app --reload --host localhost --port 7860`
 """
 import ast
-import scripts.quiz.generators as generators
-import scripts.quiz.hints as hints
-import scripts.quiz.questions as questions
-import scripts.quiz.utils as utils
+import json
+from json import JSONDecodeError
+from logging import getLogger
+import mathactive.microlessons.num_one as num_one_quiz
+import os
 import sentry_sdk
 
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-from mathtext.sentiment import sentiment
+# from mathtext.sentiment import sentiment
 from mathtext.text2int import text2int
-from pydantic import BaseModel
-
 from mathtext_fastapi.logging import prepare_message_data_for_logging
 from mathtext_fastapi.conversation_manager import manage_conversation_response
+from mathtext_fastapi.v2_conversation_manager import manage_conversation_response
 from mathtext_fastapi.nlu import evaluate_message_with_nlu
 from mathtext_fastapi.nlu import run_intent_classification
+from pydantic import BaseModel
+
+
+from dotenv import load_dotenv
+load_dotenv()
+
+log = getLogger(__name__)
 
 sentry_sdk.init(
-    dsn="https://143c9ac3f429452eb036deda0e4d5aef@o1297809.ingest.sentry.io/4504896688881664",
+    dsn=os.environ.get('SENTRY_DSN'),
 
     # Set traces_sample_rate to 1.0 to capture 100%
     # of transactions for performance monitoring.
     # We recommend adjusting this value in production,
-    traces_sample_rate=0.20,
+    traces_sample_rate=1.0,
 )
 
 app = FastAPI()
@@ -57,11 +66,11 @@ def hello(content: Text = None):
     return JSONResponse(content=content)
 
 
-@app.post("/sentiment-analysis")
-def sentiment_analysis_ep(content: Text = None):
-    ml_response = sentiment(content.content)
-    content = {"message": ml_response}
-    return JSONResponse(content=content)
+# @app.post("/sentiment-analysis")
+# def sentiment_analysis_ep(content: Text = None):
+#     ml_response = sentiment(content.content)
+#     content = {"message": ml_response}
+#     return JSONResponse(content=content)
 
 
 @app.post("/text2int")
@@ -71,7 +80,40 @@ def text2int_ep(content: Text = None):
     return JSONResponse(content=content)
 
 
-@app.post("/manager")
+@app.post("/v1/manager")
+async def programmatic_message_manager(request: Request):
+    """
+    Calls conversation management function to determine the next state
+
+    Input
+    request.body: dict - message data for the most recent user response
+    {
+        "author_id": "+47897891",
+        "contact_uuid": "j43hk26-2hjl-43jk-hnk2-k4ljl46j0ds09",
+        "author_type": "OWNER",
+        "message_body": "a test message",
+        "message_direction": "inbound",
+        "message_id": "ABJAK64jlk3-agjkl2QHFAFH",
+        "message_inserted_at": "2022-07-05T04:00:34.03352Z",
+        "message_updated_at": "2023-02-14T03:54:19.342950Z",
+    }
+
+    Output
+    context: dict - the information for the current state
+    {
+        "user": "47897891",
+        "state": "welcome-message-state",
+        "bot_message": "Welcome to Rori!",
+        "user_message": "",
+        "type": "ask"
+    }
+    """
+    data_dict = await request.json()
+    context = manage_conversation_response(data_dict)
+    return JSONResponse(context)
+
+
+@app.post("/v2/manager")
 async def programmatic_message_manager(request: Request):
     """
     Calls conversation management function to determine the next state
@@ -123,172 +165,205 @@ async def evaluate_user_message_with_nlu_api(request: Request):
       {'type':'integer', 'data': '8', 'confidence': 0}
       {'type':'sentiment', 'data': 'negative', 'confidence': 0.99}
     """
-    data_dict = await request.json()
-    message_data = data_dict.get('message_data', '')
+    log.info(f'Received request: {request}')
+    log.info(f'Request header: {request.headers}')
+    request_body = await request.body()
+    log.info(f'Request body: {request_body}')
+    request_body_str = request_body.decode()
+    log.info(f'Request_body_str: {request_body_str}')
+
+    try:
+        data_dict = await request.json()
+    except JSONDecodeError:
+        log.error(f'Request.json failed: {dir(request)}')
+        data_dict = {}
+    message_data = data_dict.get('message_data')
+    
+    if not message_data:
+        log.error(f'Data_dict: {data_dict}')
+        message_data = data_dict.get('message', {})
     nlu_response = evaluate_message_with_nlu(message_data)
     return JSONResponse(content=nlu_response)
+
+
+@app.post("/num_one")
+async def num_one(request: Request):
+    """
+    Input: 
+    {
+        "user_id": 1,
+        "message_text": 5,
+    }
+    Output:
+    {
+        'messages': 
+            ["Let's", 'practice', 'counting', '', '', '46...', '47...', '48...', '49', '', '', 'After', '49,', 'what', 'is', 'the', 'next', 'number', 'you', 'will', 'count?\n46,', '47,', '48,', '49'], 
+        'input_prompt': '50', 
+        'state': 'question'
+    }
+    """
+    data_dict = await request.json()
+    message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
+    user_id = message_data['user_id']
+    message_text = message_data['message_text']
+    return num_one_quiz.process_user_message(user_id, message_text)
     
 
-@app.post("/question")
+@app.post("/start")
 async def ask_math_question(request: Request):
-    """Generate a question and return it as response along with question data
+    """Generate a question data
     
     Input
-    request.body: json - amount of correct and incorrect answers in the account
     {
-        'number_correct': 0,
-        'number_incorrect': 0,
-        'level': 'easy'
+        'difficulty': 0.1,
+        'do_increase': True | False
     }
 
     Output
-    context: dict - the information for the current state
     {
         'text': 'What is 1+2?',
-        'question_numbers': [1,2,3], #3 numbers - current number, ordinal number, times
-        'right_answer': 3,
-        'number_correct': 0,
-        'number_incorrect': 0,
-        'hints_used': 0
+        'difficulty': 0.2,
+        'question_numbers': [3, 1, 4]
     }
     """
     data_dict = await request.json()
     message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
-    right_answers = message_data['number_correct']
-    wrong_answers = message_data['number_incorrect']
-    level = message_data['level']
+    difficulty = message_data['difficulty']
+    do_increase = message_data['do_increase']
 
-    return JSONResponse(generators.start_interactive_math(right_answers, wrong_answers, level))
+    return JSONResponse(generators.start_interactive_math(difficulty, do_increase))
 
 
 @app.post("/hint")
 async def get_hint(request: Request):
-    """Generate a hint and return it as response along with hint data
+    """Generate a hint data
     
     Input
-    request.body:
     {
-        'question_numbers': [1,2,3], #3 numbers - current number, ordinal number, times
-        'right_answer': 3,
-        'number_correct': 0,
-        'number_incorrect': 0,
-        'level': 'easy',
-        'hints_used': 0
+        'start': 5,
+        'step': 1,
+        'difficulty': 0.1
     }
 
     Output
-    context: dict - the information for the current state
     {
-        'text': 'What is 1+2?',
-        'question_numbers': [1,2,3], #2 or 3 numbers
-        'right_answer': 3,
-        'number_correct': 0,
-        'number_incorrect': 0,
-        'level': 'easy',
-        'hints_used': 0
+        'text': 'What number is greater than 4 and less than 6?',
+        'difficulty': 0.1,
+        'question_numbers': [5, 1, 6]
     }
     """
     data_dict = await request.json()
     message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
-    question_numbers = message_data['question_numbers']
-    right_answer = message_data['right_answer']
-    number_correct = message_data['number_correct']
-    number_incorrect = message_data['number_incorrect']
-    level = message_data['level']
-    hints_used = message_data['hints_used']
+    start = message_data['start']
+    step = message_data['step']
+    difficulty = message_data['difficulty']
 
-    return JSONResponse(hints.generate_hint(question_numbers, right_answer, number_correct, number_incorrect, level, hints_used))
+    return JSONResponse(hints.generate_hint(start, step, difficulty))
 
 
-@app.post("/generate_question")
-async def generate_question(request: Request):
-    """Generate a bare question and return it as response
+@app.post("/question")
+async def ask_math_question(request: Request):
+    """Generate a question data
     
     Input
-    request.body: json - level
     {
-        'level': 'easy'
+        'start': 5,
+        'step': 1,
+        'question_num': 1  # optional
     }
 
     Output
-    context: dict - the information for the current state
     {
-        "question": "Let's count up by 2s. What number is next if we start from 10?
-        6 8 10 ..."
+        'question': 'What is 1+2?',
+        'start': 5,
+        'step': 1,
+        'answer': 6
     }
     """
     data_dict = await request.json()
     message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
-    level = message_data['level']
+    start = message_data['start']
+    step = message_data['step']
+    arg_tuple = (start, step)
+    try:
+        question_num = message_data['question_num']
+        arg_tuple += (question_num,)
+    except KeyError:
+        pass
 
-    return JSONResponse(questions.generate_question_data(level)['question'])
+    return JSONResponse(questions.generate_question_data(*arg_tuple))
 
 
-@app.post("/numbers_by_level")
-async def get_numbers_by_level(request: Request):
-    """Generate three numbers and return them as response
+@app.post("/difficulty")
+async def get_hint(request: Request):
+    """Generate a number matching difficulty
     
     Input
-    request.body: json - level
     {
-        'level': 'easy'
+        'difficulty': 0.01,
+        'do_increase': True
     }
 
-    Output
-    context: dict - three generated numbers for specified level
-    {
-        "current_number": 10,
-        "ordinal_number": 2,
-        "times": 1
-    }
+    Output - value from 0.01 to 0.99 inclusively:
+    0.09
     """
     data_dict = await request.json()
     message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
-    level = message_data['level']
-    return JSONResponse(questions.generate_numbers_by_level(level))
+    difficulty = message_data['difficulty']
+    do_increase = message_data['do_increase']
+
+    return JSONResponse(utils.get_next_difficulty(difficulty, do_increase))
 
 
-@app.post("/number_sequence")
-async def get_number_sequence(request: Request):
-    """Generate a number sequence
+@app.post("/start_step")
+async def get_hint(request: Request):
+    """Generate a start and step values
     
     Input
-    request.body: json - level
     {
-        "current_number": 10,
-        "ordinal_number": 2,
-        "times": 1
+        'difficulty': 0.01,
+        'path_to_csv_file': 'scripts/quiz/data.csv'  # optional
     }
 
-    Output
-    one of following strings with (numbers differ):
-    ... 1 2 3
-    1 2 3 ...
+    Output - tuple (start, step):
+    (5, 1)
     """
     data_dict = await request.json()
     message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
-    cur_num = message_data['current_number']
-    ord_num = message_data['ordinal_number']
-    times = message_data['times']
-    return JSONResponse(questions.generate_number_sequence(cur_num, ord_num, times))
+    difficulty = message_data['difficulty']
+    arg_tuple = (difficulty,)
+    try:
+        path_to_csv_file = message_data['path_to_csv_file']
+        arg_tuple += (path_to_csv_file,)
+    except KeyError:
+        pass
 
+    return JSONResponse(utils.get_next_difficulty(*arg_tuple))
 
-@app.post("/level")
-async def get_next_level(request: Request):
-    """Depending on current level and desire to level up/down return next level
+
+@app.post("/sequence")
+async def generate_question(request: Request):
+    """Generate a sequence from start, step and optional separator parameter
     
     Input
-    request.body: json - level
     {
-        "current_level": "easy",
-        "level_up": True
+        'start': 5,
+        'step': 1,
+        'sep': ', '  # optional
     }
 
     Output
-    Literal - "easy", "medium" or "hard"
+    5, 6, 7
     """
     data_dict = await request.json()
     message_data = ast.literal_eval(data_dict.get('message_data', '').get('message_body', ''))
-    cur_level = message_data['current_level']
-    level_up = message_data['level_up']
-    return JSONResponse(utils.get_next_level(cur_level, level_up))
+    start = message_data['start']
+    step = message_data['step']
+    arg_tuple = (start, step)
+    try:
+        sep = message_data['sep']
+        arg_tuple += (sep,)
+    except KeyError:
+        pass
+
+    return JSONResponse(utils.convert_sequence_to_string(*arg_tuple))

mathtext_fastapi/conversation_manager.py

@@ -14,8 +14,8 @@ from mathtext_fastapi.math_subtraction_fsm import MathSubtractionFSM
 from supabase import create_client
 from transitions import Machine
 
-from scripts.quiz.generators import start_interactive_math
-from scripts.quiz.hints import generate_hint
+from mathactive.generators import start_interactive_math
+from mathactive.hints import generate_hint
 
 load_dotenv()
 
@@ -39,6 +39,7 @@ def create_text_message(message_text, whatsapp_id):
         "preview_url": False,
         "recipient_type": "individual",
         "to": whatsapp_id,
+        # FIXME: Better to use "message_type" (but be careful with refactor)
         "type": "text",
         "text": {
             "body": message_text
@@ -136,6 +137,9 @@ def manage_math_quiz_fsm(user_message, contact_uuid, type):
 
     # Make a completely new entry
     if fsm_check.data == []:
+        # FIXME: Try not to use the Python reserved keyword `type` as a variable name
+        #        It's better to use `kind` or `convo_type` or `convo_name`
+        #        And the variable `type` is not defined here so I don't understand how this is working at all.
         if type == 'addition':
             math_quiz_state_machine = MathQuizFSM()
         else:

mathtext_fastapi/curriculum_mapper.py

@@ -0,0 +1,183 @@
+import numpy as np
+import pandas as pd
+import re
+
+from pathlib import Path
+
+
+def read_and_preprocess_spreadsheet(file_name):
+    """ Creates a pandas dataframe from the curriculum overview spreadsheet """
+    DATA_DIR = Path(__file__).parent.parent / "mathtext_fastapi" / "data" / file_name
+    script_df = pd.read_excel(DATA_DIR, engine='openpyxl')
+    # Ensures the grade level columns are integers instead of floats
+    script_df.columns = script_df.columns[:2].tolist() + script_df.columns[2:11].astype(int).astype(str).tolist() + script_df.columns[11:].tolist()
+    script_df.fillna('', inplace=True)
+    return script_df
+
+
+def extract_skill_code(skill):
+    """ Looks within a curricular skill description for its descriptive code
+
+    Input
+    - skill: str - a brief description of a curricular skill
+
+    >>> extract_skill_code('A3.3.4 - Solve inequalities')
+    'A3.3.4'
+    >>> extract_skill_code('A3.3.2 - Graph linear equations, and identify the x- and y-intercepts or the slope of a line')
+    'A3.3.2'
+    """
+    pattern = r'[A-Z][0-9]\.\d+\.\d+'
+    result = re.search(pattern, skill)
+    return result.group()
+
+
+def build_horizontal_transitions(script_df):
+    """ Build a list of transitional relationships within a curricular skill
+
+    Inputs
+    - script_df: pandas dataframe - an overview of the curriculum skills by grade level
+
+    Output
+    - horizontal_transitions: array of arrays - transition data with label, from state, and to state
+
+    >>> script_df = read_and_preprocess_spreadsheet('curriculum_framework_for_tests.xlsx')
+    >>> build_horizontal_transitions(script_df)
+    [['right', 'N1.1.1_G1', 'N1.1.1_G2'], ['right', 'N1.1.1_G2', 'N1.1.1_G3'], ['right', 'N1.1.1_G3', 'N1.1.1_G4'], ['right', 'N1.1.1_G4', 'N1.1.1_G5'], ['right', 'N1.1.1_G5', 'N1.1.1_G6'], ['left', 'N1.1.1_G6', 'N1.1.1_G5'], ['left', 'N1.1.1_G5', 'N1.1.1_G4'], ['left', 'N1.1.1_G4', 'N1.1.1_G3'], ['left', 'N1.1.1_G3', 'N1.1.1_G2'], ['left', 'N1.1.1_G2', 'N1.1.1_G1'], ['right', 'N1.1.2_G1', 'N1.1.2_G2'], ['right', 'N1.1.2_G2', 'N1.1.2_G3'], ['right', 'N1.1.2_G3', 'N1.1.2_G4'], ['right', 'N1.1.2_G4', 'N1.1.2_G5'], ['right', 'N1.1.2_G5', 'N1.1.2_G6'], ['left', 'N1.1.2_G6', 'N1.1.2_G5'], ['left', 'N1.1.2_G5', 'N1.1.2_G4'], ['left', 'N1.1.2_G4', 'N1.1.2_G3'], ['left', 'N1.1.2_G3', 'N1.1.2_G2'], ['left', 'N1.1.2_G2', 'N1.1.2_G1']]
+    """
+    horizontal_transitions = []
+    for index, row in script_df.iterrows():     
+        skill_code = extract_skill_code(row['Knowledge or Skill'])
+
+        rightward_matches = []
+        for i in range(9):
+            # Grade column
+            current_grade = i+1
+            if row[current_grade].lower().strip() == 'x':
+                rightward_matches.append(i)
+            
+        for match in rightward_matches:
+            if rightward_matches[-1] != match:
+                horizontal_transitions.append([
+                    "right",
+                    f"{skill_code}_G{match}",
+                    f"{skill_code}_G{match+1}"
+                ])
+
+        leftward_matches = []
+        for i in reversed(range(9)):
+            current_grade = i
+            if row[current_grade].lower().strip() == 'x':
+                leftward_matches.append(i)
+
+        for match in leftward_matches:
+            if leftward_matches[0] != match:
+                horizontal_transitions.append([
+                    "left",
+                    f"{skill_code}_G{match}",
+                    f"{skill_code}_G{match-1}"
+                ])
+
+    return horizontal_transitions
+
+
+def gather_all_vertical_matches(script_df):
+    """ Build a list of transitional relationships within a grade level across skills
+
+    Inputs
+    - script_df: pandas dataframe - an overview of the curriculum skills by grade level
+
+    Output
+    - all_matches: array of arrays - represents skills at each grade level
+
+    >>> script_df = read_and_preprocess_spreadsheet('curriculum_framework_for_tests.xlsx')
+    >>> gather_all_vertical_matches(script_df)
+    [['N1.1.1', '1'], ['N1.1.2', '1'], ['N1.1.1', '2'], ['N1.1.2', '2'], ['N1.1.1', '3'], ['N1.1.2', '3'], ['N1.1.1', '4'], ['N1.1.2', '4'], ['N1.1.1', '5'], ['N1.1.2', '5'], ['N1.1.1', '6'], ['N1.1.2', '6']]
+    """
+    all_matches = []
+    columns = ['1', '2', '3', '4', '5', '6', '7', '8', '9']
+
+    for column in columns:
+        for index, value in script_df[column].iteritems():
+            row_num = index + 1
+            if value == 'x':
+                # Extract skill code
+                skill_code = extract_skill_code(
+                    script_df['Knowledge or Skill'][row_num-1]
+                )
+
+                all_matches.append([skill_code, column])
+    return all_matches
+
+
+def build_vertical_transitions(script_df):
+    """ Build a list of transitional relationships within a grade level across skills
+
+    Inputs
+    - script_df: pandas dataframe - an overview of the curriculum skills by grade level
+
+    Output
+    - vertical_transitions: array of arrays - transition data with label, from state, and to state
+
+    >>> script_df = read_and_preprocess_spreadsheet('curriculum_framework_for_tests.xlsx')
+    >>> build_vertical_transitions(script_df)
+    [['down', 'N1.1.1_G1', 'N1.1.2_G1'], ['down', 'N1.1.2_G1', 'N1.1.1_G1'], ['down', 'N1.1.1_G2', 'N1.1.2_G2'], ['down', 'N1.1.2_G2', 'N1.1.1_G2'], ['down', 'N1.1.1_G3', 'N1.1.2_G3'], ['down', 'N1.1.2_G3', 'N1.1.1_G3'], ['down', 'N1.1.1_G4', 'N1.1.2_G4'], ['down', 'N1.1.2_G4', 'N1.1.1_G4'], ['down', 'N1.1.1_G5', 'N1.1.2_G5'], ['down', 'N1.1.2_G5', 'N1.1.1_G5'], ['down', 'N1.1.1_G6', 'N1.1.2_G6'], ['up', 'N1.1.2_G6', 'N1.1.1_G6'], ['up', 'N1.1.1_G6', 'N1.1.2_G6'], ['up', 'N1.1.2_G5', 'N1.1.1_G5'], ['up', 'N1.1.1_G5', 'N1.1.2_G5'], ['up', 'N1.1.2_G4', 'N1.1.1_G4'], ['up', 'N1.1.1_G4', 'N1.1.2_G4'], ['up', 'N1.1.2_G3', 'N1.1.1_G3'], ['up', 'N1.1.1_G3', 'N1.1.2_G3'], ['up', 'N1.1.2_G2', 'N1.1.1_G2'], ['up', 'N1.1.1_G2', 'N1.1.2_G2'], ['up', 'N1.1.2_G1', 'N1.1.1_G1']]
+    """
+    vertical_transitions = []
+
+    all_matches = gather_all_vertical_matches(script_df)
+
+    # Downward
+    for index, match in enumerate(all_matches):
+        skill = match[0]
+        row_num = match[1]
+        if all_matches[-1] != match:
+            vertical_transitions.append([
+                "down",
+                f"{skill}_G{row_num}",
+                f"{all_matches[index+1][0]}_G{row_num}"
+            ])
+
+    # Upward
+    for index, match in reversed(list(enumerate(all_matches))):
+        skill = match[0]
+        row_num = match[1]
+        if all_matches[0] != match:
+            vertical_transitions.append([
+                "up",
+                f"{skill}_G{row_num}",
+                f"{all_matches[index-1][0]}_G{row_num}"
+            ])
+    
+    return vertical_transitions
+
+
+def build_all_states(all_transitions):
+    """ Creates an array with all state labels for the curriculum
+
+    Input
+    - all_transitions: list of lists - all possible up, down, left, or right transitions in curriculum
+
+    Output
+    - all_states: list - a collection of state labels (skill code and grade number)
+    
+    >>> all_transitions = [['right', 'N1.1.1_G1', 'N1.1.1_G2'], ['right', 'N1.1.1_G2', 'N1.1.1_G3'], ['right', 'N1.1.1_G3', 'N1.1.1_G4'], ['right', 'N1.1.1_G4', 'N1.1.1_G5'], ['right', 'N1.1.1_G5', 'N1.1.1_G6'], ['left', 'N1.1.1_G6', 'N1.1.1_G5'], ['left', 'N1.1.1_G5', 'N1.1.1_G4'], ['left', 'N1.1.1_G4', 'N1.1.1_G3'], ['left', 'N1.1.1_G3', 'N1.1.1_G2'], ['left', 'N1.1.1_G2', 'N1.1.1_G1'], ['right', 'N1.1.2_G1', 'N1.1.2_G2'], ['right', 'N1.1.2_G2', 'N1.1.2_G3'], ['right', 'N1.1.2_G3', 'N1.1.2_G4'], ['right', 'N1.1.2_G4', 'N1.1.2_G5'], ['right', 'N1.1.2_G5', 'N1.1.2_G6'], ['left', 'N1.1.2_G6', 'N1.1.2_G5'], ['left', 'N1.1.2_G5', 'N1.1.2_G4'], ['left', 'N1.1.2_G4', 'N1.1.2_G3'], ['left', 'N1.1.2_G3', 'N1.1.2_G2'], ['left', 'N1.1.2_G2', 'N1.1.2_G1'], ['down', 'N1.1.1_G1', 'N1.1.2_G1'], ['down', 'N1.1.2_G1', 'N1.1.1_G1'], ['down', 'N1.1.1_G2', 'N1.1.2_G2'], ['down', 'N1.1.2_G2', 'N1.1.1_G2'], ['down', 'N1.1.1_G3', 'N1.1.2_G3'], ['down', 'N1.1.2_G3', 'N1.1.1_G3'], ['down', 'N1.1.1_G4', 'N1.1.2_G4'], ['down', 'N1.1.2_G4', 'N1.1.1_G4'], ['down', 'N1.1.1_G5', 'N1.1.2_G5'], ['down', 'N1.1.2_G5', 'N1.1.1_G5'], ['down', 'N1.1.1_G6', 'N1.1.2_G6'], ['up', 'N1.1.2_G6', 'N1.1.1_G6'], ['up', 'N1.1.1_G6', 'N1.1.2_G6'], ['up', 'N1.1.2_G5', 'N1.1.1_G5'], ['up', 'N1.1.1_G5', 'N1.1.2_G5'], ['up', 'N1.1.2_G4', 'N1.1.1_G4'], ['up', 'N1.1.1_G4', 'N1.1.2_G4'], ['up', 'N1.1.2_G3', 'N1.1.1_G3'], ['up', 'N1.1.1_G3', 'N1.1.2_G3'], ['up', 'N1.1.2_G2', 'N1.1.1_G2'], ['up', 'N1.1.1_G2', 'N1.1.2_G2'], ['up', 'N1.1.2_G1', 'N1.1.1_G1']]
+    >>> build_all_states(all_transitions)
+    ['N1.1.1_G1', 'N1.1.1_G2', 'N1.1.1_G3', 'N1.1.1_G4', 'N1.1.1_G5', 'N1.1.1_G6', 'N1.1.2_G1', 'N1.1.2_G2', 'N1.1.2_G3', 'N1.1.2_G4', 'N1.1.2_G5', 'N1.1.2_G6']
+    """
+    all_states = []
+    for transition in all_transitions:
+        for index, state in enumerate(transition):
+            if index == 0:
+                continue   
+            if state not in all_states:
+                all_states.append(state)
+    return all_states
+
+
+def build_curriculum_logic():
+    script_df = read_and_preprocess_spreadsheet('Rori_Framework_v1.xlsx')
+    horizontal_transitions = build_horizontal_transitions(script_df)
+    vertical_transitions = build_vertical_transitions(script_df)
+    all_transitions = horizontal_transitions + vertical_transitions
+    all_states = build_all_states(all_transitions)
+    return all_states, all_transitions

mathtext_fastapi/data/text2int_results.csv

@@ -20,10 +20,10 @@ eight oh,80.0,8.0,False
 eighty,80.0,80.0,True
 ate,8.0,1.0,False
 double eight,88.0,8.0,False
-eight three seven five three O nine,8375309.0,8375329.0,False
+eight three seven five three O nine,8375309.0,8375319.0,False
 eight three seven five three oh nine,8375309.0,8375309.0,True
 eight three seven five three zero nine,8375309.0,8375309.0,True
-eight three seven five three oh ni-ee-ine,8375309.0,837530619.0,False
+eight three seven five three oh ni-ee-ine,8375309.0,837530111.0,False
 two eight,28.0,16.0,False
 seven oh eleven,7011.0,77.0,False
 seven elevens,77.0,77.0,True
@@ -31,10 +31,10 @@ seven eleven,711.0,77.0,False
 ninety nine oh five,9905.0,149.0,False
 seven 0 seven 0 seven 0 seven,7070707.0,7070707.0,True
 123 hundred,123000.0,223.0,False
-5 o 5,505.0,525.0,False
-15 o 5,1505.0,22.0,False
-15-o 5,1505.0,22.0,False
-15 o-5,1505.0,22.0,False
+5 o 5,505.0,515.0,False
+15 o 5,1505.0,21.0,False
+15-o 5,1505.0,21.0,False
+15 o-5,1505.0,21.0,False
 911-thousand,911000.0,911000.0,True
 twenty-two twenty-two,2222.0,44.0,False
 twenty-two twenty-twos,484.0,44.0,False

mathtext_fastapi/global_state_manager.py

@@ -0,0 +1,23 @@
+from transitions import Machine
+from mathtext_fastapi.curriculum_mapper import build_curriculum_logic
+
+all_states, all_transitions = build_curriculum_logic()
+
+class GlobalStateManager(object):
+    states = all_states
+
+    transitions = all_transitions
+
+    def __init__(
+        self,
+        initial_state='N1.1.1_G1',
+    ):
+        self.machine = Machine(
+            model=self,
+            states=GlobalStateManager.states,
+            transitions=GlobalStateManager.transitions,
+            initial=initial_state
+        )
+
+
+curriculum = GlobalStateManager()

mathtext_fastapi/nlu.py

@@ -1,15 +1,32 @@
+from collections.abc import Mapping
+from logging import getLogger
+import datetime as dt
+from dateutil.parser import isoparse
+
 from fuzzywuzzy import fuzz
+from mathtext_fastapi.intent_classification import predict_message_intent
 from mathtext_fastapi.logging import prepare_message_data_for_logging
 from mathtext.sentiment import sentiment
-from mathtext.text2int import text2int
-from mathtext_fastapi.intent_classification import create_intent_classification_model, retrieve_intent_classification_model, predict_message_intent
-import re
+from mathtext.text2int import text2int, TOKENS2INT_ERROR_INT
+
+log = getLogger(__name__)
+
+PAYLOAD_VALUE_TYPES = {
+    'author_id': str,
+    'author_type': str,
+    'contact_uuid': str,
+    'message_body': str,
+    'message_direction': str,
+    'message_id': str,
+    'message_inserted_at': str,
+    'message_updated_at': str,
+    }
 
 
-def build_nlu_response_object(type, data, confidence):
+def build_nlu_response_object(nlu_type, data, confidence):
     """ Turns nlu results into an object to send back to Turn.io
     Inputs
-    - type: str - the type of nlu run (integer or sentiment-analysis)
+    - nlu_type: str - the type of nlu run (integer or sentiment-analysis)
     - data: str/int - the student message
     - confidence: - the nlu confidence score (sentiment) or '' (integer)
 
@@ -19,7 +36,11 @@ def build_nlu_response_object(type, data, confidence):
     >>> build_nlu_response_object('sentiment', 'POSITIVE', 0.99)
     {'type': 'sentiment', 'data': 'POSITIVE', 'confidence': 0.99}
     """
-    return {'type': type, 'data': data, 'confidence': confidence}
+    return {
+        'type': nlu_type,
+        'data': data,
+        'confidence': confidence
+        }
 
 
 # def test_for_float_or_int(message_data, message_text):
@@ -107,6 +128,16 @@ def run_intent_classification(message_text):
         'hint',
         'next',
         'stop',
+        'tired',
+        'tomorrow',
+        'finished',
+        'help',
+        'please',
+        'understand',
+        'question',
+        'easier',
+        'easy',
+        'support'
     ]
     
     for command in commands:
@@ -121,6 +152,70 @@ def run_intent_classification(message_text):
     return nlu_response
 
 
+def payload_is_valid(payload_object):
+    """
+    >>> payload_is_valid({'author_id': '+5555555', 'author_type': 'OWNER', 'contact_uuid': '3246-43ad-faf7qw-zsdhg-dgGdg', 'message_body': 'thirty one', 'message_direction': 'inbound', 'message_id': 'SDFGGwafada-DFASHA4aDGA', 'message_inserted_at': '2022-07-05T04:00:34.03352Z', 'message_updated_at': '2023-04-06T10:08:23.745072Z'})
+    True
+
+    >>> payload_is_valid({"author_id": "@event.message._vnd.v1.chat.owner", "author_type": "@event.message._vnd.v1.author.type", "contact_uuid": "@event.message._vnd.v1.chat.contact_uuid", "message_body": "@event.message.text.body", "message_direction": "@event.message._vnd.v1.direction", "message_id": "@event.message.id", "message_inserted_at": "@event.message._vnd.v1.chat.inserted_at", "message_updated_at": "@event.message._vnd.v1.chat.updated_at"})
+    False
+    """
+    try:
+        isinstance(
+            isoparse(payload_object.get('message_inserted_at','')),
+            dt.datetime
+        )
+        isinstance(
+            isoparse(payload_object.get('message_updated_at','')),
+            dt.datetime
+        )
+    except ValueError:
+        return False
+    return (
+        isinstance(payload_object, Mapping) and
+        isinstance(payload_object.get('author_id'), str) and
+        isinstance(payload_object.get('author_type'), str) and
+        isinstance(payload_object.get('contact_uuid'), str) and
+        isinstance(payload_object.get('message_body'), str) and
+        isinstance(payload_object.get('message_direction'), str) and
+        isinstance(payload_object.get('message_id'), str) and
+        isinstance(payload_object.get('message_inserted_at'), str) and
+        isinstance(payload_object.get('message_updated_at'), str)    
+    )
+
+
+def log_payload_errors(payload_object):
+    errors = []
+    try:
+        assert isinstance(payload_object, Mapping)
+    except Exception as e:
+        log.error(f'Invalid HTTP request payload object: {e}')
+        errors.append(e)
+    for k, typ in PAYLOAD_VALUE_TYPES.items():
+        try:
+            assert isinstance(payload_object.get(k), typ)
+        except Exception as e:
+            log.error(f'Invalid HTTP request payload object: {e}')
+            errors.append(e)
+    try:
+        assert isinstance(
+            dt.datetime.fromisoformat(payload_object.get('message_inserted_at')),
+            dt.datetime
+        )
+    except Exception as e:
+        log.error(f'Invalid HTTP request payload object: {e}')
+        errors.append(e)
+    try: 
+        isinstance(
+            dt.datetime.fromisoformat(payload_object.get('message_updated_at')),
+            dt.datetime
+        )
+    except Exception as e:
+        log.error(f'Invalid HTTP request payload object: {e}')
+        errors.append(e)
+    return errors
+
+
 def evaluate_message_with_nlu(message_data):
     """ Process a student's message using NLU functions and send the result
     
@@ -131,20 +226,19 @@ def evaluate_message_with_nlu(message_data):
     {'type': 'sentiment', 'data': 'NEGATIVE', 'confidence': 0.9997807145118713}
     """
     # Keeps system working with two different inputs - full and filtered @event object
+    # Call validate payload
+    log.info(f'Starting evaluate message: {message_data}')
+
+    if not payload_is_valid(message_data):
+        log_payload_errors(message_data)
+        return {'type': 'error', 'data': TOKENS2INT_ERROR_INT, 'confidence': 0}
+
     try:
-        message_text = str(message_data['message_body'])
-    except KeyError:
-        message_data = {
-            'author_id': message_data['message']['_vnd']['v1']['chat']['owner'],
-            'author_type': message_data['message']['_vnd']['v1']['author']['type'],
-            'contact_uuid': message_data['message']['_vnd']['v1']['chat']['contact_uuid'],
-            'message_body': message_data['message']['text']['body'],
-            'message_direction': message_data['message']['_vnd']['v1']['direction'],
-            'message_id': message_data['message']['id'],
-            'message_inserted_at': message_data['message']['_vnd']['v1']['chat']['inserted_at'],
-            'message_updated_at': message_data['message']['_vnd']['v1']['chat']['updated_at'],
-        }
-        message_text = str(message_data['message_body'])
+        message_text = str(message_data.get('message_body', ''))
+    except:
+        log.error(f'Invalid request payload: {message_data}')
+        # use python logging system to do this//
+        return {'type': 'error', 'data': TOKENS2INT_ERROR_INT, 'confidence': 0}
 
     # Run intent classification only for keywords
     intent_api_response = run_intent_classification(message_text)
@@ -154,7 +248,7 @@ def evaluate_message_with_nlu(message_data):
 
     number_api_resp = text2int(message_text.lower())
 
-    if number_api_resp == 32202:
+    if number_api_resp == TOKENS2INT_ERROR_INT:
         # Run intent classification with logistic regression model
         predicted_label = predict_message_intent(message_text)
         if predicted_label['confidence'] > 0.01:

mathtext_fastapi/v2_conversation_manager.py

@@ -0,0 +1,271 @@
+import base64
+import copy
+import dill
+import os
+import json
+import jsonpickle
+import pickle
+import random
+import requests
+import mathtext_fastapi.global_state_manager as gsm
+
+from dotenv import load_dotenv
+from mathtext_fastapi.nlu import evaluate_message_with_nlu
+from mathtext_fastapi.math_quiz_fsm import MathQuizFSM
+from mathtext_fastapi.math_subtraction_fsm import MathSubtractionFSM
+from supabase import create_client
+from transitions import Machine
+
+from mathactive.generators import start_interactive_math
+from mathactive.hints import generate_hint
+from mathactive.microlessons import num_one
+
+load_dotenv()
+
+SUPA = create_client(
+    os.environ.get('SUPABASE_URL'),
+    os.environ.get('SUPABASE_KEY')
+)
+
+
+def pickle_and_encode_state_machine(state_machine):
+    dump = pickle.dumps(state_machine)
+    dump_encoded = base64.b64encode(dump).decode('utf-8')
+    return dump_encoded
+
+
+def manage_math_quiz_fsm(user_message, contact_uuid, type):
+    fsm_check = SUPA.table('state_machines').select("*").eq(
+        "contact_uuid",
+        contact_uuid
+    ).execute()
+
+    # This doesn't allow for when one FSM is present and the other is empty
+    """
+    1
+    data=[] count=None
+    
+    2
+    data=[{'id': 29, 'contact_uuid': 'j43hk26-2hjl-43jk-hnk2-k4ljl46j0ds09', 'addition3': None, 'subtraction': None, 'addition': 
+
+    - but problem is there is no subtraction , but it's assuming there's a subtration
+
+    Cases
+    - make a completely new record
+    - update an existing record with an existing FSM
+    - update an existing record without an existing FSM
+    """
+    print("MATH QUIZ FSM ACTIVITY")
+    print("user_message")
+    print(user_message)
+    # Make a completely new entry
+    if fsm_check.data == []:
+        if type == 'addition':
+            math_quiz_state_machine = MathQuizFSM()
+        else:
+            math_quiz_state_machine = MathSubtractionFSM()
+        messages = [math_quiz_state_machine.response_text]
+        dump_encoded = pickle_and_encode_state_machine(math_quiz_state_machine)
+
+        SUPA.table('state_machines').insert({
+            'contact_uuid': contact_uuid,
+            f'{type}': dump_encoded
+        }).execute()
+    # Update an existing record with a new state machine
+    elif not fsm_check.data[0][type]:
+        if type == 'addition':
+            math_quiz_state_machine = MathQuizFSM()
+        else:
+            math_quiz_state_machine = MathSubtractionFSM()
+        messages = [math_quiz_state_machine.response_text]
+        dump_encoded = pickle_and_encode_state_machine(math_quiz_state_machine)
+
+        SUPA.table('state_machines').update({
+            f'{type}': dump_encoded
+        }).eq(
+            "contact_uuid", contact_uuid
+        ).execute()      
+    # Update an existing record with an existing state machine
+    elif fsm_check.data[0][type]:
+        undump_encoded = base64.b64decode(
+            fsm_check.data[0][type].encode('utf-8')
+        )
+        math_quiz_state_machine = pickle.loads(undump_encoded)
+
+        math_quiz_state_machine.student_answer = user_message
+        math_quiz_state_machine.correct_answer = str(math_quiz_state_machine.correct_answer)
+        messages = math_quiz_state_machine.validate_answer()
+        dump_encoded = pickle_and_encode_state_machine(math_quiz_state_machine)          
+        SUPA.table('state_machines').update({
+            f'{type}': dump_encoded
+        }).eq(
+            "contact_uuid", contact_uuid
+        ).execute()
+    return messages
+
+
+def retrieve_microlesson_content(context_data, user_message, microlesson, contact_uuid):
+    # TODO: This is being filtered by both the local and global states, so not changing
+    if microlesson == 'addition':
+        messages = manage_math_quiz_fsm(user_message, contact_uuid, 'addition')
+
+        if user_message == 'exit':
+            state_label = 'exit'
+        else:
+            state_label = 'addition-question-sequence'
+
+        input_prompt = messages.pop()
+        message_package = {
+            'messages': messages,
+            'input_prompt': input_prompt,
+            'state': state_label
+        }
+    elif context_data['local_state'] == 'addition2' or microlesson == 'addition2':
+        if user_message == 'harder' or user_message == 'easier':
+            user_message = ''
+        message_package = num_one.process_user_message(contact_uuid, user_message)
+        message_package['state'] = 'addition2'
+        message_package['input_prompt'] = '?'
+
+    elif context_data['local_state'] == 'subtraction-question-sequence' or \
+        user_message == 'subtract' or \
+        microlesson == 'subtraction':
+        messages = manage_math_quiz_fsm(user_message, contact_uuid, 'subtraction')
+
+        if user_message == 'exit':
+            state_label = 'exit'
+        else:
+            state_label = 'subtraction-question-sequence'
+
+        input_prompt = messages.pop()
+
+        message_package = {
+            'messages': messages,
+            'input_prompt': input_prompt,
+            'state': state_label
+        }
+    print("MICROLESSON CONTENT RESPONSE")
+    print(message_package)
+    return message_package
+
+
+curriculum_lookup_table = {
+    'N1.1.1_G1': 'addition',
+    'N1.1.1_G2': 'addition2',
+    'N1.1.2_G1': 'subtraction'
+}
+
+
+def lookup_local_state(next_state):
+    microlesson = curriculum_lookup_table[next_state]
+    return microlesson
+
+
+def create_text_message(message_text, whatsapp_id):
+    """ Fills a template with input values to send a text message to Whatsapp
+
+    Inputs
+    - message_text: str - the content that the message should display
+    - whatsapp_id: str - the message recipient's phone number
+
+    Outputs
+    - message_data: dict - a preformatted template filled with inputs
+    """
+    message_data = {
+        "preview_url": False,
+        "recipient_type": "individual",
+        "to": whatsapp_id,
+        "type": "text",
+        "text": {
+            "body": message_text
+        }
+    }
+    return message_data
+
+
+def manage_conversation_response(data_json):
+    """ Calls functions necessary to determine message and context data """
+    print("V2 ENDPOINT")
+
+    # whatsapp_id = data_json['author_id']
+    message_data = data_json['message_data']
+    context_data = data_json['context_data']
+    whatsapp_id = message_data['author_id']
+    user_message = message_data['message_body']
+    print("MESSAGE DATA")
+    print(message_data)
+    print("CONTEXT DATA")
+    print(context_data)
+    print("=================")
+
+    # nlu_response = evaluate_message_with_nlu(message_data)
+
+    # context_data = {
+    #     'contact_uuid': 'abcdefg',
+    #     'current_state': 'N1.1.1_G2',
+    #     'user_message': '1',
+    #     'local_state': ''
+    # }
+    print("STEP 1")
+    print(data_json)
+    print(f"1: {context_data['current_state']}")
+    if not context_data['current_state']:
+        context_data['current_state'] = 'N1.1.1_G1'
+    print(f"2: {context_data['current_state']}")
+
+    curriculum_copy = copy.deepcopy(gsm.curriculum)
+    curriculum_copy.state = context_data['current_state']
+    print("STEP 2")
+    if user_message == 'easier':
+        curriculum_copy.left()
+        next_state = curriculum_copy.state
+    elif user_message == 'harder':
+        curriculum_copy.right()
+        next_state = curriculum_copy.state
+    else:
+        next_state = context_data['current_state']
+    print("next_state")
+    print(next_state)
+
+    print("STEP 3")
+    microlesson = lookup_local_state(next_state)
+
+    print("microlesson")
+    print(microlesson)
+
+    microlesson_content = retrieve_microlesson_content(context_data, user_message, microlesson, context_data['contact_uuid'])
+
+    headers = {
+        'Authorization': f"Bearer {os.environ.get('TURN_AUTHENTICATION_TOKEN')}",
+        'Content-Type': 'application/json'
+    }
+
+    # Send all messages for the current state before a user input prompt (text/button input request)
+    for message in microlesson_content['messages']:
+        data = create_text_message(message, whatsapp_id)
+
+        # print("data")
+        # print(data)
+
+        r = requests.post(
+            f'https://whatsapp.turn.io/v1/messages',
+            data=json.dumps(data),
+            headers=headers
+        )
+
+    print("STEP 4")
+    # combine microlesson content and context_data object
+
+    updated_context = {
+        "context": {
+            "contact_id": whatsapp_id,
+            "contact_uuid": context_data['contact_uuid'],
+            "current_state": next_state,
+            "local_state": microlesson_content['state'],
+            "bot_message": microlesson_content['input_prompt'],
+            "user_message": user_message,
+            "type": 'ask'
+        }
+    }
+    print(updated_context)
+    return updated_context

pyproject.toml

@@ -20,14 +20,17 @@ classifiers = [
 
 
 [tool.poetry.dependencies]
+mathactive = {git = "git@gitlab.com:tangibleai/community/mathactive.git", rev = "vlad"}
 mathtext = {git = "https://gitlab.com/tangibleai/community/mathtext", rev = "main"}
-fastapi = "0.74.*"
+fastapi = "^0.90.0"
 pydantic = "*"
-python = "^3.8,<3.10"
-requests = "2.27.*"
+python = "^3.8"
+requests = "2.27.*" 
 sentencepiece = "0.1.*"
 supabase = "*"
 uvicorn = "0.17.*"
+pandas = "^1.5.3"
+scipy = "^1.10.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2"

requirements.txt

@@ -3,14 +3,17 @@ en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_
 fuzzywuzzy
 jsonpickle
 mathtext @ git+https://gitlab.com/tangibleai/community/mathtext@main
-fastapi==0.74.*
-pydantic==1.10.*
+mathactive @ git+https://gitlab.com/tangibleai/community/mathactive@main
+fastapi
+pydantic
+requests
+sentencepiece
+openpyxl
 python-Levenshtein
-requests==2.27.*
-sentencepiece==0.1.*
 sentence-transformers
 sentry-sdk[fastapi]
 supabase
 transitions
-uvicorn==0.17.*
-
+uvicorn
+pandas
+scipy

scripts/bump_version.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+from pathlib import Path
+import re
+import shutil
+
+BASE_DIR = Path(__file__).parent.parent
+PYPROJECT_PATH = BASE_DIR / 'pyproject.toml'
+PATTERN = re.compile(r'(version\s*=\s*)[\'"]?(\d(\.\d+)+)[\'"]?\s*')
+
+if __name__ == '__main__':
+    verline = None
+    with PYPROJECT_PATH.open() as fin:
+        lines = []
+        verline = None
+        for line in fin:
+            lines.append(line)
+            if verline:
+                continue
+            match = PATTERN.match(line)
+            if match:
+                print(f'Found match.groups(): {dict(list(enumerate(match.groups())))}')
+                ver = [int(x) for x in match.groups()[1].split('.')]
+                print(f'             Old ver: {ver}')
+                ver[-1] += 1
+                print(f'             New ver: {ver}')
+                ver = '.'.join([str(x) for x in ver])
+                print(f'         New ver str: {ver}')
+                verline = f'version = "{ver}"\n'
+                print(f'        New ver line: {verline}')
+                lines[-1] = verline
+                print(f'        New ver line: {lines[-1]}')
+
+    if verline:
+        shutil.copy(PYPROJECT_PATH, PYPROJECT_PATH.with_suffix('.toml.bak'))
+        with PYPROJECT_PATH.open('w') as fout:
+            fout.writelines(lines)

scripts/cleanpyc.sh

@@ -0,0 +1,2 @@
+#!usr/bin/env bash
+find . | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf

scripts/make_request.py

@@ -22,7 +22,11 @@ def add_message_text_to_sample_object(message_text):
     message_data = '{' + f'"author_id": "+57787919091", "author_type": "OWNER", "contact_uuid": "j43hk26-2hjl-43jk-hnk2-k4ljl46j0ds09", "message_body": "{message_text}", "message_direction": "inbound", "message_id": "4kl209sd0-a7b8-2hj3-8563-3hu4a89b32", "message_inserted_at": "2023-01-10T02:37:28.477940Z", "message_updated_at": "2023-01-10T02:37:28.487319Z"' + '}'
     # context_data = '{' + '"user":"", "state":"addition-question-sequence", "bot_message":"", "user_message":"{message_text}"' + '}'
 
-    context_data = '{' + '"user":"", "state":"start-conversation", "bot_message":"", "user_message":"{message_text}"' + '}'
+    # V1
+    # context_data = '{' + '"user":"", "state":"start-conversation", "bot_message":"", "user_message":"{message_text}"' + '}'
+
+    #V2
+    context_data = '{' + '"contact_uuid": "j43hk26-2hjl-43jk-hnk2-k4ljl46j0ds09", "current_state":"", "local_state": "", "user_message":""' + '}'
 
     # context_data = '{' + '"user":"", "state":"addition-question-sequence", "bot_message":"", "user_message":"{message_text}","text": "What is 2+3?","question_numbers": [4,3],"right_answer": 7,"number_correct": 2, "number_incorrect": 0, "hints_used": 0, "level": "easy"' + '}'
 
@@ -40,14 +44,15 @@ def add_message_text_to_sample_object(message_text):
 # """
 
 
-def run_simulated_request(endpoint, sample_answer, context=None):
-    print(f"Case: {sample_answer}")
-    b_string = add_message_text_to_sample_object(sample_answer)
+def run_simulated_request(endpoint, sample_payload, context=None):
+    print(f"Case: {sample_payload}")
+    # Used for testing full message object - deprecated April 7
+    b_string = add_message_text_to_sample_object(sample_payload)
 
     if endpoint == 'sentiment-analysis' or endpoint == 'text2int' or endpoint =='intent-classification':
         request = requests.post(
             url=f'http://localhost:7860/{endpoint}',
-            json={'content': sample_answer}
+            json={'content': sample_payload}
         ).json()
     else:
         request = requests.post(
@@ -58,54 +63,98 @@ def run_simulated_request(endpoint, sample_answer, context=None):
     print(request)
 
 
+def run_full_nlu_endpoint_payload_test(sample_payload):
+    request = requests.post(
+        url=f'http://localhost:7860/nlu',
+        data=sample_payload
+    ).json()
+    print(request)
+
+
+# # Case: Wrong key
+run_full_nlu_endpoint_payload_test(b'{"message": {"author_id": "@event.message._vnd.v1.chat.owner", "author_type": "@event.message._vnd.v1.author.type", "contact_uuid": "@event.message._vnd.v1.chat.contact_uuid", "message_body": "@event.message.text.body", "message_direction": "@event.message._vnd.v1.direction", "message_id": "@event.message.id", "message_inserted_at": "@event.message._vnd.v1.chat.inserted_at", "message_updated_at": "@event.message._vnd.v1.chat.updated_at"}}')
+
+# # Case: Correct payload
+run_full_nlu_endpoint_payload_test(b'{"message_data": {"author_id": "57787919091", "author_type": "OWNER", "contact_uuid": "df78gsdf78df", "message_body": "8", "message_direction": "inbound", "message_id": "dfgha789789ag9ga", "message_inserted_at": "2023-01-10T02:37:28.487319Z", "message_updated_at": "2023-01-10T02:37:28.487319Z"}}')
+
+# # Case: Incorrect payload values
+run_full_nlu_endpoint_payload_test(b'{"message_data": {"author_id": "@event.message._vnd.v1.chat.owner", "author_type": "@event.message._vnd.v1.author.type", "contact_uuid": "@event.message._vnd.v1.chat.contact_uuid", "message_body": "@event.message.text.body", "message_direction": "@event.message._vnd.v1.direction", "message_id": "@event.message.id", "message_inserted_at": "@event.message._vnd.v1.chat.inserted_at", "message_updated_at": "@event.message._vnd.v1.chat.updated_at"}}')
+
+# Case: Wrong payload object
+run_full_nlu_endpoint_payload_test(b'{"message_data": {"_vnd": {"v1": {"author": {"id": 54327547257, "name": "Jin", "type": "OWNER"}, "card_uuid": None, "chat": {"assigned_to": None, "contact_uuid": "f7889-f78dfgb798-f786ah89g7-f78f9a", "inserted_at": "2023-03-28T13:21:47.581221Z", "owner": "+43789789146", "permalink": "", "state": "OPEN", "state_reason": "Re-opened by inbound message.", "unread_count": 97, "updated_at": "2023-04-07T21:05:27.389948Z", "uuid": "dfg9a78-d76a786dghas-78d9fga789g-a78d69a9"}, "direction": "inbound", "faq_uuid": None, "in_reply_to": None, "inserted_at": "2023-04-07T21:05:27.368580Z", "labels": [], "last_status": None, "last_status_timestamp": None, "on_fallback_channel": False, "rendered_content": None, "uuid": "hf78s7s89b-789fb68d9fg-789fb789dfb-f79sfb789"}}, "from": 5475248689, "id": "SBDE4zgAAy7887sfdT35SHFS", "text": {"body": 1000}, "timestamp": 1680901527, "type": "text"}, "type": "message"}')
+
+
 # run_simulated_request('intent-classification', 'exit')
 # run_simulated_request('intent-classification', "I'm not sure")
+# run_simulated_request('intent-classification', "easier")
+# run_simulated_request('intent-classification', "easy")
+# run_simulated_request('intent-classification', "harder")
+# run_simulated_request('intent-classification', "hard")
+# run_simulated_request('intent-classification', "hint")
+# run_simulated_request('intent-classification', "hin")
+# run_simulated_request('intent-classification', "hnt")
+# run_simulated_request('intent-classification', "stop")
+# run_simulated_request('intent-classification', "stp")
+# run_simulated_request('intent-classification', "sop")
+# run_simulated_request('intent-classification', "please stop")
 # run_simulated_request('sentiment-analysis', 'I reject it')
 # run_simulated_request('text2int', 'seven thousand nine hundred fifty seven')
-run_simulated_request('nlu', 'test message')
-run_simulated_request('nlu', 'eight')
-run_simulated_request('nlu', 'is it 8')
-run_simulated_request('nlu', 'can I know how its 0.5')
+# run_simulated_request('nlu', 'test message')
+# run_simulated_request('nlu', 'eight')
+# run_simulated_request('nlu', 'is it 8')
+# run_simulated_request('nlu', 'can I know how its 0.5')
 run_simulated_request('nlu', 'eight, nine, ten')
 run_simulated_request('nlu', '8, 9, 10')
 run_simulated_request('nlu', '8')
 run_simulated_request('nlu', "I don't know")
 run_simulated_request('nlu', "I don't know eight")
 run_simulated_request('nlu', "I don't 9")
-run_simulated_request('nlu', "0.2")
-run_simulated_request('nlu', 'Today is a wonderful day')
-run_simulated_request('nlu', 'IDK 5?')
+# run_simulated_request('nlu', "0.2")
+# run_simulated_request('nlu', 'Today is a wonderful day')
+# run_simulated_request('nlu', 'IDK 5?')
+# run_simulated_request('nlu', 'hin')
+# run_simulated_request('nlu', 'exi')
+# run_simulated_request('nlu', 'easier')
+# run_simulated_request('nlu', 'stp')
+# run_simulated_request('nlu', '')
 # run_simulated_request('manager', '')
 # run_simulated_request('manager', 'add')
 # run_simulated_request('manager', 'subtract')
-# run_simulated_request("question", {
-#     'number_correct': 0,
-#     'number_incorrect': 0,
-#     'level': 'easy'
+
+# run_simulated_request("start", {
+#     'difficulty': 0.04,
+#     'do_increase': True
 # })
 # run_simulated_request("hint", {
-#     'question_numbers': [1, 2, 3],
-#     'right_answer': 3,
-#     'number_correct': 0,
-#     'number_incorrect': 0,
-#     'level': 'easy',
-#     'hints_used': 0
+#     'start': 5,
+#     'step': 1,
+#     'difficulty': 0.56  # optional
 # })
-# run_simulated_request("generate_question", {
-#     'level': 'medium'
+# run_simulated_request("question", {
+#     'start': 2,
+#     'step': 1,
+#     'question_num': 2  # optional
+# })
+# run_simulated_request("difficulty", {
+#     'difficulty': 0.01,
+#     'do_increase': False  # True | False
 # })
-# run_simulated_request("numbers_by_level", {
-#     'level': 'medium'
+# Need to start with this command to populate users.json
+# If users.json is not already made
+# run_simulated_request("num_one", {
+#     "user_id": "1",
+#     "message_text": "",
 # })
-# run_simulated_request("number_sequence", {
-#     "current_number": 10,
-#     "ordinal_number": 2,
-#     "times": 1
+# run_simulated_request("num_one", {
+#     "user_id": "1",
+#     "message_text": "61",
 # })
-# run_simulated_request("level", {
-#     "current_level": "hard",
-#     "level_up": False
+# run_simulated_request("sequence", {
+#     'start': 2,
+#     'step': 1,
+#     'sep': '... '
 # })
+
 # run_simulated_request('manager', 'exit')
 
 

scripts/pin_requirements.py

@@ -0,0 +1,62 @@
+""" Parse requirements.txt and pyproject.toml and move versions to pyproject.toml """
+from pathlib import Path
+import re
+import sys
+import toml
+
+def get_requirement_versions(path='requirements.txt'):
+    """ Read requirements.txt file and return dict of package versions """
+    path = Path(path or '')
+    if path.is_dir():
+        path = next(iter(path.glob('**/requirements.txt')))
+    reqdict = {}
+    text = Path(path).open().read()
+    for line in text.splitlines():
+        if line.strip():
+            match = re.match(r'([-_a-zA-Z0-9]+)\s*([ >=<~^,.rabc0-9]+)\s*', line)
+            if match:
+                name, ver = match.groups()
+                reqdict[name] = ver
+    return reqdict  
+
+
+def normalize_name(name):
+    return str(name).strip().replace('_', '-').replace(' ', '-').lower()
+
+
+def pin_versions(pyproject='pyproject.toml', reqdict=None, overwrite=False):
+    if not reqdict or isinstance(reqdict, (str, Path)):
+        reqdict = get_requirement_versions(path=reqdict)
+    reqdict = {
+        normalize_name(k): v for (k, v) in 
+        reqdict.items()
+        }
+    
+    pyproj = toml.load(pyproject)
+    depdict = pyproj.get('tool', {}).get('poetry', {}).get('dependencies', {})
+    depdict = {
+        normalize_name(k): v for (k, v) in 
+        depdict.items()
+        }
+    
+    for name, spec in reqdict.items():
+        if name in depdict:
+            ver = depdict[name]
+            if isinstance(ver, str) and (overwrite or ver == '*'):
+                depdict[name] = spec
+
+    pyproj['tool']['poetry']['dependencies'] = depdict
+    overwrite = overwrite or (input(f'Overwrite {pyproject}?')[0].lower() == 'y')
+    if overwrite:
+        with open(pyproject, 'w') as stream:
+            toml.dump(pyproj, stream)
+    return pyproj
+
+
+if __name__ == '__main__':
+    path = 'requirements.txt'
+    if sys.argv[1:]:
+        path = sys.argv[1]
+    pyproj = pin_versions(reqdict=path)
+    print(toml.dumps(pyproj))
+

scripts/quiz/data.csv

@@ -0,0 +1,56 @@
+difficulty,start
+0.01,1
+0.02,0
+0.05,5
+0.07,10
+0.08,14
+0.1,20
+0.11,22
+0.13,27
+0.14,28
+0.16,30
+0.17,32
+0.18,34
+0.2,37
+0.21,39
+0.23,42
+0.25,43
+0.27,46
+0.3,50
+0.34,57
+0.35,64
+0.37,78
+0.39,89
+0.41,100
+0.44,112
+0.45,130
+0.48,147
+0.5,164
+0.52,180
+0.55,195
+0.58,209
+0.6,223
+0.61,236
+0.63,248
+0.64,259
+0.65,271
+0.67,284
+0.69,296
+0.7,308
+0.72,321
+0.73,333
+0.75,346
+0.78,359
+0.8,370
+0.81,385
+0.83,399
+0.84,408
+0.87,420
+0.88,435
+0.89,447
+0.9,458
+0.93,469
+0.94,483
+0.96,494
+0.97,500
+0.99,513

users.json

@@ -0,0 +1 @@
+{"1": {"skill_score": 0.04, "state": "question", "start": 1, "stop": 1, "step": 1, "answer": 2}}