ndis AI chatbot files

app.py

@@ -1,147 +1,133 @@
-import io
-import random
-from typing import List, Tuple
-
-import aiohttp
-import panel as pn
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor
-
-pn.extension(design="bootstrap", sizing_mode="stretch_width")
-
-ICON_URLS = {
-    "brand-github": "https://github.com/holoviz/panel",
-    "brand-twitter": "https://twitter.com/Panel_Org",
-    "brand-linkedin": "https://www.linkedin.com/company/panel-org",
-    "message-circle": "https://discourse.holoviz.org/",
-    "brand-discord": "https://discord.gg/AXRHnJU6sP",
-}
-
-
-async def random_url(_):
-    pet = random.choice(["cat", "dog"])
-    api_url = f"https://api.the{pet}api.com/v1/images/search"
-    async with aiohttp.ClientSession() as session:
-        async with session.get(api_url) as resp:
-            return (await resp.json())[0]["url"]
-
-
-@pn.cache
-def load_processor_model(
-    processor_name: str, model_name: str
-) -> Tuple[CLIPProcessor, CLIPModel]:
-    processor = CLIPProcessor.from_pretrained(processor_name)
-    model = CLIPModel.from_pretrained(model_name)
-    return processor, model
-
-
-async def open_image_url(image_url: str) -> Image:
-    async with aiohttp.ClientSession() as session:
-        async with session.get(image_url) as resp:
-            return Image.open(io.BytesIO(await resp.read()))
-
-
-def get_similarity_scores(class_items: List[str], image: Image) -> List[float]:
-    processor, model = load_processor_model(
-        "openai/clip-vit-base-patch32", "openai/clip-vit-base-patch32"
-    )
-    inputs = processor(
-        text=class_items,
-        images=[image],
-        return_tensors="pt",  # pytorch tensors
+import os
+import openai
+import sys
+sys.path.append('../..')
+import utils
+import panel as pn  # GUI
+pn.extension(template='bootstrap')
+
+deployment_name = os.environ['DEPLOYMENT_ID']
+
+def get_completion_from_messages(messages, engine=deployment_name, temperature=0, max_tokens=500):
+    openai.api_key = os.environ['API_KEY']
+    openai.api_base = os.environ['API_BASE']
+    openai.api_type = os.environ['API_TYPE']
+    openai.api_version = os.environ['API_VERSION']
+    response = openai.ChatCompletion.create(
+        engine=engine,
+        messages=messages,
+        temperature=temperature, 
+        max_tokens=max_tokens, 
     )
-    outputs = model(**inputs)
-    logits_per_image = outputs.logits_per_image
-    class_likelihoods = logits_per_image.softmax(dim=1).detach().numpy()
-    return class_likelihoods[0]
+    return response.choices[0].message["content"]
+
+def get_moderation_from_input(user_input):
+    openai.api_key = os.environ['MOD_API_KEY']
+    openai.api_base = os.environ['MOD_API_BASE']
+    openai.api_type = "open_ai"
+    openai.api_version = None
+    response = openai.Moderation.create(input=user_input)
+    return response
+
+def process_user_message(user_input, all_messages, debug=True):
+    delimiter = "```"
+    
+    # Step 1: Check input to see if it flags the Moderation API or is a prompt injection
+    response = get_moderation_from_input(user_input)
+    moderation_output = response["results"][0]
 
+    if moderation_output["flagged"]:
+        print("Step 1: Input flagged by Moderation API.")
+        return "Sorry, we cannot process this request."
 
-async def process_inputs(class_names: List[str], image_url: str):
+    if debug: print("Step 1: Input passed moderation check.")
+    
+    page_and_qm_response = utils.find_pages_and_qms_only(user_input)
+    #print(print(page_and_qm_response)
+    # Step 2: Extract the list of products
+    page_and_qm_list = utils.read_string_to_list(page_and_qm_response)
+    #print(page_and_qm_list)
+
+    if debug: print("Step 2: Extracted list of quality markers.")
+
+    # Step 3: If quality markers are found, look them up
+    qm_information = utils.generate_output_string(page_and_qm_list)
+    if debug: print("Step 3: Looked up quality marker information.")
+
+    # Step 4: Answer the user question
+    system_message = f"""
+    You are an experienced assistant in the domain of Behaviour Support Plans (BSPs). \
+    Respond in a friendly and helpful tone, with concise answers. \
+    Your response MUST be ONLY based on the relevant information provided to you.
+    Make sure to ask the user relevant follow-up questions.
     """
-    High level function that takes in the user inputs and returns the
-    classification results as panel objects.
+    messages = [
+        {'role': 'system', 'content': system_message},
+        {'role': 'user', 'content': f"{delimiter}{user_input}{delimiter}"},
+        {'role': 'assistant', 'content': f"Relevant information:\n{qm_information}"}
+    ]
+
+    final_response = get_completion_from_messages(all_messages + messages)
+    if debug:print("Step 4: Generated response to user question.")
+    all_messages = all_messages + messages[1:]
+
+    # Step 5: Put the answer through the Moderation API
+    response = get_moderation_from_input(final_response)
+    moderation_output = response["results"][0]
+
+    if moderation_output["flagged"]:
+        if debug: print("Step 5: Response flagged by Moderation API.")
+        return "Sorry, we cannot provide this information."
+
+    if debug: print("Step 5: Response passed moderation check.")
+
+    # Step 6: Ask the model if the response answers the initial user query well
+    user_message = f"""
+    Customer message: {delimiter}{user_input}{delimiter}
+    Agent response: {delimiter}{final_response}{delimiter}
+
+    Does the agent response sufficiently answer the customer question? \
+    Respond with a Y or N character, with no punctuation: \
+    Y - if the output sufficiently answers the question \
+    AND the response correctly uses the relevant information provided to the agent \
+    N - otherwise \
+    Output a single letter only.
     """
-    try:
-        main.disabled = True
-        if not image_url:
-            yield "##### ⚠️ Provide an image URL"
-            return
-    
-        yield "##### ⚙ Fetching image and running model..."
-        try:
-            pil_img = await open_image_url(image_url)
-            img = pn.pane.Image(pil_img, height=400, align="center")
-        except Exception as e:
-            yield f"##### 😔 Something went wrong, please try a different URL!"
-            return
-    
-        class_items = class_names.split(",")
-        class_likelihoods = get_similarity_scores(class_items, pil_img)
-    
-        # build the results column
-        results = pn.Column("##### 🎉 Here are the results!", img)
-    
-        for class_item, class_likelihood in zip(class_items, class_likelihoods):
-            row_label = pn.widgets.StaticText(
-                name=class_item.strip(), value=f"{class_likelihood:.2%}", align="center"
-            )
-            row_bar = pn.indicators.Progress(
-                value=int(class_likelihood * 100),
-                sizing_mode="stretch_width",
-                bar_color="secondary",
-                margin=(0, 10),
-                design=pn.theme.Material,
-            )
-            results.append(pn.Column(row_label, row_bar))
-        yield results
-    finally:
-        main.disabled = False
-
-
-# create widgets
-randomize_url = pn.widgets.Button(name="Randomize URL", align="end")
-
-image_url = pn.widgets.TextInput(
-    name="Image URL to classify",
-    value=pn.bind(random_url, randomize_url),
-)
-class_names = pn.widgets.TextInput(
-    name="Comma separated class names",
-    placeholder="Enter possible class names, e.g. cat, dog",
-    value="cat, dog, parrot",
-)
-
-input_widgets = pn.Column(
-    "##### 😊 Click randomize or paste a URL to start classifying!",
-    pn.Row(image_url, randomize_url),
-    class_names,
-)
-
-# add interactivity
-interactive_result = pn.panel(
-    pn.bind(process_inputs, image_url=image_url, class_names=class_names),
-    height=600,
-)
-
-# add footer
-footer_row = pn.Row(pn.Spacer(), align="center")
-for icon, url in ICON_URLS.items():
-    href_button = pn.widgets.Button(icon=icon, width=35, height=35)
-    href_button.js_on_click(code=f"window.open('{url}')")
-    footer_row.append(href_button)
-footer_row.append(pn.Spacer())
-
-# create dashboard
-main = pn.WidgetBox(
-    input_widgets,
-    interactive_result,
-    footer_row,
-)
-
-title = "Panel Demo - Image Classification"
-pn.template.BootstrapTemplate(
-    title=title,
-    main=main,
-    main_max_width="min(50%, 698px)",
-    header_background="#F08080",
-).servable(title=title)
+    messages = [
+        {'role': 'system', 'content': system_message},
+        {'role': 'user', 'content': user_message}
+    ]
+    evaluation_response = get_completion_from_messages(messages)
+    if debug: print("Step 6: Model evaluated the response.")
+
+    # Step 7: If yes, use this answer; if not, say that you will connect the user to a human
+    if "Y" in evaluation_response:  # Using "in" instead of "==" to be safer for model output variation (e.g., "Y." or "Yes")
+        if debug: print("Step 7: Model approved the response.")
+        return final_response, all_messages
+    else:
+        if debug: print("Step 7: Model disapproved the response.")
+        neg_str = "I'm unable to provide the information you're looking for. Please try asking again."
+        return neg_str, all_messages
+
+chat_box = pn.widgets.ChatBox(value=[
+                                {"You": ""},
+                                {"AI Assistant": "Greetings! Feel free to ask about the BSP summary document, such as pages, quality markers, AI models, and NLP topics."}
+                                ])
+context = [{"role": "system", "content": "You are Service Assistant"}]
+
+# Function that collects user and assistant messages over time
+def collect_messages(event) -> None:
+    global context
+    user_input = event.new[-1]
+    user_message = user_input.get("You")
+    if user_message is None or user_message == "":
+        return
+    response, context = process_user_message(user_message, context, False)
+    context.append({"role": "assistant", "content": f"{response}"})
+    chat_box.append({"AI Assistant": response})
+
+# Chat with the chatbot!
+chat_box.param.watch(collect_messages, 'value')
+
+dashboard = pn.Column(pn.pane.Markdown("# BSP Summary Document AI Assistant"), chat_box)
+dashboard.servable()

data.json

@@ -0,0 +1,298 @@
+{
+	"Q1": {
+		"page": "1",
+		"domain": "1",
+		"model_ids": [1, 2, 3, 4],
+		"topics": ["Age", "Gender", "Family History", "Diagnosed Disabilities", "Health Info", "Communication", "Likes", "Dislikes", "Sensory Experiences", "Goals & Aspirations"],
+		"text": "A summary description of the person who is the focus of the plan has been provided.",
+		"name": "Q1"
+	},
+	"Q2a": {
+		"page": "2",
+		"domain": "2",
+		"model_ids": [23, 31],
+		"topics": ["Person of Focus", "Exclusion Reason"],
+		"text": "The plan indicates that the person who is the focus of this plan was included in the assessment and data gathering processes OR an explanation has been provided as to why they were not included.",
+		"name": "Q2a"
+	},
+	"Q2b": {
+		"page": "2",
+		"domain": "2",
+		"model_ids": [5, 23, 25],
+		"topics": ["Stakeholder Interview", "File Review", "Observation / Communication", "Speak to Family", "Stakeholder Involvement", "Data Collectors", "Data Collected"],
+		"text": "The plan indicates that the focus person\u2019s family members, support staff, support providers and/or other professionals were included in the assessment and data gathering process.",
+		"name": "Q2b"
+	},
+	"Q2c": {
+		"page": "2",
+		"domain": "2",
+		"model_ids": [5, 7],
+		"topics": ["Stakeholder Interview", "File Review", "Observation / Communication", "Speak to Family", "Direct"],
+		"text": "The plan indicates that at least one direct data collection approach has been undertaken.",
+		"name": "Q2c"
+	},
+	"Q2d": {
+		"page": "2",
+		"domain": "2",
+		"model_ids": [6, 7],
+		"topics": ["Psychiatric Assessment", "Medical Assessment", "Speech and Language Assessment", "Indirect"],
+		"text": "The plan indicates that at least one indirect data collection approach has been undertaken.",
+		"name": "Q2d"
+	},
+	"Q3a": {
+		"page": "3",
+		"domain": "3",
+		"model_ids": [8],
+		"topics": ["Behaviour", "Glossary"],
+		"text": "All listed target behaviours are described in a way that would allow another person to act them out.",
+		"name": "Q3a"
+	},
+	"Q3b": {
+		"page": "3",
+		"domain": "3",
+		"model_ids": [8],
+		"topics": ["Frequency"],
+		"text": "Information has been provided about how often all the listed target behaviours occur (e.g., hourly, daily, weekly).",
+		"name": "Q3b"
+	},
+	"Q3c": {
+		"page": "3",
+		"domain": "3",
+		"model_ids": [8],
+		"topics": ["Duration"],
+		"text": "Information has been provided about how long all the listed target behaviours last for (e.g., seconds, minutes, hours, days).",
+		"name": "Q3c"
+	},
+	"Q3d": {
+		"page": "3",
+		"domain": "3",
+		"model_ids": [8],
+		"topics": ["Severity"],
+		"text": "Information has been provided about the severity of all the listed target behaviours (e.g., damage, disruption, injury, etc.).",
+		"name": "Q3d"
+	},
+	"Q3e": {
+		"page": "3",
+		"domain": "3",
+		"model_ids": [17],
+		"topics": ["Physical Signs", "Verbal Signs"],
+		"text": "All listed early warning signs are described in a way that is observable to others.",
+		"name": "Q3e"
+	},
+	"Q4a": {
+		"page": "3",
+		"domain": "4",
+		"model_ids": [9],
+		"topics": ["Triggers"],
+		"text": "The plan describes what is happening immediately before the target behaviour(s) are displayed, i.e., all identified triggers/antecedents are observable to others.",
+		"name": "Q4a"
+	},
+	"Q4b": {
+		"page": "3",
+		"domain": "4",
+		"model_ids": [9],
+		"topics": ["Consequences"],
+		"text": "The plan describes what happens immediately after the target behaviour(s) are displayed, i.e., all maintaining consequences are observable to others.",
+		"name": "Q4b"
+	},
+	"Q5b": {
+		"page": "3",
+		"domain": "5",
+		"model_ids": [12],
+		"topics": ["Background", "Contrubuting", "Sustaining", "Strength"],
+		"text": "The plan contains a formulation.",
+		"name": "Q5b"
+	},
+	"Q6b": {
+		"page": "4",
+		"domain": "6",
+		"model_ids": [15, 16],
+		"topics": ["Specific", "Measurable", "Achievable", "Relevant", "Timely", "Physical / Mental", "Relationships", "Personal Fulfillment", "Financial Stability", "Purpose / Impact"],
+		"text": "The plan contains at least one goal/objective that is related to the focus person\u2019s quality of life.",
+		"name": "Q6b"
+	},
+	"Q6c": {
+		"page": "4",
+		"domain": "6",
+		"model_ids": [15, 16],
+		"topics": ["Specific", "Measurable", "Achievable", "Relevant", "Timely", "Physical / Mental", "Relationships", "Personal Fulfillment", "Financial Stability", "Purpose / Impact"],
+		"text": "All goals are specific about who the goal/objective is for, what is to be achieved, when it will be achieved by.",
+		"name": "Q6c"
+	},
+	"Q7i": {
+		"page": "4",
+		"domain": "7",
+		"model_ids": [28],
+		"topics": ["Likes / Dislikes", "Interests", "Hobbies", "Relationships", "Employment", "Health", "Education"],
+		"text": "The plan specifies strategies that aim to enhance the focus person\u2019s quality of life and are not specifically related to the focus person\u2019s target behaviour(s).",
+		"name": "Q7i"
+	},
+	"Q7ii": {
+		"page": "4",
+		"domain": "7",
+		"model_ids": [14, 19],
+		"topics": ["Safety Strategy", "Setting Event Strategy", "Trigger Strategy", "Other Strategy"],
+		"text": "The plan specifies environmental strategies that address the identified setting events and/or triggers/antecedents.",
+		"name": "Q7ii"
+	},
+	"Q7iii": {
+		"page": "4",
+		"domain": "7",
+		"model_ids": [13, 19],
+		"topics": ["Replacement Behaviour", "Other Strategy"],
+		"text": "The plan specifies other strategies that will impact upon the focus person\u2019s target behaviour(s).",
+		"name": "Q7iii"
+	},
+	"Q7iv": {
+		"page": "4",
+		"domain": "7",
+		"model_ids": [27, 30],
+		"topics": ["Restrictive Intervention", "Physical Restraint", "Punitive Approaches"],
+		"text": "None of the proposed strategies reflect a restrictive practice (e.g., seclusion, restraint).",
+		"name": "Q7iv"
+	},
+	"Q7a": {
+		"page": "4",
+		"domain": "7",
+		"model_ids": [19, 14, 21, 29],
+		"topics": ["Safety Strategy", "Setting Event Strategy", "Trigger Strategy", "Other Strategy", "Privileges", "Breaks", "Verbal Praise", "Social Interaction", "Physical Rewards", "Attention", "When to Implement"],
+		"text": "All strategies are described in enough detail for someone to understand what needs to be done and when it needs to be implemented.",
+		"name": "Q7a"
+	},
+	"Q7b": {
+		"page": "4",
+		"domain": "7",
+		"model_ids": [13, 20, 21, 32],
+		"topics": ["Replacement Behaviour", "What", "How", "Who", "When", "Where", "Materials", "Privileges", "Breaks", "Verbal Praise", "Social Interaction", "Physical Rewards", "Attention", "Unneeded Teaching Reason"],
+		"text": "Does the plan propose to teach the person of focus alternative behaviours, functionally equivalent replacement behaviours and/or new skills?",
+		"name": "Q7b"
+	},
+	"Q8a": {
+		"page": "4",
+		"domain": "8",
+		"model_ids": [38],
+		"topics": ["Early Warning"],
+		"text": "A set of early warning signs for the target behaviour(s) are identified for the purposes of de-escalation.",
+		"name": "Q8a"
+	},
+	"Q8b": {
+		"page": "4",
+		"domain": "8",
+		"model_ids": [38],
+		"topics": ["Early Warning"],
+		"text": "At least one strategy relates to providing support to the person of focus when they start exhibiting the identified early warning signs.",
+		"name": "Q8b"
+	},
+	"Q8c": {
+		"page": "4",
+		"domain": "8",
+		"model_ids": [13],
+		"topics": ["Replacement Behaviour"],
+		"text": "At least one strategy relates to the person of focus being prompted to use an alternative response (e.g., a functionally equivalent behaviour, an alternative behaviour, moving onto another activity) when the target behaviour occurs.",
+		"name": "Q8c"
+	},
+	"Q8d": {
+		"page": "4",
+		"domain": "8",
+		"model_ids": [14],
+		"topics": ["Safety Strategy"],
+		"text": "At least one strategy relates to ensuring the safety of the person of focus and/or the people around them when the target behaviour(s) occur.",
+		"name": "Q8d"
+	},
+	"Q8e": {
+		"page": "4",
+		"domain": "8",
+		"model_ids": [18],
+		"topics": ["Debriefing Strategy"],
+		"text": "At least one strategy relates to debriefing with the person of focus and/or other relevant stakeholders at a designated time after the target behaviour(s) has subsided.",
+		"name": "Q8e"
+	},
+	"Q8f": {
+		"page": "4",
+		"domain": "8",
+		"model_ids": [27, 30],
+		"topics": ["Restrictive Intervention", "Physical Restraint", "Punitive Approaches"],
+		"text": "Restrictive interventions (e.g., seclusion, restraint) are not mentioned as a de-escalation strategy OR restrictive interventions are only discussed as a last resort strategy once all other de-escalation strategies have been implemented.",
+		"name": "Q8f"
+	},
+	"Q9a": {
+		"page": "5",
+		"domain": "9",
+		"model_ids": [5, 23, 25],
+		"topics": ["Stakeholder Interview", "Stakeholder Involvement", "Outcome Measurement Strategies"],
+		"text": "The plan describes at least one method of assessing the social validity of the proposed strategies (e.g., interview, rating scale).",
+		"name": "Q9a"
+	},
+	"Q9b": {
+		"page": "5",
+		"domain": "9",
+		"model_ids": [23, 31],
+		"topics": ["Person of Focus", "Exclusion Reason"],
+		"text": "The plan indicates that the focus person was consulted regarding the social validity of the proposed strategies OR an explanation has been provided around why they were not consulted.",
+		"name": "Q9b"
+	},
+	"Q9c": {
+		"page": "5",
+		"domain": "9",
+		"model_ids": [5, 23],
+		"topics": ["Stakeholder Interview", "Stakeholder Involvement", "Speak to Family"],
+		"text": "The plan indicates that the focus person\u2019s family members, support staff, support providers and/or other professionals were consulted regarding the social validity of the proposed strategies.",
+		"name": "Q9c"
+	},
+	"Q10a": {
+		"page": "5",
+		"domain": "10",
+		"model_ids": [24],
+		"topics": ["Implementors & Roles"],
+		"text": "At least one plan implementor and their role (e.g., parent, support staff) is identified in the plan.",
+		"name": "Q10a"
+	},
+	"Q10b": {
+		"page": "5",
+		"domain": "10",
+		"model_ids": [24],
+		"topics": ["Training Strategies"],
+		"text": "At least one strategy to train relevant plan implementors is proposed and indicates who will deliver the training.",
+		"name": "Q10b"
+	},
+	"Q10c": {
+		"page": "5",
+		"domain": "10",
+		"model_ids": [24],
+		"topics": ["Implementation Support"],
+		"text": "At least one strategy to support implementation in relevant settings (e.g., mentoring) is proposed and indicates who will deliver the strategy.",
+		"name": "Q10c"
+	},
+	"Q10d": {
+		"page": "5",
+		"domain": "10",
+		"model_ids": [24],
+		"topics": ["Communication Strategies"],
+		"text": "At least one strategy is proposed regarding how plan implementors will communicate with one another, relevant information about the plan and/or its implementation.",
+		"name": "Q10d"
+	},
+	"Q11a": {
+		"page": "5",
+		"domain": "11",
+		"model_ids": [25],
+		"topics": ["Outcome Measurement Strategies", "Data Collectors", "Data Collected"],
+		"text": "At least one outcome measurement strategy is proposed and indicates who will collect the data and what data will be collected.",
+		"name": "Q11a"
+	},
+	"Q11b": {
+		"page": "5",
+		"domain": "11",
+		"model_ids": [26],
+		"topics": ["Data Reviewer", "Reported To"],
+		"text": "The plan indicates who will review the data collected and who they will report the collated data to.",
+		"name": "Q11b"
+	},
+	"Q11c": {
+		"page": "5",
+		"domain": "11",
+		"model_ids": [26],
+		"topics": ["Review Meeting", "Review Timeframe"],
+		"text": "The plan provides a specific date or timeframe indicating when the plan will be reviewed.",
+		"name": "Q11c"
+	}
+}

requirements.txt

@@ -1,6 +1,2 @@
-panel
-jupyter
-transformers
-numpy
-torch
-aiohttp
+openai
+panel

utils.py

@@ -0,0 +1,171 @@
+import os
+import json
+import openai
+from collections import defaultdict
+
+deployment_name = "gpt-35-turbo"
+
+data_file = 'data.json'
+
+delimiter = "####"
+
+def get_completion_from_messages(messages, engine=deployment_name, temperature=0, max_tokens=500):
+    openai.api_key = os.environ['API_KEY']
+    openai.api_base = os.environ['API_BASE']
+    openai.api_type = os.environ['API_TYPE']
+    openai.api_version = os.environ['API_VERSION']
+    response = openai.ChatCompletion.create(
+        engine=engine,
+        messages=messages,
+        temperature=temperature, 
+        max_tokens=max_tokens, 
+    )
+    return response.choices[0].message["content"]
+
+def get_qms_and_pages():
+    qms = get_qms()
+    qms_by_page = defaultdict(list)
+    for qm_name, qm_info in qms.items():
+        page = qm_info.get('page')
+        if page:
+            qms_by_page[page].append(qm_name)
+    
+    return dict(qms_by_page)
+
+def get_qms():
+    with open(data_file, 'r') as file:
+        qms = json.load(file)
+    return qms
+
+def find_pages_and_qms_only(user_input):
+    delimiter = "####"
+    example_string = """
+    Customer query: What topics are related to quality markers Q10c and Q5b? \
+    Your response: [{'page': '3', 'quality_marker': ['Q5b']}, {'page': '5', 'quality_marker': ['Q10c']}] \
+    Customer query: What topics are related to page 2? \
+    Your response: [{'page': '2', 'quality_marker': ['Q2a', 'Q2b', 'Q2c', 'Q2d']}]
+    """
+    system_message = f"""
+    You will be provided with a customer query. \
+    The customer query will be delimited with {delimiter} characters. \
+    The customer is expected to query about some details related to pages or quality markers in Behaviour Support Plan (PBS) summary documents. \
+    For example, the customer may want to know which domains, models, topics, or descriptions are associated with a given page or quality marker. \
+    Your task is to output a python list of objects (dicts), where each object has the following keys: \
+    - 'page' \
+    - 'quality_marker' \
+
+    Rules: \
+    - 'page' value must be one of: 1, 2, 3, 4, 5. \
+    - 'quality_marker' value is a list of allowed quality markers \
+    - If a specific page number is mentioned, it must be associated with all the correct quality markers in the allowed quality markers list below. \
+    - If specific quality marker(s) are mentioned, the 'quality_marker' list in your response must only include the mentioned quality marker(s). \
+    - If no pages or quality markers are found, output an empty list. \
+    - Only output the list of objects (dicts), nothing else.
+        
+    Allowed quality markers: 
+    For page 1:
+    Q1
+    
+    For page 2:
+    Q2a
+    Q2b
+    Q2c
+    Q2d
+    
+    For page 3:
+    Q3a
+    Q3b
+    Q3c
+    Q3d
+    Q3e
+    Q4a
+    Q4b
+    Q5b
+    
+    For page 4:
+    Q6b
+    Q6c
+    Q7i
+    Q7ii
+    Q7iii
+    Q7iv
+    Q7a
+    Q7b
+    Q8a
+    Q8b
+    Q8c
+    Q8d
+    Q8e
+    Q8f
+    
+    For page 5:
+    Q9a
+    Q9b
+    Q9c
+    Q10a
+    Q10b
+    Q10c
+    Q10d
+    Q11a
+    Q11b
+    Q11c
+    
+    Examples:
+    {example_string}
+
+    """
+    
+    messages =  [  
+    {'role':'system', 'content': system_message},    
+    {'role':'user', 'content': f"{delimiter}{user_input}{delimiter}"},  
+    ] 
+    return get_completion_from_messages(messages)
+
+# qm look up (either by page or by qm within page)
+def get_qm_by_name(name):
+    qms = get_qms()
+    return qms.get(name, None)
+
+def get_qms_by_page(page):
+    qms = get_qms()
+    return [qm for qm in qms.values() if qm.get('page') == page]
+
+def read_string_to_list(input_string):
+    if input_string is None:
+        return None
+
+    try:
+        input_string = input_string.replace("'", "\"")  # Replace single quotes with double quotes for valid JSON
+        data = json.loads(input_string)
+        return data
+    except json.JSONDecodeError:
+        print("Error: Invalid JSON string")
+        return None
+
+def generate_output_string(data_list):
+    output_string = ""
+
+    if data_list is None:
+        return output_string
+
+    for data in data_list:
+        try:
+            if "quality_marker" in data.keys():
+                qm_list = data["quality_marker"]
+                for qm_name in qm_list:
+                    qm = get_qm_by_name(qm_name)
+                    if qm:
+                        output_string += json.dumps(qm, indent=4) + "\n"
+                    else:
+                        print(f"Error: Quality marker '{qm_name}' not found")
+            elif "page" in data:
+                page_no = data["page"]
+                page_qms = get_qms_by_page(page_no)
+                for qm in page_qms:
+                    output_string += json.dumps(qm, indent=4) + "\n"
+            else:
+                print("Error: Invalid object format")
+        except Exception as e:
+            print(f"Error: {e}")
+
+    return output_string