update yaml parser to regex

main.py

@@ -406,6 +406,8 @@ FOLLOWUP_AGENT_PROMPT = """
 You are a helpful assistant with the following skills, use them, as necessary. If the user request needs further clarification, analyze it and generate clarifying questions with options. Else respond with a helpful answer.  <response>response to user request in markdown</response> <clarification> questions:   - text: [First clarifying question]     options:       - [Option 1]       - [Option 2]       - [Option 3]       - [Option 4 (if needed)]   - text: [Second clarifying question]     options:       - [Option 1]       - [Option 2]       - [Option 3]   # Add more questions as needed    # make sure this section is in valid YAML format </clarification>
 """
 
+import re
+
 def parse_followup_response(response):
     response_parts = response.split("<response>")
     if len(response_parts) > 1:
@@ -415,14 +417,8 @@ def parse_followup_response(response):
 
     clarification_parts = response.split("<clarification>")
     if len(clarification_parts) > 1:
-        clarification_yaml = clarification_parts[1].split("</clarification>")[0].strip()
-        try:
-            # Add indentation to make it valid YAML
-            indented_yaml = "\n".join("  " + line for line in clarification_yaml.split("\n"))
-            clarification = yaml.load("questions:\n" + indented_yaml, Loader=SafeLoader)
-        except yaml.YAMLError as e:
-            logger.error(f"YAML parsing error: {e}")
-            clarification = None
+        clarification_text = clarification_parts[1].split("</clarification>")[0].strip()
+        clarification = parse_clarification(clarification_text)
     else:
         clarification = None