Update app.py

app.py

@@ -1,4 +1,5 @@
 import streamlit as st
+import requests
 import os
 import json
 from pypdf import PdfReader
@@ -65,7 +66,7 @@ def obsjsoncreate(json_template,text,ogtext):
             }
         ],
         temperature=0.21,
-        max_tokens=2048,
+        max_tokens=8000,
         top_p=1,
         stream=True,
         stop=None,
@@ -87,7 +88,7 @@ def obsjsoncreate(json_template,text,ogtext):
             }
         ],
         temperature=0.21,
-        max_tokens=2048,
+        max_tokens=8000,
         top_p=1,
         stream=True,
         stop=None,
@@ -112,7 +113,7 @@ def bizobjjsoncreate(json_template,text):
             }
         ],
         temperature=0.21,
-        max_tokens=2048,
+        max_tokens=8000,
         top_p=1,
         stream=True,
         stop=None,
@@ -155,7 +156,7 @@ def question_create(json_template):
         messages=[
             {
                 "role": "system",
-                "content": "You are an experienced writer. You will be given an array of questions. \nSome questions will ask to upload images. Ignore any of these type of questions.\nSome questions ask about different identities or descriptions of the same thing. I want you o merge the questions so as to ask input from them once.\nConvert all questions so that more of a professional but also a bit of a funny tone is maintained. \nRETURN AN ARRAY OF THE QUESTIONS ONLY. DO NOT RETURN ANYTHING ELSE. "
+                "content": "You are an experienced writer. You will be given an array of questions. \nSome questions will ask to upload images. Ignore any of these type of questions.\nSome questions ask about different identities or descriptions of the same thing. I want you to merge the questions so as to ask input from them once.\nConvert all questions so that more of a professional but also a bit of a funny tone is maintained. AVOID REDUNDANCY. DONT RETURN MORE THAN 15 QUESTIONS.\nRETURN AN ARRAY OF THE QUESTIONS ONLY. DO NOT RETURN ANYTHING ELSE. "
             },
             {
                 "role": "user",
@@ -163,7 +164,7 @@ def question_create(json_template):
             }
         ],
         temperature=0.73,
-        max_tokens=5840,
+        max_tokens=2240,
         top_p=1,
         stream=True,
         stop=None,
@@ -207,7 +208,7 @@ def answer_refill(questions,answers,obs_json_template,bizobj_json_template):
         messages=[
             {
                 "role": "system",
-                "content": "You are a helpful assistant. You will be given a Question-answer pair. You will be given a json. Some subproperties in the JSONs labelled \"User Answer\" are marked as TBD. Based on the question answer pair, I want you to fill the Answer of the question answer pair as it is into the \"User answer\" subproperty. Make sure you return the full JSON, without missing any field. Then return the final completely filled JSONs. DONT OUTPUT ANYTHING OTHER THAN THE JSONS."
+                "content": "You are a helpful assistant. You will be given a Question-answer pair. You will be given a json. Some subproperties in the JSONs labelled \"User Answer\" are marked as TBD. Based on the question answer pair, I want you to fill the Answer of the question answer pair as it is into the \"User answer\" subproperty. Make sure you return the full JSON, without missing any field. After filling, merge the two filled JSONs. Then return the final completely filled JSON. DONT OUTPUT ANYTHING OTHER THAN THE JSONS."
             },
             {
                 "role": "user",
@@ -215,7 +216,7 @@ def answer_refill(questions,answers,obs_json_template,bizobj_json_template):
             }
         ],
         temperature=1,
-        max_tokens=7610,
+        max_tokens=8000,
         top_p=1,
         stream=True,
         stop=None,
@@ -254,7 +255,89 @@ def executive_summary(json_template):
         final_summ+=chunk.choices[0].delta.content or ""
     return final_summ
 
+def chunk_data(data, chunk_size=10):
+    if isinstance(data, dict):
+        # If data is a dictionary, convert it to a list of key-value pairs
+        items = list(data.items())
+    elif isinstance(data, list):
+        items = data
+    else:
+        raise TypeError("Data must be either a dictionary or a list")
+    
+    return [dict(items[i:i + chunk_size]) for i in range(0, len(items), chunk_size)]
+
+def airtable_write(json_template):
+
+    client = Groq(api_key=os.getenv("GROQ_API_KEY"))
 
+    # Groq inference
+    completion = client.chat.completions.create(
+        model="llama-3.1-70b-versatile",
+        messages=[
+            {
+                "role": "system",
+                "content":  "You are a helpful assistant. You will be given a unstructured JSON. I want you to convert it into a fully structured JSON which will become a structured CSV. The headings of the CSV are to be \\\"Category\\\",\\\"Sub-category\\\",\\\"Description\\\" and \\\"User Answer\\\". So shuffle around the fields accordingly. \nFields marked \"Category\" are to be directly picked as the \"Category\" for the CSV. If there is \"Observation type\", then that becomes the Category.  \nDONT LEAVE ANY FIELD. MAKE SURE ALL FIELDS ARE INCLUDED IN THE RESULT. DONT OUTPUT ANYTHING OTHER THAN THE JSON. ONLY OUTPUT THE JSON.\n"
+            },
+            {
+                "role": "user",
+                "content": json_template
+            }
+        ],
+        temperature=0.25,
+        max_tokens=8000,
+        top_p=1,
+        stream=True,
+        # response_format={"type": "json_object"},
+        stop=None,
+    )
+    content=""
+    for chunk in completion:
+        content+=chunk.choices[0].delta.content or ""
+    # Get the structured JSON from Groq
+    groq_json = json.loads(content)
+    with open("groq_json.json", "w") as file:
+        json.dump(groq_json, file, indent=4)
+    API_KEY = os.getenv("AIRTABLE_KEY")
+    BASE_ID = 'appcl0egQeE4pP5ID'
+    TABLE_ID = 'tbl2AaOSxyBv6ObR5'
+    url = f'https://api.airtable.com/v0/{BASE_ID}/{TABLE_ID}'
+
+    headers = {
+        'Authorization': f'Bearer {API_KEY}',
+        'Content-Type': 'application/json'
+    }
+
+    # Chunk the data into batches of 10
+    def chunk_data(data, chunk_size=10):
+        for i in range(0, len(data), chunk_size):
+            yield data[i:i + chunk_size]
+
+    # Process each chunk and send it to Airtable
+    for batch in chunk_data(groq_json):
+        # Format the current batch for Airtable API
+        airtable_data = {
+            "records": [
+                {
+                    "fields": {
+                        "Category": item["Category"],
+                        "Sub-category": item["Sub-category"],
+                        "Description": item["Description"],
+                        "User Answer": item["User Answer"]
+                    }
+                } for item in batch
+            ]
+        }
+        
+        # Make the POST request to add records
+        response = requests.post(url, headers=headers, data=json.dumps(airtable_data))
+        
+        # Check if the request was successful
+        if response.status_code == 200:
+            print(f"Batch of {len(batch)} records added successfully!")
+        else:
+            print(f"Failed to add batch. Status code: {response.status_code}, Error: {response.text}")
+
+            
 def main():
     st.title("Qualitas Sales Data Collection Chatbot")
     st.caption("Welcome to the Qualitas Bot. First upload a PDF document which should be customer correspondence, detailing some requirements. Also sometimes the Submit button for the questions is a bit sticky. So You might have to click it twice!")
@@ -303,7 +386,18 @@ def process_document(uploaded_file):
     st.session_state.bizobj = final_bizobj_json
     questionobs = question_create(final_obs_json)
     questionbizobj = question_create(final_bizobj_json)
-    st.session_state.questions = ast.literal_eval(questionbizobj) + ast.literal_eval(questionobs)
+    while True:
+        try:
+            # Attempt to evaluate the expressions and assign them to session state
+            st.session_state.questions = ast.literal_eval(questionbizobj) + ast.literal_eval(questionobs)
+            # If successful, break out of the loop
+            break
+        except Exception as e:
+            # Print the error for debugging purposes (optional)
+            print(f"An error occurred: {e}")
+            # Wait for 1 second before trying again
+            time.sleep(1)
+            continue
     st.write(st.session_state.questions)
     # Mark file as processed
     st.session_state.file_processed = True
@@ -350,6 +444,8 @@ def show_question():
             # st.write(answers)
         
         completed_json = answer_refill(st.session_state.questions, answers, st.session_state.obs, st.session_state.bizobj)
+        # st.write(completed_json)
+        airtable_write(completed_json)
         exec_summ = executive_summary(completed_json)
         st.write(exec_summ)