Update app.py

app.py

@@ -50,6 +50,7 @@ def retrieve_thoughts(query, n):
     df = pd.DataFrame([dict(doc[0])['metadata'] for doc in docs_with_score], )
     df = pd.concat((df, pd.DataFrame([dict(doc[0])['page_content'] for doc in docs_with_score], columns = ['page_content'])), axis = 1)
     df = pd.concat((df, pd.DataFrame([doc[1] for doc in docs_with_score], columns = ['score'])), axis = 1)
+    df.sort_values("score", inplace = True)
 
   # TO-DO: What if user query doesn't match what we provide as documents
 
@@ -58,10 +59,10 @@ def retrieve_thoughts(query, n):
     # tier_2 = df[(df['score'] < 0.95) * (df["score"] > 0.7)]
 
 
-    chunks_1 = tier_1.groupby(['title', 'url']).apply(lambda x: "\n...\n".join(x.sort_values('id')['page_content'].values)).values
+    chunks_1 = tier_1.groupby(['title', 'url', ]).apply(lambda x: {f"chunk_{i}": row for i, row  in enumerate(x.sort_values('id')[['score','page_content']].to_dict('records'))}).values
     tier_1_adjusted = tier_1.groupby(['title', 'url']).first().reset_index()[['title', 'url', 'score']]
     tier_1_adjusted['ref'] = range(1, len(tier_1_adjusted) + 1 )
-    tier_1_adjusted['content'] = chunks_1
+    tier_1_adjusted['chunks'] = chunks_1
 
     # chunks_2 = tier_2.groupby(['title', 'url', '_id']).apply(lambda x: "\n...\n".join(x.sort_values('id')['page_content'].values)).values
     # tier_2_adjusted = tier_2.groupby(['title', 'url', '_id']).first().reset_index()[['_id', 'title', 'url']]
@@ -97,7 +98,7 @@ def qa_retrieve(query, llm):
     tier_1 = thoughts['tier 1']
     # tier_2 = thoughts['tier 2']
     
-    reference = tier_1[['ref', 'url', 'title', 'content','score']].to_dict('records')
+    reference = tier_1[['ref', 'url', 'title', 'chunks']].to_dict('records')
 
     # tier_1 = list(tier_1.apply(lambda x: f"[{int(x['ref'])}] title: {x['title']}\n Content: {x.content}", axis = 1).values)
     # print(len(tier_1))