Serhan Yılmaz commited on
Commit
73b3691
1 Parent(s): f77f60e
Files changed (2) hide show
  1. app.py +379 -0
  2. requirements.txt +8 -0
app.py ADDED
@@ -0,0 +1,379 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import logging
3
+ import json
4
+ import gradio as gr
5
+ import pandas as pd
6
+ from datasets import load_dataset
7
+ import random
8
+ from openai import OpenAI
9
+ from typing import List, Tuple
10
+ import numpy as np
11
+
12
+ # Set up logging
13
+ logging.basicConfig(level=logging.INFO)
14
+ logger = logging.getLogger(__name__)
15
+
16
+ # Initialize OpenAI client
17
+ client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
18
+
19
+ # Load the dataset
20
+ dataset = load_dataset("serhany/scaling-qa")
21
+
22
+ # Define sample inputs
23
+ samples = [
24
+ {
25
+ "context": "Albert Einstein is an Austrian scientist, who has completed his higher education in ETH Zurich in Zurich, Switzerland. He was later a faculty at Princeton University.",
26
+ "answer": "Switzerland"
27
+ },
28
+ {
29
+ "context": "The Eiffel Tower, located in Paris, France, is one of the most famous landmarks in the world. It was constructed in 1889 as the entrance arch to the 1889 World's Fair. The tower is 324 meters (1,063 ft) tall and is the tallest structure in Paris.",
30
+ "answer": "Paris"
31
+ },
32
+ {
33
+ "context": "The Great Wall of China is a series of fortifications and walls built across the historical northern borders of ancient Chinese states and Imperial China to protect against nomadic invasions. It is the largest man-made structure in the world, with a total length of more than 13,000 miles (21,000 kilometers).",
34
+ "answer": "China"
35
+ }
36
+ ]
37
+
38
+ def generate_questions(context: str, answer: str) -> List[str]:
39
+ try:
40
+ response = client.chat.completions.create(
41
+ model="gpt-4o-2024-08-06",
42
+ messages=[
43
+ {"role": "system", "content": "You are a helpful assistant that generates diverse questions based on given context and answer."},
44
+ {"role": "user", "content": f"Based on this context: '{context}' and answer: '{answer}', generate 5 diverse questions which when asked to the context returns the answer."}
45
+ ],
46
+ response_format={
47
+ "type": "json_schema",
48
+ "json_schema": {
49
+ "name": "question_generator",
50
+ "strict": True,
51
+ "schema": {
52
+ "type": "object",
53
+ "properties": {
54
+ "question1": {"type": "string"},
55
+ "question2": {"type": "string"},
56
+ "question3": {"type": "string"},
57
+ "question4": {"type": "string"},
58
+ "question5": {"type": "string"}
59
+ },
60
+ "required": ["question1", "question2", "question3", "question4", "question5"],
61
+ "additionalProperties": False
62
+ }
63
+ }
64
+ }
65
+ )
66
+
67
+ json_response = response.choices[0].message.content
68
+ logger.info(f"Raw JSON response: {json_response}")
69
+
70
+ parsed_response = json.loads(json_response)
71
+ questions = [parsed_response[f"question{i}"] for i in range(1, 6)]
72
+ return questions
73
+ except Exception as e:
74
+ logger.error(f"Error in generate_questions: {e}")
75
+ return [f"Failed to generate question {i}" for i in range(1, 6)]
76
+
77
+ def generate_answer(context: str, question: str) -> str:
78
+ try:
79
+ response = client.chat.completions.create(
80
+ model="gpt-4o-2024-08-06",
81
+ messages=[
82
+ {"role": "system", "content": "You are a helpful assistant that provides concise answers based on the given context."},
83
+ {"role": "user", "content": f"Context: {context}\n\nQuestion: {question}\n\nProvide a concise answer to the question based on the given context."}
84
+ ],
85
+ response_format={
86
+ "type": "json_schema",
87
+ "json_schema": {
88
+ "name": "answer_generator",
89
+ "strict": True,
90
+ "schema": {
91
+ "type": "object",
92
+ "properties": {
93
+ "answer": {"type": "string"}
94
+ },
95
+ "required": ["answer"],
96
+ "additionalProperties": False
97
+ }
98
+ }
99
+ }
100
+ )
101
+
102
+ json_response = response.choices[0].message.content
103
+ logger.info(f"Raw JSON response: {json_response}")
104
+
105
+ parsed_response = json.loads(json_response)
106
+ return parsed_response["answer"]
107
+ except Exception as e:
108
+ logger.error(f"Error in generate_answer: {e}")
109
+ return "Failed to generate answer"
110
+
111
+ def calculate_structural_diversity(questions: List[str]) -> List[float]:
112
+ try:
113
+ response = client.chat.completions.create(
114
+ model="gpt-4o-2024-08-06",
115
+ messages=[
116
+ {"role": "system", "content": "You are an expert in linguistic analysis, specializing in question structure and diversity."},
117
+ {"role": "user", "content": f"Analyze the structural diversity of the following questions and provide a diversity score for each on a scale of 0 to 1, where 1 is highly diverse:\n\n{json.dumps(questions)}"}
118
+ ],
119
+ response_format={
120
+ "type": "json_schema",
121
+ "json_schema": {
122
+ "name": "structural_diversity_analyzer",
123
+ "strict": True,
124
+ "schema": {
125
+ "type": "object",
126
+ "properties": {
127
+ "diversity_scores": {
128
+ "type": "array",
129
+ "items": {
130
+ "type": "number",
131
+ }
132
+ },
133
+ "explanation": {"type": "string"}
134
+ },
135
+ "required": ["diversity_scores", "explanation"],
136
+ "additionalProperties": False
137
+ }
138
+ }
139
+ }
140
+ )
141
+
142
+ json_response = response.choices[0].message.content
143
+ logger.info(f"Raw JSON response: {json_response}")
144
+
145
+ parsed_response = json.loads(json_response)
146
+ diversity_scores = parsed_response["diversity_scores"]
147
+ explanation = parsed_response["explanation"]
148
+
149
+ logger.info(f"Structural Diversity Explanation: {explanation}")
150
+
151
+ return diversity_scores
152
+ except Exception as e:
153
+ logger.error(f"Error in calculate_structural_diversity: {e}")
154
+ return [0.5] * len(questions) # Return neutral scores in case of error
155
+
156
+ def calculate_semantic_relevance(context: str, answer: str, questions: List[str]) -> List[float]:
157
+ try:
158
+ response = client.chat.completions.create(
159
+ model="gpt-4o-2024-08-06",
160
+ messages=[
161
+ {"role": "system", "content": "You are an expert in semantic analysis, specializing in evaluating the relevance of questions to a given context and answer."},
162
+ {"role": "user", "content": f"Analyze the semantic relevance of the following questions to the given context and answer. Provide a relevance score for each question on a scale of 0 to 1, where 1 is highly relevant:\n\nContext: {context}\nAnswer: {answer}\nQuestions: {json.dumps(questions)}"}
163
+ ],
164
+ response_format={
165
+ "type": "json_schema",
166
+ "json_schema": {
167
+ "name": "semantic_relevance_analyzer",
168
+ "strict": True,
169
+ "schema": {
170
+ "type": "object",
171
+ "properties": {
172
+ "relevance_scores": {
173
+ "type": "array",
174
+ "items": {
175
+ "type": "number",
176
+ }
177
+ },
178
+ "explanation": {"type": "string"}
179
+ },
180
+ "required": ["relevance_scores", "explanation"],
181
+ "additionalProperties": False
182
+ }
183
+ }
184
+ }
185
+ )
186
+
187
+ json_response = response.choices[0].message.content
188
+ logger.info(f"Raw JSON response: {json_response}")
189
+
190
+ parsed_response = json.loads(json_response)
191
+ relevance_scores = parsed_response["relevance_scores"]
192
+ explanation = parsed_response["explanation"]
193
+
194
+ logger.info(f"Semantic Relevance Explanation: {explanation}")
195
+
196
+ return relevance_scores
197
+ except Exception as e:
198
+ logger.error(f"Error in calculate_semantic_relevance: {e}")
199
+ return [0.5] * len(questions) # Return neutral scores in case of error
200
+
201
+ def check_answer_precision(context: str, questions: List[str], original_answer: str) -> Tuple[List[float], List[str]]:
202
+ precision_scores = []
203
+ generated_answers = []
204
+ for question in questions:
205
+ generated_answer = generate_answer(context, question)
206
+ generated_answers.append(generated_answer)
207
+
208
+ # Use OpenAI to evaluate answer precision
209
+ try:
210
+ response = client.chat.completions.create(
211
+ model="gpt-4o-2024-08-06",
212
+ messages=[
213
+ {"role": "system", "content": "You are an expert in evaluating answer precision."},
214
+ {"role": "user", "content": f"Compare the following two answers and provide a precision score from 0 to 1, where 1 means the answers are identical in meaning:\n\nOriginal Answer: {original_answer}\nGenerated Answer: {generated_answer}"}
215
+ ],
216
+ response_format={
217
+ "type": "json_schema",
218
+ "json_schema": {
219
+ "name": "answer_precision_evaluator",
220
+ "strict": True,
221
+ "schema": {
222
+ "type": "object",
223
+ "properties": {
224
+ "precision_score": {
225
+ "type": "number",
226
+ }
227
+ },
228
+ "required": ["precision_score"],
229
+ "additionalProperties": False
230
+ }
231
+ }
232
+ }
233
+ )
234
+
235
+ json_response = response.choices[0].message.content
236
+ parsed_response = json.loads(json_response)
237
+ precision_score = parsed_response["precision_score"]
238
+ precision_scores.append(precision_score)
239
+ except Exception as e:
240
+ logger.error(f"Error in evaluating answer precision: {e}")
241
+ precision_scores.append(0.5) # Neutral score in case of error
242
+
243
+ return precision_scores, generated_answers
244
+
245
+ def calculate_composite_scores(sd_scores: List[float], sr_scores: List[float], ap_scores: List[float]) -> List[float]:
246
+ return [0.3 * sd + 0.3 * sr + 0.4 * ap for sd, sr, ap in zip(sd_scores, sr_scores, ap_scores)]
247
+
248
+ def rank_questions_with_details(context: str, answer: str) -> Tuple[pd.DataFrame, List[pd.DataFrame], str]:
249
+ questions = generate_questions(context, answer)
250
+
251
+ sd_scores = calculate_structural_diversity(questions)
252
+ sr_scores = calculate_semantic_relevance(context, answer, questions)
253
+ ap_scores, generated_answers = check_answer_precision(context, questions, answer)
254
+
255
+ composite_scores = calculate_composite_scores(sd_scores, sr_scores, ap_scores)
256
+
257
+ # Create detailed scores dataframe
258
+ detailed_scores = pd.DataFrame({
259
+ 'Question': questions,
260
+ 'Answer Precision': ap_scores,
261
+ 'Composite Score': composite_scores,
262
+ 'Structural Diversity': sd_scores,
263
+ 'Semantic Relevance': sr_scores,
264
+ 'Generated Answer': generated_answers
265
+ })
266
+ detailed_scores = detailed_scores.sort_values('Composite Score', ascending=False).reset_index(drop=True)
267
+
268
+ # Create separate ranking dataframes for each metric
269
+ metrics = ['Answer Precision', 'Composite Score', 'Structural Diversity', 'Semantic Relevance']
270
+ rankings = []
271
+
272
+ for metric in metrics:
273
+ df = pd.DataFrame({
274
+ 'Rank': range(1, 6),
275
+ 'Question': [questions[i] for i in np.argsort(detailed_scores[metric])[::-1]],
276
+ f'{metric}': sorted(detailed_scores[metric], reverse=True)
277
+ })
278
+ if metric == 'Answer Precision':
279
+ df['Generated Answer'] = [generated_answers[i] for i in np.argsort(detailed_scores[metric])[::-1]]
280
+ rankings.append(df)
281
+
282
+ best_question = detailed_scores.iloc[0]['Question']
283
+
284
+ return detailed_scores, rankings, best_question
285
+
286
+ def gradio_interface(context: str, answer: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, str]:
287
+ detailed_scores, rankings, best_question = rank_questions_with_details(context, answer)
288
+ return (
289
+ detailed_scores,
290
+ rankings[0], # Answer Precision Ranking
291
+ rankings[1], # Composite Score Ranking
292
+ rankings[2], # Structural Diversity Ranking
293
+ rankings[3], # Semantic Relevance Ranking
294
+ f"Best Question: {best_question}"
295
+ )
296
+
297
+ def use_sample(sample_index: int) -> Tuple[str, str]:
298
+ return samples[sample_index]["context"], samples[sample_index]["answer"]
299
+
300
+ def get_random_entry():
301
+ random_index = random.randint(0, len(dataset['train']) - 1)
302
+ entry = dataset['train'][random_index]
303
+ return entry['context'], entry['answer'], entry['question']
304
+
305
+ # Create Gradio interface
306
+ with gr.Blocks(theme=gr.themes.Default()) as iface:
307
+ gr.Markdown("# Question Generator and Ranker")
308
+ gr.Markdown("Enter a context and an answer to generate and rank questions, use one of the sample inputs, or get a random entry from the dataset.")
309
+
310
+ with gr.Row():
311
+ with gr.Column(scale=1):
312
+ context_input = gr.Textbox(lines=5, label="Context")
313
+ answer_input = gr.Textbox(lines=2, label="Answer")
314
+ submit_button = gr.Button("Generate Questions")
315
+
316
+ with gr.Row():
317
+ sample_buttons = [gr.Button(f"Sample {i+1}") for i in range(3)]
318
+ random_button = gr.Button("Random Dataset Entry")
319
+
320
+ with gr.Column(scale=2):
321
+ original_question_output = gr.Dataframe(label="Original Question from Dataset", visible=False)
322
+ best_question_output = gr.Textbox(label="Best Generated Question")
323
+ detailed_scores_output = gr.DataFrame(label="Detailed Scores")
324
+
325
+ with gr.Row():
326
+ with gr.Column():
327
+ answer_precision_ranking_output = gr.DataFrame(label="Answer Precision Ranking")
328
+ with gr.Column():
329
+ composite_ranking_output = gr.DataFrame(label="Composite Score Ranking")
330
+
331
+ with gr.Row():
332
+ with gr.Column():
333
+ structural_diversity_ranking_output = gr.DataFrame(label="Structural Diversity Ranking")
334
+ with gr.Column():
335
+ semantic_relevance_ranking_output = gr.DataFrame(label="Semantic Relevance Ranking")
336
+
337
+ def process_random_entry():
338
+ context, answer, original_question = get_random_entry()
339
+ return (
340
+ context,
341
+ answer,
342
+ pd.DataFrame({'Original Question': [original_question]}),
343
+ gr.update(visible=True)
344
+ )
345
+
346
+ submit_button.click(
347
+ fn=gradio_interface,
348
+ inputs=[context_input, answer_input],
349
+ outputs=[
350
+ detailed_scores_output,
351
+ answer_precision_ranking_output,
352
+ composite_ranking_output,
353
+ structural_diversity_ranking_output,
354
+ semantic_relevance_ranking_output,
355
+ best_question_output
356
+ ]
357
+ )
358
+
359
+ # Set up sample button functionality
360
+ for i, button in enumerate(sample_buttons):
361
+ button.click(
362
+ fn=lambda i=i: use_sample(i),
363
+ outputs=[context_input, answer_input]
364
+ )
365
+
366
+ # Set up random button functionality
367
+ random_button.click(
368
+ fn=process_random_entry,
369
+ outputs=[
370
+ context_input,
371
+ answer_input,
372
+ original_question_output,
373
+ original_question_output
374
+ ]
375
+ )
376
+
377
+ # Launch the app
378
+ if __name__ == "__main__":
379
+ iface.launch()
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ gradio
2
+ openai
3
+ numpy
4
+ sentence-transformers
5
+ transformers
6
+ python-dotenv
7
+ pandas
8
+ datasets